Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file defines an instruction selector for the SystemZ target.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "SystemZTargetMachine.h"
14
#include "SystemZISelLowering.h"
15
#include "llvm/Analysis/AliasAnalysis.h"
16
#include "llvm/CodeGen/SelectionDAGISel.h"
17
#include "llvm/Support/Debug.h"
18
#include "llvm/Support/KnownBits.h"
19
#include "llvm/Support/raw_ostream.h"
20
21
using namespace llvm;
22
23
#define DEBUG_TYPE "systemz-isel"
24
25
namespace {
26
// Used to build addressing modes.
27
struct SystemZAddressingMode {
28
  // The shape of the address.
29
  enum AddrForm {
30
    // base+displacement
31
    FormBD,
32
33
    // base+displacement+index for load and store operands
34
    FormBDXNormal,
35
36
    // base+displacement+index for load address operands
37
    FormBDXLA,
38
39
    // base+displacement+index+ADJDYNALLOC
40
    FormBDXDynAlloc
41
  };
42
  AddrForm Form;
43
44
  // The type of displacement.  The enum names here correspond directly
45
  // to the definitions in SystemZOperand.td.  We could split them into
46
  // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
47
  enum DispRange {
48
    Disp12Only,
49
    Disp12Pair,
50
    Disp20Only,
51
    Disp20Only128,
52
    Disp20Pair
53
  };
54
  DispRange DR;
55
56
  // The parts of the address.  The address is equivalent to:
57
  //
58
  //     Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
59
  SDValue Base;
60
  int64_t Disp;
61
  SDValue Index;
62
  bool IncludesDynAlloc;
63
64
  SystemZAddressingMode(AddrForm form, DispRange dr)
65
    : Form(form), DR(dr), Base(), Disp(0), Index(),
66
24.7k
      IncludesDynAlloc(false) {}
67
68
  // True if the address can have an index register.
69
1.38k
  bool hasIndexField() { return Form != FormBD; }
70
71
  // True if the address can (and must) include ADJDYNALLOC.
72
22.3k
  bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73
74
0
  void dump(const llvm::SelectionDAG *DAG) {
75
0
    errs() << "SystemZAddressingMode " << this << '\n';
76
0
77
0
    errs() << " Base ";
78
0
    if (Base.getNode())
79
0
      Base.getNode()->dump(DAG);
80
0
    else
81
0
      errs() << "null\n";
82
0
83
0
    if (hasIndexField()) {
84
0
      errs() << " Index ";
85
0
      if (Index.getNode())
86
0
        Index.getNode()->dump(DAG);
87
0
      else
88
0
        errs() << "null\n";
89
0
    }
90
0
91
0
    errs() << " Disp " << Disp;
92
0
    if (IncludesDynAlloc)
93
0
      errs() << " + ADJDYNALLOC";
94
0
    errs() << '\n';
95
0
  }
96
};
97
98
// Return a mask with Count low bits set.
99
7.17k
static uint64_t allOnes(unsigned int Count) {
100
7.17k
  assert(Count <= 64);
101
7.17k
  if (Count > 63)
102
2.12k
    return UINT64_MAX;
103
5.05k
  return (uint64_t(1) << Count) - 1;
104
5.05k
}
105
106
// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107
// given by Opcode.  The operands are: Input (R2), Start (I3), End (I4) and
108
// Rotate (I5).  The combined operand value is effectively:
109
//
110
//   (or (rotl Input, Rotate), ~Mask)
111
//
112
// for RNSBG and:
113
//
114
//   (and (rotl Input, Rotate), Mask)
115
//
116
// otherwise.  The output value has BitSize bits, although Input may be
117
// narrower (in which case the upper bits are don't care), or wider (in which
118
// case the result will be truncated as part of the operation).
119
struct RxSBGOperands {
120
  RxSBGOperands(unsigned Op, SDValue N)
121
    : Opcode(Op), BitSize(N.getValueSizeInBits()),
122
      Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123
4.62k
      Rotate(0) {}
124
125
  unsigned Opcode;
126
  unsigned BitSize;
127
  uint64_t Mask;
128
  SDValue Input;
129
  unsigned Start;
130
  unsigned End;
131
  unsigned Rotate;
132
};
133
134
class SystemZDAGToDAGISel : public SelectionDAGISel {
135
  const SystemZSubtarget *Subtarget;
136
137
  // Used by SystemZOperands.td to create integer constants.
138
0
  inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139
0
    return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140
0
  }
141
142
0
  const SystemZTargetMachine &getTargetMachine() const {
143
0
    return static_cast<const SystemZTargetMachine &>(TM);
144
0
  }
145
146
4.20k
  const SystemZInstrInfo *getInstrInfo() const {
147
4.20k
    return Subtarget->getInstrInfo();
148
4.20k
  }
149
150
  // Try to fold more of the base or index of AM into AM, where IsBase
151
  // selects between the base and index.
152
  bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153
154
  // Try to describe N in AM, returning true on success.
155
  bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156
157
  // Extract individual target operands from matched address AM.
158
  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159
                          SDValue &Base, SDValue &Disp) const;
160
  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161
                          SDValue &Base, SDValue &Disp, SDValue &Index) const;
162
163
  // Try to match Addr as a FormBD address with displacement type DR.
164
  // Return true on success, storing the base and displacement in
165
  // Base and Disp respectively.
166
  bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167
                    SDValue &Base, SDValue &Disp) const;
168
169
  // Try to match Addr as a FormBDX address with displacement type DR.
170
  // Return true on success and if the result had no index.  Store the
171
  // base and displacement in Base and Disp respectively.
172
  bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173
                     SDValue &Base, SDValue &Disp) const;
174
175
  // Try to match Addr as a FormBDX* address of form Form with
176
  // displacement type DR.  Return true on success, storing the base,
177
  // displacement and index in Base, Disp and Index respectively.
178
  bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179
                     SystemZAddressingMode::DispRange DR, SDValue Addr,
180
                     SDValue &Base, SDValue &Disp, SDValue &Index) const;
181
182
  // PC-relative address matching routines used by SystemZOperands.td.
183
7.47k
  bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184
7.47k
    if (SystemZISD::isPCREL(Addr.getOpcode())) {
185
2.53k
      Target = Addr.getOperand(0);
186
2.53k
      return true;
187
2.53k
    }
188
4.94k
    return false;
189
4.94k
  }
190
191
  // BD matching routines used by SystemZOperands.td.
192
1.82k
  bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193
1.82k
    return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194
1.82k
  }
195
158
  bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196
158
    return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197
158
  }
198
1.83k
  bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199
1.83k
    return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200
1.83k
  }
201
28
  bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202
28
    return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203
28
  }
204
205
  // MVI matching routines used by SystemZOperands.td.
206
529
  bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207
529
    return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208
529
  }
209
34
  bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210
34
    return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211
34
  }
212
213
  // BDX matching routines used by SystemZOperands.td.
214
  bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215
7.43k
                           SDValue &Index) const {
216
7.43k
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217
7.43k
                         SystemZAddressingMode::Disp12Only,
218
7.43k
                         Addr, Base, Disp, Index);
219
7.43k
  }
220
  bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221
4.42k
                           SDValue &Index) const {
222
4.42k
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223
4.42k
                         SystemZAddressingMode::Disp12Pair,
224
4.42k
                         Addr, Base, Disp, Index);
225
4.42k
  }
226
  bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227
1.36k
                            SDValue &Index) const {
228
1.36k
    return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229
1.36k
                         SystemZAddressingMode::Disp12Only,
230
1.36k
                         Addr, Base, Disp, Index);
231
1.36k
  }
232
  bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233
3.87k
                           SDValue &Index) const {
234
3.87k
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235
3.87k
                         SystemZAddressingMode::Disp20Only,
236
3.87k
                         Addr, Base, Disp, Index);
237
3.87k
  }
238
  bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239
482
                              SDValue &Index) const {
240
482
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241
482
                         SystemZAddressingMode::Disp20Only128,
242
482
                         Addr, Base, Disp, Index);
243
482
  }
244
  bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245
398
                           SDValue &Index) const {
246
398
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247
398
                         SystemZAddressingMode::Disp20Pair,
248
398
                         Addr, Base, Disp, Index);
249
398
  }
250
  bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251
1.31k
                          SDValue &Index) const {
252
1.31k
    return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253
1.31k
                         SystemZAddressingMode::Disp12Pair,
254
1.31k
                         Addr, Base, Disp, Index);
255
1.31k
  }
256
  bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257
1.00k
                          SDValue &Index) const {
258
1.00k
    return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259
1.00k
                         SystemZAddressingMode::Disp20Pair,
260
1.00k
                         Addr, Base, Disp, Index);
261
1.00k
  }
262
263
  // Try to match Addr as an address with a base, 12-bit displacement
264
  // and index, where the index is element Elem of a vector.
265
  // Return true on success, storing the base, displacement and vector
266
  // in Base, Disp and Index respectively.
267
  bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268
                           SDValue &Disp, SDValue &Index) const;
269
270
  // Check whether (or Op (and X InsertMask)) is effectively an insertion
271
  // of X into bits InsertMask of some Y != Op.  Return true if so and
272
  // set Op to that Y.
273
  bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274
275
  // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276
  // Return true on success.
277
  bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278
279
  // Try to fold some of RxSBG.Input into other fields of RxSBG.
280
  // Return true on success.
281
  bool expandRxSBG(RxSBGOperands &RxSBG) const;
282
283
  // Return an undefined value of type VT.
284
  SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285
286
  // Convert N to VT, if it isn't already.
287
  SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288
289
  // Try to implement AND or shift node N using RISBG with the zero flag set.
290
  // Return the selected node on success, otherwise return null.
291
  bool tryRISBGZero(SDNode *N);
292
293
  // Try to use RISBG or Opcode to implement OR or XOR node N.
294
  // Return the selected node on success, otherwise return null.
295
  bool tryRxSBG(SDNode *N, unsigned Opcode);
296
297
  // If Op0 is null, then Node is a constant that can be loaded using:
298
  //
299
  //   (Opcode UpperVal LowerVal)
300
  //
301
  // If Op0 is nonnull, then Node can be implemented using:
302
  //
303
  //   (Opcode (Opcode Op0 UpperVal) LowerVal)
304
  void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305
                           uint64_t UpperVal, uint64_t LowerVal);
306
307
  void loadVectorConstant(const SystemZVectorConstantInfo &VCI,
308
                          SDNode *Node);
309
310
  // Try to use gather instruction Opcode to implement vector insertion N.
311
  bool tryGather(SDNode *N, unsigned Opcode);
312
313
  // Try to use scatter instruction Opcode to implement store Store.
314
  bool tryScatter(StoreSDNode *Store, unsigned Opcode);
315
316
  // Change a chain of {load; op; store} of the same value into a simple op
317
  // through memory of that value, if the uses of the modified value and its
318
  // address are suitable.
319
  bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);
320
321
  // Return true if Load and Store are loads and stores of the same size
322
  // and are guaranteed not to overlap.  Such operations can be implemented
323
  // using block (SS-format) instructions.
324
  //
325
  // Partial overlap would lead to incorrect code, since the block operations
326
  // are logically bytewise, even though they have a fast path for the
327
  // non-overlapping case.  We also need to avoid full overlap (i.e. two
328
  // addresses that might be equal at run time) because although that case
329
  // would be handled correctly, it might be implemented by millicode.
330
  bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
331
332
  // N is a (store (load Y), X) pattern.  Return true if it can use an MVC
333
  // from Y to X.
334
  bool storeLoadCanUseMVC(SDNode *N) const;
335
336
  // N is a (store (op (load A[0]), (load A[1])), X) pattern.  Return true
337
  // if A[1 - I] == X and if N can use a block operation like NC from A[I]
338
  // to X.
339
  bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
340
341
  // Try to expand a boolean SELECT_CCMASK using an IPM sequence.
342
  SDValue expandSelectBoolean(SDNode *Node);
343
344
public:
345
  SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
346
1.01k
      : SelectionDAGISel(TM, OptLevel) {}
347
348
8.11k
  bool runOnMachineFunction(MachineFunction &MF) override {
349
8.11k
    Subtarget = &MF.getSubtarget<SystemZSubtarget>();
350
8.11k
    return SelectionDAGISel::runOnMachineFunction(MF);
351
8.11k
  }
352
353
  // Override MachineFunctionPass.
354
8.11k
  StringRef getPassName() const override {
355
8.11k
    return "SystemZ DAG->DAG Pattern Instruction Selection";
356
8.11k
  }
357
358
  // Override SelectionDAGISel.
359
  void Select(SDNode *Node) override;
360
  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
361
                                    std::vector<SDValue> &OutOps) override;
362
  bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
363
  void PreprocessISelDAG() override;
364
365
  // Include the pieces autogenerated from the target description.
366
  #include "SystemZGenDAGISel.inc"
367
};
368
} // end anonymous namespace
369
370
FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM,
371
1.01k
                                         CodeGenOpt::Level OptLevel) {
372
1.01k
  return new SystemZDAGToDAGISel(TM, OptLevel);
373
1.01k
}
374
375
// Return true if Val should be selected as a displacement for an address
376
// with range DR.  Here we're interested in the range of both the instruction
377
// described by DR and of any pairing instruction.
378
8.79k
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
379
8.79k
  switch (DR) {
380
8.79k
  case SystemZAddressingMode::Disp12Only:
381
2.80k
    return isUInt<12>(Val);
382
8.79k
383
8.79k
  case SystemZAddressingMode::Disp12Pair:
384
5.96k
  case SystemZAddressingMode::Disp20Only:
385
5.96k
  case SystemZAddressingMode::Disp20Pair:
386
5.96k
    return isInt<20>(Val);
387
5.96k
388
5.96k
  case SystemZAddressingMode::Disp20Only128:
389
20
    return isInt<20>(Val) && 
isInt<20>(Val + 8)18
;
390
0
  }
391
0
  llvm_unreachable("Unhandled displacement range");
392
0
}
393
394
// Change the base or index in AM to Value, where IsBase selects
395
// between the base and index.
396
static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
397
6.91k
                            SDValue Value) {
398
6.91k
  if (IsBase)
399
6.88k
    AM.Base = Value;
400
24
  else
401
24
    AM.Index = Value;
402
6.91k
}
403
404
// The base or index of AM is equivalent to Value + ADJDYNALLOC,
405
// where IsBase selects between the base and index.  Try to fold the
406
// ADJDYNALLOC into AM.
407
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
408
99
                              SDValue Value) {
409
99
  if (AM.isDynAlloc() && 
!AM.IncludesDynAlloc46
) {
410
46
    changeComponent(AM, IsBase, Value);
411
46
    AM.IncludesDynAlloc = true;
412
46
    return true;
413
46
  }
414
53
  return false;
415
53
}
416
417
// The base of AM is equivalent to Base + Index.  Try to use Index as
418
// the index register.
419
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
420
1.38k
                        SDValue Index) {
421
1.38k
  if (AM.hasIndexField() && 
!AM.Index.getNode()1.32k
) {
422
1.17k
    AM.Base = Base;
423
1.17k
    AM.Index = Index;
424
1.17k
    return true;
425
1.17k
  }
426
209
  return false;
427
209
}
428
429
// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
430
// between the base and index.  Try to fold Op1 into AM's displacement.
431
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
432
8.79k
                       SDValue Op0, uint64_t Op1) {
433
8.79k
  // First try adjusting the displacement.
434
8.79k
  int64_t TestDisp = AM.Disp + Op1;
435
8.79k
  if (selectDisp(AM.DR, TestDisp)) {
436
6.86k
    changeComponent(AM, IsBase, Op0);
437
6.86k
    AM.Disp = TestDisp;
438
6.86k
    return true;
439
6.86k
  }
440
1.92k
441
1.92k
  // We could consider forcing the displacement into a register and
442
1.92k
  // using it as an index, but it would need to be carefully tuned.
443
1.92k
  return false;
444
1.92k
}
445
446
bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
447
29.8k
                                        bool IsBase) const {
448
29.8k
  SDValue N = IsBase ? 
AM.Base28.6k
:
AM.Index1.19k
;
449
29.8k
  unsigned Opcode = N.getOpcode();
450
29.8k
  if (Opcode == ISD::TRUNCATE) {
451
43
    N = N.getOperand(0);
452
43
    Opcode = N.getOpcode();
453
43
  }
454
29.8k
  if (Opcode == ISD::ADD || 
CurDAG->isBaseWithConstantOffset(N)21.7k
) {
455
8.19k
    SDValue Op0 = N.getOperand(0);
456
8.19k
    SDValue Op1 = N.getOperand(1);
457
8.19k
458
8.19k
    unsigned Op0Code = Op0->getOpcode();
459
8.19k
    unsigned Op1Code = Op1->getOpcode();
460
8.19k
461
8.19k
    if (Op0Code == SystemZISD::ADJDYNALLOC)
462
1
      return expandAdjDynAlloc(AM, IsBase, Op1);
463
8.19k
    if (Op1Code == SystemZISD::ADJDYNALLOC)
464
96
      return expandAdjDynAlloc(AM, IsBase, Op0);
465
8.09k
466
8.09k
    if (Op0Code == ISD::Constant)
467
2
      return expandDisp(AM, IsBase, Op1,
468
2
                        cast<ConstantSDNode>(Op0)->getSExtValue());
469
8.09k
    if (Op1Code == ISD::Constant)
470
6.70k
      return expandDisp(AM, IsBase, Op0,
471
6.70k
                        cast<ConstantSDNode>(Op1)->getSExtValue());
472
1.38k
473
1.38k
    if (IsBase && 
expandIndex(AM, Op0, Op1)1.38k
)
474
1.17k
      return true;
475
21.8k
  }
476
21.8k
  if (Opcode == SystemZISD::PCREL_OFFSET) {
477
9
    SDValue Full = N.getOperand(0);
478
9
    SDValue Base = N.getOperand(1);
479
9
    SDValue Anchor = Base.getOperand(0);
480
9
    uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
481
9
                       cast<GlobalAddressSDNode>(Anchor)->getOffset());
482
9
    return expandDisp(AM, IsBase, Base, Offset);
483
9
  }
484
21.8k
  return false;
485
21.8k
}
486
487
// Return true if an instruction with displacement range DR should be
488
// used for displacement value Val.  selectDisp(DR, Val) must already hold.
489
22.8k
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
490
22.8k
  assert(selectDisp(DR, Val) && "Invalid displacement");
491
22.8k
  switch (DR) {
492
22.8k
  case SystemZAddressingMode::Disp12Only:
493
16.8k
  case SystemZAddressingMode::Disp20Only:
494
16.8k
  case SystemZAddressingMode::Disp20Only128:
495
16.8k
    return true;
496
16.8k
497
16.8k
  case SystemZAddressingMode::Disp12Pair:
498
5.46k
    // Use the other instruction if the displacement is too large.
499
5.46k
    return isUInt<12>(Val);
500
16.8k
501
16.8k
  case SystemZAddressingMode::Disp20Pair:
502
504
    // Use the other instruction if the displacement is small enough.
503
504
    return !isUInt<12>(Val);
504
0
  }
505
0
  llvm_unreachable("Unhandled displacement range");
506
0
}
507
508
// Return true if Base + Disp + Index should be performed by LA(Y).
509
2.32k
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
510
2.32k
  // Don't use LA(Y) for constants.
511
2.32k
  if (!Base)
512
0
    return false;
513
2.32k
514
2.32k
  // Always use LA(Y) for frame addresses, since we know that the destination
515
2.32k
  // register is almost always (perhaps always) going to be different from
516
2.32k
  // the frame register.
517
2.32k
  if (Base->getOpcode() == ISD::FrameIndex)
518
218
    return true;
519
2.10k
520
2.10k
  if (Disp) {
521
669
    // Always use LA(Y) if there is a base, displacement and index.
522
669
    if (Index)
523
42
      return true;
524
627
525
627
    // Always use LA if the displacement is small enough.  It should always
526
627
    // be no worse than AGHI (and better if it avoids a move).
527
627
    if (isUInt<12>(Disp))
528
83
      return true;
529
544
530
544
    // For similar reasons, always use LAY if the constant is too big for AGHI.
531
544
    // LAY should be no worse than AGFI.
532
544
    if (!isInt<16>(Disp))
533
24
      return true;
534
1.43k
  } else {
535
1.43k
    // Don't use LA for plain registers.
536
1.43k
    if (!Index)
537
1.10k
      return false;
538
331
539
331
    // Don't use LA for plain addition if the index operand is only used
540
331
    // once.  It should be a natural two-operand addition in that case.
541
331
    if (Index->hasOneUse())
542
238
      return false;
543
93
544
93
    // Prefer addition if the second operation is sign-extended, in the
545
93
    // hope of using AGF.
546
93
    unsigned IndexOpcode = Index->getOpcode();
547
93
    if (IndexOpcode == ISD::SIGN_EXTEND ||
548
93
        IndexOpcode == ISD::SIGN_EXTEND_INREG)
549
0
      return false;
550
613
  }
551
613
552
613
  // Don't use LA for two-operand addition if either operand is only
553
613
  // used once.  The addition instructions are better in that case.
554
613
  if (Base->hasOneUse())
555
582
    return false;
556
31
557
31
  return true;
558
31
}
559
560
// Return true if Addr is suitable for AM, updating AM if so.
561
bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
562
24.7k
                                        SystemZAddressingMode &AM) const {
563
24.7k
  // Start out assuming that the address will need to be loaded separately,
564
24.7k
  // then try to extend it as much as we can.
565
24.7k
  AM.Base = Addr;
566
24.7k
567
24.7k
  // First try treating the address as a constant.
568
24.7k
  if (Addr.getOpcode() == ISD::Constant &&
569
24.7k
      expandDisp(AM, true, SDValue(),
570
2.07k
                 cast<ConstantSDNode>(Addr)->getSExtValue()))
571
2.07k
    ;
572
22.6k
  // Also see if it's a bare ADJDYNALLOC.
573
22.6k
  else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
574
22.6k
           
expandAdjDynAlloc(AM, true, SDValue())2
)
575
2
    ;
576
22.6k
  else
577
22.6k
    // Otherwise try expanding each component.
578
28.6k
    
while (22.6k
expandAddress(AM, true) ||
579
28.6k
           
(22.6k
AM.Index.getNode()22.6k
&&
expandAddress(AM, false)1.19k
))
580
6.00k
      continue;
581
24.7k
582
24.7k
  // Reject cases where it isn't profitable to use LA(Y).
583
24.7k
  if (AM.Form == SystemZAddressingMode::FormBDXLA &&
584
24.7k
      
!shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode())2.32k
)
585
1.92k
    return false;
586
22.8k
587
22.8k
  // Reject cases where the other instruction in a pair should be used.
588
22.8k
  if (!isValidDisp(AM.DR, AM.Disp))
589
529
    return false;
590
22.2k
591
22.2k
  // Make sure that ADJDYNALLOC is included where necessary.
592
22.2k
  if (AM.isDynAlloc() && 
!AM.IncludesDynAlloc1.36k
)
593
1.31k
    return false;
594
20.9k
595
20.9k
  LLVM_DEBUG(AM.dump(CurDAG));
596
20.9k
  return true;
597
20.9k
}
598
599
// Insert a node into the DAG at least before Pos.  This will reposition
600
// the node as needed, and will assign it a node ID that is <= Pos's ID.
601
// Note that this does *not* preserve the uniqueness of node IDs!
602
// The selection DAG must no longer depend on their uniqueness when this
603
// function is used.
604
206
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
605
206
  if (N->getNodeId() == -1 ||
606
206
      (SelectionDAGISel::getUninvalidatedNodeId(N.getNode()) >
607
194
       SelectionDAGISel::getUninvalidatedNodeId(Pos))) {
608
194
    DAG->RepositionNode(Pos->getIterator(), N.getNode());
609
194
    // Mark Node as invalid for pruning as after this it may be a successor to a
610
194
    // selected node but otherwise be in the same position of Pos.
611
194
    // Conservatively mark it with the same -abs(Id) to assure node id
612
194
    // invariant is preserved.
613
194
    N->setNodeId(Pos->getNodeId());
614
194
    SelectionDAGISel::InvalidateNodeId(N.getNode());
615
194
  }
616
206
}
617
618
void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
619
                                             EVT VT, SDValue &Base,
620
20.9k
                                             SDValue &Disp) const {
621
20.9k
  Base = AM.Base;
622
20.9k
  if (!Base.getNode())
623
2.07k
    // Register 0 means "no base".  This is mostly useful for shifts.
624
2.07k
    Base = CurDAG->getRegister(0, VT);
625
18.8k
  else if (Base.getOpcode() == ISD::FrameIndex) {
626
764
    // Lower a FrameIndex to a TargetFrameIndex.
627
764
    int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
628
764
    Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
629
18.1k
  } else if (Base.getValueType() != VT) {
630
0
    // Truncate values from i64 to i32, for shifts.
631
0
    assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
632
0
           "Unexpected truncation");
633
0
    SDLoc DL(Base);
634
0
    SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
635
0
    insertDAGNode(CurDAG, Base.getNode(), Trunc);
636
0
    Base = Trunc;
637
0
  }
638
20.9k
639
20.9k
  // Lower the displacement to a TargetConstant.
640
20.9k
  Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
641
20.9k
}
642
643
void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
644
                                             EVT VT, SDValue &Base,
645
                                             SDValue &Disp,
646
16.6k
                                             SDValue &Index) const {
647
16.6k
  getAddressOperands(AM, VT, Base, Disp);
648
16.6k
649
16.6k
  Index = AM.Index;
650
16.6k
  if (!Index.getNode())
651
16.0k
    // Register 0 means "no index".
652
16.0k
    Index = CurDAG->getRegister(0, VT);
653
16.6k
}
654
655
bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
656
                                       SDValue Addr, SDValue &Base,
657
3.84k
                                       SDValue &Disp) const {
658
3.84k
  SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
659
3.84k
  if (!selectAddress(Addr, AM))
660
28
    return false;
661
3.81k
662
3.81k
  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
663
3.81k
  return true;
664
3.81k
}
665
666
bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
667
                                        SDValue Addr, SDValue &Base,
668
563
                                        SDValue &Disp) const {
669
563
  SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
670
563
  if (!selectAddress(Addr, AM) || 
AM.Index.getNode()519
)
671
61
    return false;
672
502
673
502
  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
674
502
  return true;
675
502
}
676
677
bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
678
                                        SystemZAddressingMode::DispRange DR,
679
                                        SDValue Addr, SDValue &Base,
680
20.3k
                                        SDValue &Disp, SDValue &Index) const {
681
20.3k
  SystemZAddressingMode AM(Form, DR);
682
20.3k
  if (!selectAddress(Addr, AM))
683
3.69k
    return false;
684
16.6k
685
16.6k
  getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
686
16.6k
  return true;
687
16.6k
}
688
689
bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
690
                                              SDValue &Base,
691
                                              SDValue &Disp,
692
170
                                              SDValue &Index) const {
693
170
  SDValue Regs[2];
694
170
  if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
695
170
      Regs[0].getNode() && Regs[1].getNode()) {
696
418
    for (unsigned int I = 0; I < 2; 
++I248
) {
697
320
      Base = Regs[I];
698
320
      Index = Regs[1 - I];
699
320
      // We can't tell here whether the index vector has the right type
700
320
      // for the access; the caller needs to do that instead.
701
320
      if (Index.getOpcode() == ISD::ZERO_EXTEND)
702
43
        Index = Index.getOperand(0);
703
320
      if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
704
320
          
Index.getOperand(1) == Elem72
) {
705
72
        Index = Index.getOperand(0);
706
72
        return true;
707
72
      }
708
320
    }
709
170
  }
710
170
  
return false98
;
711
170
}
712
713
bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
714
402
                                               uint64_t InsertMask) const {
715
402
  // We're only interested in cases where the insertion is into some operand
716
402
  // of Op, rather than into Op itself.  The only useful case is an AND.
717
402
  if (Op.getOpcode() != ISD::AND)
718
383
    return false;
719
19
720
19
  // We need a constant mask.
721
19
  auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
722
19
  if (!MaskNode)
723
0
    return false;
724
19
725
19
  // It's not an insertion of Op.getOperand(0) if the two masks overlap.
726
19
  uint64_t AndMask = MaskNode->getZExtValue();
727
19
  if (InsertMask & AndMask)
728
7
    return false;
729
12
730
12
  // It's only an insertion if all bits are covered or are known to be zero.
731
12
  // The inner check covers all cases but is more expensive.
732
12
  uint64_t Used = allOnes(Op.getValueSizeInBits());
733
12
  if (Used != (AndMask | InsertMask)) {
734
1
    KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
735
1
    if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
736
1
      return false;
737
11
  }
738
11
739
11
  Op = Op.getOperand(0);
740
11
  return true;
741
11
}
742
743
bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
744
4.20k
                                          uint64_t Mask) const {
745
4.20k
  const SystemZInstrInfo *TII = getInstrInfo();
746
4.20k
  if (RxSBG.Rotate != 0)
747
414
    Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
748
4.20k
  Mask &= RxSBG.Mask;
749
4.20k
  if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {
750
4.08k
    RxSBG.Mask = Mask;
751
4.08k
    return true;
752
4.08k
  }
753
126
  return false;
754
126
}
755
756
// Return true if any bits of (RxSBG.Input & Mask) are significant.
757
54
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
758
54
  // Rotate the mask in the same way as RxSBG.Input is rotated.
759
54
  if (RxSBG.Rotate != 0)
760
21
    Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
761
54
  return (Mask & RxSBG.Mask) != 0;
762
54
}
763
764
9.18k
bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
765
9.18k
  SDValue N = RxSBG.Input;
766
9.18k
  unsigned Opcode = N.getOpcode();
767
9.18k
  switch (Opcode) {
768
9.18k
  case ISD::TRUNCATE: {
769
313
    if (RxSBG.Opcode == SystemZ::RNSBG)
770
0
      return false;
771
313
    uint64_t BitSize = N.getValueSizeInBits();
772
313
    uint64_t Mask = allOnes(BitSize);
773
313
    if (!refineRxSBGMask(RxSBG, Mask))
774
0
      return false;
775
313
    RxSBG.Input = N.getOperand(0);
776
313
    return true;
777
313
  }
778
1.79k
  case ISD::AND: {
779
1.79k
    if (RxSBG.Opcode == SystemZ::RNSBG)
780
36
      return false;
781
1.76k
782
1.76k
    auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
783
1.76k
    if (!MaskNode)
784
119
      return false;
785
1.64k
786
1.64k
    SDValue Input = N.getOperand(0);
787
1.64k
    uint64_t Mask = MaskNode->getZExtValue();
788
1.64k
    if (!refineRxSBGMask(RxSBG, Mask)) {
789
63
      // If some bits of Input are already known zeros, those bits will have
790
63
      // been removed from the mask.  See if adding them back in makes the
791
63
      // mask suitable.
792
63
      KnownBits Known = CurDAG->computeKnownBits(Input);
793
63
      Mask |= Known.Zero.getZExtValue();
794
63
      if (!refineRxSBGMask(RxSBG, Mask))
795
55
        return false;
796
1.58k
    }
797
1.58k
    RxSBG.Input = Input;
798
1.58k
    return true;
799
1.58k
  }
800
1.58k
801
1.58k
  case ISD::OR: {
802
422
    if (RxSBG.Opcode != SystemZ::RNSBG)
803
405
      return false;
804
17
805
17
    auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
806
17
    if (!MaskNode)
807
0
      return false;
808
17
809
17
    SDValue Input = N.getOperand(0);
810
17
    uint64_t Mask = ~MaskNode->getZExtValue();
811
17
    if (!refineRxSBGMask(RxSBG, Mask)) {
812
0
      // If some bits of Input are already known ones, those bits will have
813
0
      // been removed from the mask.  See if adding them back in makes the
814
0
      // mask suitable.
815
0
      KnownBits Known = CurDAG->computeKnownBits(Input);
816
0
      Mask &= ~Known.One.getZExtValue();
817
0
      if (!refineRxSBGMask(RxSBG, Mask))
818
0
        return false;
819
17
    }
820
17
    RxSBG.Input = Input;
821
17
    return true;
822
17
  }
823
17
824
337
  case ISD::ROTL: {
825
337
    // Any 64-bit rotate left can be merged into the RxSBG.
826
337
    if (RxSBG.BitSize != 64 || 
N.getValueType() != MVT::i6426
)
827
311
      return false;
828
26
    auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
829
26
    if (!CountNode)
830
14
      return false;
831
12
832
12
    RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
833
12
    RxSBG.Input = N.getOperand(0);
834
12
    return true;
835
12
  }
836
12
837
457
  case ISD::ANY_EXTEND:
838
457
    // Bits above the extended operand are don't-care.
839
457
    RxSBG.Input = N.getOperand(0);
840
457
    return true;
841
12
842
144
  case ISD::ZERO_EXTEND:
843
144
    if (RxSBG.Opcode != SystemZ::RNSBG) {
844
143
      // Restrict the mask to the extended operand.
845
143
      unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
846
143
      if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
847
0
        return false;
848
143
849
143
      RxSBG.Input = N.getOperand(0);
850
143
      return true;
851
143
    }
852
1
    LLVM_FALLTHROUGH;
853
1
854
4
  case ISD::SIGN_EXTEND: {
855
4
    // Check that the extension bits are don't-care (i.e. are masked out
856
4
    // by the final mask).
857
4
    unsigned BitSize = N.getValueSizeInBits();
858
4
    unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
859
4
    if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) {
860
4
      // In the case where only the sign bit is active, increase Rotate with
861
4
      // the extension width.
862
4
      if (RxSBG.Mask == 1 && 
RxSBG.Rotate == 11
)
863
1
        RxSBG.Rotate += (BitSize - InnerBitSize);
864
3
      else
865
3
        return false;
866
1
    }
867
1
868
1
    RxSBG.Input = N.getOperand(0);
869
1
    return true;
870
1
  }
871
1
872
788
  case ISD::SHL: {
873
788
    auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
874
788
    if (!CountNode)
875
37
      return false;
876
751
877
751
    uint64_t Count = CountNode->getZExtValue();
878
751
    unsigned BitSize = N.getValueSizeInBits();
879
751
    if (Count < 1 || Count >= BitSize)
880
0
      return false;
881
751
882
751
    if (RxSBG.Opcode == SystemZ::RNSBG) {
883
11
      // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
884
11
      // count bits from RxSBG.Input are ignored.
885
11
      if (maskMatters(RxSBG, allOnes(Count)))
886
4
        return false;
887
740
    } else {
888
740
      // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
889
740
      if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
890
4
        return false;
891
743
    }
892
743
893
743
    RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
894
743
    RxSBG.Input = N.getOperand(0);
895
743
    return true;
896
743
  }
897
743
898
1.35k
  case ISD::SRL:
899
1.35k
  case ISD::SRA: {
900
1.35k
    auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
901
1.35k
    if (!CountNode)
902
31
      return false;
903
1.32k
904
1.32k
    uint64_t Count = CountNode->getZExtValue();
905
1.32k
    unsigned BitSize = N.getValueSizeInBits();
906
1.32k
    if (Count < 1 || Count >= BitSize)
907
0
      return false;
908
1.32k
909
1.32k
    if (RxSBG.Opcode == SystemZ::RNSBG || 
Opcode == ISD::SRA1.31k
) {
910
39
      // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
911
39
      // count bits from RxSBG.Input are ignored.
912
39
      if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
913
29
        return false;
914
1.28k
    } else {
915
1.28k
      // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
916
1.28k
      // which is similar to SLL above.
917
1.28k
      if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
918
4
        return false;
919
1.29k
    }
920
1.29k
921
1.29k
    RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
922
1.29k
    RxSBG.Input = N.getOperand(0);
923
1.29k
    return true;
924
1.29k
  }
925
3.56k
  default:
926
3.56k
    return false;
927
9.18k
  }
928
9.18k
}
929
930
1.95k
SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
931
1.95k
  SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
932
1.95k
  return SDValue(N, 0);
933
1.95k
}
934
935
SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
936
3.32k
                                       SDValue N) const {
937
3.32k
  if (N.getValueType() == MVT::i32 && 
VT == MVT::i641.72k
)
938
1.35k
    return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
939
1.35k
                                         DL, VT, getUNDEF(DL, MVT::i64), N);
940
1.96k
  if (N.getValueType() == MVT::i64 && 
VT == MVT::i321.60k
)
941
459
    return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
942
1.50k
  assert(N.getValueType() == VT && "Unexpected value types");
943
1.50k
  return N;
944
1.50k
}
945
946
3.15k
bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
947
3.15k
  SDLoc DL(N);
948
3.15k
  EVT VT = N->getValueType(0);
949
3.15k
  if (!VT.isInteger() || VT.getSizeInBits() > 64)
950
134
    return false;
951
3.01k
  RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
952
3.01k
  unsigned Count = 0;
953
6.88k
  while (expandRxSBG(RISBG))
954
3.86k
    // The widening or narrowing is expected to be free.
955
3.86k
    // Counting widening or narrowing as a saved operation will result in
956
3.86k
    // preferring an R*SBG over a simple shift/logical instruction.
957
3.86k
    if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
958
3.86k
        
RISBG.Input.getOpcode() != ISD::TRUNCATE3.42k
)
959
3.11k
      Count += 1;
960
3.01k
  if (Count == 0)
961
550
    return false;
962
2.46k
963
2.46k
  // Prefer to use normal shift instructions over RISBG, since they can handle
964
2.46k
  // all cases and are sometimes shorter.
965
2.46k
  if (Count == 1 && 
N->getOpcode() != ISD::AND1.83k
)
966
1.07k
    return false;
967
1.39k
968
1.39k
  // Prefer register extensions like LLC over RISBG.  Also prefer to start
969
1.39k
  // out with normal ANDs if one instruction would be enough.  We can convert
970
1.39k
  // these ANDs into an RISBG later if a three-address instruction is useful.
971
1.39k
  if (RISBG.Rotate == 0) {
972
845
    bool PreferAnd = false;
973
845
    // Prefer AND for any 32-bit and-immediate operation.
974
845
    if (VT == MVT::i32)
975
294
      PreferAnd = true;
976
551
    // As well as for any 64-bit operation that can be implemented via LLC(R),
977
551
    // LLH(R), LLGT(R), or one of the and-immediate instructions.
978
551
    else if (RISBG.Mask == 0xff ||
979
551
             
RISBG.Mask == 0xffff473
||
980
551
             
RISBG.Mask == 0x7fffffff420
||
981
551
             
SystemZ::isImmLF(~RISBG.Mask)405
||
982
551
             
SystemZ::isImmHF(~RISBG.Mask)70
)
983
492
     PreferAnd = true;
984
59
    // And likewise for the LLZRGF instruction, which doesn't have a register
985
59
    // to register version.
986
59
    else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) {
987
14
      if (Load->getMemoryVT() == MVT::i32 &&
988
14
          
(8
Load->getExtensionType() == ISD::EXTLOAD8
||
989
8
           Load->getExtensionType() == ISD::ZEXTLOAD) &&
990
14
          
RISBG.Mask == 0xffffff008
&&
991
14
          
Subtarget->hasLoadAndZeroRightmostByte()8
)
992
8
      PreferAnd = true;
993
14
    }
994
845
    if (PreferAnd) {
995
794
      // Replace the current node with an AND.  Note that the current node
996
794
      // might already be that same AND, in which case it is already CSE'd
997
794
      // with it, and we must not call ReplaceNode.
998
794
      SDValue In = convertTo(DL, VT, RISBG.Input);
999
794
      SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
1000
794
      SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
1001
794
      if (N != New.getNode()) {
1002
103
        insertDAGNode(CurDAG, N, Mask);
1003
103
        insertDAGNode(CurDAG, N, New);
1004
103
        ReplaceNode(N, New.getNode());
1005
103
        N = New.getNode();
1006
103
      }
1007
794
      // Now, select the machine opcode to implement this operation.
1008
794
      if (!N->isMachineOpcode())
1009
794
        SelectCode(N);
1010
794
      return true;
1011
794
    }
1012
602
  }
1013
602
1014
602
  unsigned Opcode = SystemZ::RISBG;
1015
602
  // Prefer RISBGN if available, since it does not clobber CC.
1016
602
  if (Subtarget->hasMiscellaneousExtensions())
1017
71
    Opcode = SystemZ::RISBGN;
1018
602
  EVT OpcodeVT = MVT::i64;
1019
602
  if (VT == MVT::i32 && 
Subtarget->hasHighWord()98
&&
1020
602
      // We can only use the 32-bit instructions if all source bits are
1021
602
      // in the low 32 bits without wrapping, both after rotation (because
1022
602
      // of the smaller range for Start and End) and before rotation
1023
602
      // (because the input value is truncated).
1024
602
      
RISBG.Start >= 3240
&&
RISBG.End >= RISBG.Start40
&&
1025
602
      
((RISBG.Start + RISBG.Rotate) & 63) >= 3240
&&
1026
602
      ((RISBG.End + RISBG.Rotate) & 63) >=
1027
39
      ((RISBG.Start + RISBG.Rotate) & 63)) {
1028
39
    Opcode = SystemZ::RISBMux;
1029
39
    OpcodeVT = MVT::i32;
1030
39
    RISBG.Start &= 31;
1031
39
    RISBG.End &= 31;
1032
39
  }
1033
602
  SDValue Ops[5] = {
1034
602
    getUNDEF(DL, OpcodeVT),
1035
602
    convertTo(DL, OpcodeVT, RISBG.Input),
1036
602
    CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1037
602
    CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1038
602
    CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1039
602
  };
1040
602
  SDValue New = convertTo(
1041
602
      DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1042
602
  ReplaceNode(N, New.getNode());
1043
602
  return true;
1044
602
}
1045
1046
1.25k
bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1047
1.25k
  SDLoc DL(N);
1048
1.25k
  EVT VT = N->getValueType(0);
1049
1.25k
  if (!VT.isInteger() || VT.getSizeInBits() > 64)
1050
450
    return false;
1051
801
  // Try treating each operand of N as the second operand of the RxSBG
1052
801
  // and see which goes deepest.
1053
801
  RxSBGOperands RxSBG[] = {
1054
801
    RxSBGOperands(Opcode, N->getOperand(0)),
1055
801
    RxSBGOperands(Opcode, N->getOperand(1))
1056
801
  };
1057
801
  unsigned Count[] = { 0, 0 };
1058
2.40k
  for (unsigned I = 0; I < 2; 
++I1.60k
)
1059
2.30k
    
while (1.60k
expandRxSBG(RxSBG[I]))
1060
704
      // The widening or narrowing is expected to be free.
1061
704
      // Counting widening or narrowing as a saved operation will result in
1062
704
      // preferring an R*SBG over a simple shift/logical instruction.
1063
704
      if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1064
704
          
RxSBG[I].Input.getOpcode() != ISD::TRUNCATE688
)
1065
684
        Count[I] += 1;
1066
801
1067
801
  // Do nothing if neither operand is suitable.
1068
801
  if (Count[0] == 0 && 
Count[1] == 0726
)
1069
330
    return false;
1070
471
1071
471
  // Pick the deepest second operand.
1072
471
  unsigned I = Count[0] > Count[1] ? 
021
:
1450
;
1073
471
  SDValue Op0 = N->getOperand(I ^ 1);
1074
471
1075
471
  // Prefer IC for character insertions from memory.
1076
471
  if (Opcode == SystemZ::ROSBG && 
(RxSBG[I].Mask & 0xff) == 0432
)
1077
55
    if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode()))
1078
31
      if (Load->getMemoryVT() == MVT::i8)
1079
30
        return false;
1080
441
1081
441
  // See whether we can avoid an AND in the first operand by converting
1082
441
  // ROSBG to RISBG.
1083
441
  if (Opcode == SystemZ::ROSBG && 
detectOrAndInsertion(Op0, RxSBG[I].Mask)402
) {
1084
11
    Opcode = SystemZ::RISBG;
1085
11
    // Prefer RISBGN if available, since it does not clobber CC.
1086
11
    if (Subtarget->hasMiscellaneousExtensions())
1087
2
      Opcode = SystemZ::RISBGN;
1088
11
  }
1089
441
1090
441
  SDValue Ops[5] = {
1091
441
    convertTo(DL, MVT::i64, Op0),
1092
441
    convertTo(DL, MVT::i64, RxSBG[I].Input),
1093
441
    CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1094
441
    CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1095
441
    CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1096
441
  };
1097
441
  SDValue New = convertTo(
1098
441
      DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1099
441
  ReplaceNode(N, New.getNode());
1100
441
  return true;
1101
441
}
1102
1103
void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1104
                                              SDValue Op0, uint64_t UpperVal,
1105
49
                                              uint64_t LowerVal) {
1106
49
  EVT VT = Node->getValueType(0);
1107
49
  SDLoc DL(Node);
1108
49
  SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1109
49
  if (Op0.getNode())
1110
10
    Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1111
49
1112
49
  {
1113
49
    // When we haven't passed in Op0, Upper will be a constant. In order to
1114
49
    // prevent folding back to the large immediate in `Or = getNode(...)` we run
1115
49
    // SelectCode first and end up with an opaque machine node. This means that
1116
49
    // we need to use a handle to keep track of Upper in case it gets CSE'd by
1117
49
    // SelectCode.
1118
49
    //
1119
49
    // Note that in the case where Op0 is passed in we could just call
1120
49
    // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1121
49
    // the handle at all, but it's fine to do it here.
1122
49
    //
1123
49
    // TODO: This is a pretty hacky way to do this. Can we do something that
1124
49
    // doesn't require a two paragraph explanation?
1125
49
    HandleSDNode Handle(Upper);
1126
49
    SelectCode(Upper.getNode());
1127
49
    Upper = Handle.getValue();
1128
49
  }
1129
49
1130
49
  SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1131
49
  SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1132
49
1133
49
  ReplaceNode(Node, Or.getNode());
1134
49
1135
49
  SelectCode(Or.getNode());
1136
49
}
1137
1138
void SystemZDAGToDAGISel::loadVectorConstant(
1139
529
    const SystemZVectorConstantInfo &VCI, SDNode *Node) {
1140
529
  assert((VCI.Opcode == SystemZISD::BYTE_MASK ||
1141
529
          VCI.Opcode == SystemZISD::REPLICATE ||
1142
529
          VCI.Opcode == SystemZISD::ROTATE_MASK) &&
1143
529
         "Bad opcode!");
1144
529
  assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type");
1145
529
  EVT VT = Node->getValueType(0);
1146
529
  SDLoc DL(Node);
1147
529
  SmallVector<SDValue, 2> Ops;
1148
529
  for (unsigned OpVal : VCI.OpVals)
1149
687
    Ops.push_back(CurDAG->getConstant(OpVal, DL, MVT::i32));
1150
529
  SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
1151
529
1152
529
  if (VCI.VecVT == VT.getSimpleVT())
1153
184
    ReplaceNode(Node, Op.getNode());
1154
345
  else if (VT.getSizeInBits() == 128) {
1155
292
    SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);
1156
292
    ReplaceNode(Node, BitCast.getNode());
1157
292
    SelectCode(BitCast.getNode());
1158
292
  } else { // float or double
1159
53
    unsigned SubRegIdx =
1160
53
        (VT.getSizeInBits() == 32 ? 
SystemZ::subreg_h3237
:
SystemZ::subreg_h6416
);
1161
53
    ReplaceNode(
1162
53
        Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());
1163
53
  }
1164
529
  SelectCode(Op.getNode());
1165
529
}
1166
1167
215
bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1168
215
  SDValue ElemV = N->getOperand(2);
1169
215
  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1170
215
  if (!ElemN)
1171
28
    return false;
1172
187
1173
187
  unsigned Elem = ElemN->getZExtValue();
1174
187
  EVT VT = N->getValueType(0);
1175
187
  if (Elem >= VT.getVectorNumElements())
1176
0
    return false;
1177
187
1178
187
  auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1179
187
  if (!Load || 
!Load->hasNUsesOfValue(1, 0)100
)
1180
88
    return false;
1181
99
  if (Load->getMemoryVT().getSizeInBits() !=
1182
99
      Load->getValueType(0).getSizeInBits())
1183
0
    return false;
1184
99
1185
99
  SDValue Base, Disp, Index;
1186
99
  if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1187
99
      
Index.getValueType() != VT.changeVectorElementTypeToInteger()37
)
1188
62
    return false;
1189
37
1190
37
  SDLoc DL(Load);
1191
37
  SDValue Ops[] = {
1192
37
    N->getOperand(0), Base, Disp, Index,
1193
37
    CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1194
37
  };
1195
37
  SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1196
37
  ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1197
37
  ReplaceNode(N, Res);
1198
37
  return true;
1199
37
}
1200
1201
4.56k
bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1202
4.56k
  SDValue Value = Store->getValue();
1203
4.56k
  if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1204
4.44k
    return false;
1205
114
  if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())
1206
30
    return false;
1207
84
1208
84
  SDValue ElemV = Value.getOperand(1);
1209
84
  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1210
84
  if (!ElemN)
1211
13
    return false;
1212
71
1213
71
  SDValue Vec = Value.getOperand(0);
1214
71
  EVT VT = Vec.getValueType();
1215
71
  unsigned Elem = ElemN->getZExtValue();
1216
71
  if (Elem >= VT.getVectorNumElements())
1217
0
    return false;
1218
71
1219
71
  SDValue Base, Disp, Index;
1220
71
  if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1221
71
      
Index.getValueType() != VT.changeVectorElementTypeToInteger()35
)
1222
36
    return false;
1223
35
1224
35
  SDLoc DL(Store);
1225
35
  SDValue Ops[] = {
1226
35
    Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1227
35
    Store->getChain()
1228
35
  };
1229
35
  ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1230
35
  return true;
1231
35
}
1232
1233
// Check whether or not the chain ending in StoreNode is suitable for doing
1234
// the {load; op; store} to modify transformation.
1235
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
1236
                                        SDValue StoredVal, SelectionDAG *CurDAG,
1237
                                        LoadSDNode *&LoadNode,
1238
495
                                        SDValue &InputChain) {
1239
495
  // Is the stored value result 0 of the operation?
1240
495
  if (StoredVal.getResNo() != 0)
1241
0
    return false;
1242
495
1243
495
  // Are there other uses of the loaded value than the operation?
1244
495
  if (!StoredVal.getNode()->hasNUsesOfValue(1, 0))
1245
0
    return false;
1246
495
1247
495
  // Is the store non-extending and non-indexed?
1248
495
  if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
1249
0
    return false;
1250
495
1251
495
  SDValue Load = StoredVal->getOperand(0);
1252
495
  // Is the stored value a non-extending and non-indexed load?
1253
495
  if (!ISD::isNormalLoad(Load.getNode()))
1254
367
    return false;
1255
128
1256
128
  // Return LoadNode by reference.
1257
128
  LoadNode = cast<LoadSDNode>(Load);
1258
128
1259
128
  // Is store the only read of the loaded value?
1260
128
  if (!Load.hasOneUse())
1261
5
    return false;
1262
123
1263
123
  // Is the address of the store the same as the load?
1264
123
  if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
1265
123
      LoadNode->getOffset() != StoreNode->getOffset())
1266
0
    return false;
1267
123
1268
123
  // Check if the chain is produced by the load or is a TokenFactor with
1269
123
  // the load output chain as an operand. Return InputChain by reference.
1270
123
  SDValue Chain = StoreNode->getChain();
1271
123
1272
123
  bool ChainCheck = false;
1273
123
  if (Chain == Load.getValue(1)) {
1274
110
    ChainCheck = true;
1275
110
    InputChain = LoadNode->getChain();
1276
110
  } else 
if (13
Chain.getOpcode() == ISD::TokenFactor13
) {
1277
13
    SmallVector<SDValue, 4> ChainOps;
1278
13
    SmallVector<const SDNode *, 4> LoopWorklist;
1279
13
    SmallPtrSet<const SDNode *, 16> Visited;
1280
13
    const unsigned int Max = 1024;
1281
39
    for (unsigned i = 0, e = Chain.getNumOperands(); i != e; 
++i26
) {
1282
26
      SDValue Op = Chain.getOperand(i);
1283
26
      if (Op == Load.getValue(1)) {
1284
1
        ChainCheck = true;
1285
1
        // Drop Load, but keep its chain. No cycle check necessary.
1286
1
        ChainOps.push_back(Load.getOperand(0));
1287
1
        continue;
1288
1
      }
1289
25
      LoopWorklist.push_back(Op.getNode());
1290
25
      ChainOps.push_back(Op);
1291
25
    }
1292
13
1293
13
    if (ChainCheck) {
1294
1
      // Add the other operand of StoredVal to worklist.
1295
1
      for (SDValue Op : StoredVal->ops())
1296
2
        if (Op.getNode() != LoadNode)
1297
1
          LoopWorklist.push_back(Op.getNode());
1298
1
1299
1
      // Check if Load is reachable from any of the nodes in the worklist.
1300
1
      if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max,
1301
1
                                       true))
1302
1
        return false;
1303
0
1304
0
      // Make a new TokenFactor with all the other input chains except
1305
0
      // for the load.
1306
0
      InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain),
1307
0
                                   MVT::Other, ChainOps);
1308
0
    }
1309
13
  }
1310
123
  
if (122
!ChainCheck122
)
1311
12
    return false;
1312
110
1313
110
  return true;
1314
110
}
1315
1316
// Change a chain of {load; op; store} of the same value into a simple op
1317
// through memory of that value, if the uses of the modified value and its
1318
// address are suitable.
1319
//
1320
// The tablegen pattern memory operand pattern is currently not able to match
1321
// the case where the CC on the original operation are used.
1322
//
1323
// See the equivalent routine in X86ISelDAGToDAG for further comments.
1324
6.63k
bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {
1325
6.63k
  StoreSDNode *StoreNode = cast<StoreSDNode>(Node);
1326
6.63k
  SDValue StoredVal = StoreNode->getOperand(1);
1327
6.63k
  unsigned Opc = StoredVal->getOpcode();
1328
6.63k
  SDLoc DL(StoreNode);
1329
6.63k
1330
6.63k
  // Before we try to select anything, make sure this is memory operand size
1331
6.63k
  // and opcode we can handle. Note that this must match the code below that
1332
6.63k
  // actually lowers the opcodes.
1333
6.63k
  EVT MemVT = StoreNode->getMemoryVT();
1334
6.63k
  unsigned NewOpc = 0;
1335
6.63k
  bool NegateOperand = false;
1336
6.63k
  switch (Opc) {
1337
6.63k
  default:
1338
6.13k
    return false;
1339
6.63k
  case SystemZISD::SSUBO:
1340
58
    NegateOperand = true;
1341
58
    LLVM_FALLTHROUGH;
1342
262
  case SystemZISD::SADDO:
1343
262
    if (MemVT == MVT::i32)
1344
126
      NewOpc = SystemZ::ASI;
1345
136
    else if (MemVT == MVT::i64)
1346
136
      NewOpc = SystemZ::AGSI;
1347
0
    else
1348
0
      return false;
1349
262
    break;
1350
262
  case SystemZISD::USUBO:
1351
110
    NegateOperand = true;
1352
110
    LLVM_FALLTHROUGH;
1353
233
  case SystemZISD::UADDO:
1354
233
    if (MemVT == MVT::i32)
1355
76
      NewOpc = SystemZ::ALSI;
1356
157
    else if (MemVT == MVT::i64)
1357
157
      NewOpc = SystemZ::ALGSI;
1358
0
    else
1359
0
      return false;
1360
233
    break;
1361
495
  }
1362
495
1363
495
  LoadSDNode *LoadNode = nullptr;
1364
495
  SDValue InputChain;
1365
495
  if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode,
1366
495
                                   InputChain))
1367
385
    return false;
1368
110
1369
110
  SDValue Operand = StoredVal.getOperand(1);
1370
110
  auto *OperandC = dyn_cast<ConstantSDNode>(Operand);
1371
110
  if (!OperandC)
1372
14
    return false;
1373
96
  auto OperandV = OperandC->getAPIntValue();
1374
96
  if (NegateOperand)
1375
24
    OperandV = -OperandV;
1376
96
  if (OperandV.getMinSignedBits() > 8)
1377
16
    return false;
1378
80
  Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);
1379
80
1380
80
  SDValue Base, Disp;
1381
80
  if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))
1382
0
    return false;
1383
80
1384
80
  SDValue Ops[] = { Base, Disp, Operand, InputChain };
1385
80
  MachineSDNode *Result =
1386
80
    CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);
1387
80
  CurDAG->setNodeMemRefs(
1388
80
      Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()});
1389
80
1390
80
  ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
1391
80
  ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0));
1392
80
  CurDAG->RemoveDeadNode(Node);
1393
80
  return true;
1394
80
}
1395
1396
bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1397
710
                                               LoadSDNode *Load) const {
1398
710
  // Check that the two memory operands have the same size.
1399
710
  if (Load->getMemoryVT() != Store->getMemoryVT())
1400
0
    return false;
1401
710
1402
710
  // Volatility stops an access from being decomposed.
1403
710
  if (Load->isVolatile() || 
Store->isVolatile()194
)
1404
519
    return false;
1405
191
1406
191
  // There's no chance of overlap if the load is invariant.
1407
191
  if (Load->isInvariant() && 
Load->isDereferenceable()0
)
1408
0
    return true;
1409
191
1410
191
  // Otherwise we need to check whether there's an alias.
1411
191
  const Value *V1 = Load->getMemOperand()->getValue();
1412
191
  const Value *V2 = Store->getMemOperand()->getValue();
1413
191
  if (!V1 || 
!V2135
)
1414
66
    return false;
1415
125
1416
125
  // Reject equality.
1417
125
  uint64_t Size = Load->getMemoryVT().getStoreSize();
1418
125
  int64_t End1 = Load->getSrcValueOffset() + Size;
1419
125
  int64_t End2 = Store->getSrcValueOffset() + Size;
1420
125
  if (V1 == V2 && 
End1 == End251
)
1421
51
    return false;
1422
74
1423
74
  return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()),
1424
74
                    MemoryLocation(V2, End2, Store->getAAInfo()));
1425
74
}
1426
1427
716
bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1428
716
  auto *Store = cast<StoreSDNode>(N);
1429
716
  auto *Load = cast<LoadSDNode>(Store->getValue());
1430
716
1431
716
  // Prefer not to use MVC if either address can use ... RELATIVE LONG
1432
716
  // instructions.
1433
716
  uint64_t Size = Load->getMemoryVT().getStoreSize();
1434
716
  if (Size > 1 && 
Size <= 8705
) {
1435
630
    // Prefer LHRL, LRL and LGRL.
1436
630
    if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1437
44
      return false;
1438
586
    // Prefer STHRL, STRL and STGRL.
1439
586
    if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode()))
1440
3
      return false;
1441
669
  }
1442
669
1443
669
  return canUseBlockOperation(Store, Load);
1444
669
}
1445
1446
bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1447
43
                                                     unsigned I) const {
1448
43
  auto *StoreA = cast<StoreSDNode>(N);
1449
43
  auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1450
43
  auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1451
43
  return !LoadA->isVolatile() && 
canUseBlockOperation(StoreA, LoadB)41
;
1452
43
}
1453
1454
123k
void SystemZDAGToDAGISel::Select(SDNode *Node) {
1455
123k
  // If we have a custom node, we already have selected!
1456
123k
  if (Node->isMachineOpcode()) {
1457
498
    LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1458
498
    Node->setNodeId(-1);
1459
498
    return;
1460
498
  }
1461
122k
1462
122k
  unsigned Opcode = Node->getOpcode();
1463
122k
  switch (Opcode) {
1464
122k
  case ISD::OR:
1465
762
    if (Node->getOperand(1).getOpcode() != ISD::Constant)
1466
665
      if (tryRxSBG(Node, SystemZ::ROSBG))
1467
402
        return;
1468
360
    goto or_xor;
1469
360
1470
410
  case ISD::XOR:
1471
410
    if (Node->getOperand(1).getOpcode() != ISD::Constant)
1472
323
      if (tryRxSBG(Node, SystemZ::RXSBG))
1473
21
        return;
1474
749
    // Fall through.
1475
749
  or_xor:
1476
749
    // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1477
749
    // split the operation into two.  If both operands here happen to be
1478
749
    // constant, leave this to common code to optimize.
1479
749
    if (Node->getValueType(0) == MVT::i64 &&
1480
749
        
Node->getOperand(0).getOpcode() != ISD::Constant210
)
1481
209
      if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1482
93
        uint64_t Val = Op1->getZExtValue();
1483
93
        // Don't split the operation if we can match one of the combined
1484
93
        // logical operations provided by miscellaneous-extensions-3.
1485
93
        if (Subtarget->hasMiscellaneousExtensions3()) {
1486
4
          unsigned ChildOpcode = Node->getOperand(0).getOpcode();
1487
4
          // Check whether this expression matches NAND/NOR/NXOR.
1488
4
          if (Val == (uint64_t)-1 && 
Opcode == ISD::XOR3
)
1489
3
            if (ChildOpcode == ISD::AND || 
ChildOpcode == ISD::OR2
||
1490
3
                
ChildOpcode == ISD::XOR1
)
1491
3
              break;
1492
1
          // Check whether this expression matches OR-with-complement.
1493
1
          if (Opcode == ISD::OR && ChildOpcode == ISD::XOR) {
1494
1
            auto Op0 = Node->getOperand(0);
1495
1
            if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1)))
1496
1
              if (Op0Op1->getZExtValue() == (uint64_t)-1)
1497
1
                break;
1498
89
          }
1499
1
        }
1500
89
        if (!SystemZ::isImmLF(Val) && 
!SystemZ::isImmHF(Val)37
) {
1501
10
          splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1502
10
                              Val - uint32_t(Val), uint32_t(Val));
1503
10
          return;
1504
10
        }
1505
735
      }
1506
735
    break;
1507
735
1508
1.58k
  case ISD::AND:
1509
1.58k
    if (Node->getOperand(1).getOpcode() != ISD::Constant)
1510
263
      if (tryRxSBG(Node, SystemZ::RNSBG))
1511
18
        return;
1512
1.56k
    LLVM_FALLTHROUGH;
1513
3.15k
  case ISD::ROTL:
1514
3.15k
  case ISD::SHL:
1515
3.15k
  case ISD::SRL:
1516
3.15k
  case ISD::ZERO_EXTEND:
1517
3.15k
    if (tryRISBGZero(Node))
1518
1.39k
      return;
1519
1.75k
    break;
1520
1.75k
1521
1.75k
  case ISD::Constant:
1522
1.08k
    // If this is a 64-bit constant that is out of the range of LLILF,
1523
1.08k
    // LLIHF and LGFI, split it into two 32-bit pieces.
1524
1.08k
    if (Node->getValueType(0) == MVT::i64) {
1525
513
      uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1526
513
      if (!SystemZ::isImmLF(Val) && 
!SystemZ::isImmHF(Val)137
&&
!isInt<32>(Val)109
) {
1527
39
        splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1528
39
                            uint32_t(Val));
1529
39
        return;
1530
39
      }
1531
1.04k
    }
1532
1.04k
    break;
1533
1.04k
1534
1.15k
  case SystemZISD::SELECT_CCMASK: {
1535
1.15k
    SDValue Op0 = Node->getOperand(0);
1536
1.15k
    SDValue Op1 = Node->getOperand(1);
1537
1.15k
    // Prefer to put any load first, so that it can be matched as a
1538
1.15k
    // conditional load.  Likewise for constants in range for LOCHI.
1539
1.15k
    if ((Op1.getOpcode() == ISD::LOAD && 
Op0.getOpcode() != ISD::LOAD70
) ||
1540
1.15k
        
(1.08k
Subtarget->hasLoadStoreOnCond2()1.08k
&&
1541
1.08k
         
Node->getValueType(0).isInteger()565
&&
1542
1.08k
         
Op1.getOpcode() == ISD::Constant549
&&
1543
1.08k
         
isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue())487
&&
1544
1.08k
         
!(487
Op0.getOpcode() == ISD::Constant487
&&
1545
487
           
isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())479
))) {
1546
78
      SDValue CCValid = Node->getOperand(2);
1547
78
      SDValue CCMask = Node->getOperand(3);
1548
78
      uint64_t ConstCCValid =
1549
78
        cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1550
78
      uint64_t ConstCCMask =
1551
78
        cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1552
78
      // Invert the condition.
1553
78
      CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node),
1554
78
                                   CCMask.getValueType());
1555
78
      SDValue Op4 = Node->getOperand(4);
1556
78
      SDNode *UpdatedNode =
1557
78
        CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1558
78
      if (UpdatedNode != Node) {
1559
1
        // In case this node already exists then replace Node with it.
1560
1
        ReplaceNode(Node, UpdatedNode);
1561
1
        Node = UpdatedNode;
1562
1
      }
1563
78
    }
1564
1.15k
    break;
1565
1.04k
  }
1566
1.04k
1567
1.04k
  case ISD::INSERT_VECTOR_ELT: {
1568
488
    EVT VT = Node->getValueType(0);
1569
488
    unsigned ElemBitSize = VT.getScalarSizeInBits();
1570
488
    if (ElemBitSize == 32) {
1571
127
      if (tryGather(Node, SystemZ::VGEF))
1572
22
        return;
1573
361
    } else if (ElemBitSize == 64) {
1574
88
      if (tryGather(Node, SystemZ::VGEG))
1575
15
        return;
1576
451
    }
1577
451
    break;
1578
451
  }
1579
451
1580
470
  case ISD::BUILD_VECTOR: {
1581
470
    auto *BVN = cast<BuildVectorSDNode>(Node);
1582
470
    SystemZVectorConstantInfo VCI(BVN);
1583
470
    if (VCI.isVectorConstantLegal(*Subtarget)) {
1584
470
      loadVectorConstant(VCI, Node);
1585
470
      return;
1586
470
    }
1587
0
    break;
1588
0
  }
1589
0
1590
214
  case ISD::ConstantFP: {
1591
214
    APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
1592
214
    if (Imm.isZero() || 
Imm.isNegZero()59
)
1593
155
      break;
1594
59
    SystemZVectorConstantInfo VCI(Imm);
1595
59
    bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success;
1596
59
    assert(Success && "Expected legal FP immediate");
1597
59
    loadVectorConstant(VCI, Node);
1598
59
    return;
1599
59
  }
1600
59
1601
6.63k
  case ISD::STORE: {
1602
6.63k
    if (tryFoldLoadStoreIntoMemOperand(Node))
1603
80
      return;
1604
6.55k
    auto *Store = cast<StoreSDNode>(Node);
1605
6.55k
    unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1606
6.55k
    if (ElemBitSize == 32) {
1607
2.73k
      if (tryScatter(Store, SystemZ::VSCEF))
1608
21
        return;
1609
3.81k
    } else if (ElemBitSize == 64) {
1610
1.82k
      if (tryScatter(Store, SystemZ::VSCEG))
1611
14
        return;
1612
6.51k
    }
1613
6.51k
    break;
1614
6.51k
  }
1615
119k
  }
1616
119k
1617
119k
  SelectCode(Node);
1618
119k
}
1619
1620
bool SystemZDAGToDAGISel::
1621
SelectInlineAsmMemoryOperand(const SDValue &Op,
1622
                             unsigned ConstraintID,
1623
24
                             std::vector<SDValue> &OutOps) {
1624
24
  SystemZAddressingMode::AddrForm Form;
1625
24
  SystemZAddressingMode::DispRange DispRange;
1626
24
  SDValue Base, Disp, Index;
1627
24
1628
24
  switch(ConstraintID) {
1629
24
  default:
1630
0
    llvm_unreachable("Unexpected asm memory constraint");
1631
24
  case InlineAsm::Constraint_i:
1632
5
  case InlineAsm::Constraint_Q:
1633
5
    // Accept an address with a short displacement, but no index.
1634
5
    Form = SystemZAddressingMode::FormBD;
1635
5
    DispRange = SystemZAddressingMode::Disp12Only;
1636
5
    break;
1637
7
  case InlineAsm::Constraint_R:
1638
7
    // Accept an address with a short displacement and an index.
1639
7
    Form = SystemZAddressingMode::FormBDXNormal;
1640
7
    DispRange = SystemZAddressingMode::Disp12Only;
1641
7
    break;
1642
5
  case InlineAsm::Constraint_S:
1643
4
    // Accept an address with a long displacement, but no index.
1644
4
    Form = SystemZAddressingMode::FormBD;
1645
4
    DispRange = SystemZAddressingMode::Disp20Only;
1646
4
    break;
1647
8
  case InlineAsm::Constraint_T:
1648
8
  case InlineAsm::Constraint_m:
1649
8
  case InlineAsm::Constraint_o:
1650
8
    // Accept an address with a long displacement and an index.
1651
8
    // m works the same as T, as this is the most general case.
1652
8
    // We don't really have any special handling of "offsettable"
1653
8
    // memory addresses, so just treat o the same as m.
1654
8
    Form = SystemZAddressingMode::FormBDXNormal;
1655
8
    DispRange = SystemZAddressingMode::Disp20Only;
1656
8
    break;
1657
24
  }
1658
24
1659
24
  if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {
1660
24
    const TargetRegisterClass *TRC =
1661
24
      Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1662
24
    SDLoc DL(Base);
1663
24
    SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1664
24
1665
24
    // Make sure that the base address doesn't go into %r0.
1666
24
    // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1667
24
    if (Base.getOpcode() != ISD::TargetFrameIndex &&
1668
24
        Base.getOpcode() != ISD::Register) {
1669
24
      Base =
1670
24
        SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1671
24
                                       DL, Base.getValueType(),
1672
24
                                       Base, RC), 0);
1673
24
    }
1674
24
1675
24
    // Make sure that the index register isn't assigned to %r0 either.
1676
24
    if (Index.getOpcode() != ISD::Register) {
1677
4
      Index =
1678
4
        SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1679
4
                                       DL, Index.getValueType(),
1680
4
                                       Index, RC), 0);
1681
4
    }
1682
24
1683
24
    OutOps.push_back(Base);
1684
24
    OutOps.push_back(Disp);
1685
24
    OutOps.push_back(Index);
1686
24
    return false;
1687
24
  }
1688
0
1689
0
  return true;
1690
0
}
1691
1692
// IsProfitableToFold - Returns true if is profitable to fold the specific
1693
// operand node N of U during instruction selection that starts at Root.
1694
bool
1695
SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,
1696
7.76k
                                        SDNode *Root) const {
1697
7.76k
  // We want to avoid folding a LOAD into an ICMP node if as a result
1698
7.76k
  // we would be forced to spill the condition code into a GPR.
1699
7.76k
  if (N.getOpcode() == ISD::LOAD && 
U->getOpcode() == SystemZISD::ICMP7.67k
) {
1700
909
    if (!N.hasOneUse() || 
!U->hasOneUse()836
)
1701
73
      return false;
1702
836
1703
836
    // The user of the CC value will usually be a CopyToReg into the
1704
836
    // physical CC register, which in turn is glued and chained to the
1705
836
    // actual instruction that uses the CC value.  Bail out if we have
1706
836
    // anything else than that.
1707
836
    SDNode *CCUser = *U->use_begin();
1708
836
    SDNode *CCRegUser = nullptr;
1709
836
    if (CCUser->getOpcode() == ISD::CopyToReg ||
1710
836
        
cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC0
) {
1711
1.00k
      for (auto *U : CCUser->uses()) {
1712
1.00k
        if (CCRegUser == nullptr)
1713
836
          CCRegUser = U;
1714
167
        else if (CCRegUser != U)
1715
0
          return false;
1716
1.00k
      }
1717
836
    }
1718
836
    if (CCRegUser == nullptr)
1719
0
      return false;
1720
836
1721
836
    // If the actual instruction is a branch, the only thing that remains to be
1722
836
    // checked is whether the CCUser chain is a predecessor of the load.
1723
836
    if (CCRegUser->isMachineOpcode() &&
1724
836
        CCRegUser->getMachineOpcode() == SystemZ::BRC)
1725
167
      return !N->isPredecessorOf(CCUser->getOperand(0).getNode());
1726
669
1727
669
    // Otherwise, the instruction may have multiple operands, and we need to
1728
669
    // verify that none of them are a predecessor of the load.  This is exactly
1729
669
    // the same check that would be done by common code if the CC setter were
1730
669
    // glued to the CC user, so simply invoke that check here.
1731
669
    if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))
1732
4
      return false;
1733
7.51k
  }
1734
7.51k
1735
7.51k
  return true;
1736
7.51k
}
1737
1738
namespace {
1739
// Represents a sequence for extracting a 0/1 value from an IPM result:
1740
// (((X ^ XORValue) + AddValue) >> Bit)
1741
struct IPMConversion {
1742
  IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)
1743
840
    : XORValue(xorValue), AddValue(addValue), Bit(bit) {}
1744
1745
  int64_t XORValue;
1746
  int64_t AddValue;
1747
  unsigned Bit;
1748
};
1749
} // end anonymous namespace
1750
1751
// Return a sequence for getting a 1 from an IPM result when CC has a
1752
// value in CCMask and a 0 when CC has a value in CCValid & ~CCMask.
1753
// The handling of CC values outside CCValid doesn't matter.
1754
840
static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) {
1755
840
  // Deal with cases where the result can be taken directly from a bit
1756
840
  // of the IPM result.
1757
840
  if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3)))
1758
40
    return IPMConversion(0, 0, SystemZ::IPM_CC);
1759
800
  if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3)))
1760
167
    return IPMConversion(0, 0, SystemZ::IPM_CC + 1);
1761
633
1762
633
  // Deal with cases where we can add a value to force the sign bit
1763
633
  // to contain the right value.  Putting the bit in 31 means we can
1764
633
  // use SRL rather than RISBG(L), and also makes it easier to get a
1765
633
  // 0/-1 value, so it has priority over the other tests below.
1766
633
  //
1767
633
  // These sequences rely on the fact that the upper two bits of the
1768
633
  // IPM result are zero.
1769
633
  uint64_t TopBit = uint64_t(1) << 31;
1770
633
  if (CCMask == (CCValid & SystemZ::CCMASK_0))
1771
21
    return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31);
1772
612
  if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1)))
1773
156
    return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31);
1774
456
  if (CCMask == (CCValid & (SystemZ::CCMASK_0
1775
456
                            | SystemZ::CCMASK_1
1776
456
                            | SystemZ::CCMASK_2)))
1777
5
    return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31);
1778
451
  if (CCMask == (CCValid & SystemZ::CCMASK_3))
1779
403
    return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);
1780
48
  if (CCMask == (CCValid & (SystemZ::CCMASK_1
1781
48
                            | SystemZ::CCMASK_2
1782
48
                            | SystemZ::CCMASK_3)))
1783
11
    return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);
1784
37
1785
37
  // Next try inverting the value and testing a bit.  0/1 could be
1786
37
  // handled this way too, but we dealt with that case above.
1787
37
  if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2)))
1788
7
    return IPMConversion(-1, 0, SystemZ::IPM_CC);
1789
30
1790
30
  // Handle cases where adding a value forces a non-sign bit to contain
1791
30
  // the right value.
1792
30
  if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2)))
1793
5
    return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1);
1794
25
  if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3)))
1795
5
    return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1);
1796
20
1797
20
  // The remaining cases are 1, 2, 0/1/3 and 0/2/3.  All these are
1798
20
  // can be done by inverting the low CC bit and applying one of the
1799
20
  // sign-based extractions above.
1800
20
  if (CCMask == (CCValid & SystemZ::CCMASK_1))
1801
5
    return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31);
1802
15
  if (CCMask == (CCValid & SystemZ::CCMASK_2))
1803
5
    return IPMConversion(1 << SystemZ::IPM_CC,
1804
5
                         TopBit - (3 << SystemZ::IPM_CC), 31);
1805
10
  if (CCMask == (CCValid & (SystemZ::CCMASK_0
1806
10
                            | SystemZ::CCMASK_1
1807
10
                            | SystemZ::CCMASK_3)))
1808
5
    return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31);
1809
5
  if (CCMask == (CCValid & (SystemZ::CCMASK_0
1810
5
                            | SystemZ::CCMASK_2
1811
5
                            | SystemZ::CCMASK_3)))
1812
5
    return IPMConversion(1 << SystemZ::IPM_CC,
1813
5
                         TopBit - (1 << SystemZ::IPM_CC), 31);
1814
0
1815
0
  llvm_unreachable("Unexpected CC combination");
1816
0
}
1817
1818
1.49k
SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {
1819
1.49k
  auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0));
1820
1.49k
  auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1821
1.49k
  if (!TrueOp || 
!FalseOp866
)
1822
631
    return SDValue();
1823
865
  if (FalseOp->getZExtValue() != 0)
1824
24
    return SDValue();
1825
841
  if (TrueOp->getSExtValue() != 1 && 
TrueOp->getSExtValue() != -152
)
1826
1
    return SDValue();
1827
840
1828
840
  auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1829
840
  auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3));
1830
840
  if (!CCValidOp || !CCMaskOp)
1831
0
    return SDValue();
1832
840
  int CCValid = CCValidOp->getZExtValue();
1833
840
  int CCMask = CCMaskOp->getZExtValue();
1834
840
1835
840
  SDLoc DL(Node);
1836
840
  SDValue CCReg = Node->getOperand(4);
1837
840
  IPMConversion IPM = getIPMConversion(CCValid, CCMask);
1838
840
  SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
1839
840
1840
840
  if (IPM.XORValue)
1841
27
    Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result,
1842
27
                             CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));
1843
840
1844
840
  if (IPM.AddValue)
1845
626
    Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,
1846
626
                             CurDAG->getConstant(IPM.AddValue, DL, MVT::i32));
1847
840
1848
840
  EVT VT = Node->getValueType(0);
1849
840
  if (VT == MVT::i32 && 
IPM.Bit == 31461
) {
1850
318
    unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? 
ISD::SRL296
:
ISD::SRA22
;
1851
318
    Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,
1852
318
                             CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1853
522
  } else {
1854
522
    if (VT != MVT::i32)
1855
379
      Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result);
1856
522
1857
522
    if (TrueOp->getSExtValue() == 1) {
1858
493
      // The SHR/AND sequence should get optimized to an RISBG.
1859
493
      Result = CurDAG->getNode(ISD::SRL, DL, VT, Result,
1860
493
                               CurDAG->getConstant(IPM.Bit, DL, MVT::i32));
1861
493
      Result = CurDAG->getNode(ISD::AND, DL, VT, Result,
1862
493
                               CurDAG->getConstant(1, DL, VT));
1863
493
    } else {
1864
29
      // Sign-extend from IPM.Bit using a pair of shifts.
1865
29
      int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit;
1866
29
      int SraAmt = VT.getSizeInBits() - 1;
1867
29
      Result = CurDAG->getNode(ISD::SHL, DL, VT, Result,
1868
29
                               CurDAG->getConstant(ShlAmt, DL, MVT::i32));
1869
29
      Result = CurDAG->getNode(ISD::SRA, DL, VT, Result,
1870
29
                               CurDAG->getConstant(SraAmt, DL, MVT::i32));
1871
29
    }
1872
522
  }
1873
840
1874
840
  return Result;
1875
840
}
1876
1877
9.55k
void SystemZDAGToDAGISel::PreprocessISelDAG() {
1878
9.55k
  // If we have conditional immediate loads, we always prefer
1879
9.55k
  // using those over an IPM sequence.
1880
9.55k
  if (Subtarget->hasLoadStoreOnCond2())
1881
3.38k
    return;
1882
6.16k
1883
6.16k
  bool MadeChange = false;
1884
6.16k
1885
6.16k
  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
1886
6.16k
                                       E = CurDAG->allnodes_end();
1887
101k
       I != E;) {
1888
95.5k
    SDNode *N = &*I++;
1889
95.5k
    if (N->use_empty())
1890
6.16k
      continue;
1891
89.3k
1892
89.3k
    SDValue Res;
1893
89.3k
    switch (N->getOpcode()) {
1894
89.3k
    
default: break87.8k
;
1895
89.3k
    case SystemZISD::SELECT_CCMASK:
1896
1.49k
      Res = expandSelectBoolean(N);
1897
1.49k
      break;
1898
89.3k
    }
1899
89.3k
1900
89.3k
    if (Res) {
1901
840
      LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld:    ");
1902
840
      LLVM_DEBUG(N->dump(CurDAG));
1903
840
      LLVM_DEBUG(dbgs() << "\nNew: ");
1904
840
      LLVM_DEBUG(Res.getNode()->dump(CurDAG));
1905
840
      LLVM_DEBUG(dbgs() << "\n");
1906
840
1907
840
      CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1908
840
      MadeChange = true;
1909
840
    }
1910
89.3k
  }
1911
6.16k
1912
6.16k
  if (MadeChange)
1913
507
    CurDAG->RemoveDeadNodes();
1914
6.16k
}