Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// This file defines an instruction selector for the SystemZ target.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "SystemZTargetMachine.h"
15
#include "llvm/Analysis/AliasAnalysis.h"
16
#include "llvm/CodeGen/SelectionDAGISel.h"
17
#include "llvm/Support/Debug.h"
18
#include "llvm/Support/KnownBits.h"
19
#include "llvm/Support/raw_ostream.h"
20
21
using namespace llvm;
22
23
#define DEBUG_TYPE "systemz-isel"
24
25
namespace {
26
// Used to build addressing modes.
27
struct SystemZAddressingMode {
28
  // The shape of the address.
29
  enum AddrForm {
30
    // base+displacement
31
    FormBD,
32
33
    // base+displacement+index for load and store operands
34
    FormBDXNormal,
35
36
    // base+displacement+index for load address operands
37
    FormBDXLA,
38
39
    // base+displacement+index+ADJDYNALLOC
40
    FormBDXDynAlloc
41
  };
42
  AddrForm Form;
43
44
  // The type of displacement.  The enum names here correspond directly
45
  // to the definitions in SystemZOperand.td.  We could split them into
46
  // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it.
47
  enum DispRange {
48
    Disp12Only,
49
    Disp12Pair,
50
    Disp20Only,
51
    Disp20Only128,
52
    Disp20Pair
53
  };
54
  DispRange DR;
55
56
  // The parts of the address.  The address is equivalent to:
57
  //
58
  //     Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0)
59
  SDValue Base;
60
  int64_t Disp;
61
  SDValue Index;
62
  bool IncludesDynAlloc;
63
64
  SystemZAddressingMode(AddrForm form, DispRange dr)
65
    : Form(form), DR(dr), Base(), Disp(0), Index(),
66
15.6k
      IncludesDynAlloc(false) {}
67
68
  // True if the address can have an index register.
69
1.00k
  bool hasIndexField() { return Form != FormBD; }
70
71
  // True if the address can (and must) include ADJDYNALLOC.
72
13.7k
  bool isDynAlloc() { return Form == FormBDXDynAlloc; }
73
74
0
  void dump() {
75
0
    errs() << "SystemZAddressingMode " << this << '\n';
76
0
77
0
    errs() << " Base ";
78
0
    if (Base.getNode())
79
0
      Base.getNode()->dump();
80
0
    else
81
0
      errs() << "null\n";
82
0
83
0
    if (hasIndexField()) {
84
0
      errs() << " Index ";
85
0
      if (Index.getNode())
86
0
        Index.getNode()->dump();
87
0
      else
88
0
        errs() << "null\n";
89
0
    }
90
0
91
0
    errs() << " Disp " << Disp;
92
0
    if (IncludesDynAlloc)
93
0
      errs() << " + ADJDYNALLOC";
94
0
    errs() << '\n';
95
0
  }
96
};
97
98
// Return a mask with Count low bits set.
99
4.11k
static uint64_t allOnes(unsigned int Count) {
100
4.11k
  assert(Count <= 64);
101
4.11k
  if (Count > 63)
102
1.24k
    return UINT64_MAX;
103
2.86k
  return (uint64_t(1) << Count) - 1;
104
2.86k
}
105
106
// Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation
107
// given by Opcode.  The operands are: Input (R2), Start (I3), End (I4) and
108
// Rotate (I5).  The combined operand value is effectively:
109
//
110
//   (or (rotl Input, Rotate), ~Mask)
111
//
112
// for RNSBG and:
113
//
114
//   (and (rotl Input, Rotate), Mask)
115
//
116
// otherwise.  The output value has BitSize bits, although Input may be
117
// narrower (in which case the upper bits are don't care), or wider (in which
118
// case the result will be truncated as part of the operation).
119
struct RxSBGOperands {
120
  RxSBGOperands(unsigned Op, SDValue N)
121
    : Opcode(Op), BitSize(N.getValueSizeInBits()),
122
      Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),
123
2.85k
      Rotate(0) {}
124
125
  unsigned Opcode;
126
  unsigned BitSize;
127
  uint64_t Mask;
128
  SDValue Input;
129
  unsigned Start;
130
  unsigned End;
131
  unsigned Rotate;
132
};
133
134
class SystemZDAGToDAGISel : public SelectionDAGISel {
135
  const SystemZSubtarget *Subtarget;
136
137
  // Used by SystemZOperands.td to create integer constants.
138
0
  inline SDValue getImm(const SDNode *Node, uint64_t Imm) const {
139
0
    return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));
140
0
  }
141
142
0
  const SystemZTargetMachine &getTargetMachine() const {
143
0
    return static_cast<const SystemZTargetMachine &>(TM);
144
0
  }
145
146
2.12k
  const SystemZInstrInfo *getInstrInfo() const {
147
2.12k
    return Subtarget->getInstrInfo();
148
2.12k
  }
149
150
  // Try to fold more of the base or index of AM into AM, where IsBase
151
  // selects between the base and index.
152
  bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;
153
154
  // Try to describe N in AM, returning true on success.
155
  bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;
156
157
  // Extract individual target operands from matched address AM.
158
  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
159
                          SDValue &Base, SDValue &Disp) const;
160
  void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,
161
                          SDValue &Base, SDValue &Disp, SDValue &Index) const;
162
163
  // Try to match Addr as a FormBD address with displacement type DR.
164
  // Return true on success, storing the base and displacement in
165
  // Base and Disp respectively.
166
  bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
167
                    SDValue &Base, SDValue &Disp) const;
168
169
  // Try to match Addr as a FormBDX address with displacement type DR.
170
  // Return true on success and if the result had no index.  Store the
171
  // base and displacement in Base and Disp respectively.
172
  bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,
173
                     SDValue &Base, SDValue &Disp) const;
174
175
  // Try to match Addr as a FormBDX* address of form Form with
176
  // displacement type DR.  Return true on success, storing the base,
177
  // displacement and index in Base, Disp and Index respectively.
178
  bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,
179
                     SystemZAddressingMode::DispRange DR, SDValue Addr,
180
                     SDValue &Base, SDValue &Disp, SDValue &Index) const;
181
182
  // PC-relative address matching routines used by SystemZOperands.td.
183
4.30k
  bool selectPCRelAddress(SDValue Addr, SDValue &Target) const {
184
4.30k
    if (
SystemZISD::isPCREL(Addr.getOpcode())4.30k
) {
185
1.14k
      Target = Addr.getOperand(0);
186
1.14k
      return true;
187
1.14k
    }
188
3.15k
    return false;
189
3.15k
  }
190
191
  // BD matching routines used by SystemZOperands.td.
192
1.48k
  bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
193
1.48k
    return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);
194
1.48k
  }
195
154
  bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
196
154
    return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
197
154
  }
198
1.23k
  bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const {
199
1.23k
    return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);
200
1.23k
  }
201
28
  bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
202
28
    return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
203
28
  }
204
205
  // MVI matching routines used by SystemZOperands.td.
206
516
  bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
207
516
    return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);
208
516
  }
209
34
  bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const {
210
34
    return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);
211
34
  }
212
213
  // BDX matching routines used by SystemZOperands.td.
214
  bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
215
2.73k
                           SDValue &Index) const {
216
2.73k
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
217
2.73k
                         SystemZAddressingMode::Disp12Only,
218
2.73k
                         Addr, Base, Disp, Index);
219
2.73k
  }
220
  bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
221
3.19k
                           SDValue &Index) const {
222
3.19k
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
223
3.19k
                         SystemZAddressingMode::Disp12Pair,
224
3.19k
                         Addr, Base, Disp, Index);
225
3.19k
  }
226
  bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp,
227
1.13k
                            SDValue &Index) const {
228
1.13k
    return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,
229
1.13k
                         SystemZAddressingMode::Disp12Only,
230
1.13k
                         Addr, Base, Disp, Index);
231
1.13k
  }
232
  bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp,
233
2.56k
                           SDValue &Index) const {
234
2.56k
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
235
2.56k
                         SystemZAddressingMode::Disp20Only,
236
2.56k
                         Addr, Base, Disp, Index);
237
2.56k
  }
238
  bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp,
239
380
                              SDValue &Index) const {
240
380
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
241
380
                         SystemZAddressingMode::Disp20Only128,
242
380
                         Addr, Base, Disp, Index);
243
380
  }
244
  bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
245
307
                           SDValue &Index) const {
246
307
    return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,
247
307
                         SystemZAddressingMode::Disp20Pair,
248
307
                         Addr, Base, Disp, Index);
249
307
  }
250
  bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
251
1.08k
                          SDValue &Index) const {
252
1.08k
    return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
253
1.08k
                         SystemZAddressingMode::Disp12Pair,
254
1.08k
                         Addr, Base, Disp, Index);
255
1.08k
  }
256
  bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp,
257
807
                          SDValue &Index) const {
258
807
    return selectBDXAddr(SystemZAddressingMode::FormBDXLA,
259
807
                         SystemZAddressingMode::Disp20Pair,
260
807
                         Addr, Base, Disp, Index);
261
807
  }
262
263
  // Try to match Addr as an address with a base, 12-bit displacement
264
  // and index, where the index is element Elem of a vector.
265
  // Return true on success, storing the base, displacement and vector
266
  // in Base, Disp and Index respectively.
267
  bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base,
268
                           SDValue &Disp, SDValue &Index) const;
269
270
  // Check whether (or Op (and X InsertMask)) is effectively an insertion
271
  // of X into bits InsertMask of some Y != Op.  Return true if so and
272
  // set Op to that Y.
273
  bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;
274
275
  // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used.
276
  // Return true on success.
277
  bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;
278
279
  // Try to fold some of RxSBG.Input into other fields of RxSBG.
280
  // Return true on success.
281
  bool expandRxSBG(RxSBGOperands &RxSBG) const;
282
283
  // Return an undefined value of type VT.
284
  SDValue getUNDEF(const SDLoc &DL, EVT VT) const;
285
286
  // Convert N to VT, if it isn't already.
287
  SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const;
288
289
  // Try to implement AND or shift node N using RISBG with the zero flag set.
290
  // Return the selected node on success, otherwise return null.
291
  bool tryRISBGZero(SDNode *N);
292
293
  // Try to use RISBG or Opcode to implement OR or XOR node N.
294
  // Return the selected node on success, otherwise return null.
295
  bool tryRxSBG(SDNode *N, unsigned Opcode);
296
297
  // If Op0 is null, then Node is a constant that can be loaded using:
298
  //
299
  //   (Opcode UpperVal LowerVal)
300
  //
301
  // If Op0 is nonnull, then Node can be implemented using:
302
  //
303
  //   (Opcode (Opcode Op0 UpperVal) LowerVal)
304
  void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,
305
                           uint64_t UpperVal, uint64_t LowerVal);
306
307
  // Try to use gather instruction Opcode to implement vector insertion N.
308
  bool tryGather(SDNode *N, unsigned Opcode);
309
310
  // Try to use scatter instruction Opcode to implement store Store.
311
  bool tryScatter(StoreSDNode *Store, unsigned Opcode);
312
313
  // Return true if Load and Store are loads and stores of the same size
314
  // and are guaranteed not to overlap.  Such operations can be implemented
315
  // using block (SS-format) instructions.
316
  //
317
  // Partial overlap would lead to incorrect code, since the block operations
318
  // are logically bytewise, even though they have a fast path for the
319
  // non-overlapping case.  We also need to avoid full overlap (i.e. two
320
  // addresses that might be equal at run time) because although that case
321
  // would be handled correctly, it might be implemented by millicode.
322
  bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const;
323
324
  // N is a (store (load Y), X) pattern.  Return true if it can use an MVC
325
  // from Y to X.
326
  bool storeLoadCanUseMVC(SDNode *N) const;
327
328
  // N is a (store (op (load A[0]), (load A[1])), X) pattern.  Return true
329
  // if A[1 - I] == X and if N can use a block operation like NC from A[I]
330
  // to X.
331
  bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;
332
333
public:
334
  SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel)
335
796
      : SelectionDAGISel(TM, OptLevel) {}
336
337
6.58k
  bool runOnMachineFunction(MachineFunction &MF) override {
338
6.58k
    Subtarget = &MF.getSubtarget<SystemZSubtarget>();
339
6.58k
    return SelectionDAGISel::runOnMachineFunction(MF);
340
6.58k
  }
341
342
  // Override MachineFunctionPass.
343
0
  StringRef getPassName() const override {
344
0
    return "SystemZ DAG->DAG Pattern Instruction Selection";
345
0
  }
346
347
  // Override SelectionDAGISel.
348
  void Select(SDNode *Node) override;
349
  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
350
                                    std::vector<SDValue> &OutOps) override;
351
352
  // Include the pieces autogenerated from the target description.
353
  #include "SystemZGenDAGISel.inc"
354
};
355
} // end anonymous namespace
356
357
FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM,
358
796
                                         CodeGenOpt::Level OptLevel) {
359
796
  return new SystemZDAGToDAGISel(TM, OptLevel);
360
796
}
361
362
// Return true if Val should be selected as a displacement for an address
363
// with range DR.  Here we're interested in the range of both the instruction
364
// described by DR and of any pairing instruction.
365
7.00k
static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
366
7.00k
  switch (DR) {
367
2.27k
  case SystemZAddressingMode::Disp12Only:
368
2.27k
    return isUInt<12>(Val);
369
7.00k
370
4.71k
  case SystemZAddressingMode::Disp12Pair:
371
4.71k
  case SystemZAddressingMode::Disp20Only:
372
4.71k
  case SystemZAddressingMode::Disp20Pair:
373
4.71k
    return isInt<20>(Val);
374
4.71k
375
20
  case SystemZAddressingMode::Disp20Only128:
376
18
    return isInt<20>(Val) && isInt<20>(Val + 8);
377
0
  }
378
0
  
llvm_unreachable0
("Unhandled displacement range");
379
0
}
380
381
// Change the base or index in AM to Value, where IsBase selects
382
// between the base and index.
383
static void changeComponent(SystemZAddressingMode &AM, bool IsBase,
384
5.45k
                            SDValue Value) {
385
5.45k
  if (IsBase)
386
5.45k
    AM.Base = Value;
387
5.45k
  else
388
4
    AM.Index = Value;
389
5.45k
}
390
391
// The base or index of AM is equivalent to Value + ADJDYNALLOC,
392
// where IsBase selects between the base and index.  Try to fold the
393
// ADJDYNALLOC into AM.
394
static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase,
395
98
                              SDValue Value) {
396
98
  if (
AM.isDynAlloc() && 98
!AM.IncludesDynAlloc45
) {
397
45
    changeComponent(AM, IsBase, Value);
398
45
    AM.IncludesDynAlloc = true;
399
45
    return true;
400
45
  }
401
53
  return false;
402
53
}
403
404
// The base of AM is equivalent to Base + Index.  Try to use Index as
405
// the index register.
406
static bool expandIndex(SystemZAddressingMode &AM, SDValue Base,
407
1.00k
                        SDValue Index) {
408
1.00k
  if (
AM.hasIndexField() && 1.00k
!AM.Index.getNode()956
) {
409
830
    AM.Base = Base;
410
830
    AM.Index = Index;
411
830
    return true;
412
830
  }
413
176
  return false;
414
176
}
415
416
// The base or index of AM is equivalent to Op0 + Op1, where IsBase selects
417
// between the base and index.  Try to fold Op1 into AM's displacement.
418
static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,
419
7.00k
                       SDValue Op0, uint64_t Op1) {
420
7.00k
  // First try adjusting the displacement.
421
7.00k
  int64_t TestDisp = AM.Disp + Op1;
422
7.00k
  if (
selectDisp(AM.DR, TestDisp)7.00k
) {
423
5.41k
    changeComponent(AM, IsBase, Op0);
424
5.41k
    AM.Disp = TestDisp;
425
5.41k
    return true;
426
5.41k
  }
427
1.59k
428
1.59k
  // We could consider forcing the displacement into a register and
429
1.59k
  // using it as an index, but it would need to be carefully tuned.
430
1.59k
  return false;
431
1.59k
}
432
433
bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,
434
20.1k
                                        bool IsBase) const {
435
20.1k
  SDValue N = IsBase ? 
AM.Base19.3k
:
AM.Index834
;
436
20.1k
  unsigned Opcode = N.getOpcode();
437
20.1k
  if (
Opcode == ISD::TRUNCATE20.1k
) {
438
42
    N = N.getOperand(0);
439
42
    Opcode = N.getOpcode();
440
42
  }
441
20.1k
  if (
Opcode == ISD::ADD || 20.1k
CurDAG->isBaseWithConstantOffset(N)13.4k
) {
442
6.80k
    SDValue Op0 = N.getOperand(0);
443
6.80k
    SDValue Op1 = N.getOperand(1);
444
6.80k
445
6.80k
    unsigned Op0Code = Op0->getOpcode();
446
6.80k
    unsigned Op1Code = Op1->getOpcode();
447
6.80k
448
6.80k
    if (Op0Code == SystemZISD::ADJDYNALLOC)
449
1
      return expandAdjDynAlloc(AM, IsBase, Op1);
450
6.80k
    
if (6.80k
Op1Code == SystemZISD::ADJDYNALLOC6.80k
)
451
95
      return expandAdjDynAlloc(AM, IsBase, Op0);
452
6.70k
453
6.70k
    
if (6.70k
Op0Code == ISD::Constant6.70k
)
454
0
      return expandDisp(AM, IsBase, Op1,
455
0
                        cast<ConstantSDNode>(Op0)->getSExtValue());
456
6.70k
    
if (6.70k
Op1Code == ISD::Constant6.70k
)
457
5.68k
      return expandDisp(AM, IsBase, Op0,
458
5.68k
                        cast<ConstantSDNode>(Op1)->getSExtValue());
459
1.02k
460
1.02k
    
if (1.02k
IsBase && 1.02k
expandIndex(AM, Op0, Op1)1.00k
)
461
830
      return true;
462
13.5k
  }
463
13.5k
  
if (13.5k
Opcode == SystemZISD::PCREL_OFFSET13.5k
) {
464
1
    SDValue Full = N.getOperand(0);
465
1
    SDValue Base = N.getOperand(1);
466
1
    SDValue Anchor = Base.getOperand(0);
467
1
    uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() -
468
1
                       cast<GlobalAddressSDNode>(Anchor)->getOffset());
469
1
    return expandDisp(AM, IsBase, Base, Offset);
470
1
  }
471
13.5k
  return false;
472
13.5k
}
473
474
// Return true if an instruction with displacement range DR should be
475
// used for displacement value Val.  selectDisp(DR, Val) must already hold.
476
14.1k
static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {
477
14.1k
  assert(selectDisp(DR, Val) && "Invalid displacement");
478
14.1k
  switch (DR) {
479
9.55k
  case SystemZAddressingMode::Disp12Only:
480
9.55k
  case SystemZAddressingMode::Disp20Only:
481
9.55k
  case SystemZAddressingMode::Disp20Only128:
482
9.55k
    return true;
483
9.55k
484
4.16k
  case SystemZAddressingMode::Disp12Pair:
485
4.16k
    // Use the other instruction if the displacement is too large.
486
4.16k
    return isUInt<12>(Val);
487
9.55k
488
394
  case SystemZAddressingMode::Disp20Pair:
489
394
    // Use the other instruction if the displacement is small enough.
490
394
    return !isUInt<12>(Val);
491
0
  }
492
0
  
llvm_unreachable0
("Unhandled displacement range");
493
0
}
494
495
// Return true if Base + Disp + Index should be performed by LA(Y).
496
1.89k
static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) {
497
1.89k
  // Don't use LA(Y) for constants.
498
1.89k
  if (!Base)
499
0
    return false;
500
1.89k
501
1.89k
  // Always use LA(Y) for frame addresses, since we know that the destination
502
1.89k
  // register is almost always (perhaps always) going to be different from
503
1.89k
  // the frame register.
504
1.89k
  
if (1.89k
Base->getOpcode() == ISD::FrameIndex1.89k
)
505
195
    return true;
506
1.69k
507
1.69k
  
if (1.69k
Disp1.69k
) {
508
438
    // Always use LA(Y) if there is a base, displacement and index.
509
438
    if (Index)
510
25
      return true;
511
413
512
413
    // Always use LA if the displacement is small enough.  It should always
513
413
    // be no worse than AGHI (and better if it avoids a move).
514
413
    
if (413
isUInt<12>(Disp)413
)
515
73
      return true;
516
340
517
340
    // For similar reasons, always use LAY if the constant is too big for AGHI.
518
340
    // LAY should be no worse than AGFI.
519
340
    
if (340
!isInt<16>(Disp)340
)
520
12
      return true;
521
1.26k
  } else {
522
1.26k
    // Don't use LA for plain registers.
523
1.26k
    if (!Index)
524
992
      return false;
525
268
526
268
    // Don't use LA for plain addition if the index operand is only used
527
268
    // once.  It should be a natural two-operand addition in that case.
528
268
    
if (268
Index->hasOneUse()268
)
529
174
      return false;
530
94
531
94
    // Prefer addition if the second operation is sign-extended, in the
532
94
    // hope of using AGF.
533
94
    unsigned IndexOpcode = Index->getOpcode();
534
94
    if (IndexOpcode == ISD::SIGN_EXTEND ||
535
94
        IndexOpcode == ISD::SIGN_EXTEND_INREG)
536
0
      return false;
537
422
  }
538
422
539
422
  // Don't use LA for two-operand addition if either operand is only
540
422
  // used once.  The addition instructions are better in that case.
541
422
  
if (422
Base->hasOneUse()422
)
542
398
    return false;
543
24
544
24
  return true;
545
24
}
546
547
// Return true if Addr is suitable for AM, updating AM if so.
548
bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,
549
15.6k
                                        SystemZAddressingMode &AM) const {
550
15.6k
  // Start out assuming that the address will need to be loaded separately,
551
15.6k
  // then try to extend it as much as we can.
552
15.6k
  AM.Base = Addr;
553
15.6k
554
15.6k
  // First try treating the address as a constant.
555
15.6k
  if (Addr.getOpcode() == ISD::Constant &&
556
1.32k
      expandDisp(AM, true, SDValue(),
557
1.32k
                 cast<ConstantSDNode>(Addr)->getSExtValue()))
558
1.31k
    ;
559
15.6k
  // Also see if it's a bare ADJDYNALLOC.
560
14.3k
  else 
if (14.3k
Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&
561
2
           expandAdjDynAlloc(AM, true, SDValue()))
562
2
    ;
563
14.3k
  else
564
14.3k
    // Otherwise try expanding each component.
565
14.3k
    
while (14.3k
expandAddress(AM, true) ||
566
19.3k
           
(AM.Index.getNode() && 14.3k
expandAddress(AM, false)834
))
567
4.96k
      continue;
568
15.6k
569
15.6k
  // Reject cases where it isn't profitable to use LA(Y).
570
15.6k
  if (AM.Form == SystemZAddressingMode::FormBDXLA &&
571
1.89k
      !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode()))
572
1.56k
    return false;
573
14.1k
574
14.1k
  // Reject cases where the other instruction in a pair should be used.
575
14.1k
  
if (14.1k
!isValidDisp(AM.DR, AM.Disp)14.1k
)
576
419
    return false;
577
13.6k
578
13.6k
  // Make sure that ADJDYNALLOC is included where necessary.
579
13.6k
  
if (13.6k
AM.isDynAlloc() && 13.6k
!AM.IncludesDynAlloc1.13k
)
580
1.08k
    return false;
581
12.6k
582
12.6k
  
DEBUG12.6k
(AM.dump());
583
12.6k
  return true;
584
12.6k
}
585
586
// Insert a node into the DAG at least before Pos.  This will reposition
587
// the node as needed, and will assign it a node ID that is <= Pos's ID.
588
// Note that this does *not* preserve the uniqueness of node IDs!
589
// The selection DAG must no longer depend on their uniqueness when this
590
// function is used.
591
52
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) {
592
52
  if (N.getNode()->getNodeId() == -1 ||
593
52
      
N.getNode()->getNodeId() > Pos->getNodeId()7
) {
594
45
    DAG->RepositionNode(Pos->getIterator(), N.getNode());
595
45
    N.getNode()->setNodeId(Pos->getNodeId());
596
45
  }
597
52
}
598
599
void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
600
                                             EVT VT, SDValue &Base,
601
12.5k
                                             SDValue &Disp) const {
602
12.5k
  Base = AM.Base;
603
12.5k
  if (!Base.getNode())
604
12.5k
    // Register 0 means "no base".  This is mostly useful for shifts.
605
1.32k
    Base = CurDAG->getRegister(0, VT);
606
11.2k
  else 
if (11.2k
Base.getOpcode() == ISD::FrameIndex11.2k
) {
607
1.69k
    // Lower a FrameIndex to a TargetFrameIndex.
608
1.69k
    int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex();
609
1.69k
    Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);
610
11.2k
  } else 
if (9.57k
Base.getValueType() != VT9.57k
) {
611
22
    // Truncate values from i64 to i32, for shifts.
612
22
    assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&
613
22
           "Unexpected truncation");
614
22
    SDLoc DL(Base);
615
22
    SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base);
616
22
    insertDAGNode(CurDAG, Base.getNode(), Trunc);
617
22
    Base = Trunc;
618
22
  }
619
12.5k
620
12.5k
  // Lower the displacement to a TargetConstant.
621
12.5k
  Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT);
622
12.5k
}
623
624
void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,
625
                                             EVT VT, SDValue &Base,
626
                                             SDValue &Disp,
627
9.23k
                                             SDValue &Index) const {
628
9.23k
  getAddressOperands(AM, VT, Base, Disp);
629
9.23k
630
9.23k
  Index = AM.Index;
631
9.23k
  if (!Index.getNode())
632
9.23k
    // Register 0 means "no index".
633
8.88k
    Index = CurDAG->getRegister(0, VT);
634
9.23k
}
635
636
bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,
637
                                       SDValue Addr, SDValue &Base,
638
2.89k
                                       SDValue &Disp) const {
639
2.89k
  SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);
640
2.89k
  if (!selectAddress(Addr, AM))
641
28
    return false;
642
2.87k
643
2.87k
  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
644
2.87k
  return true;
645
2.87k
}
646
647
bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,
648
                                        SDValue Addr, SDValue &Base,
649
550
                                        SDValue &Disp) const {
650
550
  SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);
651
550
  if (
!selectAddress(Addr, AM) || 550
AM.Index.getNode()506
)
652
61
    return false;
653
489
654
489
  getAddressOperands(AM, Addr.getValueType(), Base, Disp);
655
489
  return true;
656
489
}
657
658
bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,
659
                                        SystemZAddressingMode::DispRange DR,
660
                                        SDValue Addr, SDValue &Base,
661
12.2k
                                        SDValue &Disp, SDValue &Index) const {
662
12.2k
  SystemZAddressingMode AM(Form, DR);
663
12.2k
  if (!selectAddress(Addr, AM))
664
2.99k
    return false;
665
9.23k
666
9.23k
  getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);
667
9.23k
  return true;
668
9.23k
}
669
670
bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,
671
                                              SDValue &Base,
672
                                              SDValue &Disp,
673
88
                                              SDValue &Index) const {
674
88
  SDValue Regs[2];
675
88
  if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&
676
88
      
Regs[0].getNode()88
&&
Regs[1].getNode()88
) {
677
228
    for (unsigned int I = 0; 
I < 2228
;
++I140
) {
678
158
      Base = Regs[I];
679
158
      Index = Regs[1 - I];
680
158
      // We can't tell here whether the index vector has the right type
681
158
      // for the access; the caller needs to do that instead.
682
158
      if (Index.getOpcode() == ISD::ZERO_EXTEND)
683
10
        Index = Index.getOperand(0);
684
158
      if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
685
158
          
Index.getOperand(1) == Elem18
) {
686
18
        Index = Index.getOperand(0);
687
18
        return true;
688
18
      }
689
158
    }
690
88
  }
691
70
  return false;
692
88
}
693
694
bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
695
25
                                               uint64_t InsertMask) const {
696
25
  // We're only interested in cases where the insertion is into some operand
697
25
  // of Op, rather than into Op itself.  The only useful case is an AND.
698
25
  if (Op.getOpcode() != ISD::AND)
699
12
    return false;
700
13
701
13
  // We need a constant mask.
702
13
  auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode());
703
13
  if (!MaskNode)
704
0
    return false;
705
13
706
13
  // It's not an insertion of Op.getOperand(0) if the two masks overlap.
707
13
  uint64_t AndMask = MaskNode->getZExtValue();
708
13
  if (InsertMask & AndMask)
709
1
    return false;
710
12
711
12
  // It's only an insertion if all bits are covered or are known to be zero.
712
12
  // The inner check covers all cases but is more expensive.
713
12
  uint64_t Used = allOnes(Op.getValueSizeInBits());
714
12
  if (
Used != (AndMask | InsertMask)12
) {
715
1
    KnownBits Known;
716
1
    CurDAG->computeKnownBits(Op.getOperand(0), Known);
717
1
    if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
718
1
      return false;
719
11
  }
720
11
721
11
  Op = Op.getOperand(0);
722
11
  return true;
723
11
}
724
725
bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,
726
2.12k
                                          uint64_t Mask) const {
727
2.12k
  const SystemZInstrInfo *TII = getInstrInfo();
728
2.12k
  if (RxSBG.Rotate != 0)
729
310
    Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));
730
2.12k
  Mask &= RxSBG.Mask;
731
2.12k
  if (
TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)2.12k
) {
732
2.03k
    RxSBG.Mask = Mask;
733
2.03k
    return true;
734
2.03k
  }
735
88
  return false;
736
88
}
737
738
// Return true if any bits of (RxSBG.Input & Mask) are significant.
739
42
static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) {
740
42
  // Rotate the mask in the same way as RxSBG.Input is rotated.
741
42
  if (RxSBG.Rotate != 0)
742
10
    Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));
743
42
  return (Mask & RxSBG.Mask) != 0;
744
42
}
745
746
4.95k
bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
747
4.95k
  SDValue N = RxSBG.Input;
748
4.95k
  unsigned Opcode = N.getOpcode();
749
4.95k
  switch (Opcode) {
750
298
  case ISD::TRUNCATE: {
751
298
    if (RxSBG.Opcode == SystemZ::RNSBG)
752
0
      return false;
753
298
    uint64_t BitSize = N.getValueSizeInBits();
754
298
    uint64_t Mask = allOnes(BitSize);
755
298
    if (!refineRxSBGMask(RxSBG, Mask))
756
0
      return false;
757
298
    RxSBG.Input = N.getOperand(0);
758
298
    return true;
759
298
  }
760
1.01k
  case ISD::AND: {
761
1.01k
    if (RxSBG.Opcode == SystemZ::RNSBG)
762
36
      return false;
763
982
764
982
    auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
765
982
    if (!MaskNode)
766
117
      return false;
767
865
768
865
    SDValue Input = N.getOperand(0);
769
865
    uint64_t Mask = MaskNode->getZExtValue();
770
865
    if (
!refineRxSBGMask(RxSBG, Mask)865
) {
771
44
      // If some bits of Input are already known zeros, those bits will have
772
44
      // been removed from the mask.  See if adding them back in makes the
773
44
      // mask suitable.
774
44
      KnownBits Known;
775
44
      CurDAG->computeKnownBits(Input, Known);
776
44
      Mask |= Known.Zero.getZExtValue();
777
44
      if (!refineRxSBGMask(RxSBG, Mask))
778
40
        return false;
779
825
    }
780
825
    RxSBG.Input = Input;
781
825
    return true;
782
825
  }
783
825
784
63
  case ISD::OR: {
785
63
    if (RxSBG.Opcode != SystemZ::RNSBG)
786
46
      return false;
787
17
788
17
    auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
789
17
    if (!MaskNode)
790
0
      return false;
791
17
792
17
    SDValue Input = N.getOperand(0);
793
17
    uint64_t Mask = ~MaskNode->getZExtValue();
794
17
    if (
!refineRxSBGMask(RxSBG, Mask)17
) {
795
0
      // If some bits of Input are already known ones, those bits will have
796
0
      // been removed from the mask.  See if adding them back in makes the
797
0
      // mask suitable.
798
0
      KnownBits Known;
799
0
      CurDAG->computeKnownBits(Input, Known);
800
0
      Mask &= ~Known.One.getZExtValue();
801
0
      if (!refineRxSBGMask(RxSBG, Mask))
802
0
        return false;
803
17
    }
804
17
    RxSBG.Input = Input;
805
17
    return true;
806
17
  }
807
17
808
344
  case ISD::ROTL: {
809
344
    // Any 64-bit rotate left can be merged into the RxSBG.
810
344
    if (
RxSBG.BitSize != 64 || 344
N.getValueType() != MVT::i6422
)
811
322
      return false;
812
22
    auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
813
22
    if (!CountNode)
814
13
      return false;
815
9
816
9
    RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;
817
9
    RxSBG.Input = N.getOperand(0);
818
9
    return true;
819
9
  }
820
9
821
42
  case ISD::ANY_EXTEND:
822
42
    // Bits above the extended operand are don't-care.
823
42
    RxSBG.Input = N.getOperand(0);
824
42
    return true;
825
9
826
38
  case ISD::ZERO_EXTEND:
827
38
    if (
RxSBG.Opcode != SystemZ::RNSBG38
) {
828
37
      // Restrict the mask to the extended operand.
829
37
      unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
830
37
      if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))
831
0
        return false;
832
37
833
37
      RxSBG.Input = N.getOperand(0);
834
37
      return true;
835
37
    }
836
1
    
LLVM_FALLTHROUGH1
;
837
1
838
1
  case ISD::SIGN_EXTEND: {
839
1
    // Check that the extension bits are don't-care (i.e. are masked out
840
1
    // by the final mask).
841
1
    unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();
842
1
    if (maskMatters(RxSBG, allOnes(RxSBG.BitSize) - allOnes(InnerBitSize)))
843
1
      return false;
844
0
845
0
    RxSBG.Input = N.getOperand(0);
846
0
    return true;
847
0
  }
848
0
849
612
  case ISD::SHL: {
850
612
    auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
851
612
    if (!CountNode)
852
33
      return false;
853
579
854
579
    uint64_t Count = CountNode->getZExtValue();
855
579
    unsigned BitSize = N.getValueSizeInBits();
856
579
    if (
Count < 1 || 579
Count >= BitSize579
)
857
0
      return false;
858
579
859
579
    
if (579
RxSBG.Opcode == SystemZ::RNSBG579
) {
860
11
      // Treat (shl X, count) as (rotl X, size-count) as long as the bottom
861
11
      // count bits from RxSBG.Input are ignored.
862
11
      if (maskMatters(RxSBG, allOnes(Count)))
863
4
        return false;
864
568
    } else {
865
568
      // Treat (shl X, count) as (and (rotl X, count), ~0<<count).
866
568
      if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count))
867
2
        return false;
868
573
    }
869
573
870
573
    RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;
871
573
    RxSBG.Input = N.getOperand(0);
872
573
    return true;
873
573
  }
874
573
875
355
  case ISD::SRL:
876
355
  case ISD::SRA: {
877
355
    auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode());
878
355
    if (!CountNode)
879
27
      return false;
880
328
881
328
    uint64_t Count = CountNode->getZExtValue();
882
328
    unsigned BitSize = N.getValueSizeInBits();
883
328
    if (
Count < 1 || 328
Count >= BitSize328
)
884
0
      return false;
885
328
886
328
    
if (328
RxSBG.Opcode == SystemZ::RNSBG || 328
Opcode == ISD::SRA314
) {
887
30
      // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top
888
30
      // count bits from RxSBG.Input are ignored.
889
30
      if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count)))
890
23
        return false;
891
298
    } else {
892
298
      // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count),
893
298
      // which is similar to SLL above.
894
298
      if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))
895
2
        return false;
896
303
    }
897
303
898
303
    RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;
899
303
    RxSBG.Input = N.getOperand(0);
900
303
    return true;
901
303
  }
902
2.18k
  default:
903
2.18k
    return false;
904
0
  }
905
0
}
906
907
283
SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {
908
283
  SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
909
283
  return SDValue(N, 0);
910
283
}
911
912
SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,
913
1.10k
                                       SDValue N) const {
914
1.10k
  if (
N.getValueType() == MVT::i32 && 1.10k
VT == MVT::i64374
)
915
116
    return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,
916
116
                                         DL, VT, getUNDEF(DL, MVT::i64), N);
917
985
  
if (985
N.getValueType() == MVT::i64 && 985
VT == MVT::i32727
)
918
78
    return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);
919
985
  assert(N.getValueType() == VT && "Unexpected value types");
920
907
  return N;
921
907
}
922
923
2.18k
bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {
924
2.18k
  SDLoc DL(N);
925
2.18k
  EVT VT = N->getValueType(0);
926
2.18k
  if (
!VT.isInteger() || 2.18k
VT.getSizeInBits() > 642.18k
)
927
140
    return false;
928
2.04k
  RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));
929
2.04k
  unsigned Count = 0;
930
4.02k
  while (expandRxSBG(RISBG))
931
2.04k
    // The widening or narrowing is expected to be free.
932
2.04k
    // Counting widening or narrowing as a saved operation will result in
933
2.04k
    // preferring an R*SBG over a simple shift/logical instruction.
934
1.98k
    
if (1.98k
RISBG.Input.getOpcode() != ISD::ANY_EXTEND &&
935
1.94k
        RISBG.Input.getOpcode() != ISD::TRUNCATE)
936
1.65k
      Count += 1;
937
2.04k
  if (Count == 0)
938
542
    return false;
939
1.49k
940
1.49k
  // Prefer to use normal shift instructions over RISBG, since they can handle
941
1.49k
  // all cases and are sometimes shorter.
942
1.49k
  
if (1.49k
Count == 1 && 1.49k
N->getOpcode() != ISD::AND1.34k
)
943
723
    return false;
944
775
945
775
  // Prefer register extensions like LLC over RISBG.  Also prefer to start
946
775
  // out with normal ANDs if one instruction would be enough.  We can convert
947
775
  // these ANDs into an RISBG later if a three-address instruction is useful.
948
775
  
if (775
RISBG.Rotate == 0775
) {
949
631
    bool PreferAnd = false;
950
631
    // Prefer AND for any 32-bit and-immediate operation.
951
631
    if (VT == MVT::i32)
952
212
      PreferAnd = true;
953
631
    // As well as for any 64-bit operation that can be implemented via LLC(R),
954
631
    // LLH(R), LLGT(R), or one of the and-immediate instructions.
955
419
    else 
if (419
RISBG.Mask == 0xff ||
956
397
             RISBG.Mask == 0xffff ||
957
374
             RISBG.Mask == 0x7fffffff ||
958
359
             SystemZ::isImmLF(~RISBG.Mask) ||
959
39
             SystemZ::isImmHF(~RISBG.Mask))
960
388
     PreferAnd = true;
961
419
    // And likewise for the LLZRGF instruction, which doesn't have a register
962
419
    // to register version.
963
31
    else 
if (auto *31
Load31
= dyn_cast<LoadSDNode>(RISBG.Input)) {
964
14
      if (Load->getMemoryVT() == MVT::i32 &&
965
8
          (Load->getExtensionType() == ISD::EXTLOAD ||
966
8
           Load->getExtensionType() == ISD::ZEXTLOAD) &&
967
8
          RISBG.Mask == 0xffffff00 &&
968
8
          Subtarget->hasLoadAndZeroRightmostByte())
969
8
      PreferAnd = true;
970
419
    }
971
631
    if (
PreferAnd631
) {
972
608
      // Replace the current node with an AND.  Note that the current node
973
608
      // might already be that same AND, in which case it is already CSE'd
974
608
      // with it, and we must not call ReplaceNode.
975
608
      SDValue In = convertTo(DL, VT, RISBG.Input);
976
608
      SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);
977
608
      SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask);
978
608
      if (
N != New.getNode()608
) {
979
15
        insertDAGNode(CurDAG, N, Mask);
980
15
        insertDAGNode(CurDAG, N, New);
981
15
        ReplaceNode(N, New.getNode());
982
15
        N = New.getNode();
983
15
      }
984
608
      // Now, select the machine opcode to implement this operation.
985
608
      SelectCode(N);
986
608
      return true;
987
608
    }
988
167
  }
989
167
990
167
  unsigned Opcode = SystemZ::RISBG;
991
167
  // Prefer RISBGN if available, since it does not clobber CC.
992
167
  if (Subtarget->hasMiscellaneousExtensions())
993
58
    Opcode = SystemZ::RISBGN;
994
167
  EVT OpcodeVT = MVT::i64;
995
167
  if (
VT == MVT::i32 && 167
Subtarget->hasHighWord()77
) {
996
25
    Opcode = SystemZ::RISBMux;
997
25
    OpcodeVT = MVT::i32;
998
25
    RISBG.Start &= 31;
999
25
    RISBG.End &= 31;
1000
25
  }
1001
2.18k
  SDValue Ops[5] = {
1002
2.18k
    getUNDEF(DL, OpcodeVT),
1003
2.18k
    convertTo(DL, OpcodeVT, RISBG.Input),
1004
2.18k
    CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),
1005
2.18k
    CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),
1006
2.18k
    CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)
1007
2.18k
  };
1008
2.18k
  SDValue New = convertTo(
1009
2.18k
      DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));
1010
2.18k
  ReplaceUses(N, New.getNode());
1011
2.18k
  CurDAG->RemoveDeadNode(N);
1012
2.18k
  return true;
1013
2.18k
}
1014
1015
874
bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {
1016
874
  SDLoc DL(N);
1017
874
  EVT VT = N->getValueType(0);
1018
874
  if (
!VT.isInteger() || 874
VT.getSizeInBits() > 64874
)
1019
467
    return false;
1020
407
  // Try treating each operand of N as the second operand of the RxSBG
1021
407
  // and see which goes deepest.
1022
407
  RxSBGOperands RxSBG[] = {
1023
407
    RxSBGOperands(Opcode, N->getOperand(0)),
1024
407
    RxSBGOperands(Opcode, N->getOperand(1))
1025
407
  };
1026
407
  unsigned Count[] = { 0, 0 };
1027
1.22k
  for (unsigned I = 0; 
I < 21.22k
;
++I814
)
1028
932
    
while (814
expandRxSBG(RxSBG[I])932
)
1029
814
      // The widening or narrowing is expected to be free.
1030
814
      // Counting widening or narrowing as a saved operation will result in
1031
814
      // preferring an R*SBG over a simple shift/logical instruction.
1032
118
      
if (118
RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND &&
1033
118
          RxSBG[I].Input.getOpcode() != ISD::TRUNCATE)
1034
116
        Count[I] += 1;
1035
407
1036
407
  // Do nothing if neither operand is suitable.
1037
407
  if (
Count[0] == 0 && 407
Count[1] == 0386
)
1038
324
    return false;
1039
83
1040
83
  // Pick the deepest second operand.
1041
83
  
unsigned I = Count[0] > Count[1] ? 83
010
:
173
;
1042
83
  SDValue Op0 = N->getOperand(I ^ 1);
1043
83
1044
83
  // Prefer IC for character insertions from memory.
1045
83
  if (
Opcode == SystemZ::ROSBG && 83
(RxSBG[I].Mask & 0xff) == 055
)
1046
34
    
if (auto *34
Load34
= dyn_cast<LoadSDNode>(Op0.getNode()))
1047
30
      
if (30
Load->getMemoryVT() == MVT::i830
)
1048
30
        return false;
1049
53
1050
53
  // See whether we can avoid an AND in the first operand by converting
1051
53
  // ROSBG to RISBG.
1052
53
  
if (53
Opcode == SystemZ::ROSBG && 53
detectOrAndInsertion(Op0, RxSBG[I].Mask)25
) {
1053
11
    Opcode = SystemZ::RISBG;
1054
11
    // Prefer RISBGN if available, since it does not clobber CC.
1055
11
    if (Subtarget->hasMiscellaneousExtensions())
1056
2
      Opcode = SystemZ::RISBGN;
1057
11
  }
1058
874
1059
874
  SDValue Ops[5] = {
1060
874
    convertTo(DL, MVT::i64, Op0),
1061
874
    convertTo(DL, MVT::i64, RxSBG[I].Input),
1062
874
    CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),
1063
874
    CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),
1064
874
    CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)
1065
874
  };
1066
874
  SDValue New = convertTo(
1067
874
      DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));
1068
874
  ReplaceNode(N, New.getNode());
1069
874
  return true;
1070
874
}
1071
1072
void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,
1073
                                              SDValue Op0, uint64_t UpperVal,
1074
34
                                              uint64_t LowerVal) {
1075
34
  EVT VT = Node->getValueType(0);
1076
34
  SDLoc DL(Node);
1077
34
  SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);
1078
34
  if (Op0.getNode())
1079
10
    Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);
1080
34
1081
34
  {
1082
34
    // When we haven't passed in Op0, Upper will be a constant. In order to
1083
34
    // prevent folding back to the large immediate in `Or = getNode(...)` we run
1084
34
    // SelectCode first and end up with an opaque machine node. This means that
1085
34
    // we need to use a handle to keep track of Upper in case it gets CSE'd by
1086
34
    // SelectCode.
1087
34
    //
1088
34
    // Note that in the case where Op0 is passed in we could just call
1089
34
    // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing
1090
34
    // the handle at all, but it's fine to do it here.
1091
34
    //
1092
34
    // TODO: This is a pretty hacky way to do this. Can we do something that
1093
34
    // doesn't require a two paragraph explanation?
1094
34
    HandleSDNode Handle(Upper);
1095
34
    SelectCode(Upper.getNode());
1096
34
    Upper = Handle.getValue();
1097
34
  }
1098
34
1099
34
  SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);
1100
34
  SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower);
1101
34
1102
34
  ReplaceUses(Node, Or.getNode());
1103
34
  CurDAG->RemoveDeadNode(Node);
1104
34
1105
34
  SelectCode(Or.getNode());
1106
34
}
1107
1108
115
bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
1109
115
  SDValue ElemV = N->getOperand(2);
1110
115
  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1111
115
  if (!ElemN)
1112
10
    return false;
1113
105
1114
105
  unsigned Elem = ElemN->getZExtValue();
1115
105
  EVT VT = N->getValueType(0);
1116
105
  if (Elem >= VT.getVectorNumElements())
1117
0
    return false;
1118
105
1119
105
  auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
1120
105
  if (
!Load || 105
!Load->hasOneUse()49
)
1121
58
    return false;
1122
47
  
if (47
Load->getMemoryVT().getSizeInBits() !=
1123
47
      Load->getValueType(0).getSizeInBits())
1124
0
    return false;
1125
47
1126
47
  SDValue Base, Disp, Index;
1127
47
  if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||
1128
9
      Index.getValueType() != VT.changeVectorElementTypeToInteger())
1129
38
    return false;
1130
9
1131
9
  SDLoc DL(Load);
1132
9
  SDValue Ops[] = {
1133
9
    N->getOperand(0), Base, Disp, Index,
1134
9
    CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()
1135
9
  };
1136
9
  SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);
1137
9
  ReplaceUses(SDValue(Load, 1), SDValue(Res, 1));
1138
9
  ReplaceNode(N, Res);
1139
9
  return true;
1140
9
}
1141
1142
3.40k
bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {
1143
3.40k
  SDValue Value = Store->getValue();
1144
3.40k
  if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1145
3.33k
    return false;
1146
69
  
if (69
Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits()69
)
1147
24
    return false;
1148
45
1149
45
  SDValue ElemV = Value.getOperand(1);
1150
45
  auto *ElemN = dyn_cast<ConstantSDNode>(ElemV);
1151
45
  if (!ElemN)
1152
4
    return false;
1153
41
1154
41
  SDValue Vec = Value.getOperand(0);
1155
41
  EVT VT = Vec.getValueType();
1156
41
  unsigned Elem = ElemN->getZExtValue();
1157
41
  if (Elem >= VT.getVectorNumElements())
1158
0
    return false;
1159
41
1160
41
  SDValue Base, Disp, Index;
1161
41
  if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||
1162
9
      Index.getValueType() != VT.changeVectorElementTypeToInteger())
1163
32
    return false;
1164
9
1165
9
  SDLoc DL(Store);
1166
9
  SDValue Ops[] = {
1167
9
    Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),
1168
9
    Store->getChain()
1169
9
  };
1170
9
  ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
1171
9
  return true;
1172
9
}
1173
1174
bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,
1175
380
                                               LoadSDNode *Load) const {
1176
380
  // Check that the two memory operands have the same size.
1177
380
  if (Load->getMemoryVT() != Store->getMemoryVT())
1178
0
    return false;
1179
380
1180
380
  // Volatility stops an access from being decomposed.
1181
380
  
if (380
Load->isVolatile() || 380
Store->isVolatile()161
)
1182
222
    return false;
1183
158
1184
158
  // There's no chance of overlap if the load is invariant.
1185
158
  
if (158
Load->isInvariant() && 158
Load->isDereferenceable()0
)
1186
0
    return true;
1187
158
1188
158
  // Otherwise we need to check whether there's an alias.
1189
158
  const Value *V1 = Load->getMemOperand()->getValue();
1190
158
  const Value *V2 = Store->getMemOperand()->getValue();
1191
158
  if (
!V1 || 158
!V2120
)
1192
38
    return false;
1193
120
1194
120
  // Reject equality.
1195
120
  uint64_t Size = Load->getMemoryVT().getStoreSize();
1196
120
  int64_t End1 = Load->getSrcValueOffset() + Size;
1197
120
  int64_t End2 = Store->getSrcValueOffset() + Size;
1198
120
  if (
V1 == V2 && 120
End1 == End251
)
1199
51
    return false;
1200
69
1201
69
  return !AA->alias(MemoryLocation(V1, End1, Load->getAAInfo()),
1202
69
                    MemoryLocation(V2, End2, Store->getAAInfo()));
1203
69
}
1204
1205
369
bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {
1206
369
  auto *Store = cast<StoreSDNode>(N);
1207
369
  auto *Load = cast<LoadSDNode>(Store->getValue());
1208
369
1209
369
  // Prefer not to use MVC if either address can use ... RELATIVE LONG
1210
369
  // instructions.
1211
369
  uint64_t Size = Load->getMemoryVT().getStoreSize();
1212
369
  if (
Size > 1 && 369
Size <= 8358
) {
1213
310
    // Prefer LHRL, LRL and LGRL.
1214
310
    if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode()))
1215
28
      return false;
1216
282
    // Prefer STHRL, STRL and STGRL.
1217
282
    
if (282
SystemZISD::isPCREL(Store->getBasePtr().getOpcode())282
)
1218
2
      return false;
1219
339
  }
1220
339
1221
339
  return canUseBlockOperation(Store, Load);
1222
339
}
1223
1224
bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
1225
43
                                                     unsigned I) const {
1226
43
  auto *StoreA = cast<StoreSDNode>(N);
1227
43
  auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I));
1228
43
  auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I));
1229
41
  return !LoadA->isVolatile() && canUseBlockOperation(StoreA, LoadB);
1230
43
}
1231
1232
93.8k
void SystemZDAGToDAGISel::Select(SDNode *Node) {
1233
93.8k
  // Dump information about the Node being selected
1234
93.8k
  DEBUG(errs() << "Selecting: "; Node->dump(CurDAG); errs() << "\n");
1235
93.8k
1236
93.8k
  // If we have a custom node, we already have selected!
1237
93.8k
  if (
Node->isMachineOpcode()93.8k
) {
1238
362
    DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
1239
362
    Node->setNodeId(-1);
1240
362
    return;
1241
362
  }
1242
93.4k
1243
93.4k
  unsigned Opcode = Node->getOpcode();
1244
93.4k
  switch (Opcode) {
1245
413
  case ISD::OR:
1246
413
    if (Node->getOperand(1).getOpcode() != ISD::Constant)
1247
320
      
if (320
tryRxSBG(Node, SystemZ::ROSBG)320
)
1248
25
        return;
1249
388
    goto or_xor;
1250
388
1251
357
  case ISD::XOR:
1252
357
    if (Node->getOperand(1).getOpcode() != ISD::Constant)
1253
287
      
if (287
tryRxSBG(Node, SystemZ::RXSBG)287
)
1254
10
        return;
1255
735
    // Fall through.
1256
735
  or_xor:
1257
735
    // If this is a 64-bit operation in which both 32-bit halves are nonzero,
1258
735
    // split the operation into two.
1259
735
    if (Node->getValueType(0) == MVT::i64)
1260
200
      
if (auto *200
Op1200
= dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
1261
88
        uint64_t Val = Op1->getZExtValue();
1262
88
        if (
!SystemZ::isImmLF(Val) && 88
!SystemZ::isImmHF(Val)37
) {
1263
10
          splitLargeImmediate(Opcode, Node, Node->getOperand(0),
1264
10
                              Val - uint32_t(Val), uint32_t(Val));
1265
10
          return;
1266
10
        }
1267
725
      }
1268
725
    break;
1269
725
1270
1.05k
  case ISD::AND:
1271
1.05k
    if (Node->getOperand(1).getOpcode() != ISD::Constant)
1272
267
      
if (267
tryRxSBG(Node, SystemZ::RNSBG)267
)
1273
18
        return;
1274
1.03k
    
LLVM_FALLTHROUGH1.03k
;
1275
2.18k
  case ISD::ROTL:
1276
2.18k
  case ISD::SHL:
1277
2.18k
  case ISD::SRL:
1278
2.18k
  case ISD::ZERO_EXTEND:
1279
2.18k
    if (tryRISBGZero(Node))
1280
775
      return;
1281
1.40k
    break;
1282
1.40k
1283
831
  case ISD::Constant:
1284
831
    // If this is a 64-bit constant that is out of the range of LLILF,
1285
831
    // LLIHF and LGFI, split it into two 32-bit pieces.
1286
831
    if (
Node->getValueType(0) == MVT::i64831
) {
1287
398
      uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue();
1288
398
      if (
!SystemZ::isImmLF(Val) && 398
!SystemZ::isImmHF(Val)106
&&
!isInt<32>(Val)82
) {
1289
24
        splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),
1290
24
                            uint32_t(Val));
1291
24
        return;
1292
24
      }
1293
807
    }
1294
807
    break;
1295
807
1296
659
  case SystemZISD::SELECT_CCMASK: {
1297
659
    SDValue Op0 = Node->getOperand(0);
1298
659
    SDValue Op1 = Node->getOperand(1);
1299
659
    // Prefer to put any load first, so that it can be matched as a
1300
659
    // conditional load.  Likewise for constants in range for LOCHI.
1301
659
    if (
(Op1.getOpcode() == ISD::LOAD && 659
Op0.getOpcode() != ISD::LOAD75
) ||
1302
584
        (Subtarget->hasLoadStoreOnCond2() &&
1303
71
         Node->getValueType(0).isInteger() &&
1304
58
         Op1.getOpcode() == ISD::Constant &&
1305
10
         isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) &&
1306
10
         !(Op0.getOpcode() == ISD::Constant &&
1307
659
           
isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())6
))) {
1308
79
      SDValue CCValid = Node->getOperand(2);
1309
79
      SDValue CCMask = Node->getOperand(3);
1310
79
      uint64_t ConstCCValid =
1311
79
        cast<ConstantSDNode>(CCValid.getNode())->getZExtValue();
1312
79
      uint64_t ConstCCMask =
1313
79
        cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
1314
79
      // Invert the condition.
1315
79
      CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node),
1316
79
                                   CCMask.getValueType());
1317
79
      SDValue Op4 = Node->getOperand(4);
1318
79
      Node = CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
1319
79
    }
1320
659
    break;
1321
807
  }
1322
807
1323
242
  case ISD::INSERT_VECTOR_ELT: {
1324
242
    EVT VT = Node->getValueType(0);
1325
242
    unsigned ElemBitSize = VT.getScalarSizeInBits();
1326
242
    if (
ElemBitSize == 32242
) {
1327
66
      if (tryGather(Node, SystemZ::VGEF))
1328
5
        return;
1329
176
    } else 
if (176
ElemBitSize == 64176
) {
1330
49
      if (tryGather(Node, SystemZ::VGEG))
1331
4
        return;
1332
233
    }
1333
233
    break;
1334
233
  }
1335
233
1336
3.83k
  case ISD::STORE: {
1337
3.83k
    auto *Store = cast<StoreSDNode>(Node);
1338
3.83k
    unsigned ElemBitSize = Store->getValue().getValueSizeInBits();
1339
3.83k
    if (
ElemBitSize == 323.83k
) {
1340
1.96k
      if (tryScatter(Store, SystemZ::VSCEF))
1341
5
        return;
1342
1.87k
    } else 
if (1.87k
ElemBitSize == 641.87k
) {
1343
1.44k
      if (tryScatter(Store, SystemZ::VSCEG))
1344
4
        return;
1345
3.82k
    }
1346
3.82k
    break;
1347
3.82k
  }
1348
92.5k
  }
1349
92.5k
1350
92.5k
  SelectCode(Node);
1351
92.5k
}
1352
1353
bool SystemZDAGToDAGISel::
1354
SelectInlineAsmMemoryOperand(const SDValue &Op,
1355
                             unsigned ConstraintID,
1356
23
                             std::vector<SDValue> &OutOps) {
1357
23
  SystemZAddressingMode::AddrForm Form;
1358
23
  SystemZAddressingMode::DispRange DispRange;
1359
23
  SDValue Base, Disp, Index;
1360
23
1361
23
  switch(ConstraintID) {
1362
0
  default:
1363
0
    llvm_unreachable("Unexpected asm memory constraint");
1364
5
  case InlineAsm::Constraint_i:
1365
5
  case InlineAsm::Constraint_Q:
1366
5
    // Accept an address with a short displacement, but no index.
1367
5
    Form = SystemZAddressingMode::FormBD;
1368
5
    DispRange = SystemZAddressingMode::Disp12Only;
1369
5
    break;
1370
7
  case InlineAsm::Constraint_R:
1371
7
    // Accept an address with a short displacement and an index.
1372
7
    Form = SystemZAddressingMode::FormBDXNormal;
1373
7
    DispRange = SystemZAddressingMode::Disp12Only;
1374
7
    break;
1375
4
  case InlineAsm::Constraint_S:
1376
4
    // Accept an address with a long displacement, but no index.
1377
4
    Form = SystemZAddressingMode::FormBD;
1378
4
    DispRange = SystemZAddressingMode::Disp20Only;
1379
4
    break;
1380
7
  case InlineAsm::Constraint_T:
1381
7
  case InlineAsm::Constraint_m:
1382
7
    // Accept an address with a long displacement and an index.
1383
7
    // m works the same as T, as this is the most general case.
1384
7
    Form = SystemZAddressingMode::FormBDXNormal;
1385
7
    DispRange = SystemZAddressingMode::Disp20Only;
1386
7
    break;
1387
23
  }
1388
23
1389
23
  
if (23
selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)23
) {
1390
23
    const TargetRegisterClass *TRC =
1391
23
      Subtarget->getRegisterInfo()->getPointerRegClass(*MF);
1392
23
    SDLoc DL(Base);
1393
23
    SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);
1394
23
1395
23
    // Make sure that the base address doesn't go into %r0.
1396
23
    // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything.
1397
23
    if (Base.getOpcode() != ISD::TargetFrameIndex &&
1398
23
        
Base.getOpcode() != ISD::Register23
) {
1399
23
      Base =
1400
23
        SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1401
23
                                       DL, Base.getValueType(),
1402
23
                                       Base, RC), 0);
1403
23
    }
1404
23
1405
23
    // Make sure that the index register isn't assigned to %r0 either.
1406
23
    if (
Index.getOpcode() != ISD::Register23
) {
1407
4
      Index =
1408
4
        SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1409
4
                                       DL, Index.getValueType(),
1410
4
                                       Index, RC), 0);
1411
4
    }
1412
23
1413
23
    OutOps.push_back(Base);
1414
23
    OutOps.push_back(Disp);
1415
23
    OutOps.push_back(Index);
1416
23
    return false;
1417
23
  }
1418
0
1419
0
  return true;
1420
0
}