Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file defines an instruction selector for the ARM target.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "ARM.h"
14
#include "ARMBaseInstrInfo.h"
15
#include "ARMTargetMachine.h"
16
#include "MCTargetDesc/ARMAddressingModes.h"
17
#include "Utils/ARMBaseInfo.h"
18
#include "llvm/ADT/StringSwitch.h"
19
#include "llvm/CodeGen/MachineFrameInfo.h"
20
#include "llvm/CodeGen/MachineFunction.h"
21
#include "llvm/CodeGen/MachineInstrBuilder.h"
22
#include "llvm/CodeGen/MachineRegisterInfo.h"
23
#include "llvm/CodeGen/SelectionDAG.h"
24
#include "llvm/CodeGen/SelectionDAGISel.h"
25
#include "llvm/CodeGen/TargetLowering.h"
26
#include "llvm/IR/CallingConv.h"
27
#include "llvm/IR/Constants.h"
28
#include "llvm/IR/DerivedTypes.h"
29
#include "llvm/IR/Function.h"
30
#include "llvm/IR/Intrinsics.h"
31
#include "llvm/IR/LLVMContext.h"
32
#include "llvm/Support/CommandLine.h"
33
#include "llvm/Support/Debug.h"
34
#include "llvm/Support/ErrorHandling.h"
35
#include "llvm/Target/TargetOptions.h"
36
37
using namespace llvm;
38
39
#define DEBUG_TYPE "arm-isel"
40
41
static cl::opt<bool>
42
DisableShifterOp("disable-shifter-op", cl::Hidden,
43
  cl::desc("Disable isel of shifter-op"),
44
  cl::init(false));
45
46
//===--------------------------------------------------------------------===//
47
/// ARMDAGToDAGISel - ARM specific code to select ARM machine
48
/// instructions for SelectionDAG operations.
49
///
50
namespace {
51
52
class ARMDAGToDAGISel : public SelectionDAGISel {
53
  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
54
  /// make the right decision when generating code for different targets.
55
  const ARMSubtarget *Subtarget;
56
57
public:
58
  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
59
5.14k
      : SelectionDAGISel(tm, OptLevel) {}
60
61
26.3k
  bool runOnMachineFunction(MachineFunction &MF) override {
62
26.3k
    // Reset the subtarget each time through.
63
26.3k
    Subtarget = &MF.getSubtarget<ARMSubtarget>();
64
26.3k
    SelectionDAGISel::runOnMachineFunction(MF);
65
26.3k
    return true;
66
26.3k
  }
67
68
26.3k
  StringRef getPassName() const override { return "ARM Instruction Selection"; }
69
70
  void PreprocessISelDAG() override;
71
72
  /// getI32Imm - Return a target constant of type i32 with the specified
73
  /// value.
74
120
  inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
75
120
    return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
76
120
  }
77
78
  void Select(SDNode *N) override;
79
80
  bool hasNoVMLxHazardUse(SDNode *N) const;
81
  bool isShifterOpProfitable(const SDValue &Shift,
82
                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
83
  bool SelectRegShifterOperand(SDValue N, SDValue &A,
84
                               SDValue &B, SDValue &C,
85
                               bool CheckProfitability = true);
86
  bool SelectImmShifterOperand(SDValue N, SDValue &A,
87
                               SDValue &B, bool CheckProfitability = true);
88
  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
89
608
                                    SDValue &B, SDValue &C) {
90
608
    // Don't apply the profitability check
91
608
    return SelectRegShifterOperand(N, A, B, C, false);
92
608
  }
93
  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
94
188k
                                    SDValue &B) {
95
188k
    // Don't apply the profitability check
96
188k
    return SelectImmShifterOperand(N, A, B, false);
97
188k
  }
98
99
  bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
100
101
  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
102
  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
103
104
7.37k
  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
105
7.37k
    const ConstantSDNode *CN = cast<ConstantSDNode>(N);
106
7.37k
    Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
107
7.37k
    Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
108
7.37k
    return true;
109
7.37k
  }
110
111
  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
112
                             SDValue &Offset, SDValue &Opc);
113
  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
114
                             SDValue &Offset, SDValue &Opc);
115
  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
116
                             SDValue &Offset, SDValue &Opc);
117
  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
118
  bool SelectAddrMode3(SDValue N, SDValue &Base,
119
                       SDValue &Offset, SDValue &Opc);
120
  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
121
                             SDValue &Offset, SDValue &Opc);
122
  bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, bool FP16);
123
  bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
124
  bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
125
  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
126
  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
127
128
  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
129
130
  // Thumb Addressing Modes:
131
  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
132
  bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
133
  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
134
                                SDValue &OffImm);
135
  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
136
                                 SDValue &OffImm);
137
  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
138
                                 SDValue &OffImm);
139
  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
140
                                 SDValue &OffImm);
141
  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
142
143
  // Thumb 2 Addressing Modes:
144
  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
145
  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
146
                            SDValue &OffImm);
147
  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
148
                                 SDValue &OffImm);
149
  template<unsigned Shift>
150
  bool SelectT2AddrModeImm7(SDValue N, SDValue &Base,
151
                            SDValue &OffImm);
152
  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
153
                             SDValue &OffReg, SDValue &ShImm);
154
  bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
155
156
0
  inline bool is_so_imm(unsigned Imm) const {
157
0
    return ARM_AM::getSOImmVal(Imm) != -1;
158
0
  }
159
160
0
  inline bool is_so_imm_not(unsigned Imm) const {
161
0
    return ARM_AM::getSOImmVal(~Imm) != -1;
162
0
  }
163
164
4.96k
  inline bool is_t2_so_imm(unsigned Imm) const {
165
4.96k
    return ARM_AM::getT2SOImmVal(Imm) != -1;
166
4.96k
  }
167
168
1.95k
  inline bool is_t2_so_imm_not(unsigned Imm) const {
169
1.95k
    return ARM_AM::getT2SOImmVal(~Imm) != -1;
170
1.95k
  }
171
172
  // Include the pieces autogenerated from the target description.
173
#include "ARMGenDAGISel.inc"
174
175
private:
176
  void transferMemOperands(SDNode *Src, SDNode *Dst);
177
178
  /// Indexed (pre/post inc/dec) load matching code for ARM.
179
  bool tryARMIndexedLoad(SDNode *N);
180
  bool tryT1IndexedLoad(SDNode *N);
181
  bool tryT2IndexedLoad(SDNode *N);
182
183
  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
184
  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
185
  /// loads of D registers and even subregs and odd subregs of Q registers.
186
  /// For NumVecs <= 2, QOpcodes1 is not used.
187
  void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
188
                 const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
189
                 const uint16_t *QOpcodes1);
190
191
  /// SelectVST - Select NEON store intrinsics.  NumVecs should
192
  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
193
  /// stores of D registers and even subregs and odd subregs of Q registers.
194
  /// For NumVecs <= 2, QOpcodes1 is not used.
195
  void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
196
                 const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
197
                 const uint16_t *QOpcodes1);
198
199
  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
200
  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
201
  /// load/store of D registers and Q registers.
202
  void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
203
                       unsigned NumVecs, const uint16_t *DOpcodes,
204
                       const uint16_t *QOpcodes);
205
206
  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
207
  /// should be 1, 2, 3 or 4.  The opcode array specifies the instructions used
208
  /// for loading D registers.
209
  void SelectVLDDup(SDNode *N, bool IsIntrinsic, bool isUpdating,
210
                    unsigned NumVecs, const uint16_t *DOpcodes,
211
                    const uint16_t *QOpcodes0 = nullptr,
212
                    const uint16_t *QOpcodes1 = nullptr);
213
214
  /// Try to select SBFX/UBFX instructions for ARM.
215
  bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
216
217
  // Select special operations if node forms integer ABS pattern
218
  bool tryABSOp(SDNode *N);
219
220
  bool tryReadRegister(SDNode *N);
221
  bool tryWriteRegister(SDNode *N);
222
223
  bool tryInlineAsm(SDNode *N);
224
225
  void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
226
227
  void SelectCMP_SWAP(SDNode *N);
228
229
  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
230
  /// inline asm expressions.
231
  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
232
                                    std::vector<SDValue> &OutOps) override;
233
234
  // Form pairs of consecutive R, S, D, or Q registers.
235
  SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1);
236
  SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
237
  SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
238
  SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
239
240
  // Form sequences of 4 consecutive S, D, or Q registers.
241
  SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
242
  SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
243
  SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
244
245
  // Get the alignment operand for a NEON VLD or VST instruction.
246
  SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
247
                        bool is64BitVector);
248
249
  /// Returns the number of instructions required to materialize the given
250
  /// constant in a register, or 3 if a literal pool load is needed.
251
  unsigned ConstantMaterializationCost(unsigned Val) const;
252
253
  /// Checks if N is a multiplication by a constant where we can extract out a
254
  /// power of two from the constant so that it can be used in a shift, but only
255
  /// if it simplifies the materialization of the constant. Returns true if it
256
  /// is, and assigns to PowerOfTwo the power of two that should be extracted
257
  /// out and to NewMulConst the new constant to be multiplied by.
258
  bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
259
                              unsigned &PowerOfTwo, SDValue &NewMulConst) const;
260
261
  /// Replace N with M in CurDAG, in a way that also ensures that M gets
262
  /// selected when N would have been selected.
263
  void replaceDAGValue(const SDValue &N, SDValue M);
264
};
265
}
266
267
/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
268
/// operand. If so Imm will receive the 32-bit value.
269
10.3k
static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
270
10.3k
  if (N->getOpcode() == ISD::Constant && 
N->getValueType(0) == MVT::i328.41k
) {
271
8.41k
    Imm = cast<ConstantSDNode>(N)->getZExtValue();
272
8.41k
    return true;
273
8.41k
  }
274
1.91k
  return false;
275
1.91k
}
276
277
// isInt32Immediate - This method tests to see if a constant operand.
278
// If so Imm will receive the 32 bit value.
279
69
static bool isInt32Immediate(SDValue N, unsigned &Imm) {
280
69
  return isInt32Immediate(N.getNode(), Imm);
281
69
}
282
283
// isOpcWithIntImmediate - This method tests to see if the node is a specific
284
// opcode and that it has a immediate integer right operand.
285
// If so Imm will receive the 32 bit value.
286
222k
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
287
222k
  return N->getOpcode() == Opc &&
288
222k
         
isInt32Immediate(N->getOperand(1).getNode(), Imm)10.2k
;
289
222k
}
290
291
/// Check whether a particular node is a constant value representable as
292
/// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
293
///
294
/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
295
static bool isScaledConstantInRange(SDValue Node, int Scale,
296
                                    int RangeMin, int RangeMax,
297
6.99k
                                    int &ScaledConstant) {
298
6.99k
  assert(Scale > 0 && "Invalid scale!");
299
6.99k
300
6.99k
  // Check that this is a constant.
301
6.99k
  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
302
6.99k
  if (!C)
303
189
    return false;
304
6.80k
305
6.80k
  ScaledConstant = (int) C->getZExtValue();
306
6.80k
  if ((ScaledConstant % Scale) != 0)
307
58
    return false;
308
6.74k
309
6.74k
  ScaledConstant /= Scale;
310
6.74k
  return ScaledConstant >= RangeMin && 
ScaledConstant < RangeMax6.73k
;
311
6.74k
}
312
313
107k
void ARMDAGToDAGISel::PreprocessISelDAG() {
314
107k
  if (!Subtarget->hasV6T2Ops())
315
8.36k
    return;
316
99.3k
317
99.3k
  bool isThumb2 = Subtarget->isThumb();
318
99.3k
  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
319
2.31M
       E = CurDAG->allnodes_end(); I != E; ) {
320
2.21M
    SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
321
2.21M
322
2.21M
    if (N->getOpcode() != ISD::ADD)
323
2.11M
      continue;
324
98.0k
325
98.0k
    // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
326
98.0k
    // leading zeros, followed by consecutive set bits, followed by 1 or 2
327
98.0k
    // trailing zeros, e.g. 1020.
328
98.0k
    // Transform the expression to
329
98.0k
    // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
330
98.0k
    // of trailing zeros of c2. The left shift would be folded as an shifter
331
98.0k
    // operand of 'add' and the 'and' and 'srl' would become a bits extraction
332
98.0k
    // node (UBFX).
333
98.0k
334
98.0k
    SDValue N0 = N->getOperand(0);
335
98.0k
    SDValue N1 = N->getOperand(1);
336
98.0k
    unsigned And_imm = 0;
337
98.0k
    if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
338
97.6k
      if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
339
540
        std::swap(N0, N1);
340
97.6k
    }
341
98.0k
    if (!And_imm)
342
97.1k
      continue;
343
886
344
886
    // Check if the AND mask is an immediate of the form: 000.....1111111100
345
886
    unsigned TZ = countTrailingZeros(And_imm);
346
886
    if (TZ != 1 && 
TZ != 2884
)
347
855
      // Be conservative here. Shifter operands aren't always free. e.g. On
348
855
      // Swift, left shifter operand of 1 / 2 for free but others are not.
349
855
      // e.g.
350
855
      //  ubfx   r3, r1, #16, #8
351
855
      //  ldr.w  r3, [r0, r3, lsl #2]
352
855
      // vs.
353
855
      //  mov.w  r9, #1020
354
855
      //  and.w  r2, r9, r1, lsr #14
355
855
      //  ldr    r2, [r0, r2]
356
855
      continue;
357
31
    And_imm >>= TZ;
358
31
    if (And_imm & (And_imm + 1))
359
0
      continue;
360
31
361
31
    // Look for (and (srl X, c1), c2).
362
31
    SDValue Srl = N1.getOperand(0);
363
31
    unsigned Srl_imm = 0;
364
31
    if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
365
31
        
(Srl_imm <= 2)3
)
366
28
      continue;
367
3
368
3
    // Make sure first operand is not a shifter operand which would prevent
369
3
    // folding of the left shift.
370
3
    SDValue CPTmp0;
371
3
    SDValue CPTmp1;
372
3
    SDValue CPTmp2;
373
3
    if (isThumb2) {
374
0
      if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
375
0
        continue;
376
3
    } else {
377
3
      if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
378
3
          SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
379
0
        continue;
380
3
    }
381
3
382
3
    // Now make the transformation.
383
3
    Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
384
3
                          Srl.getOperand(0),
385
3
                          CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
386
3
                                              MVT::i32));
387
3
    N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
388
3
                         Srl,
389
3
                         CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
390
3
    N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
391
3
                         N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
392
3
    CurDAG->UpdateNodeOperands(N, N0, N1);
393
3
  }
394
99.3k
}
395
396
/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
397
/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
398
/// least on current ARM implementations) which should be avoidded.
399
1.14k
bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
400
1.14k
  if (OptLevel == CodeGenOpt::None)
401
16
    return true;
402
1.12k
403
1.12k
  if (!Subtarget->hasVMLxHazards())
404
556
    return true;
405
572
406
572
  if (!N->hasOneUse())
407
55
    return false;
408
517
409
517
  SDNode *Use = *N->use_begin();
410
517
  if (Use->getOpcode() == ISD::CopyToReg)
411
11
    return true;
412
506
  if (Use->isMachineOpcode()) {
413
506
    const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
414
506
        CurDAG->getSubtarget().getInstrInfo());
415
506
416
506
    const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
417
506
    if (MCID.mayStore())
418
14
      return true;
419
492
    unsigned Opcode = MCID.getOpcode();
420
492
    if (Opcode == ARM::VMOVRS || 
Opcode == ARM::VMOVRRD452
)
421
75
      return true;
422
417
    // vmlx feeding into another vmlx. We actually want to unfold
423
417
    // the use later in the MLxExpansion pass. e.g.
424
417
    // vmla
425
417
    // vmla (stall 8 cycles)
426
417
    //
427
417
    // vmul (5 cycles)
428
417
    // vadd (5 cycles)
429
417
    // vmla
430
417
    // This adds up to about 18 - 19 cycles.
431
417
    //
432
417
    // vmla
433
417
    // vmul (stall 4 cycles)
434
417
    // vadd adds up to about 14 cycles.
435
417
    return TII->isFpMLxInstruction(Opcode);
436
417
  }
437
0
438
0
  return false;
439
0
}
440
441
bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
442
                                            ARM_AM::ShiftOpc ShOpcVal,
443
1.74k
                                            unsigned ShAmt) {
444
1.74k
  if (!Subtarget->isLikeA9() && 
!Subtarget->isSwift()1.73k
)
445
1.30k
    return true;
446
440
  if (Shift.hasOneUse())
447
422
    return true;
448
18
  // R << 2 is free.
449
18
  return ShOpcVal == ARM_AM::lsl &&
450
18
         (ShAmt == 2 || 
(2
Subtarget->isSwift()2
&&
ShAmt == 10
));
451
18
}
452
453
55.8k
unsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const {
454
55.8k
  if (Subtarget->isThumb()) {
455
52.9k
    if (Val <= 255) 
return 137.1k
; // MOV
456
15.8k
    if (Subtarget->hasV6T2Ops() &&
457
15.8k
        
(14.3k
Val <= 0xffff14.3k
|| // MOV
458
14.3k
         
ARM_AM::getT2SOImmVal(Val) != -16.17k
|| // MOVW
459
14.3k
         
ARM_AM::getT2SOImmVal(~Val) != -14.20k
)) // MVN
460
10.7k
      return 1;
461
5.10k
    if (Val <= 510) 
return 246
; // MOV + ADDi8
462
5.05k
    if (~Val <= 255) 
return 2205
; // MOV + MVN
463
4.85k
    if (ARM_AM::isThumbImmShiftedVal(Val)) 
return 2364
; // MOV + LSL
464
2.90k
  } else {
465
2.90k
    if (ARM_AM::getSOImmVal(Val) != -1) 
return 12.18k
; // MOV
466
719
    if (ARM_AM::getSOImmVal(~Val) != -1) 
return 1124
; // MVN
467
595
    if (Subtarget->hasV6T2Ops() && 
Val <= 0xffff294
)
return 175
; // MOVW
468
520
    if (ARM_AM::isSOImmTwoPartVal(Val)) 
return 2229
; // two instrs
469
4.78k
  }
470
4.78k
  if (Subtarget->useMovt()) 
return 22.53k
; // MOVW + MOVT
471
2.24k
  return 3; // Literal pool load
472
2.24k
}
473
474
bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
475
                                             unsigned MaxShift,
476
                                             unsigned &PowerOfTwo,
477
998
                                             SDValue &NewMulConst) const {
478
998
  assert(N.getOpcode() == ISD::MUL);
479
998
  assert(MaxShift > 0);
480
998
481
998
  // If the multiply is used in more than one place then changing the constant
482
998
  // will make other uses incorrect, so don't.
483
998
  if (!N.hasOneUse()) 
return false48
;
484
950
  // Check if the multiply is by a constant
485
950
  ConstantSDNode *MulConst = dyn_cast<ConstantSDNode>(N.getOperand(1));
486
950
  if (!MulConst) 
return false876
;
487
74
  // If the constant is used in more than one place then modifying it will mean
488
74
  // we need to materialize two constants instead of one, which is a bad idea.
489
74
  if (!MulConst->hasOneUse()) 
return false8
;
490
66
  unsigned MulConstVal = MulConst->getZExtValue();
491
66
  if (MulConstVal == 0) 
return false0
;
492
66
493
66
  // Find the largest power of 2 that MulConstVal is a multiple of
494
66
  PowerOfTwo = MaxShift;
495
1.61k
  while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
496
1.55k
    --PowerOfTwo;
497
1.55k
    if (PowerOfTwo == 0) 
return false13
;
498
1.55k
  }
499
66
500
66
  // Only optimise if the new cost is better
501
66
  unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
502
53
  NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32);
503
53
  unsigned OldCost = ConstantMaterializationCost(MulConstVal);
504
53
  unsigned NewCost = ConstantMaterializationCost(NewMulConstVal);
505
53
  return NewCost < OldCost;
506
66
}
507
508
33
void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
509
33
  CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode());
510
33
  ReplaceUses(N, M);
511
33
}
512
513
bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
514
                                              SDValue &BaseReg,
515
                                              SDValue &Opc,
516
197k
                                              bool CheckProfitability) {
517
197k
  if (DisableShifterOp)
518
0
    return false;
519
197k
520
197k
  // If N is a multiply-by-constant and it's profitable to extract a shift and
521
197k
  // use it in a shifted operand do so.
522
197k
  if (N.getOpcode() == ISD::MUL) {
523
970
    unsigned PowerOfTwo = 0;
524
970
    SDValue NewMulConst;
525
970
    if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) {
526
18
      HandleSDNode Handle(N);
527
18
      SDLoc Loc(N);
528
18
      replaceDAGValue(N.getOperand(1), NewMulConst);
529
18
      BaseReg = Handle.getValue();
530
18
      Opc = CurDAG->getTargetConstant(
531
18
          ARM_AM::getSORegOpc(ARM_AM::lsl, PowerOfTwo), Loc, MVT::i32);
532
18
      return true;
533
18
    }
534
197k
  }
535
197k
536
197k
  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
537
197k
538
197k
  // Don't match base register only case. That is matched to a separate
539
197k
  // lower complexity pattern with explicit register operand.
540
197k
  if (ShOpcVal == ARM_AM::no_shift) 
return false186k
;
541
10.6k
542
10.6k
  BaseReg = N.getOperand(0);
543
10.6k
  unsigned ShImmVal = 0;
544
10.6k
  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
545
10.6k
  if (!RHS) 
return false1.77k
;
546
8.92k
  ShImmVal = RHS->getZExtValue() & 31;
547
8.92k
  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
548
8.92k
                                  SDLoc(N), MVT::i32);
549
8.92k
  return true;
550
8.92k
}
551
552
bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
553
                                              SDValue &BaseReg,
554
                                              SDValue &ShReg,
555
                                              SDValue &Opc,
556
10.3k
                                              bool CheckProfitability) {
557
10.3k
  if (DisableShifterOp)
558
0
    return false;
559
10.3k
560
10.3k
  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
561
10.3k
562
10.3k
  // Don't match base register only case. That is matched to a separate
563
10.3k
  // lower complexity pattern with explicit register operand.
564
10.3k
  if (ShOpcVal == ARM_AM::no_shift) 
return false9.12k
;
565
1.21k
566
1.21k
  BaseReg = N.getOperand(0);
567
1.21k
  unsigned ShImmVal = 0;
568
1.21k
  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
569
1.21k
  if (RHS) 
return false1.01k
;
570
200
571
200
  ShReg = N.getOperand(1);
572
200
  if (CheckProfitability && 
!isShifterOpProfitable(N, ShOpcVal, ShImmVal)156
)
573
0
    return false;
574
200
  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
575
200
                                  SDLoc(N), MVT::i32);
576
200
  return true;
577
200
}
578
579
// Determine whether an ISD::OR's operands are suitable to turn the operation
580
// into an addition, which often has more compact encodings.
581
6.97k
bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) {
582
6.97k
  assert(Parent->getOpcode() == ISD::OR && "unexpected parent");
583
6.97k
  Out = N;
584
6.97k
  return CurDAG->haveNoCommonBitsSet(N, Parent->getOperand(1));
585
6.97k
}
586
587
588
bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
589
                                          SDValue &Base,
590
6.57k
                                          SDValue &OffImm) {
591
6.57k
  // Match simple R + imm12 operands.
592
6.57k
593
6.57k
  // Base only.
594
6.57k
  if (N.getOpcode() != ISD::ADD && 
N.getOpcode() != ISD::SUB3.57k
&&
595
6.57k
      
!CurDAG->isBaseWithConstantOffset(N)3.57k
) {
596
3.40k
    if (N.getOpcode() == ISD::FrameIndex) {
597
1.78k
      // Match frame index.
598
1.78k
      int FI = cast<FrameIndexSDNode>(N)->getIndex();
599
1.78k
      Base = CurDAG->getTargetFrameIndex(
600
1.78k
          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
601
1.78k
      OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
602
1.78k
      return true;
603
1.78k
    }
604
1.62k
605
1.62k
    if (N.getOpcode() == ARMISD::Wrapper &&
606
1.62k
        
N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress427
&&
607
1.62k
        
N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol287
&&
608
1.62k
        
N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress287
) {
609
281
      Base = N.getOperand(0);
610
281
    } else
611
1.34k
      Base = N;
612
1.62k
    OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
613
1.62k
    return true;
614
1.62k
  }
615
3.16k
616
3.16k
  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
617
3.16k
    int RHSC = (int)RHS->getSExtValue();
618
3.16k
    if (N.getOpcode() == ISD::SUB)
619
0
      RHSC = -RHSC;
620
3.16k
621
3.16k
    if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
622
3.16k
      Base   = N.getOperand(0);
623
3.16k
      if (Base.getOpcode() == ISD::FrameIndex) {
624
1.35k
        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
625
1.35k
        Base = CurDAG->getTargetFrameIndex(
626
1.35k
            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
627
1.35k
      }
628
3.16k
      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
629
3.16k
      return true;
630
3.16k
    }
631
0
  }
632
0
633
0
  // Base only.
634
0
  Base = N;
635
0
  OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
636
0
  return true;
637
0
}
638
639
640
641
bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
642
6.73k
                                      SDValue &Opc) {
643
6.73k
  if (N.getOpcode() == ISD::MUL &&
644
6.73k
      
(0
(0
!Subtarget->isLikeA9()0
&&
!Subtarget->isSwift()0
) ||
N.hasOneUse()0
)) {
645
0
    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
646
0
      // X * [3,5,9] -> X + X * [2,4,8] etc.
647
0
      int RHSC = (int)RHS->getZExtValue();
648
0
      if (RHSC & 1) {
649
0
        RHSC = RHSC & ~1;
650
0
        ARM_AM::AddrOpc AddSub = ARM_AM::add;
651
0
        if (RHSC < 0) {
652
0
          AddSub = ARM_AM::sub;
653
0
          RHSC = - RHSC;
654
0
        }
655
0
        if (isPowerOf2_32(RHSC)) {
656
0
          unsigned ShAmt = Log2_32(RHSC);
657
0
          Base = Offset = N.getOperand(0);
658
0
          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
659
0
                                                            ARM_AM::lsl),
660
0
                                          SDLoc(N), MVT::i32);
661
0
          return true;
662
0
        }
663
6.73k
      }
664
0
    }
665
0
  }
666
6.73k
667
6.73k
  if (N.getOpcode() != ISD::ADD && 
N.getOpcode() != ISD::SUB3.59k
&&
668
6.73k
      // ISD::OR that is equivalent to an ISD::ADD.
669
6.73k
      
!CurDAG->isBaseWithConstantOffset(N)3.58k
)
670
3.42k
    return false;
671
3.31k
672
3.31k
  // Leave simple R +/- imm12 operands for LDRi12
673
3.31k
  if (N.getOpcode() == ISD::ADD || 
N.getOpcode() == ISD::OR171
) {
674
3.30k
    int RHSC;
675
3.30k
    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
676
3.30k
                                -0x1000+1, 0x1000, RHSC)) // 12 bits.
677
3.16k
      return false;
678
148
  }
679
148
680
148
  // Otherwise this is R +/- [possibly shifted] R.
681
148
  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? 
ARM_AM::sub5
:
ARM_AM::add143
;
682
148
  ARM_AM::ShiftOpc ShOpcVal =
683
148
    ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
684
148
  unsigned ShAmt = 0;
685
148
686
148
  Base   = N.getOperand(0);
687
148
  Offset = N.getOperand(1);
688
148
689
148
  if (ShOpcVal != ARM_AM::no_shift) {
690
71
    // Check to see if the RHS of the shift is a constant, if not, we can't fold
691
71
    // it.
692
71
    if (ConstantSDNode *Sh =
693
71
           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
694
71
      ShAmt = Sh->getZExtValue();
695
71
      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
696
71
        Offset = N.getOperand(1).getOperand(0);
697
0
      else {
698
0
        ShAmt = 0;
699
0
        ShOpcVal = ARM_AM::no_shift;
700
0
      }
701
71
    } else {
702
0
      ShOpcVal = ARM_AM::no_shift;
703
0
    }
704
71
  }
705
148
706
148
  // Try matching (R shl C) + (R).
707
148
  if (N.getOpcode() != ISD::SUB && 
ShOpcVal == ARM_AM::no_shift143
&&
708
148
      
!(74
Subtarget->isLikeA9()74
||
Subtarget->isSwift()61
||
709
74
        
N.getOperand(0).hasOneUse()61
)) {
710
6
    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
711
6
    if (ShOpcVal != ARM_AM::no_shift) {
712
0
      // Check to see if the RHS of the shift is a constant, if not, we can't
713
0
      // fold it.
714
0
      if (ConstantSDNode *Sh =
715
0
          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
716
0
        ShAmt = Sh->getZExtValue();
717
0
        if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
718
0
          Offset = N.getOperand(0).getOperand(0);
719
0
          Base = N.getOperand(1);
720
0
        } else {
721
0
          ShAmt = 0;
722
0
          ShOpcVal = ARM_AM::no_shift;
723
0
        }
724
0
      } else {
725
0
        ShOpcVal = ARM_AM::no_shift;
726
0
      }
727
0
    }
728
6
  }
729
148
730
148
  // If Offset is a multiply-by-constant and it's profitable to extract a shift
731
148
  // and use it in a shifted operand do so.
732
148
  if (Offset.getOpcode() == ISD::MUL && 
N.hasOneUse()17
) {
733
17
    unsigned PowerOfTwo = 0;
734
17
    SDValue NewMulConst;
735
17
    if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
736
11
      HandleSDNode Handle(Offset);
737
11
      replaceDAGValue(Offset.getOperand(1), NewMulConst);
738
11
      Offset = Handle.getValue();
739
11
      ShAmt = PowerOfTwo;
740
11
      ShOpcVal = ARM_AM::lsl;
741
11
    }
742
17
  }
743
148
744
148
  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
745
148
                                  SDLoc(N), MVT::i32);
746
148
  return true;
747
148
}
748
749
bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
750
27
                                            SDValue &Offset, SDValue &Opc) {
751
27
  unsigned Opcode = Op->getOpcode();
752
27
  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
753
27
    ? 
cast<LoadSDNode>(Op)->getAddressingMode()14
754
27
    : 
cast<StoreSDNode>(Op)->getAddressingMode()13
;
755
27
  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || 
AM == ISD::POST_INC18
)
756
27
    ? 
ARM_AM::add19
:
ARM_AM::sub8
;
757
27
  int Val;
758
27
  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
759
8
    return false;
760
19
761
19
  Offset = N;
762
19
  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
763
19
  unsigned ShAmt = 0;
764
19
  if (ShOpcVal != ARM_AM::no_shift) {
765
7
    // Check to see if the RHS of the shift is a constant, if not, we can't fold
766
7
    // it.
767
7
    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
768
7
      ShAmt = Sh->getZExtValue();
769
7
      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
770
5
        Offset = N.getOperand(0);
771
2
      else {
772
2
        ShAmt = 0;
773
2
        ShOpcVal = ARM_AM::no_shift;
774
2
      }
775
7
    } else {
776
0
      ShOpcVal = ARM_AM::no_shift;
777
0
    }
778
7
  }
779
19
780
19
  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
781
19
                                  SDLoc(N), MVT::i32);
782
19
  return true;
783
19
}
784
785
bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
786
23
                                            SDValue &Offset, SDValue &Opc) {
787
23
  unsigned Opcode = Op->getOpcode();
788
23
  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
789
23
    ? cast<LoadSDNode>(Op)->getAddressingMode()
790
23
    : 
cast<StoreSDNode>(Op)->getAddressingMode()0
;
791
23
  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || 
AM == ISD::POST_INC6
)
792
23
    ? 
ARM_AM::add17
:
ARM_AM::sub6
;
793
23
  int Val;
794
23
  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
795
14
    if (AddSub == ARM_AM::sub) 
Val *= -14
;
796
14
    Offset = CurDAG->getRegister(0, MVT::i32);
797
14
    Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
798
14
    return true;
799
14
  }
800
9
801
9
  return false;
802
9
}
803
804
805
bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
806
39
                                            SDValue &Offset, SDValue &Opc) {
807
39
  unsigned Opcode = Op->getOpcode();
808
39
  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
809
39
    ? 
cast<LoadSDNode>(Op)->getAddressingMode()14
810
39
    : 
cast<StoreSDNode>(Op)->getAddressingMode()25
;
811
39
  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || 
AM == ISD::POST_INC24
)
812
39
    ? 
ARM_AM::add28
:
ARM_AM::sub11
;
813
39
  int Val;
814
39
  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
815
32
    Offset = CurDAG->getRegister(0, MVT::i32);
816
32
    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
817
32
                                                      ARM_AM::no_shift),
818
32
                                    SDLoc(Op), MVT::i32);
819
32
    return true;
820
32
  }
821
7
822
7
  return false;
823
7
}
824
825
1.61k
bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
826
1.61k
  Base = N;
827
1.61k
  return true;
828
1.61k
}
829
830
bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
831
                                      SDValue &Base, SDValue &Offset,
832
745
                                      SDValue &Opc) {
833
745
  if (N.getOpcode() == ISD::SUB) {
834
0
    // X - C  is canonicalize to X + -C, no need to handle it here.
835
0
    Base = N.getOperand(0);
836
0
    Offset = N.getOperand(1);
837
0
    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N),
838
0
                                    MVT::i32);
839
0
    return true;
840
0
  }
841
745
842
745
  if (!CurDAG->isBaseWithConstantOffset(N)) {
843
647
    Base = N;
844
647
    if (N.getOpcode() == ISD::FrameIndex) {
845
80
      int FI = cast<FrameIndexSDNode>(N)->getIndex();
846
80
      Base = CurDAG->getTargetFrameIndex(
847
80
          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
848
80
    }
849
647
    Offset = CurDAG->getRegister(0, MVT::i32);
850
647
    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
851
647
                                    MVT::i32);
852
647
    return true;
853
647
  }
854
98
855
98
  // If the RHS is +/- imm8, fold into addr mode.
856
98
  int RHSC;
857
98
  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
858
98
                              -256 + 1, 256, RHSC)) { // 8 bits.
859
98
    Base = N.getOperand(0);
860
98
    if (Base.getOpcode() == ISD::FrameIndex) {
861
9
      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
862
9
      Base = CurDAG->getTargetFrameIndex(
863
9
          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
864
9
    }
865
98
    Offset = CurDAG->getRegister(0, MVT::i32);
866
98
867
98
    ARM_AM::AddrOpc AddSub = ARM_AM::add;
868
98
    if (RHSC < 0) {
869
0
      AddSub = ARM_AM::sub;
870
0
      RHSC = -RHSC;
871
0
    }
872
98
    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N),
873
98
                                    MVT::i32);
874
98
    return true;
875
98
  }
876
0
877
0
  Base = N.getOperand(0);
878
0
  Offset = N.getOperand(1);
879
0
  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
880
0
                                  MVT::i32);
881
0
  return true;
882
0
}
883
884
bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
885
14
                                            SDValue &Offset, SDValue &Opc) {
886
14
  unsigned Opcode = Op->getOpcode();
887
14
  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
888
14
    ? 
cast<LoadSDNode>(Op)->getAddressingMode()9
889
14
    : 
cast<StoreSDNode>(Op)->getAddressingMode()5
;
890
14
  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || 
AM == ISD::POST_INC9
)
891
14
    ? 
ARM_AM::add11
:
ARM_AM::sub3
;
892
14
  int Val;
893
14
  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
894
14
    Offset = CurDAG->getRegister(0, MVT::i32);
895
14
    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op),
896
14
                                    MVT::i32);
897
14
    return true;
898
14
  }
899
0
900
0
  Offset = N;
901
0
  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op),
902
0
                                  MVT::i32);
903
0
  return true;
904
0
}
905
906
bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
907
5.46k
                                        bool FP16) {
908
5.46k
  if (!CurDAG->isBaseWithConstantOffset(N)) {
909
5.13k
    Base = N;
910
5.13k
    if (N.getOpcode() == ISD::FrameIndex) {
911
832
      int FI = cast<FrameIndexSDNode>(N)->getIndex();
912
832
      Base = CurDAG->getTargetFrameIndex(
913
832
          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
914
4.30k
    } else if (N.getOpcode() == ARMISD::Wrapper &&
915
4.30k
               
N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress1.89k
&&
916
4.30k
               
N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol1.74k
&&
917
4.30k
               
N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress1.74k
) {
918
1.74k
      Base = N.getOperand(0);
919
1.74k
    }
920
5.13k
    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
921
5.13k
                                       SDLoc(N), MVT::i32);
922
5.13k
    return true;
923
5.13k
  }
924
334
925
334
  // If the RHS is +/- imm8, fold into addr mode.
926
334
  int RHSC;
927
334
  const int Scale = FP16 ? 
213
:
4321
;
928
334
929
334
  if (isScaledConstantInRange(N.getOperand(1), Scale, -255, 256, RHSC)) {
930
330
    Base = N.getOperand(0);
931
330
    if (Base.getOpcode() == ISD::FrameIndex) {
932
2
      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
933
2
      Base = CurDAG->getTargetFrameIndex(
934
2
          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
935
2
    }
936
330
937
330
    ARM_AM::AddrOpc AddSub = ARM_AM::add;
938
330
    if (RHSC < 0) {
939
8
      AddSub = ARM_AM::sub;
940
8
      RHSC = -RHSC;
941
8
    }
942
330
943
330
    if (FP16)
944
9
      Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(AddSub, RHSC),
945
9
                                         SDLoc(N), MVT::i32);
946
321
    else
947
321
      Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
948
321
                                         SDLoc(N), MVT::i32);
949
330
950
330
    return true;
951
330
  }
952
4
953
4
  Base = N;
954
4
955
4
  if (FP16)
956
4
    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(ARM_AM::add, 0),
957
4
                                       SDLoc(N), MVT::i32);
958
0
  else
959
0
    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
960
0
                                       SDLoc(N), MVT::i32);
961
4
962
4
  return true;
963
4
}
964
965
bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
966
4.61k
                                      SDValue &Base, SDValue &Offset) {
967
4.61k
  return IsAddressingMode5(N, Base, Offset, /*FP16=*/ false);
968
4.61k
}
969
970
bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N,
971
850
                                          SDValue &Base, SDValue &Offset) {
972
850
  return IsAddressingMode5(N, Base, Offset, /*FP16=*/ true);
973
850
}
974
975
bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
976
6.26k
                                      SDValue &Align) {
977
6.26k
  Addr = N;
978
6.26k
979
6.26k
  unsigned Alignment = 0;
980
6.26k
981
6.26k
  MemSDNode *MemN = cast<MemSDNode>(Parent);
982
6.26k
983
6.26k
  if (isa<LSBaseSDNode>(MemN) ||
984
6.26k
      
(1.28k
(1.28k
MemN->getOpcode() == ARMISD::VST1_UPD1.28k
||
985
1.28k
        
MemN->getOpcode() == ARMISD::VLD1_UPD851
) &&
986
5.78k
       
MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1801
)) {
987
5.78k
    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
988
5.78k
    // The maximum alignment is equal to the memory size being referenced.
989
5.78k
    unsigned MMOAlign = MemN->getAlignment();
990
5.78k
    unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8;
991
5.78k
    if (MMOAlign >= MemSize && 
MemSize > 12.03k
)
992
2.02k
      Alignment = MemSize;
993
5.78k
  } else {
994
484
    // All other uses of addrmode6 are for intrinsics.  For now just record
995
484
    // the raw alignment value; it will be refined later based on the legal
996
484
    // alignment operands for the intrinsic.
997
484
    Alignment = MemN->getAlignment();
998
484
  }
999
6.26k
1000
6.26k
  Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
1001
6.26k
  return true;
1002
6.26k
}
1003
1004
bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1005
4
                                            SDValue &Offset) {
1006
4
  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
1007
4
  ISD::MemIndexedMode AM = LdSt->getAddressingMode();
1008
4
  if (AM != ISD::POST_INC)
1009
0
    return false;
1010
4
  Offset = N;
1011
4
  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1012
4
    if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1013
3
      Offset = CurDAG->getRegister(0, MVT::i32);
1014
4
  }
1015
4
  return true;
1016
4
}
1017
1018
bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1019
7.52k
                                       SDValue &Offset, SDValue &Label) {
1020
7.52k
  if (N.getOpcode() == ARMISD::PIC_ADD && 
N.hasOneUse()12
) {
1021
12
    Offset = N.getOperand(0);
1022
12
    SDValue N1 = N.getOperand(1);
1023
12
    Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1024
12
                                      SDLoc(N), MVT::i32);
1025
12
    return true;
1026
12
  }
1027
7.51k
1028
7.51k
  return false;
1029
7.51k
}
1030
1031
1032
//===----------------------------------------------------------------------===//
1033
//                         Thumb Addressing Modes
1034
//===----------------------------------------------------------------------===//
1035
1036
1.76k
static bool shouldUseZeroOffsetLdSt(SDValue N) {
1037
1.76k
  // Negative numbers are difficult to materialise in thumb1. If we are
1038
1.76k
  // selecting the add of a negative, instead try to select ri with a zero
1039
1.76k
  // offset, so create the add node directly which will become a sub.
1040
1.76k
  if (N.getOpcode() != ISD::ADD)
1041
765
    return false;
1042
999
1043
999
  // Look for an imm which is not legal for ld/st, but is legal for sub.
1044
999
  if (auto C = dyn_cast<ConstantSDNode>(N.getOperand(1)))
1045
587
    return C->getSExtValue() < 0 && 
C->getSExtValue() >= -25524
;
1046
412
1047
412
  return false;
1048
412
}
1049
1050
bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base,
1051
373
                                                SDValue &Offset) {
1052
373
  if (N.getOpcode() != ISD::ADD && 
!CurDAG->isBaseWithConstantOffset(N)38
) {
1053
38
    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
1054
38
    if (!NC || 
!NC->isNullValue()2
)
1055
36
      return false;
1056
2
1057
2
    Base = Offset = N;
1058
2
    return true;
1059
2
  }
1060
335
1061
335
  Base = N.getOperand(0);
1062
335
  Offset = N.getOperand(1);
1063
335
  return true;
1064
335
}
1065
1066
bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, SDValue &Base,
1067
289
                                            SDValue &Offset) {
1068
289
  if (shouldUseZeroOffsetLdSt(N))
1069
0
    return false; // Select ri instead
1070
289
  return SelectThumbAddrModeRRSext(N, Base, Offset);
1071
289
}
1072
1073
bool
1074
ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1075
1.47k
                                          SDValue &Base, SDValue &OffImm) {
1076
1.47k
  if (shouldUseZeroOffsetLdSt(N)) {
1077
12
    Base = N;
1078
12
    OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1079
12
    return true;
1080
12
  }
1081
1.46k
1082
1.46k
  if (!CurDAG->isBaseWithConstantOffset(N)) {
1083
970
    if (N.getOpcode() == ISD::ADD) {
1084
206
      return false; // We want to select register offset instead
1085
764
    } else if (N.getOpcode() == ARMISD::Wrapper &&
1086
764
        
N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress20
&&
1087
764
        
N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol10
&&
1088
764
        
N.getOperand(0).getOpcode() != ISD::TargetConstantPool10
&&
1089
764
        
N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress0
) {
1090
0
      Base = N.getOperand(0);
1091
764
    } else {
1092
764
      Base = N;
1093
764
    }
1094
970
1095
970
    OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1096
764
    return true;
1097
493
  }
1098
493
1099
493
  // If the RHS is + imm5 * scale, fold into addr mode.
1100
493
  int RHSC;
1101
493
  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1102
410
    Base = N.getOperand(0);
1103
410
    OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1104
410
    return true;
1105
410
  }
1106
83
1107
83
  // Offset is too large, so use register offset instead.
1108
83
  return false;
1109
83
}
1110
1111
bool
1112
ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1113
1.08k
                                           SDValue &OffImm) {
1114
1.08k
  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1115
1.08k
}
1116
1117
bool
1118
ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1119
170
                                           SDValue &OffImm) {
1120
170
  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1121
170
}
1122
1123
bool
1124
ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1125
216
                                           SDValue &OffImm) {
1126
216
  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1127
216
}
1128
1129
bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1130
2.29k
                                            SDValue &Base, SDValue &OffImm) {
1131
2.29k
  if (N.getOpcode() == ISD::FrameIndex) {
1132
616
    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1133
616
    // Only multiples of 4 are allowed for the offset, so the frame object
1134
616
    // alignment must be at least 4.
1135
616
    MachineFrameInfo &MFI = MF->getFrameInfo();
1136
616
    if (MFI.getObjectAlignment(FI) < 4)
1137
9
      MFI.setObjectAlignment(FI, 4);
1138
616
    Base = CurDAG->getTargetFrameIndex(
1139
616
        FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1140
616
    OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1141
616
    return true;
1142
616
  }
1143
1.68k
1144
1.68k
  if (!CurDAG->isBaseWithConstantOffset(N))
1145
746
    return false;
1146
934
1147
934
  if (N.getOperand(0).getOpcode() == ISD::FrameIndex) {
1148
574
    // If the RHS is + imm8 * scale, fold into addr mode.
1149
574
    int RHSC;
1150
574
    if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1151
574
      Base = N.getOperand(0);
1152
574
      int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1153
574
      // Make sure the offset is inside the object, or we might fail to
1154
574
      // allocate an emergency spill slot. (An out-of-range access is UB, but
1155
574
      // it could show up anyway.)
1156
574
      MachineFrameInfo &MFI = MF->getFrameInfo();
1157
574
      if (RHSC * 4 < MFI.getObjectSize(FI)) {
1158
564
        // For LHS+RHS to result in an offset that's a multiple of 4 the object
1159
564
        // indexed by the LHS must be 4-byte aligned.
1160
564
        if (!MFI.isFixedObjectIndex(FI) && 
MFI.getObjectAlignment(FI) < 4522
)
1161
0
          MFI.setObjectAlignment(FI, 4);
1162
564
        if (MFI.getObjectAlignment(FI) >= 4) {
1163
564
          Base = CurDAG->getTargetFrameIndex(
1164
564
              FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1165
564
          OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1166
564
          return true;
1167
564
        }
1168
370
      }
1169
574
    }
1170
574
  }
1171
370
1172
370
  return false;
1173
370
}
1174
1175
1176
//===----------------------------------------------------------------------===//
1177
//                        Thumb 2 Addressing Modes
1178
//===----------------------------------------------------------------------===//
1179
1180
1181
bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1182
110k
                                            SDValue &Base, SDValue &OffImm) {
1183
110k
  // Match simple R + imm12 operands.
1184
110k
1185
110k
  // Base only.
1186
110k
  if (N.getOpcode() != ISD::ADD && 
N.getOpcode() != ISD::SUB47.1k
&&
1187
110k
      
!CurDAG->isBaseWithConstantOffset(N)47.1k
) {
1188
46.4k
    if (N.getOpcode() == ISD::FrameIndex) {
1189
9.74k
      // Match frame index.
1190
9.74k
      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1191
9.74k
      Base = CurDAG->getTargetFrameIndex(
1192
9.74k
          FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1193
9.74k
      OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1194
9.74k
      return true;
1195
9.74k
    }
1196
36.7k
1197
36.7k
    if (N.getOpcode() == ARMISD::Wrapper &&
1198
36.7k
        
N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress291
&&
1199
36.7k
        
N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol86
&&
1200
36.7k
        
N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress79
) {
1201
73
      Base = N.getOperand(0);
1202
73
      if (Base.getOpcode() == ISD::TargetConstantPool)
1203
73
        return false;  // We want to select t2LDRpci instead.
1204
36.6k
    } else
1205
36.6k
      Base = N;
1206
36.7k
    OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1207
36.6k
    return true;
1208
63.9k
  }
1209
63.9k
1210
63.9k
  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1211
63.9k
    if (SelectT2AddrModeImm8(N, Base, OffImm))
1212
637
      // Let t2LDRi8 handle (R - imm8).
1213
637
      return false;
1214
63.3k
1215
63.3k
    int RHSC = (int)RHS->getZExtValue();
1216
63.3k
    if (N.getOpcode() == ISD::SUB)
1217
0
      RHSC = -RHSC;
1218
63.3k
1219
63.3k
    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1220
63.3k
      Base   = N.getOperand(0);
1221
63.3k
      if (Base.getOpcode() == ISD::FrameIndex) {
1222
17.7k
        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1223
17.7k
        Base = CurDAG->getTargetFrameIndex(
1224
17.7k
            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1225
17.7k
      }
1226
63.3k
      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1227
63.3k
      return true;
1228
63.3k
    }
1229
15
  }
1230
15
1231
15
  // Base only.
1232
15
  Base = N;
1233
15
  OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1234
15
  return true;
1235
15
}
1236
1237
bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1238
64.6k
                                           SDValue &Base, SDValue &OffImm) {
1239
64.6k
  // Match simple R - imm8 operands.
1240
64.6k
  if (N.getOpcode() != ISD::ADD && 
N.getOpcode() != ISD::SUB765
&&
1241
64.6k
      
!CurDAG->isBaseWithConstantOffset(N)765
)
1242
73
    return false;
1243
64.6k
1244
64.6k
  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1245
64.6k
    int RHSC = (int)RHS->getSExtValue();
1246
64.6k
    if (N.getOpcode() == ISD::SUB)
1247
0
      RHSC = -RHSC;
1248
64.6k
1249
64.6k
    if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1250
1.27k
      Base = N.getOperand(0);
1251
1.27k
      if (Base.getOpcode() == ISD::FrameIndex) {
1252
0
        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1253
0
        Base = CurDAG->getTargetFrameIndex(
1254
0
            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1255
0
      }
1256
1.27k
      OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1257
1.27k
      return true;
1258
1.27k
    }
1259
63.3k
  }
1260
63.3k
1261
63.3k
  return false;
1262
63.3k
}
1263
1264
bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1265
1.72k
                                                 SDValue &OffImm){
1266
1.72k
  unsigned Opcode = Op->getOpcode();
1267
1.72k
  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1268
1.72k
    ? 
cast<LoadSDNode>(Op)->getAddressingMode()1.20k
1269
1.72k
    : 
cast<StoreSDNode>(Op)->getAddressingMode()527
;
1270
1.72k
  int RHSC;
1271
1.72k
  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1272
1.72k
    OffImm = ((AM == ISD::PRE_INC) || 
(AM == ISD::POST_INC)895
)
1273
1.72k
      ? 
CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)1.65k
1274
1.72k
      : 
CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32)71
;
1275
1.72k
    return true;
1276
1.72k
  }
1277
0
1278
0
  return false;
1279
0
}
1280
1281
template<unsigned Shift>
1282
bool ARMDAGToDAGISel::SelectT2AddrModeImm7(SDValue N,
1283
559
                                           SDValue &Base, SDValue &OffImm) {
1284
559
  if (N.getOpcode() == ISD::SUB ||
1285
559
      CurDAG->isBaseWithConstantOffset(N)) {
1286
168
    if (auto RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1287
168
      int RHSC = (int)RHS->getZExtValue();
1288
168
      if (N.getOpcode() == ISD::SUB)
1289
0
        RHSC = -RHSC;
1290
168
1291
168
      if (isShiftedInt<7, Shift>(RHSC)) {
1292
62
        Base = N.getOperand(0);
1293
62
        if (Base.getOpcode() == ISD::FrameIndex) {
1294
6
          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1295
6
          Base = CurDAG->getTargetFrameIndex(
1296
6
            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1297
6
        }
1298
62
        OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1299
62
        return true;
1300
62
      }
1301
497
    }
1302
168
  }
1303
497
1304
497
  // Base only.
1305
497
  Base = N;
1306
497
  OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1307
497
  return true;
1308
497
}
ARMISelDAGToDAG.cpp:bool (anonymous namespace)::ARMDAGToDAGISel::SelectT2AddrModeImm7<0u>(llvm::SDValue, llvm::SDValue&, llvm::SDValue&)
Line
Count
Source
1283
9
                                           SDValue &Base, SDValue &OffImm) {
1284
9
  if (N.getOpcode() == ISD::SUB ||
1285
9
      CurDAG->isBaseWithConstantOffset(N)) {
1286
4
    if (auto RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1287
4
      int RHSC = (int)RHS->getZExtValue();
1288
4
      if (N.getOpcode() == ISD::SUB)
1289
0
        RHSC = -RHSC;
1290
4
1291
4
      if (isShiftedInt<7, Shift>(RHSC)) {
1292
4
        Base = N.getOperand(0);
1293
4
        if (Base.getOpcode() == ISD::FrameIndex) {
1294
2
          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1295
2
          Base = CurDAG->getTargetFrameIndex(
1296
2
            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1297
2
        }
1298
4
        OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1299
4
        return true;
1300
4
      }
1301
5
    }
1302
4
  }
1303
5
1304
5
  // Base only.
1305
5
  Base = N;
1306
5
  OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1307
5
  return true;
1308
5
}
ARMISelDAGToDAG.cpp:bool (anonymous namespace)::ARMDAGToDAGISel::SelectT2AddrModeImm7<1u>(llvm::SDValue, llvm::SDValue&, llvm::SDValue&)
Line
Count
Source
1283
118
                                           SDValue &Base, SDValue &OffImm) {
1284
118
  if (N.getOpcode() == ISD::SUB ||
1285
118
      CurDAG->isBaseWithConstantOffset(N)) {
1286
54
    if (auto RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1287
54
      int RHSC = (int)RHS->getZExtValue();
1288
54
      if (N.getOpcode() == ISD::SUB)
1289
0
        RHSC = -RHSC;
1290
54
1291
54
      if (isShiftedInt<7, Shift>(RHSC)) {
1292
18
        Base = N.getOperand(0);
1293
18
        if (Base.getOpcode() == ISD::FrameIndex) {
1294
2
          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1295
2
          Base = CurDAG->getTargetFrameIndex(
1296
2
            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1297
2
        }
1298
18
        OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1299
18
        return true;
1300
18
      }
1301
100
    }
1302
54
  }
1303
100
1304
100
  // Base only.
1305
100
  Base = N;
1306
100
  OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1307
100
  return true;
1308
100
}
ARMISelDAGToDAG.cpp:bool (anonymous namespace)::ARMDAGToDAGISel::SelectT2AddrModeImm7<2u>(llvm::SDValue, llvm::SDValue&, llvm::SDValue&)
Line
Count
Source
1283
432
                                           SDValue &Base, SDValue &OffImm) {
1284
432
  if (N.getOpcode() == ISD::SUB ||
1285
432
      CurDAG->isBaseWithConstantOffset(N)) {
1286
110
    if (auto RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1287
110
      int RHSC = (int)RHS->getZExtValue();
1288
110
      if (N.getOpcode() == ISD::SUB)
1289
0
        RHSC = -RHSC;
1290
110
1291
110
      if (isShiftedInt<7, Shift>(RHSC)) {
1292
40
        Base = N.getOperand(0);
1293
40
        if (Base.getOpcode() == ISD::FrameIndex) {
1294
2
          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1295
2
          Base = CurDAG->getTargetFrameIndex(
1296
2
            FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1297
2
        }
1298
40
        OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1299
40
        return true;
1300
40
      }
1301
392
    }
1302
110
  }
1303
392
1304
392
  // Base only.
1305
392
  Base = N;
1306
392
  OffImm  = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1307
392
  return true;
1308
392
}
1309
1310
bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1311
                                            SDValue &Base,
1312
115k
                                            SDValue &OffReg, SDValue &ShImm) {
1313
115k
  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1314
115k
  if (N.getOpcode() != ISD::ADD && 
!CurDAG->isBaseWithConstantOffset(N)47.1k
)
1315
46.4k
    return false;
1316
69.1k
1317
69.1k
  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1318
69.1k
  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1319
64.6k
    int RHSC = (int)RHS->getZExtValue();
1320
64.6k
    if (RHSC >= 0 && 
RHSC < 0x100064.0k
) // 12 bits (unsigned)
1321
63.3k
      return false;
1322
1.32k
    else if (RHSC < 0 && 
RHSC >= -255646
) // 8 bits
1323
637
      return false;
1324
5.20k
  }
1325
5.20k
1326
5.20k
  // Look for (R + R) or (R + (R << [1,2,3])).
1327
5.20k
  unsigned ShAmt = 0;
1328
5.20k
  Base   = N.getOperand(0);
1329
5.20k
  OffReg = N.getOperand(1);
1330
5.20k
1331
5.20k
  // Swap if it is ((R << c) + R).
1332
5.20k
  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1333
5.20k
  if (ShOpcVal != ARM_AM::lsl) {
1334
3.64k
    ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1335
3.64k
    if (ShOpcVal == ARM_AM::lsl)
1336
0
      std::swap(Base, OffReg);
1337
3.64k
  }
1338
5.20k
1339
5.20k
  if (ShOpcVal == ARM_AM::lsl) {
1340
1.56k
    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1341
1.56k
    // it.
1342
1.56k
    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1343
1.56k
      ShAmt = Sh->getZExtValue();
1344
1.56k
      if (ShAmt < 4 && 
isShifterOpProfitable(OffReg, ShOpcVal, ShAmt)1.51k
)
1345
1.51k
        OffReg = OffReg.getOperand(0);
1346
49
      else {
1347
49
        ShAmt = 0;
1348
49
      }
1349
1.56k
    }
1350
1.56k
  }
1351
5.20k
1352
5.20k
  // If OffReg is a multiply-by-constant and it's profitable to extract a shift
1353
5.20k
  // and use it in a shifted operand do so.
1354
5.20k
  if (OffReg.getOpcode() == ISD::MUL && 
N.hasOneUse()12
) {
1355
11
    unsigned PowerOfTwo = 0;
1356
11
    SDValue NewMulConst;
1357
11
    if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1358
4
      HandleSDNode Handle(OffReg);
1359
4
      replaceDAGValue(OffReg.getOperand(1), NewMulConst);
1360
4
      OffReg = Handle.getValue();
1361
4
      ShAmt = PowerOfTwo;
1362
4
    }
1363
11
  }
1364
5.20k
1365
5.20k
  ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
1366
5.20k
1367
5.20k
  return true;
1368
5.20k
}
1369
1370
bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1371
561
                                                SDValue &OffImm) {
1372
561
  // This *must* succeed since it's used for the irreplaceable ldrex and strex
1373
561
  // instructions.
1374
561
  Base = N;
1375
561
  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1376
561
1377
561
  if (N.getOpcode() != ISD::ADD || 
!CurDAG->isBaseWithConstantOffset(N)107
)
1378
454
    return true;
1379
107
1380
107
  ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1381
107
  if (!RHS)
1382
0
    return true;
1383
107
1384
107
  uint32_t RHSC = (int)RHS->getZExtValue();
1385
107
  if (RHSC > 1020 || 
RHSC % 4 != 0101
)
1386
10
    return true;
1387
97
1388
97
  Base = N.getOperand(0);
1389
97
  if (Base.getOpcode() == ISD::FrameIndex) {
1390
8
    int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1391
8
    Base = CurDAG->getTargetFrameIndex(
1392
8
        FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1393
8
  }
1394
97
1395
97
  OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
1396
97
  return true;
1397
97
}
1398
1399
//===--------------------------------------------------------------------===//
1400
1401
/// getAL - Returns a ARMCC::AL immediate node.
1402
17.7k
static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) {
1403
17.7k
  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32);
1404
17.7k
}
1405
1406
1.25k
void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
1407
1.25k
  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
1408
1.25k
  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
1409
1.25k
}
1410
1411
8.08k
bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) {
1412
8.08k
  LoadSDNode *LD = cast<LoadSDNode>(N);
1413
8.08k
  ISD::MemIndexedMode AM = LD->getAddressingMode();
1414
8.08k
  if (AM == ISD::UNINDEXED)
1415
8.03k
    return false;
1416
46
1417
46
  EVT LoadedVT = LD->getMemoryVT();
1418
46
  SDValue Offset, AMOpc;
1419
46
  bool isPre = (AM == ISD::PRE_INC) || 
(AM == ISD::PRE_DEC)25
;
1420
46
  unsigned Opcode = 0;
1421
46
  bool Match = false;
1422
46
  if (LoadedVT == MVT::i32 && 
isPre30
&&
1423
46
      
SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)16
) {
1424
7
    Opcode = ARM::LDR_PRE_IMM;
1425
7
    Match = true;
1426
39
  } else if (LoadedVT == MVT::i32 && 
!isPre23
&&
1427
39
      
SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)14
) {
1428
9
    Opcode = ARM::LDR_POST_IMM;
1429
9
    Match = true;
1430
30
  } else if (LoadedVT == MVT::i32 &&
1431
30
      
SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)14
) {
1432
14
    Opcode = isPre ? 
ARM::LDR_PRE_REG9
:
ARM::LDR_POST_REG5
;
1433
14
    Match = true;
1434
14
1435
16
  } else if (LoadedVT == MVT::i16 &&
1436
16
             
SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)4
) {
1437
4
    Match = true;
1438
4
    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1439
4
      ? 
(isPre 0
?
ARM::LDRSH_PRE0
:
ARM::LDRSH_POST0
)
1440
4
      : (isPre ? ARM::LDRH_PRE : 
ARM::LDRH_POST0
);
1441
12
  } else if (LoadedVT == MVT::i8 || 
LoadedVT == MVT::i10
) {
1442
12
    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1443
5
      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1444
5
        Match = true;
1445
5
        Opcode = isPre ? 
ARM::LDRSB_PRE0
: ARM::LDRSB_POST;
1446
5
      }
1447
7
    } else {
1448
7
      if (isPre &&
1449
7
          SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1450
7
        Match = true;
1451
7
        Opcode = ARM::LDRB_PRE_IMM;
1452
7
      } else 
if (0
!isPre0
&&
1453
0
                  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1454
0
        Match = true;
1455
0
        Opcode = ARM::LDRB_POST_IMM;
1456
0
      } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1457
0
        Match = true;
1458
0
        Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1459
0
      }
1460
7
    }
1461
12
  }
1462
46
1463
46
  if (Match) {
1464
46
    if (Opcode == ARM::LDR_PRE_IMM || 
Opcode == ARM::LDRB_PRE_IMM39
) {
1465
14
      SDValue Chain = LD->getChain();
1466
14
      SDValue Base = LD->getBasePtr();
1467
14
      SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
1468
14
                       CurDAG->getRegister(0, MVT::i32), Chain };
1469
14
      SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1470
14
                                           MVT::Other, Ops);
1471
14
      transferMemOperands(N, New);
1472
14
      ReplaceNode(N, New);
1473
14
      return true;
1474
32
    } else {
1475
32
      SDValue Chain = LD->getChain();
1476
32
      SDValue Base = LD->getBasePtr();
1477
32
      SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
1478
32
                       CurDAG->getRegister(0, MVT::i32), Chain };
1479
32
      SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1480
32
                                           MVT::Other, Ops);
1481
32
      transferMemOperands(N, New);
1482
32
      ReplaceNode(N, New);
1483
32
      return true;
1484
32
    }
1485
0
  }
1486
0
1487
0
  return false;
1488
0
}
1489
1490
2.11k
bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) {
1491
2.11k
  LoadSDNode *LD = cast<LoadSDNode>(N);
1492
2.11k
  EVT LoadedVT = LD->getMemoryVT();
1493
2.11k
  ISD::MemIndexedMode AM = LD->getAddressingMode();
1494
2.11k
  if (AM != ISD::POST_INC || 
LD->getExtensionType() != ISD::NON_EXTLOAD5
||
1495
2.11k
      
LoadedVT.getSimpleVT().SimpleTy != MVT::i325
)
1496
2.10k
    return false;
1497
5
1498
5
  auto *COffs = dyn_cast<ConstantSDNode>(LD->getOffset());
1499
5
  if (!COffs || COffs->getZExtValue() != 4)
1500
0
    return false;
1501
5
1502
5
  // A T1 post-indexed load is just a single register LDM: LDM r0!, {r1}.
1503
5
  // The encoding of LDM is not how the rest of ISel expects a post-inc load to
1504
5
  // look however, so we use a pseudo here and switch it for a tLDMIA_UPD after
1505
5
  // ISel.
1506
5
  SDValue Chain = LD->getChain();
1507
5
  SDValue Base = LD->getBasePtr();
1508
5
  SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)),
1509
5
                   CurDAG->getRegister(0, MVT::i32), Chain };
1510
5
  SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32,
1511
5
                                       MVT::i32, MVT::Other, Ops);
1512
5
  transferMemOperands(N, New);
1513
5
  ReplaceNode(N, New);
1514
5
  return true;
1515
5
}
1516
1517
53.8k
bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) {
1518
53.8k
  LoadSDNode *LD = cast<LoadSDNode>(N);
1519
53.8k
  ISD::MemIndexedMode AM = LD->getAddressingMode();
1520
53.8k
  if (AM == ISD::UNINDEXED)
1521
52.6k
    return false;
1522
1.20k
1523
1.20k
  EVT LoadedVT = LD->getMemoryVT();
1524
1.20k
  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1525
1.20k
  SDValue Offset;
1526
1.20k
  bool isPre = (AM == ISD::PRE_INC) || 
(AM == ISD::PRE_DEC)606
;
1527
1.20k
  unsigned Opcode = 0;
1528
1.20k
  bool Match = false;
1529
1.20k
  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1530
1.20k
    switch (LoadedVT.getSimpleVT().SimpleTy) {
1531
1.20k
    case MVT::i32:
1532
610
      Opcode = isPre ? 
ARM::t2LDR_PRE403
:
ARM::t2LDR_POST207
;
1533
610
      break;
1534
1.20k
    case MVT::i16:
1535
63
      if (isSExtLd)
1536
51
        Opcode = isPre ? 
ARM::t2LDRSH_PRE35
:
ARM::t2LDRSH_POST16
;
1537
12
      else
1538
12
        Opcode = isPre ? 
ARM::t2LDRH_PRE0
: ARM::t2LDRH_POST;
1539
63
      break;
1540
1.20k
    case MVT::i8:
1541
528
    case MVT::i1:
1542
528
      if (isSExtLd)
1543
95
        Opcode = isPre ? 
ARM::t2LDRSB_PRE52
:
ARM::t2LDRSB_POST43
;
1544
433
      else
1545
433
        Opcode = isPre ? 
ARM::t2LDRB_PRE165
:
ARM::t2LDRB_POST268
;
1546
528
      break;
1547
528
    default:
1548
0
      return false;
1549
1.20k
    }
1550
1.20k
    Match = true;
1551
1.20k
  }
1552
1.20k
1553
1.20k
  if (Match) {
1554
1.20k
    SDValue Chain = LD->getChain();
1555
1.20k
    SDValue Base = LD->getBasePtr();
1556
1.20k
    SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
1557
1.20k
                     CurDAG->getRegister(0, MVT::i32), Chain };
1558
1.20k
    SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1559
1.20k
                                         MVT::Other, Ops);
1560
1.20k
    transferMemOperands(N, New);
1561
1.20k
    ReplaceNode(N, New);
1562
1.20k
    return true;
1563
1.20k
  }
1564
0
1565
0
  return false;
1566
0
}
1567
1568
/// Form a GPRPair pseudo register from a pair of GPR regs.
1569
117
SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
1570
117
  SDLoc dl(V0.getNode());
1571
117
  SDValue RegClass =
1572
117
    CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1573
117
  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1574
117
  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1575
117
  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1576
117
  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1577
117
}
1578
1579
/// Form a D register from a pair of S registers.
1580
32
SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1581
32
  SDLoc dl(V0.getNode());
1582
32
  SDValue RegClass =
1583
32
    CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1584
32
  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1585
32
  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1586
32
  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1587
32
  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1588
32
}
1589
1590
/// Form a quad register from a pair of D registers.
1591
508
SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1592
508
  SDLoc dl(V0.getNode());
1593
508
  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1594
508
                                               MVT::i32);
1595
508
  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1596
508
  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1597
508
  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1598
508
  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1599
508
}
1600
1601
/// Form 4 consecutive D registers from a pair of Q registers.
1602
32
SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1603
32
  SDLoc dl(V0.getNode());
1604
32
  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1605
32
                                               MVT::i32);
1606
32
  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1607
32
  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1608
32
  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1609
32
  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1610
32
}
1611
1612
/// Form 4 consecutive S registers.
1613
SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1614
119
                                   SDValue V2, SDValue V3) {
1615
119
  SDLoc dl(V0.getNode());
1616
119
  SDValue RegClass =
1617
119
    CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1618
119
  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1619
119
  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1620
119
  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1621
119
  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1622
119
  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1623
119
                                    V2, SubReg2, V3, SubReg3 };
1624
119
  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1625
119
}
1626
1627
/// Form 4 consecutive D registers.
1628
SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1629
75
                                   SDValue V2, SDValue V3) {
1630
75
  SDLoc dl(V0.getNode());
1631
75
  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1632
75
                                               MVT::i32);
1633
75
  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1634
75
  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1635
75
  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1636
75
  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1637
75
  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1638
75
                                    V2, SubReg2, V3, SubReg3 };
1639
75
  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1640
75
}
1641
1642
/// Form 4 consecutive Q registers.
1643
SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1644
54
                                   SDValue V2, SDValue V3) {
1645
54
  SDLoc dl(V0.getNode());
1646
54
  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1647
54
                                               MVT::i32);
1648
54
  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1649
54
  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1650
54
  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1651
54
  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1652
54
  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1653
54
                                    V2, SubReg2, V3, SubReg3 };
1654
54
  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1655
54
}
1656
1657
/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1658
/// of a NEON VLD or VST instruction.  The supported values depend on the
1659
/// number of registers being loaded.
1660
SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1661
1.11k
                                       unsigned NumVecs, bool is64BitVector) {
1662
1.11k
  unsigned NumRegs = NumVecs;
1663
1.11k
  if (!is64BitVector && 
NumVecs < 3899
)
1664
845
    NumRegs *= 2;
1665
1.11k
1666
1.11k
  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1667
1.11k
  if (Alignment >= 32 && 
NumRegs == 463
)
1668
34
    Alignment = 32;
1669
1.07k
  else if (Alignment >= 16 && 
(155
NumRegs == 2155
||
NumRegs == 442
))
1670
123
    Alignment = 16;
1671
956
  else if (Alignment >= 8)
1672
93
    Alignment = 8;
1673
863
  else
1674
863
    Alignment = 0;
1675
1.11k
1676
1.11k
  return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1677
1.11k
}
1678
1679
static bool isVLDfixed(unsigned Opc)
1680
387
{
1681
387
  switch (Opc) {
1682
387
  
default: return false3
;
1683
387
  
case ARM::VLD1d8wb_fixed : return true20
;
1684
387
  
case ARM::VLD1d16wb_fixed : return true3
;
1685
387
  
case ARM::VLD1d64Qwb_fixed : return true0
;
1686
387
  
case ARM::VLD1d32wb_fixed : return true2
;
1687
387
  
case ARM::VLD1d64wb_fixed : return true8
;
1688
387
  
case ARM::VLD1d64TPseudoWB_fixed : return true4
;
1689
387
  
case ARM::VLD1d64QPseudoWB_fixed : return true2
;
1690
387
  
case ARM::VLD1q8wb_fixed : return true143
;
1691
387
  
case ARM::VLD1q16wb_fixed : return true7
;
1692
387
  
case ARM::VLD1q32wb_fixed : return true165
;
1693
387
  
case ARM::VLD1q64wb_fixed : return true22
;
1694
387
  
case ARM::VLD1DUPd8wb_fixed : return true0
;
1695
387
  
case ARM::VLD1DUPd16wb_fixed : return true0
;
1696
387
  
case ARM::VLD1DUPd32wb_fixed : return true0
;
1697
387
  
case ARM::VLD1DUPq8wb_fixed : return true0
;
1698
387
  
case ARM::VLD1DUPq16wb_fixed : return true0
;
1699
387
  
case ARM::VLD1DUPq32wb_fixed : return true0
;
1700
387
  
case ARM::VLD2d8wb_fixed : return true0
;
1701
387
  
case ARM::VLD2d16wb_fixed : return true0
;
1702
387
  
case ARM::VLD2d32wb_fixed : return true2
;
1703
387
  
case ARM::VLD2q8PseudoWB_fixed : return true1
;
1704
387
  
case ARM::VLD2q16PseudoWB_fixed : return true0
;
1705
387
  
case ARM::VLD2q32PseudoWB_fixed : return true5
;
1706
387
  
case ARM::VLD2DUPd8wb_fixed : return true0
;
1707
387
  
case ARM::VLD2DUPd16wb_fixed : return true0
;
1708
387
  
case ARM::VLD2DUPd32wb_fixed : return true0
;
1709
387
  }
1710
387
}
1711
1712
static bool isVSTfixed(unsigned Opc)
1713
442
{
1714
442
  switch (Opc) {
1715
442
  
default: return false2
;
1716
442
  
case ARM::VST1d8wb_fixed : return true18
;
1717
442
  
case ARM::VST1d16wb_fixed : return true8
;
1718
442
  
case ARM::VST1d32wb_fixed : return true12
;
1719
442
  
case ARM::VST1d64wb_fixed : return true4
;
1720
442
  
case ARM::VST1q8wb_fixed : return true149
;
1721
442
  
case ARM::VST1q16wb_fixed : return true18
;
1722
442
  
case ARM::VST1q32wb_fixed : return true141
;
1723
442
  
case ARM::VST1q64wb_fixed : return true82
;
1724
442
  
case ARM::VST1d64TPseudoWB_fixed : return true2
;
1725
442
  
case ARM::VST1d64QPseudoWB_fixed : return true2
;
1726
442
  
case ARM::VST2d8wb_fixed : return true1
;
1727
442
  
case ARM::VST2d16wb_fixed : return true1
;
1728
442
  
case ARM::VST2d32wb_fixed : return true0
;
1729
442
  
case ARM::VST2q8PseudoWB_fixed : return true0
;
1730
442
  
case ARM::VST2q16PseudoWB_fixed : return true0
;
1731
442
  
case ARM::VST2q32PseudoWB_fixed : return true2
;
1732
442
  }
1733
442
}
1734
1735
// Get the register stride update opcode of a VLD/VST instruction that
1736
// is otherwise equivalent to the given fixed stride updating instruction.
1737
486
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1738
486
  assert((isVLDfixed(Opc) || isVSTfixed(Opc))
1739
486
    && "Incorrect fixed stride updating instruction.");
1740
486
  switch (Opc) {
1741
486
  
default: break0
;
1742
486
  
case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register13
;
1743
486
  
case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register0
;
1744
486
  
case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register2
;
1745
486
  
case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register8
;
1746
486
  
case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register83
;
1747
486
  
case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register0
;
1748
486
  
case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register107
;
1749
486
  
case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register5
;
1750
486
  
case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register0
;
1751
486
  
case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register0
;
1752
486
  
case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register2
;
1753
486
  
case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register1
;
1754
486
  
case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register1
;
1755
486
  
case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register0
;
1756
486
  
case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register0
;
1757
486
  
case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register1
;
1758
486
  
case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register0
;
1759
486
  
case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register0
;
1760
486
1761
486
  
case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register10
;
1762
486
  
case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register8
;
1763
486
  
case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register10
;
1764
486
  
case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register4
;
1765
486
  
case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register84
;
1766
486
  
case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register5
;
1767
486
  
case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register77
;
1768
486
  
case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register60
;
1769
486
  
case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register1
;
1770
486
  
case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register1
;
1771
486
1772
486
  
case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register0
;
1773
486
  
case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register0
;
1774
486
  
case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register0
;
1775
486
  
case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register1
;
1776
486
  
case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register0
;
1777
486
  
case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register0
;
1778
486
1779
486
  
case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register1
;
1780
486
  
case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register0
;
1781
486
  
case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register0
;
1782
486
  
case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register0
;
1783
486
  
case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register0
;
1784
486
  
case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register0
;
1785
486
1786
486
  
case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register1
;
1787
486
  
case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register0
;
1788
486
  
case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register0
;
1789
0
  }
1790
0
  return Opc; // If not one we handle, return it unchanged.
1791
0
}
1792
1793
/// Returns true if the given increment is a Constant known to be equal to the
1794
/// access size performed by a NEON load/store. This means the "[rN]!" form can
1795
/// be used.
1796
852
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs) {
1797
852
  auto C = dyn_cast<ConstantSDNode>(Inc);
1798
852
  return C && 
C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs810
;
1799
852
}
1800
1801
void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1802
                                const uint16_t *DOpcodes,
1803
                                const uint16_t *QOpcodes0,
1804
550
                                const uint16_t *QOpcodes1) {
1805
550
  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1806
550
  SDLoc dl(N);
1807
550
1808
550
  SDValue MemAddr, Align;
1809
550
  bool IsIntrinsic = !isUpdating;  // By coincidence, all supported updating
1810
550
                                   // nodes are not intrinsics.
1811
550
  unsigned AddrOpIdx = IsIntrinsic ? 
2159
:
1391
;
1812
550
  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1813
0
    return;
1814
550
1815
550
  SDValue Chain = N->getOperand(0);
1816
550
  EVT VT = N->getValueType(0);
1817
550
  bool is64BitVector = VT.is64BitVector();
1818
550
  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1819
550
1820
550
  unsigned OpcodeIndex;
1821
550
  switch (VT.getSimpleVT().SimpleTy) {
1822
550
  
default: 0
llvm_unreachable0
("unhandled vld type");
1823
550
    // Double-register operations:
1824
550
  
case MVT::v8i8: OpcodeIndex = 0; break43
;
1825
550
  case MVT::v4f16:
1826
14
  case MVT::v4i16: OpcodeIndex = 1; break;
1827
29
  case MVT::v2f32:
1828
29
  case MVT::v2i32: OpcodeIndex = 2; break;
1829
29
  
case MVT::v1i64: OpcodeIndex = 3; break23
;
1830
29
    // Quad-register operations:
1831
163
  case MVT::v16i8: OpcodeIndex = 0; break;
1832
29
  case MVT::v8f16:
1833
21
  case MVT::v8i16: OpcodeIndex = 1; break;
1834
227
  case MVT::v4f32:
1835
227
  case MVT::v4i32: OpcodeIndex = 2; break;
1836
227
  case MVT::v2f64:
1837
30
  case MVT::v2i64: OpcodeIndex = 3; break;
1838
550
  }
1839
550
1840
550
  EVT ResTy;
1841
550
  if (NumVecs == 1)
1842
438
    ResTy = VT;
1843
112
  else {
1844
112
    unsigned ResTyElts = (NumVecs == 3) ? 
454
:
NumVecs58
;
1845
112
    if (!is64BitVector)
1846
48
      ResTyElts *= 2;
1847
112
    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1848
112
  }
1849
550
  std::vector<EVT> ResTys;
1850
550
  ResTys.push_back(ResTy);
1851
550
  if (isUpdating)
1852
391
    ResTys.push_back(MVT::i32);
1853
550
  ResTys.push_back(MVT::Other);
1854
550
1855
550
  SDValue Pred = getAL(CurDAG, dl);
1856
550
  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1857
550
  SDNode *VLd;
1858
550
  SmallVector<SDValue, 7> Ops;
1859
550
1860
550
  // Double registers and VLD1/VLD2 quad registers are directly supported.
1861
550
  if (is64BitVector || 
NumVecs <= 2441
) {
1862
522
    unsigned Opc = (is64BitVector ? 
DOpcodes[OpcodeIndex]109
:
1863
522
                    
QOpcodes0[OpcodeIndex]413
);
1864
522
    Ops.push_back(MemAddr);
1865
522
    Ops.push_back(Align);
1866
522
    if (isUpdating) {
1867
387
      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1868
387
      bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1869
387
      if (!IsImmUpdate) {
1870
225
        // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1871
225
        // check for the opcode rather than the number of vector elements.
1872
225
        if (isVLDfixed(Opc))
1873
222
          Opc = getVLDSTRegisterUpdateOpcode(Opc);
1874
225
        Ops.push_back(Inc);
1875
225
      // VLD1/VLD2 fixed increment does not need Reg0 so only include it in
1876
225
      // the operands if not such an opcode.
1877
225
      } else 
if (162
!isVLDfixed(Opc)162
)
1878
0
        Ops.push_back(Reg0);
1879
387
    }
1880
522
    Ops.push_back(Pred);
1881
522
    Ops.push_back(Reg0);
1882
522
    Ops.push_back(Chain);
1883
522
    VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1884
522
1885
522
  } else {
1886
28
    // Otherwise, quad registers are loaded with two separate instructions,
1887
28
    // where one loads the even registers and the other loads the odd registers.
1888
28
    EVT AddrTy = MemAddr.getValueType();
1889
28
1890
28
    // Load the even subregs.  This is always an updating load, so that it
1891
28
    // provides the address to the second load for the odd subregs.
1892
28
    SDValue ImplDef =
1893
28
      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1894
28
    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1895
28
    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1896
28
                                          ResTy, AddrTy, MVT::Other, OpsA);
1897
28
    Chain = SDValue(VLdA, 2);
1898
28
1899
28
    // Load the odd subregs.
1900
28
    Ops.push_back(SDValue(VLdA, 1));
1901
28
    Ops.push_back(Align);
1902
28
    if (isUpdating) {
1903
4
      SDValue Inc = N->getOperand(AddrOpIdx + 1);
1904
4
      assert(isa<ConstantSDNode>(Inc.getNode()) &&
1905
4
             "only constant post-increment update allowed for VLD3/4");
1906
4
      (void)Inc;
1907
4
      Ops.push_back(Reg0);
1908
4
    }
1909
28
    Ops.push_back(SDValue(VLdA, 0));
1910
28
    Ops.push_back(Pred);
1911
28
    Ops.push_back(Reg0);
1912
28
    Ops.push_back(Chain);
1913
28
    VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
1914
28
  }
1915
550
1916
550
  // Transfer memoperands.
1917
550
  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1918
550
  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {MemOp});
1919
550
1920
550
  if (NumVecs == 1) {
1921
438
    ReplaceNode(N, VLd);
1922
438
    return;
1923
438
  }
1924
112
1925
112
  // Extract out the subregisters.
1926
112
  SDValue SuperReg = SDValue(VLd, 0);
1927
112
  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
1928
112
                    ARM::qsub_3 == ARM::qsub_0 + 3,
1929
112
                "Unexpected subreg numbering");
1930
112
  unsigned Sub0 = (is64BitVector ? 
ARM::dsub_064
:
ARM::qsub_048
);
1931
436
  for (unsigned Vec = 0; Vec < NumVecs; 
++Vec324
)
1932
324
    ReplaceUses(SDValue(N, Vec),
1933
324
                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1934
112
  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1935
112
  if (isUpdating)
1936
21
    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1937
112
  CurDAG->RemoveDeadNode(N);
1938
112
}
1939
1940
void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1941
                                const uint16_t *DOpcodes,
1942
                                const uint16_t *QOpcodes0,
1943
563
                                const uint16_t *QOpcodes1) {
1944
563
  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1945
563
  SDLoc dl(N);
1946
563
1947
563
  SDValue MemAddr, Align;
1948
563
  bool IsIntrinsic = !isUpdating;  // By coincidence, all supported updating
1949
563
                                   // nodes are not intrinsics.
1950
563
  unsigned AddrOpIdx = IsIntrinsic ? 
2119
:
1444
;
1951
563
  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1952
563
  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1953
0
    return;
1954
563
1955
563
  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1956
563
1957
563
  SDValue Chain = N->getOperand(0);
1958
563
  EVT VT = N->getOperand(Vec0Idx).getValueType();
1959
563
  bool is64BitVector = VT.is64BitVector();
1960
563
  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1961
563
1962
563
  unsigned OpcodeIndex;
1963
563
  switch (VT.getSimpleVT().SimpleTy) {
1964
563
  
default: 0
llvm_unreachable0
("unhandled vst type");
1965
563
    // Double-register operations:
1966
563
  
case MVT::v8i8: OpcodeIndex = 0; break40
;
1967
563
  case MVT::v4f16:
1968
17
  case MVT::v4i16: OpcodeIndex = 1; break;
1969
31
  case MVT::v2f32:
1970
31
  case MVT::v2i32: OpcodeIndex = 2; break;
1971
31
  
case MVT::v1i64: OpcodeIndex = 3; break17
;
1972
31
    // Quad-register operations:
1973
159
  case MVT::v16i8: OpcodeIndex = 0; break;
1974
31
  case MVT::v8f16:
1975
31
  case MVT::v8i16: OpcodeIndex = 1; break;
1976
182
  case MVT::v4f32:
1977
182
  case MVT::v4i32: OpcodeIndex = 2; break;
1978
182
  case MVT::v2f64:
1979
86
  case MVT::v2i64: OpcodeIndex = 3; break;
1980
563
  }
1981
563
1982
563
  std::vector<EVT> ResTys;
1983
563
  if (isUpdating)
1984
444
    ResTys.push_back(MVT::i32);
1985
563
  ResTys.push_back(MVT::Other);
1986
563
1987
563
  SDValue Pred = getAL(CurDAG, dl);
1988
563
  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1989
563
  SmallVector<SDValue, 7> Ops;
1990
563
1991
563
  // Double registers and VST1/VST2 quad registers are directly supported.
1992
563
  if (is64BitVector || 
NumVecs <= 2458
) {
1993
537
    SDValue SrcReg;
1994
537
    if (NumVecs == 1) {
1995
468
      SrcReg = N->getOperand(Vec0Idx);
1996
468
    } else 
if (69
is64BitVector69
) {
1997
50
      // Form a REG_SEQUENCE to force register allocation.
1998
50
      SDValue V0 = N->getOperand(Vec0Idx + 0);
1999
50
      SDValue V1 = N->getOperand(Vec0Idx + 1);
2000
50
      if (NumVecs == 2)
2001
14
        SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2002
36
      else {
2003
36
        SDValue V2 = N->getOperand(Vec0Idx + 2);
2004
36
        // If it's a vst3, form a quad D-register and leave the last part as
2005
36
        // an undef.
2006
36
        SDValue V3 = (NumVecs == 3)
2007
36
          ? 
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)22
2008
36
          : 
N->getOperand(Vec0Idx + 3)14
;
2009
36
        SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2010
36
      }
2011
50
    } else {
2012
19
      // Form a QQ register.
2013
19
      SDValue Q0 = N->getOperand(Vec0Idx);
2014
19
      SDValue Q1 = N->getOperand(Vec0Idx + 1);
2015
19
      SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
2016
19
    }
2017
537
2018
537
    unsigned Opc = (is64BitVector ? 
DOpcodes[OpcodeIndex]105
:
2019
537
                    
QOpcodes0[OpcodeIndex]432
);
2020
537
    Ops.push_back(MemAddr);
2021
537
    Ops.push_back(Align);
2022
537
    if (isUpdating) {
2023
442
      SDValue Inc = N->getOperand(AddrOpIdx + 1);
2024
442
      bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
2025
442
      if (!IsImmUpdate) {
2026
262
        // We use a VST1 for v1i64 even if the pseudo says VST2/3/4, so
2027
262
        // check for the opcode rather than the number of vector elements.
2028
262
        if (isVSTfixed(Opc))
2029
261
          Opc = getVLDSTRegisterUpdateOpcode(Opc);
2030
262
        Ops.push_back(Inc);
2031
262
      }
2032
180
      // VST1/VST2 fixed increment does not need Reg0 so only include it in
2033
180
      // the operands if not such an opcode.
2034
180
      else if (!isVSTfixed(Opc))
2035
1
        Ops.push_back(Reg0);
2036
442
    }
2037
537
    Ops.push_back(SrcReg);
2038
537
    Ops.push_back(Pred);
2039
537
    Ops.push_back(Reg0);
2040
537
    Ops.push_back(Chain);
2041
537
    SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2042
537
2043
537
    // Transfer memoperands.
2044
537
    CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {MemOp});
2045
537
2046
537
    ReplaceNode(N, VSt);
2047
537
    return;
2048
537
  }
2049
26
2050
26
  // Otherwise, quad registers are stored with two separate instructions,
2051
26
  // where one stores the even registers and the other stores the odd registers.
2052
26
2053
26
  // Form the QQQQ REG_SEQUENCE.
2054
26
  SDValue V0 = N->getOperand(Vec0Idx + 0);
2055
26
  SDValue V1 = N->getOperand(Vec0Idx + 1);
2056
26
  SDValue V2 = N->getOperand(Vec0Idx + 2);
2057
26
  SDValue V3 = (NumVecs == 3)
2058
26
    ? 
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)9
2059
26
    : 
N->getOperand(Vec0Idx + 3)17
;
2060
26
  SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2061
26
2062
26
  // Store the even D registers.  This is always an updating store, so that it
2063
26
  // provides the address to the second store for the odd subregs.
2064
26
  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2065
26
  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2066
26
                                        MemAddr.getValueType(),
2067
26
                                        MVT::Other, OpsA);
2068
26
  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {MemOp});
2069
26
  Chain = SDValue(VStA, 1);
2070
26
2071
26
  // Store the odd D registers.
2072
26
  Ops.push_back(SDValue(VStA, 0));
2073
26
  Ops.push_back(Align);
2074
26
  if (isUpdating) {
2075
2
    SDValue Inc = N->getOperand(AddrOpIdx + 1);
2076
2
    assert(isa<ConstantSDNode>(Inc.getNode()) &&
2077
2
           "only constant post-increment update allowed for VST3/4");
2078
2
    (void)Inc;
2079
2
    Ops.push_back(Reg0);
2080
2
  }
2081
26
  Ops.push_back(RegSeq);
2082
26
  Ops.push_back(Pred);
2083
26
  Ops.push_back(Reg0);
2084
26
  Ops.push_back(Chain);
2085
26
  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2086
26
                                        Ops);
2087
26
  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {MemOp});
2088
26
  ReplaceNode(N, VStB);
2089
26
}
2090
2091
void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
2092
                                      unsigned NumVecs,
2093
                                      const uint16_t *DOpcodes,
2094
108
                                      const uint16_t *QOpcodes) {
2095
108
  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2096
108
  SDLoc dl(N);
2097
108
2098
108
  SDValue MemAddr, Align;
2099
108
  bool IsIntrinsic = !isUpdating;  // By coincidence, all supported updating
2100
108
                                   // nodes are not intrinsics.
2101
108
  unsigned AddrOpIdx = IsIntrinsic ? 
293
:
115
;
2102
108
  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2103
108
  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2104
0
    return;
2105
108
2106
108
  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2107
108
2108
108
  SDValue Chain = N->getOperand(0);
2109
108
  unsigned Lane =
2110
108
    cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2111
108
  EVT VT = N->getOperand(Vec0Idx).getValueType();
2112
108
  bool is64BitVector = VT.is64BitVector();
2113
108
2114
108
  unsigned Alignment = 0;
2115
108
  if (NumVecs != 3) {
2116
73
    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2117
73
    unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2118
73
    if (Alignment > NumBytes)
2119
22
      Alignment = NumBytes;
2120
73
    if (Alignment < 8 && 
Alignment < NumBytes60
)
2121
47
      Alignment = 0;
2122
73
    // Alignment must be a power of two; make sure of that.
2123
73
    Alignment = (Alignment & -Alignment);
2124
73
    if (Alignment == 1)
2125
0
      Alignment = 0;
2126
73
  }
2127
108
  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2128
108
2129
108
  unsigned OpcodeIndex;
2130
108
  switch (VT.getSimpleVT().SimpleTy) {
2131
108
  
default: 0
llvm_unreachable0
("unhandled vld/vst lane type");
2132
108
    // Double-register operations:
2133
108
  
case MVT::v8i8: OpcodeIndex = 0; break16
;
2134
108
  case MVT::v4f16:
2135
21
  case MVT::v4i16: OpcodeIndex = 1; break;
2136
30
  case MVT::v2f32:
2137
30
  case MVT::v2i32: OpcodeIndex = 2; break;
2138
30
    // Quad-register operations:
2139
30
  case MVT::v8f16:
2140
22
  case MVT::v8i16: OpcodeIndex = 0; break;
2141
22
  case MVT::v4f32:
2142
19
  case MVT::v4i32: OpcodeIndex = 1; break;
2143
108
  }
2144
108
2145
108
  std::vector<EVT> ResTys;
2146
108
  if (IsLoad) {
2147
75
    unsigned ResTyElts = (NumVecs == 3) ? 
424
:
NumVecs51
;
2148
75
    if (!is64BitVector)
2149
28
      ResTyElts *= 2;
2150
75
    ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2151
75
                                      MVT::i64, ResTyElts));
2152
75
  }
2153
108
  if (isUpdating)
2154
15
    ResTys.push_back(MVT::i32);
2155
108
  ResTys.push_back(MVT::Other);
2156
108
2157
108
  SDValue Pred = getAL(CurDAG, dl);
2158
108
  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2159
108
2160
108
  SmallVector<SDValue, 8> Ops;
2161
108
  Ops.push_back(MemAddr);
2162
108
  Ops.push_back(Align);
2163
108
  if (isUpdating) {
2164
15
    SDValue Inc = N->getOperand(AddrOpIdx + 1);
2165
15
    bool IsImmUpdate =
2166
15
        isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2167
15
    Ops.push_back(IsImmUpdate ? 
Reg08
:
Inc7
);
2168
15
  }
2169
108
2170
108
  SDValue SuperReg;
2171
108
  SDValue V0 = N->getOperand(Vec0Idx + 0);
2172
108
  SDValue V1 = N->getOperand(Vec0Idx + 1);
2173
108
  if (NumVecs == 2) {
2174
41
    if (is64BitVector)
2175
28
      SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2176
13
    else
2177
13
      SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2178
67
  } else {
2179
67
    SDValue V2 = N->getOperand(Vec0Idx + 2);
2180
67
    SDValue V3 = (NumVecs == 3)
2181
67
      ? 
SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)35
2182
67
      : 
N->getOperand(Vec0Idx + 3)32
;
2183
67
    if (is64BitVector)
2184
39
      SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2185
28
    else
2186
28
      SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2187
67
  }
2188
108
  Ops.push_back(SuperReg);
2189
108
  Ops.push_back(getI32Imm(Lane, dl));
2190
108
  Ops.push_back(Pred);
2191
108
  Ops.push_back(Reg0);
2192
108
  Ops.push_back(Chain);
2193
108
2194
108
  unsigned Opc = (is64BitVector ? 
DOpcodes[OpcodeIndex]67
:
2195
108
                                  
QOpcodes[OpcodeIndex]41
);
2196
108
  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2197
108
  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {MemOp});
2198
108
  if (!IsLoad) {
2199
33
    ReplaceNode(N, VLdLn);
2200
33
    return;
2201
33
  }
2202
75
2203
75
  // Extract the subregisters.
2204
75
  SuperReg = SDValue(VLdLn, 0);
2205
75
  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2206
75
                    ARM::qsub_3 == ARM::qsub_0 + 3,
2207
75
                "Unexpected subreg numbering");
2208
75
  unsigned Sub0 = is64BitVector ? 
ARM::dsub_047
:
ARM::qsub_028
;
2209
291
  for (unsigned Vec = 0; Vec < NumVecs; 
++Vec216
)
2210
216
    ReplaceUses(SDValue(N, Vec),
2211
216
                CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2212
75
  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2213
75
  if (isUpdating)
2214
12
    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2215
75
  CurDAG->RemoveDeadNode(N);
2216
75
}
2217
2218
void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic,
2219
                                   bool isUpdating, unsigned NumVecs,
2220
                                   const uint16_t *DOpcodes,
2221
                                   const uint16_t *QOpcodes0,
2222
61
                                   const uint16_t *QOpcodes1) {
2223
61
  assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2224
61
  SDLoc dl(N);
2225
61
2226
61
  SDValue MemAddr, Align;
2227
61
  unsigned AddrOpIdx = IsIntrinsic ? 
224
:
137
;
2228
61
  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2229
0
    return;
2230
61
2231
61
  SDValue Chain = N->getOperand(0);
2232
61
  EVT VT = N->getValueType(0);
2233
61
  bool is64BitVector = VT.is64BitVector();
2234
61
2235
61
  unsigned Alignment = 0;
2236
61
  if (NumVecs != 3) {
2237
53
    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2238
53
    unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2239
53
    if (Alignment > NumBytes)
2240
4
      Alignment = NumBytes;
2241
53
    if (Alignment < 8 && 
Alignment < NumBytes51
)
2242
18
      Alignment = 0;
2243
53
    // Alignment must be a power of two; make sure of that.
2244
53
    Alignment = (Alignment & -Alignment);
2245
53
    if (Alignment == 1)
2246
11
      Alignment = 0;
2247
53
  }
2248
61
  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2249
61
2250
61
  unsigned OpcodeIndex;
2251
61
  switch (VT.getSimpleVT().SimpleTy) {
2252
61
  
default: 0
llvm_unreachable0
("unhandled vld-dup type");
2253
61
  case MVT::v8i8:
2254
20
  case MVT::v16i8: OpcodeIndex = 0; break;
2255
20
  case MVT::v4i16:
2256
15
  case MVT::v8i16:
2257
15
  case MVT::v4f16:
2258
15
  case MVT::v8f16:
2259
15
                  OpcodeIndex = 1; break;
2260
23
  case MVT::v2f32:
2261
23
  case MVT::v2i32:
2262
23
  case MVT::v4f32:
2263
23
  case MVT::v4i32: OpcodeIndex = 2; break;
2264
23
  case MVT::v1f64:
2265
3
  case MVT::v1i64: OpcodeIndex = 3; break;
2266
61
  }
2267
61
2268
61
  unsigned ResTyElts = (NumVecs == 3) ? 
48
:
NumVecs53
;
2269
61
  if (!is64BitVector)
2270
22
    ResTyElts *= 2;
2271
61
  EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
2272
61
2273
61
  std::vector<EVT> ResTys;
2274
61
  ResTys.push_back(ResTy);
2275
61
  if (isUpdating)
2276
8
    ResTys.push_back(MVT::i32);
2277
61
  ResTys.push_back(MVT::Other);
2278
61
2279
61
  SDValue Pred = getAL(CurDAG, dl);
2280
61
  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2281
61
2282
61
  SDNode *VLdDup;
2283
61
  if (is64BitVector || 
NumVecs == 122
) {
2284
52
    SmallVector<SDValue, 6> Ops;
2285
52
    Ops.push_back(MemAddr);
2286
52
    Ops.push_back(Align);
2287
52
    unsigned Opc = is64BitVector ? 
DOpcodes[OpcodeIndex]39
:
2288
52
                                   
QOpcodes0[OpcodeIndex]13
;
2289
52
    if (isUpdating) {
2290
8
      // fixed-stride update instructions don't have an explicit writeback
2291
8
      // operand. It's implicit in the opcode itself.
2292
8
      SDValue Inc = N->getOperand(2);
2293
8
      bool IsImmUpdate =
2294
8
          isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2295
8
      if (NumVecs <= 2 && !IsImmUpdate)
2296
3
        Opc = getVLDSTRegisterUpdateOpcode(Opc);
2297
8
      if (!IsImmUpdate)
2298
3
        Ops.push_back(Inc);
2299
5
      // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2300
5
      else if (NumVecs > 2)
2301
0
        Ops.push_back(Reg0);
2302
8
    }
2303
52
    Ops.push_back(Pred);
2304
52
    Ops.push_back(Reg0);
2305
52
    Ops.push_back(Chain);
2306
52
    VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2307
52
  } else 
if (9
NumVecs == 29
) {
2308
3
    const SDValue OpsA[] = { MemAddr, Align, Pred, Reg0, Chain };
2309
3
    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
2310
3
                                          dl, ResTys, OpsA);
2311
3
2312
3
    Chain = SDValue(VLdA, 1);
2313
3
    const SDValue OpsB[] = { MemAddr, Align, Pred, Reg0, Chain };
2314
3
    VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
2315
6
  } else {
2316
6
    SDValue ImplDef =
2317
6
      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2318
6
    const SDValue OpsA[] = { MemAddr, Align, ImplDef, Pred, Reg0, Chain };
2319
6
    SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
2320
6
                                          dl, ResTys, OpsA);
2321
6
2322
6
    SDValue SuperReg = SDValue(VLdA, 0);
2323
6
    Chain = SDValue(VLdA, 1);
2324
6
    const SDValue OpsB[] = { MemAddr, Align, SuperReg, Pred, Reg0, Chain };
2325
6
    VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
2326
6
  }
2327
61
2328
61
  // Transfer memoperands.
2329
61
  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2330
61
  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {MemOp});
2331
61
2332
61
  // Extract the subregisters.
2333
61
  if (NumVecs == 1) {
2334
34
    ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
2335
34
  } else {
2336
27
    SDValue SuperReg = SDValue(VLdDup, 0);
2337
27
    static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
2338
27
    unsigned SubIdx = is64BitVector ? 
ARM::dsub_018
:
ARM::qsub_09
;
2339
105
    for (unsigned Vec = 0; Vec != NumVecs; 
++Vec78
) {
2340
78
      ReplaceUses(SDValue(N, Vec),
2341
78
                  CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2342
78
    }
2343
27
  }
2344
61
  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2345
61
  if (isUpdating)
2346
8
    ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2347
61
  CurDAG->RemoveDeadNode(N);
2348
61
}
2349
2350
16.7k
bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) {
2351
16.7k
  if (!Subtarget->hasV6T2Ops())
2352
2.50k
    return false;
2353
14.2k
2354
14.2k
  unsigned Opc = isSigned
2355
14.2k
    ? 
(Subtarget->isThumb() 1.12k
?
ARM::t2SBFX1.02k
:
ARM::SBFX102
)
2356
14.2k
    : 
(Subtarget->isThumb() 13.1k
?
ARM::t2UBFX12.6k
:
ARM::UBFX557
);
2357
14.2k
  SDLoc dl(N);
2358
14.2k
2359
14.2k
  // For unsigned extracts, check for a shift right and mask
2360
14.2k
  unsigned And_imm = 0;
2361
14.2k
  if (N->getOpcode() == ISD::AND) {
2362
8.27k
    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2363
6.73k
2364
6.73k
      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2365
6.73k
      if (And_imm & (And_imm + 1))
2366
1.65k
        return false;
2367
5.08k
2368
5.08k
      unsigned Srl_imm = 0;
2369
5.08k
      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
2370
5.08k
                                Srl_imm)) {
2371
630
        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2372
630
2373
630
        // Mask off the unnecessary bits of the AND immediate; normally
2374
630
        // DAGCombine will do this, but that might not happen if
2375
630
        // targetShrinkDemandedConstant chooses a different immediate.
2376
630
        And_imm &= -1U >> Srl_imm;
2377
630
2378
630
        // Note: The width operand is encoded as width-1.
2379
630
        unsigned Width = countTrailingOnes(And_imm) - 1;
2380
630
        unsigned LSB = Srl_imm;
2381
630
2382
630
        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2383
630
2384
630
        if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
2385
1
          // It's cheaper to use a right shift to extract the top bits.
2386
1
          if (Subtarget->isThumb()) {
2387
0
            Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
2388
0
            SDValue Ops[] = { N->getOperand(0).getOperand(0),
2389
0
                              CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2390
0
                              getAL(CurDAG, dl), Reg0, Reg0 };
2391
0
            CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2392
0
            return true;
2393
0
          }
2394
1
2395
1
          // ARM models shift instructions as MOVsi with shifter operand.
2396
1
          ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL);
2397
1
          SDValue ShOpc =
2398
1
            CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl,
2399
1
                                      MVT::i32);
2400
1
          SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2401
1
                            getAL(CurDAG, dl), Reg0, Reg0 };
2402
1
          CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
2403
1
          return true;
2404
1
        }
2405
629
2406
629
        assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2407
629
        SDValue Ops[] = { N->getOperand(0).getOperand(0),
2408
629
                          CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2409
629
                          CurDAG->getTargetConstant(Width, dl, MVT::i32),
2410
629
                          getAL(CurDAG, dl), Reg0 };
2411
629
        CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2412
629
        return true;
2413
629
      }
2414
5.08k
    }
2415
5.98k
    return false;
2416
5.98k
  }
2417
6.01k
2418
6.01k
  // Otherwise, we're looking for a shift of a shift
2419
6.01k
  unsigned Shl_imm = 0;
2420
6.01k
  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2421
12
    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2422
12
    unsigned Srl_imm = 0;
2423
12
    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2424
12
      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2425
12
      // Note: The width operand is encoded as width-1.
2426
12
      unsigned Width = 32 - Srl_imm - 1;
2427
12
      int LSB = Srl_imm - Shl_imm;
2428
12
      if (LSB < 0)
2429
1
        return false;
2430
11
      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2431
11
      assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2432
11
      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2433
11
                        CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2434
11
                        CurDAG->getTargetConstant(Width, dl, MVT::i32),
2435
11
                        getAL(CurDAG, dl), Reg0 };
2436
11
      CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2437
11
      return true;
2438
11
    }
2439
12
  }
2440
6.00k
2441
6.00k
  // Or we are looking for a shift of an and, with a mask operand
2442
6.00k
  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) &&
2443
6.00k
      
isShiftedMask_32(And_imm)67
) {
2444
57
    unsigned Srl_imm = 0;
2445
57
    unsigned LSB = countTrailingZeros(And_imm);
2446
57
    // Shift must be the same as the ands lsb
2447
57
    if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) {
2448
39
      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2449
39
      unsigned MSB = 31 - countLeadingZeros(And_imm);
2450
39
      // Note: The width operand is encoded as width-1.
2451
39
      unsigned Width = MSB - LSB;
2452
39
      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2453
39
      assert(Srl_imm + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2454
39
      SDValue Ops[] = { N->getOperand(0).getOperand(0),
2455
39
                        CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
2456
39
                        CurDAG->getTargetConstant(Width, dl, MVT::i32),
2457
39
                        getAL(CurDAG, dl), Reg0 };
2458
39
      CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2459
39
      return true;
2460
39
    }
2461
5.96k
  }
2462
5.96k
2463
5.96k
  if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
2464
547
    unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2465
547
    unsigned LSB = 0;
2466
547
    if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
2467
547
        
!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB)534
)
2468
534
      return false;
2469
13
2470
13
    if (LSB + Width > 32)
2471
0
      return false;
2472
13
2473
13
    SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2474
13
    assert(LSB + Width <= 32 && "Shouldn't create an invalid ubfx");
2475
13
    SDValue Ops[] = { N->getOperand(0).getOperand(0),
2476
13
                      CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2477
13
                      CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
2478
13
                      getAL(CurDAG, dl), Reg0 };
2479
13
    CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2480
13
    return true;
2481
13
  }
2482
5.41k
2483
5.41k
  return false;
2484
5.41k
}
2485
2486
/// Target-specific DAG combining for ISD::XOR.
2487
/// Target-independent combining lowers SELECT_CC nodes of the form
2488
/// select_cc setg[ge] X,  0,  X, -X
2489
/// select_cc setgt    X, -1,  X, -X
2490
/// select_cc setl[te] X,  0, -X,  X
2491
/// select_cc setlt    X,  1, -X,  X
2492
/// which represent Integer ABS into:
2493
/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2494
/// ARM instruction selection detects the latter and matches it to
2495
/// ARM::ABS or ARM::t2ABS machine node.
2496
3.31k
bool ARMDAGToDAGISel::tryABSOp(SDNode *N){
2497
3.31k
  SDValue XORSrc0 = N->getOperand(0);
2498
3.31k
  SDValue XORSrc1 = N->getOperand(1);
2499
3.31k
  EVT VT = N->getValueType(0);
2500
3.31k
2501
3.31k
  if (Subtarget->isThumb1Only())
2502
362
    return false;
2503
2.95k
2504
2.95k
  if (XORSrc0.getOpcode() != ISD::ADD || 
XORSrc1.getOpcode() != ISD::SRA67
)
2505
2.90k
    return false;
2506
50
2507
50
  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2508
50
  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2509
50
  SDValue SRASrc0 = XORSrc1.getOperand(0);
2510
50
  SDValue SRASrc1 = XORSrc1.getOperand(1);
2511
50
  ConstantSDNode *SRAConstant =  dyn_cast<ConstantSDNode>(SRASrc1);
2512
50
  EVT XType = SRASrc0.getValueType();
2513
50
  unsigned Size = XType.getSizeInBits() - 1;
2514
50
2515
50
  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2516
50
      XType.isInteger() && SRAConstant != nullptr &&
2517
50
      Size == SRAConstant->getZExtValue()) {
2518
50
    unsigned Opcode = Subtarget->isThumb2() ? 
ARM::t2ABS47
:
ARM::ABS3
;
2519
50
    CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2520
50
    return true;
2521
50
  }
2522
0
2523
0
  return false;
2524
0
}
2525
2526
/// We've got special pseudo-instructions for these
2527
6
void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
2528
6
  unsigned Opcode;
2529
6
  EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
2530
6
  if (MemTy == MVT::i8)
2531
2
    Opcode = ARM::CMP_SWAP_8;
2532
4
  else if (MemTy == MVT::i16)
2533
2
    Opcode = ARM::CMP_SWAP_16;
2534
2
  else if (MemTy == MVT::i32)
2535
2
    Opcode = ARM::CMP_SWAP_32;
2536
2
  else
2537
2
    
llvm_unreachable0
("Unknown AtomicCmpSwap type");
2538
6
2539
6
  SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2540
6
                   N->getOperand(0)};
2541
6
  SDNode *CmpSwap = CurDAG->getMachineNode(
2542
6
      Opcode, SDLoc(N),
2543
6
      CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
2544
6
2545
6
  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
2546
6
  CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
2547
6
2548
6
  ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
2549
6
  ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
2550
6
  CurDAG->RemoveDeadNode(N);
2551
6
}
2552
2553
static Optional<std::pair<unsigned, unsigned>>
2554
791
getContiguousRangeOfSetBits(const APInt &A) {
2555
791
  unsigned FirstOne = A.getBitWidth() - A.countLeadingZeros() - 1;
2556
791
  unsigned LastOne = A.countTrailingZeros();
2557
791
  if (A.countPopulation() != (FirstOne - LastOne + 1))
2558
32
    return Optional<std::pair<unsigned,unsigned>>();
2559
759
  return std::make_pair(FirstOne, LastOne);
2560
759
}
2561
2562
38.1k
void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
2563
38.1k
  assert(N->getOpcode() == ARMISD::CMPZ);
2564
38.1k
  SwitchEQNEToPLMI = false;
2565
38.1k
2566
38.1k
  if (!Subtarget->isThumb())
2567
846
    // FIXME: Work out whether it is profitable to do this in A32 mode - LSL and
2568
846
    // LSR don't exist as standalone instructions - they need the barrel shifter.
2569
846
    return;
2570
37.3k
2571
37.3k
  // select (cmpz (and X, C), #0) -> (LSLS X) or (LSRS X) or (LSRS (LSLS X))
2572
37.3k
  SDValue And = N->getOperand(0);
2573
37.3k
  if (!And->hasOneUse())
2574
15.9k
    return;
2575
21.4k
2576
21.4k
  SDValue Zero = N->getOperand(1);
2577
21.4k
  if (!isa<ConstantSDNode>(Zero) || 
!cast<ConstantSDNode>(Zero)->isNullValue()19.8k
||
2578
21.4k
      
And->getOpcode() != ISD::AND17.7k
)
2579
19.7k
    return;
2580
1.67k
  SDValue X = And.getOperand(0);
2581
1.67k
  auto C = dyn_cast<ConstantSDNode>(And.getOperand(1));
2582
1.67k
2583
1.67k
  if (!C)
2584
884
    return;
2585
791
  auto Range = getContiguousRangeOfSetBits(C->getAPIntValue());
2586
791
  if (!Range)
2587
32
    return;
2588
759
2589
759
  // There are several ways to lower this:
2590
759
  SDNode *NewN;
2591
759
  SDLoc dl(N);
2592
759
2593
759
  auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* {
2594
756
    if (Subtarget->isThumb2()) {
2595
714
      Opc = (Opc == ARM::tLSLri) ? 
ARM::t2LSLri707
:
ARM::t2LSRri7
;
2596
714
      SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2597
714
                        getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2598
714
                        CurDAG->getRegister(0, MVT::i32) };
2599
714
      return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2600
714
    } else {
2601
42
      SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
2602
42
                       CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2603
42
                       getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
2604
42
      return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2605
42
    }
2606
756
  };
2607
759
2608
759
  if (Range->second == 0) {
2609
456
    //  1. Mask includes the LSB -> Simply shift the top N bits off
2610
456
    NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2611
456
    ReplaceNode(And.getNode(), NewN);
2612
456
  } else 
if (303
Range->first == 31303
) {
2613
8
    //  2. Mask includes the MSB -> Simply shift the bottom N bits off
2614
8
    NewN = EmitShift(ARM::tLSRri, X, Range->second);
2615
8
    ReplaceNode(And.getNode(), NewN);
2616
295
  } else if (Range->first == Range->second) {
2617
290
    //  3. Only one bit is set. We can shift this into the sign bit and use a
2618
290
    //     PL/MI comparison.
2619
290
    NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2620
290
    ReplaceNode(And.getNode(), NewN);
2621
290
2622
290
    SwitchEQNEToPLMI = true;
2623
290
  } else 
if (5
!Subtarget->hasV6T2Ops()5
) {
2624
1
    //  4. Do a double shift to clear bottom and top bits, but only in
2625
1
    //     thumb-1 mode as in thumb-2 we can use UBFX.
2626
1
    NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2627
1
    NewN = EmitShift(ARM::tLSRri, SDValue(NewN, 0),
2628
1
                     Range->second + (31 - Range->first));
2629
1
    ReplaceNode(And.getNode(), NewN);
2630
1
  }
2631
759
2632
759
}
2633
2634
2.03M
void ARMDAGToDAGISel::Select(SDNode *N) {
2635
2.03M
  SDLoc dl(N);
2636
2.03M
2637
2.03M
  if (N->isMachineOpcode()) {
2638
187
    N->setNodeId(-1);
2639
187
    return;   // Already selected.
2640
187
  }
2641
2.03M
2642
2.03M
  switch (N->getOpcode()) {
2643
2.03M
  
default: break1.70M
;
2644
2.03M
  case ISD::STORE: {
2645
74.7k
    // For Thumb1, match an sp-relative store in C++. This is a little
2646
74.7k
    // unfortunate, but I don't think I can make the chain check work
2647
74.7k
    // otherwise.  (The chain of the store has to be the same as the chain
2648
74.7k
    // of the CopyFromReg, or else we can't replace the CopyFromReg with
2649
74.7k
    // a direct reference to "SP".)
2650
74.7k
    //
2651
74.7k
    // This is only necessary on Thumb1 because Thumb1 sp-relative stores use
2652
74.7k
    // a different addressing mode from other four-byte stores.
2653
74.7k
    //
2654
74.7k
    // This pattern usually comes up with call arguments.
2655
74.7k
    StoreSDNode *ST = cast<StoreSDNode>(N);
2656
74.7k
    SDValue Ptr = ST->getBasePtr();
2657
74.7k
    if (Subtarget->isThumb1Only() && 
ST->isUnindexed()977
) {
2658
966
      int RHSC = 0;
2659
966
      if (Ptr.getOpcode() == ISD::ADD &&
2660
966
          
isScaledConstantInRange(Ptr.getOperand(1), /*Scale=*/4, 0, 256, RHSC)354
)
2661
276
        Ptr = Ptr.getOperand(0);
2662
966
2663
966
      if (Ptr.getOpcode() == ISD::CopyFromReg &&
2664
966
          
cast<RegisterSDNode>(Ptr.getOperand(1))->getReg() == ARM::SP330
&&
2665
966
          
Ptr.getOperand(0) == ST->getChain()86
) {
2666
86
        SDValue Ops[] = {ST->getValue(),
2667
86
                         CurDAG->getRegister(ARM::SP, MVT::i32),
2668
86
                         CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
2669
86
                         getAL(CurDAG, dl),
2670
86
                         CurDAG->getRegister(0, MVT::i32),
2671
86
                         ST->getChain()};
2672
86
        MachineSDNode *ResNode =
2673
86
            CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
2674
86
        MachineMemOperand *MemOp = ST->getMemOperand();
2675
86
        CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
2676
86
        ReplaceNode(N, ResNode);
2677
86
        return;
2678
86
      }
2679
74.6k
    }
2680
74.6k
    break;
2681
74.6k
  }
2682
74.6k
  case ISD::WRITE_REGISTER:
2683
135
    if (tryWriteRegister(N))
2684
132
      return;
2685
3
    break;
2686
95
  case ISD::READ_REGISTER:
2687
95
    if (tryReadRegister(N))
2688
85
      return;
2689
10
    break;
2690
4.95k
  case ISD::INLINEASM:
2691
4.95k
  case ISD::INLINEASM_BR:
2692
4.95k
    if (tryInlineAsm(N))
2693
59
      return;
2694
4.89k
    break;
2695
4.89k
  case ISD::XOR:
2696
3.31k
    // Select special operations if XOR node forms integer ABS pattern
2697
3.31k
    if (tryABSOp(N))
2698
50
      return;
2699
3.26k
    // Other cases are autogenerated.
2700
3.26k
    break;
2701
53.2k
  case ISD::Constant: {
2702
53.2k
    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2703
53.2k
    // If we can't materialize the constant we need to use a literal pool
2704
53.2k
    if (ConstantMaterializationCost(Val) > 2) {
2705
1.62k
      SDValue CPIdx = CurDAG->getTargetConstantPool(
2706
1.62k
          ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
2707
1.62k
          TLI->getPointerTy(CurDAG->getDataLayout()));
2708
1.62k
2709
1.62k
      SDNode *ResNode;
2710
1.62k
      if (Subtarget->isThumb()) {
2711
1.50k
        SDValue Ops[] = {
2712
1.50k
          CPIdx,
2713
1.50k
          getAL(CurDAG, dl),
2714
1.50k
          CurDAG->getRegister(0, MVT::i32),
2715
1.50k
          CurDAG->getEntryNode()
2716
1.50k
        };
2717
1.50k
        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2718
1.50k
                                         Ops);
2719
1.50k
      } else {
2720
115
        SDValue Ops[] = {
2721
115
          CPIdx,
2722
115
          CurDAG->getTargetConstant(0, dl, MVT::i32),
2723
115
          getAL(CurDAG, dl),
2724
115
          CurDAG->getRegister(0, MVT::i32),
2725
115
          CurDAG->getEntryNode()
2726
115
        };
2727
115
        ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2728
115
                                         Ops);
2729
115
      }
2730
1.62k
      // Annotate the Node with memory operand information so that MachineInstr
2731
1.62k
      // queries work properly. This e.g. gives the register allocation the
2732
1.62k
      // required information for rematerialization.
2733
1.62k
      MachineFunction& MF = CurDAG->getMachineFunction();
2734
1.62k
      MachineMemOperand *MemOp =
2735
1.62k
          MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),
2736
1.62k
                                  MachineMemOperand::MOLoad, 4, 4);
2737
1.62k
2738
1.62k
      CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
2739
1.62k
2740
1.62k
      ReplaceNode(N, ResNode);
2741
1.62k
      return;
2742
1.62k
    }
2743
51.6k
2744
51.6k
    // Other cases are autogenerated.
2745
51.6k
    break;
2746
51.6k
  }
2747
51.6k
  case ISD::FrameIndex: {
2748
10.8k
    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2749
10.8k
    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2750
10.8k
    SDValue TFI = CurDAG->getTargetFrameIndex(
2751
10.8k
        FI, TLI->getPointerTy(CurDAG->getDataLayout()));
2752
10.8k
    if (Subtarget->isThumb1Only()) {
2753
331
      // Set the alignment of the frame object to 4, to avoid having to generate
2754
331
      // more than one ADD
2755
331
      MachineFrameInfo &MFI = MF->getFrameInfo();
2756
331
      if (MFI.getObjectAlignment(FI) < 4)
2757
42
        MFI.setObjectAlignment(FI, 4);
2758
331
      CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
2759
331
                           CurDAG->getTargetConstant(0, dl, MVT::i32));
2760
331
      return;
2761
10.5k
    } else {
2762
10.5k
      unsigned Opc = ((Subtarget->isThumb() && 
Subtarget->hasThumb2()9.34k
) ?
2763
9.34k
                      ARM::t2ADDri : 
ARM::ADDri1.17k
);
2764
10.5k
      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
2765
10.5k
                        getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2766
10.5k
                        CurDAG->getRegister(0, MVT::i32) };
2767
10.5k
      CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2768
10.5k
      return;
2769
10.5k
    }
2770
0
  }
2771
5.56k
  case ISD::SRL:
2772
5.56k
    if (tryV6T2BitfieldExtractOp(N, false))
2773
40
      return;
2774
5.52k
    break;
2775
5.52k
  case ISD::SIGN_EXTEND_INREG:
2776
1.48k
  case ISD::SRA:
2777
1.48k
    if (tryV6T2BitfieldExtractOp(N, true))
2778
23
      return;
2779
1.46k
    break;
2780
1.46k
  case ISD::MUL:
2781
971
    if (Subtarget->isThumb1Only())
2782
158
      break;
2783
813
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2784
100
      unsigned RHSV = C->getZExtValue();
2785
100
      if (!RHSV) 
break0
;
2786
100
      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2787
0
        unsigned ShImm = Log2_32(RHSV-1);
2788
0
        if (ShImm >= 32)
2789
0
          break;
2790
0
        SDValue V = N->getOperand(0);
2791
0
        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2792
0
        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2793
0
        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2794
0
        if (Subtarget->isThumb()) {
2795
0
          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2796
0
          CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
2797
0
          return;
2798
0
        } else {
2799
0
          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2800
0
                            Reg0 };
2801
0
          CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
2802
0
          return;
2803
0
        }
2804
100
      }
2805
100
      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2806
0
        unsigned ShImm = Log2_32(RHSV+1);
2807
0
        if (ShImm >= 32)
2808
0
          break;
2809
0
        SDValue V = N->getOperand(0);
2810
0
        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2811
0
        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2812
0
        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2813
0
        if (Subtarget->isThumb()) {
2814
0
          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2815
0
          CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
2816
0
          return;
2817
0
        } else {
2818
0
          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2819
0
                            Reg0 };
2820
0
          CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
2821
0
          return;
2822
0
        }
2823
813
      }
2824
100
    }
2825
813
    break;
2826
9.74k
  case ISD::AND: {
2827
9.74k
    // Check for unsigned bitfield extract
2828
9.74k
    if (tryV6T2BitfieldExtractOp(N, false))
2829
630
      return;
2830
9.11k
2831
9.11k
    // If an immediate is used in an AND node, it is possible that the immediate
2832
9.11k
    // can be more optimally materialized when negated. If this is the case we
2833
9.11k
    // can negate the immediate and use a BIC instead.
2834
9.11k
    auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2835
9.11k
    if (N1C && 
N1C->hasOneUse()7.19k
&&
Subtarget->isThumb()5.64k
) {
2836
5.23k
      uint32_t Imm = (uint32_t) N1C->getZExtValue();
2837
5.23k
2838
5.23k
      // In Thumb2 mode, an AND can take a 12-bit immediate. If this
2839
5.23k
      // immediate can be negated and fit in the immediate operand of
2840
5.23k
      // a t2BIC, don't do any manual transform here as this can be
2841
5.23k
      // handled by the generic ISel machinery.
2842
5.23k
      bool PreferImmediateEncoding =
2843
5.23k
        Subtarget->hasThumb2() && 
(4.96k
is_t2_so_imm(Imm)4.96k
||
is_t2_so_imm_not(Imm)1.95k
);
2844
5.23k
      if (!PreferImmediateEncoding &&
2845
5.23k
          ConstantMaterializationCost(Imm) >
2846
1.25k
              ConstantMaterializationCost(~Imm)) {
2847
228
        // The current immediate costs more to materialize than a negated
2848
228
        // immediate, so negate the immediate and use a BIC.
2849
228
        SDValue NewImm =
2850
228
          CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
2851
228
        // If the new constant didn't exist before, reposition it in the topological
2852
228
        // ordering so it is just before N. Otherwise, don't touch its location.
2853
228
        if (NewImm->getNodeId() == -1)
2854
217
          CurDAG->RepositionNode(N->getIterator(), NewImm.getNode());
2855
228
2856
228
        if (!Subtarget->hasThumb2()) {
2857
39
          SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
2858
39
                           N->getOperand(0), NewImm, getAL(CurDAG, dl),
2859
39
                           CurDAG->getRegister(0, MVT::i32)};
2860
39
          ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
2861
39
          return;
2862
189
        } else {
2863
189
          SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
2864
189
                           CurDAG->getRegister(0, MVT::i32),
2865
189
                           CurDAG->getRegister(0, MVT::i32)};
2866
189
          ReplaceNode(N,
2867
189
                      CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
2868
189
          return;
2869
189
        }
2870
8.88k
      }
2871
5.23k
    }
2872
8.88k
2873
8.88k
    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2874
8.88k
    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2875
8.88k
    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2876
8.88k
    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2877
8.88k
    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2878
8.88k
    EVT VT = N->getValueType(0);
2879
8.88k
    if (VT != MVT::i32)
2880
93
      break;
2881
8.79k
    unsigned Opc = (Subtarget->isThumb() && 
Subtarget->hasThumb2()7.95k
)
2882
8.79k
      ? 
ARM::t2MOVTi167.06k
2883
8.79k
      : 
(Subtarget->hasV6T2Ops() 1.73k
?
ARM::MOVTi16308
:
01.42k
);
2884
8.79k
    if (!Opc)
2885
1.42k
      break;
2886
7.37k
    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2887
7.37k
    N1C = dyn_cast<ConstantSDNode>(N1);
2888
7.37k
    if (!N1C)
2889
1.45k
      break;
2890
5.92k
    if (N0.getOpcode() == ISD::OR && 
N0.getNode()->hasOneUse()70
) {
2891
66
      SDValue N2 = N0.getOperand(1);
2892
66
      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2893
66
      if (!N2C)
2894
59
        break;
2895
7
      unsigned N1CVal = N1C->getZExtValue();
2896
7
      unsigned N2CVal = N2C->getZExtValue();
2897
7
      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2898
7
          (N1CVal & 0xffffU) == 0xffffU &&
2899
7
          (N2CVal & 0xffffU) == 0x0U) {
2900
3
        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2901
3
                                                  dl, MVT::i32);
2902
3
        SDValue Ops[] = { N0.getOperand(0), Imm16,
2903
3
                          getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
2904
3
        ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
2905
3
        return;
2906
3
      }
2907
5.85k
    }
2908
5.85k
2909
5.85k
    break;
2910
5.85k
  }
2911
5.85k
  case ARMISD::UMAAL: {
2912
48
    unsigned Opc = Subtarget->isThumb() ? 
ARM::t2UMAAL44
:
ARM::UMAAL4
;
2913
48
    SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2914
48
                      N->getOperand(2), N->getOperand(3),
2915
48
                      getAL(CurDAG, dl),
2916
48
                      CurDAG->getRegister(0, MVT::i32) };
2917
48
    ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
2918
48
    return;
2919
5.85k
  }
2920
5.85k
  case ARMISD::UMLAL:{
2921
63
    if (Subtarget->isThumb()) {
2922
47
      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2923
47
                        N->getOperand(3), getAL(CurDAG, dl),
2924
47
                        CurDAG->getRegister(0, MVT::i32)};
2925
47
      ReplaceNode(
2926
47
          N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
2927
47
      return;
2928
47
    }else{
2929
16
      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2930
16
                        N->getOperand(3), getAL(CurDAG, dl),
2931
16
                        CurDAG->getRegister(0, MVT::i32),
2932
16
                        CurDAG->getRegister(0, MVT::i32) };
2933
16
      ReplaceNode(N, CurDAG->getMachineNode(
2934
16
                         Subtarget->hasV6Ops() ? 
ARM::UMLAL4
:
ARM::UMLALv512
, dl,
2935
16
                         MVT::i32, MVT::i32, Ops));
2936
16
      return;
2937
16
    }
2938
0
  }
2939
46
  case ARMISD::SMLAL:{
2940
46
    if (Subtarget->isThumb()) {
2941
31
      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2942
31
                        N->getOperand(3), getAL(CurDAG, dl),
2943
31
                        CurDAG->getRegister(0, MVT::i32)};
2944
31
      ReplaceNode(
2945
31
          N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
2946
31
      return;
2947
31
    }else{
2948
15
      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2949
15
                        N->getOperand(3), getAL(CurDAG, dl),
2950
15
                        CurDAG->getRegister(0, MVT::i32),
2951
15
                        CurDAG->getRegister(0, MVT::i32) };
2952
15
      ReplaceNode(N, CurDAG->getMachineNode(
2953
15
                         Subtarget->hasV6Ops() ? 
ARM::SMLAL6
:
ARM::SMLALv59
, dl,
2954
15
                         MVT::i32, MVT::i32, Ops));
2955
15
      return;
2956
15
    }
2957
0
  }
2958
1.63k
  case ARMISD::SUBE: {
2959
1.63k
    if (!Subtarget->hasV6Ops() || 
!Subtarget->hasDSP()1.57k
)
2960
393
      break;
2961
1.24k
    // Look for a pattern to match SMMLS
2962
1.24k
    // (sube a, (smul_loHi a, b), (subc 0, (smul_LOhi(a, b))))
2963
1.24k
    if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI ||
2964
1.24k
        
N->getOperand(2).getOpcode() != ARMISD::SUBC12
||
2965
1.24k
        
!SDValue(N, 1).use_empty()12
)
2966
1.23k
      break;
2967
7
2968
7
    if (Subtarget->isThumb())
2969
7
      assert(Subtarget->hasThumb2() &&
2970
7
             "This pattern should not be generated for Thumb");
2971
7
2972
7
    SDValue SmulLoHi = N->getOperand(1);
2973
7
    SDValue Subc = N->getOperand(2);
2974
7
    auto *Zero = dyn_cast<ConstantSDNode>(Subc.getOperand(0));
2975
7
2976
7
    if (!Zero || Zero->getZExtValue() != 0 ||
2977
7
        Subc.getOperand(1) != SmulLoHi.getValue(0) ||
2978
7
        N->getOperand(1) != SmulLoHi.getValue(1) ||
2979
7
        N->getOperand(2) != Subc.getValue(1))
2980
0
      break;
2981
7
2982
7
    unsigned Opc = Subtarget->isThumb2() ? 
ARM::t2SMMLS4
:
ARM::SMMLS3
;
2983
7
    SDValue Ops[] = { SmulLoHi.getOperand(0), SmulLoHi.getOperand(1),
2984
7
                      N->getOperand(0), getAL(CurDAG, dl),
2985
7
                      CurDAG->getRegister(0, MVT::i32) };
2986
7
    ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
2987
7
    return;
2988
7
  }
2989
64.0k
  case ISD::LOAD: {
2990
64.0k
    if (Subtarget->isThumb() && 
Subtarget->hasThumb2()55.9k
) {
2991
53.8k
      if (tryT2IndexedLoad(N))
2992
1.20k
        return;
2993
10.1k
    } else if (Subtarget->isThumb()) {
2994
2.11k
      if (tryT1IndexedLoad(N))
2995
5
        return;
2996
8.08k
    } else if (tryARMIndexedLoad(N))
2997
46
      return;
2998
62.7k
    // Other cases are autogenerated.
2999
62.7k
    break;
3000
62.7k
  }
3001
62.7k
  case ARMISD::WLS:
3002
76
  case ARMISD::LE: {
3003
76
    SDValue Ops[] = { N->getOperand(1),
3004
76
                      N->getOperand(2),
3005
76
                      N->getOperand(0) };
3006
76
    unsigned Opc = N->getOpcode() == ARMISD::WLS ?
3007
59
      
ARM::t2WhileLoopStart17
: ARM::t2LoopEnd;
3008
76
    SDNode *New = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
3009
76
    ReplaceUses(N, New);
3010
76
    CurDAG->RemoveDeadNode(N);
3011
76
    return;
3012
76
  }
3013
76
  case ARMISD::LOOP_DEC: {
3014
59
    SDValue Ops[] = { N->getOperand(1),
3015
59
                      N->getOperand(2),
3016
59
                      N->getOperand(0) };
3017
59
    SDNode *Dec =
3018
59
      CurDAG->getMachineNode(ARM::t2LoopDec, dl,
3019
59
                             CurDAG->getVTList(MVT::i32, MVT::Other), Ops);
3020
59
    ReplaceUses(N, Dec);
3021
59
    CurDAG->RemoveDeadNode(N);
3022
59
    return;
3023
76
  }
3024
48.4k
  case ARMISD::BRCOND: {
3025
48.4k
    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
3026
48.4k
    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
3027
48.4k
    // Pattern complexity = 6  cost = 1  size = 0
3028
48.4k
3029
48.4k
    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
3030
48.4k
    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
3031
48.4k
    // Pattern complexity = 6  cost = 1  size = 0
3032
48.4k
3033
48.4k
    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
3034
48.4k
    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
3035
48.4k
    // Pattern complexity = 6  cost = 1  size = 0
3036
48.4k
3037
48.4k
    unsigned Opc = Subtarget->isThumb() ?
3038
47.6k
      ((Subtarget->hasThumb2()) ? 
ARM::t2Bcc46.5k
:
ARM::tBcc1.08k
) :
ARM::Bcc739
;
3039
48.4k
    SDValue Chain = N->getOperand(0);
3040
48.4k
    SDValue N1 = N->getOperand(1);
3041
48.4k
    SDValue N2 = N->getOperand(2);
3042
48.4k
    SDValue N3 = N->getOperand(3);
3043
48.4k
    SDValue InFlag = N->getOperand(4);
3044
48.4k
    assert(N1.getOpcode() == ISD::BasicBlock);
3045
48.4k
    assert(N2.getOpcode() == ISD::Constant);
3046
48.4k
    assert(N3.getOpcode() == ISD::Register);
3047
48.4k
3048
48.4k
    unsigned CC = (unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
3049
48.4k
3050
48.4k
    if (InFlag.getOpcode() == ARMISD::CMPZ) {
3051
34.5k
      if (InFlag.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN) {
3052
1.12k
        SDValue Int = InFlag.getOperand(0);
3053
1.12k
        uint64_t ID = cast<ConstantSDNode>(Int->getOperand(1))->getZExtValue();
3054
1.12k
3055
1.12k
        // Handle low-overhead loops.
3056
1.12k
        if (ID == Intrinsic::loop_decrement_reg) {
3057
0
          SDValue Elements = Int.getOperand(2);
3058
0
          SDValue Size = CurDAG->getTargetConstant(
3059
0
            cast<ConstantSDNode>(Int.getOperand(3))->getZExtValue(), dl,
3060
0
                                 MVT::i32);
3061
0
3062
0
          SDValue Args[] = { Elements, Size, Int.getOperand(0) };
3063
0
          SDNode *LoopDec =
3064
0
            CurDAG->getMachineNode(ARM::t2LoopDec, dl,
3065
0
                                   CurDAG->getVTList(MVT::i32, MVT::Other),
3066
0
                                   Args);
3067
0
          ReplaceUses(Int.getNode(), LoopDec);
3068
0
3069
0
          SDValue EndArgs[] = { SDValue(LoopDec, 0), N1, Chain };
3070
0
          SDNode *LoopEnd =
3071
0
            CurDAG->getMachineNode(ARM::t2LoopEnd, dl, MVT::Other, EndArgs);
3072
0
3073
0
          ReplaceUses(N, LoopEnd);
3074
0
          CurDAG->RemoveDeadNode(N);
3075
0
          CurDAG->RemoveDeadNode(InFlag.getNode());
3076
0
          CurDAG->RemoveDeadNode(Int.getNode());
3077
0
          return;
3078
0
        }
3079
34.5k
      }
3080
34.5k
3081
34.5k
      bool SwitchEQNEToPLMI;
3082
34.5k
      SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
3083
34.5k
      InFlag = N->getOperand(4);
3084
34.5k
3085
34.5k
      if (SwitchEQNEToPLMI) {
3086
287
        switch ((ARMCC::CondCodes)CC) {
3087
287
        
default: 0
llvm_unreachable0
("CMPZ must be either NE or EQ!");
3088
287
        case ARMCC::NE:
3089
17
          CC = (unsigned)ARMCC::MI;
3090
17
          break;
3091
287
        case ARMCC::EQ:
3092
270
          CC = (unsigned)ARMCC::PL;
3093
270
          break;
3094
48.4k
        }
3095
48.4k
      }
3096
34.5k
    }
3097
48.4k
3098
48.4k
    SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
3099
48.4k
    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
3100
48.4k
    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
3101
48.4k
                                             MVT::Glue, Ops);
3102
48.4k
    Chain = SDValue(ResNode, 0);
3103
48.4k
    if (N->getNumValues() == 2) {
3104
587
      InFlag = SDValue(ResNode, 1);
3105
587
      ReplaceUses(SDValue(N, 1), InFlag);
3106
587
    }
3107
48.4k
    ReplaceUses(SDValue(N, 0),
3108
48.4k
                SDValue(Chain.getNode(), Chain.getResNo()));
3109
48.4k
    CurDAG->RemoveDeadNode(N);
3110
48.4k
    return;
3111
48.4k
  }
3112
48.4k
3113
48.4k
  case ARMISD::CMPZ: {
3114
38.1k
    // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0)
3115
38.1k
    //   This allows us to avoid materializing the expensive negative constant.
3116
38.1k
    //   The CMPZ #0 is useless and will be peepholed away but we need to keep it
3117
38.1k
    //   for its glue output.
3118
38.1k
    SDValue X = N->getOperand(0);
3119
38.1k
    auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode());
3120
38.1k
    if (C && 
C->getSExtValue() < 035.5k
&&
Subtarget->isThumb()753
) {
3121
745
      int64_t Addend = -C->getSExtValue();
3122
745
3123
745
      SDNode *Add = nullptr;
3124
745
      // ADDS can be better than CMN if the immediate fits in a
3125
745
      // 16-bit ADDS, which means either [0,256) for tADDi8 or [0,8) for tADDi3.
3126
745
      // Outside that range we can just use a CMN which is 32-bit but has a
3127
745
      // 12-bit immediate range.
3128
745
      if (Addend < 1<<8) {
3129
550
        if (Subtarget->isThumb2()) {
3130
548
          SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
3131
548
                            getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3132
548
                            CurDAG->getRegister(0, MVT::i32) };
3133
548
          Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
3134
548
        } else {
3135
2
          unsigned Opc = (Addend < 1<<3) ? 
ARM::tADDi31
:
ARM::tADDi81
;
3136
2
          SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
3137
2
                           CurDAG->getTargetConstant(Addend, dl, MVT::i32),
3138
2
                           getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3139
2
          Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3140
2
        }
3141
550
      }
3142
745
      if (Add) {
3143
550
        SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
3144
550
        CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
3145
550
      }
3146
745
    }
3147
38.1k
    // Other cases are autogenerated.
3148
38.1k
    break;
3149
48.4k
  }
3150
48.4k
3151
48.4k
  case ARMISD::CMOV: {
3152
7.63k
    SDValue InFlag = N->getOperand(4);
3153
7.63k
3154
7.63k
    if (InFlag.getOpcode() == ARMISD::CMPZ) {
3155
3.61k
      bool SwitchEQNEToPLMI;
3156
3.61k
      SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
3157
3.61k
3158
3.61k
      if (SwitchEQNEToPLMI) {
3159
3
        SDValue ARMcc = N->getOperand(2);
3160
3
        ARMCC::CondCodes CC =
3161
3
          (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
3162
3
3163
3
        switch (CC) {
3164
3
        
default: 0
llvm_unreachable0
("CMPZ must be either NE or EQ!");
3165
3
        case ARMCC::NE:
3166
0
          CC = ARMCC::MI;
3167
0
          break;
3168
3
        case ARMCC::EQ:
3169
3
          CC = ARMCC::PL;
3170
3
          break;
3171
3
        }
3172
3
        SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
3173
3
        SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
3174
3
                         N->getOperand(3), N->getOperand(4)};
3175
3
        CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
3176
3
      }
3177
3.61k
3178
3.61k
    }
3179
7.63k
    // Other cases are autogenerated.
3180
7.63k
    break;
3181
7.63k
  }
3182
7.63k
3183
7.63k
  case ARMISD::VZIP: {
3184
28
    unsigned Opc = 0;
3185
28
    EVT VT = N->getValueType(0);
3186
28
    switch (VT.getSimpleVT().SimpleTy) {
3187
28
    
default: return0
;
3188
28
    
case MVT::v8i8: Opc = ARM::VZIPd8; break5
;
3189
28
    case MVT::v4f16:
3190
8
    case MVT::v4i16: Opc = ARM::VZIPd16; break;
3191
8
    case MVT::v2f32:
3192
0
    // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3193
0
    case MVT::v2i32: Opc = ARM::VTRNd32; break;
3194
4
    case MVT::v16i8: Opc = ARM::VZIPq8; break;
3195
3
    case MVT::v8f16:
3196
3
    case MVT::v8i16: Opc = ARM::VZIPq16; break;
3197
8
    case MVT::v4f32:
3198
8
    case MVT::v4i32: Opc = ARM::VZIPq32; break;
3199
28
    }
3200
28
    SDValue Pred = getAL(CurDAG, dl);
3201
28
    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3202
28
    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3203
28
    ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3204
28
    return;
3205
28
  }
3206
50
  case ARMISD::VUZP: {
3207
50
    unsigned Opc = 0;
3208
50
    EVT VT = N->getValueType(0);
3209
50
    switch (VT.getSimpleVT().SimpleTy) {
3210
50
    
default: return0
;
3211
50
    
case MVT::v8i8: Opc = ARM::VUZPd8; break19
;
3212
50
    case MVT::v4f16:
3213
17
    case MVT::v4i16: Opc = ARM::VUZPd16; break;
3214
17
    case MVT::v2f32:
3215
0
    // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3216
0
    case MVT::v2i32: Opc = ARM::VTRNd32; break;
3217
2
    case MVT::v16i8: Opc = ARM::VUZPq8; break;
3218
6
    case MVT::v8f16:
3219
6
    case MVT::v8i16: Opc = ARM::VUZPq16; break;
3220
6
    case MVT::v4f32:
3221
6
    case MVT::v4i32: Opc = ARM::VUZPq32; break;
3222
50
    }
3223
50
    SDValue Pred = getAL(CurDAG, dl);
3224
50
    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3225
50
    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3226
50
    ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3227
50
    return;
3228
50
  }
3229
50
  case ARMISD::VTRN: {
3230
35
    unsigned Opc = 0;
3231
35
    EVT VT = N->getValueType(0);
3232
35
    switch (VT.getSimpleVT().SimpleTy) {
3233
35
    
default: return0
;
3234
35
    
case MVT::v8i8: Opc = ARM::VTRNd8; break6
;
3235
35
    case MVT::v4f16:
3236
8
    case MVT::v4i16: Opc = ARM::VTRNd16; break;
3237
8
    case MVT::v2f32:
3238
7
    case MVT::v2i32: Opc = ARM::VTRNd32; break;
3239
7
    
case MVT::v16i8: Opc = ARM::VTRNq8; break2
;
3240
7
    case MVT::v8f16:
3241
7
    case MVT::v8i16: Opc = ARM::VTRNq16; break;
3242
7
    case MVT::v4f32:
3243
5
    case MVT::v4i32: Opc = ARM::VTRNq32; break;
3244
35
    }
3245
35
    SDValue Pred = getAL(CurDAG, dl);
3246
35
    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3247
35
    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3248
35
    ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3249
35
    return;
3250
35
  }
3251
617
  case ARMISD::BUILD_VECTOR: {
3252
617
    EVT VecVT = N->getValueType(0);
3253
617
    EVT EltVT = VecVT.getVectorElementType();
3254
617
    unsigned NumElts = VecVT.getVectorNumElements();
3255
617
    if (EltVT == MVT::f64) {
3256
466
      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
3257
466
      ReplaceNode(
3258
466
          N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3259
466
      return;
3260
466
    }
3261
151
    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
3262
151
    if (NumElts == 2) {
3263
32
      ReplaceNode(
3264
32
          N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3265
32
      return;
3266
32
    }
3267
119
    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
3268
119
    ReplaceNode(N,
3269
119
                createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
3270
119
                                    N->getOperand(2), N->getOperand(3)));
3271
119
    return;
3272
119
  }
3273
119
3274
119
  case ARMISD::VLD1DUP: {
3275
28
    static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
3276
28
                                         ARM::VLD1DUPd32 };
3277
28
    static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
3278
28
                                         ARM::VLD1DUPq32 };
3279
28
    SelectVLDDup(N, /* IsIntrinsic= */ false, false, 1, DOpcodes, QOpcodes);
3280
28
    return;
3281
119
  }
3282
119
3283
119
  case ARMISD::VLD2DUP: {
3284
1
    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3285
1
                                        ARM::VLD2DUPd32 };
3286
1
    SelectVLDDup(N, /* IsIntrinsic= */ false, false, 2, Opcodes);
3287
1
    return;
3288
119
  }
3289
119
3290
119
  case ARMISD::VLD3DUP: {
3291
0
    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
3292
0
                                        ARM::VLD3DUPd16Pseudo,
3293
0
                                        ARM::VLD3DUPd32Pseudo };
3294
0
    SelectVLDDup(N, /* IsIntrinsic= */ false, false, 3, Opcodes);
3295
0
    return;
3296
119
  }
3297
119
3298
119
  case ARMISD::VLD4DUP: {
3299
0
    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
3300
0
                                        ARM::VLD4DUPd16Pseudo,
3301
0
                                        ARM::VLD4DUPd32Pseudo };
3302
0
    SelectVLDDup(N, /* IsIntrinsic= */ false, false, 4, Opcodes);
3303
0
    return;
3304
119
  }
3305
119
3306
119
  case ARMISD::VLD1DUP_UPD: {
3307
6
    static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
3308
6
                                         ARM::VLD1DUPd16wb_fixed,
3309
6
                                         ARM::VLD1DUPd32wb_fixed };
3310
6
    static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
3311
6
                                         ARM::VLD1DUPq16wb_fixed,
3312
6
                                         ARM::VLD1DUPq32wb_fixed };
3313
6
    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 1, DOpcodes, QOpcodes);
3314
6
    return;
3315
119
  }
3316
119
3317
119
  case ARMISD::VLD2DUP_UPD: {
3318
2
    static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
3319
2
                                        ARM::VLD2DUPd16wb_fixed,
3320
2
                                        ARM::VLD2DUPd32wb_fixed };
3321
2
    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 2, Opcodes);
3322
2
    return;
3323
119
  }
3324
119
3325
119
  case ARMISD::VLD3DUP_UPD: {
3326
0
    static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
3327
0
                                        ARM::VLD3DUPd16Pseudo_UPD,
3328
0
                                        ARM::VLD3DUPd32Pseudo_UPD };
3329
0
    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 3, Opcodes);
3330
0
    return;
3331
119
  }
3332
119
3333
119
  case ARMISD::VLD4DUP_UPD: {
3334
0
    static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
3335
0
                                        ARM::VLD4DUPd16Pseudo_UPD,
3336
0
                                        ARM::VLD4DUPd32Pseudo_UPD };
3337
0
    SelectVLDDup(N, /* IsIntrinsic= */ false, true, 4, Opcodes);
3338
0
    return;
3339
119
  }
3340
119
3341
370
  case ARMISD::VLD1_UPD: {
3342
370
    static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
3343
370
                                         ARM::VLD1d16wb_fixed,
3344
370
                                         ARM::VLD1d32wb_fixed,
3345
370
                                         ARM::VLD1d64wb_fixed };
3346
370
    static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
3347
370
                                         ARM::VLD1q16wb_fixed,
3348
370
                                         ARM::VLD1q32wb_fixed,
3349
370
                                         ARM::VLD1q64wb_fixed };
3350
370
    SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
3351
370
    return;
3352
119
  }
3353
119
3354
119
  case ARMISD::VLD2_UPD: {
3355
8
    static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
3356
8
                                         ARM::VLD2d16wb_fixed,
3357
8
                                         ARM::VLD2d32wb_fixed,
3358
8
                                         ARM::VLD1q64wb_fixed};
3359
8
    static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
3360
8
                                         ARM::VLD2q16PseudoWB_fixed,
3361
8
                                         ARM::VLD2q32PseudoWB_fixed };
3362
8
    SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
3363
8
    return;
3364
119
  }
3365
119
3366
119
  case ARMISD::VLD3_UPD: {
3367
9
    static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
3368
9
                                         ARM::VLD3d16Pseudo_UPD,
3369
9
                                         ARM::VLD3d32Pseudo_UPD,
3370
9
                                         ARM::VLD1d64TPseudoWB_fixed};
3371
9
    static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3372
9
                                          ARM::VLD3q16Pseudo_UPD,
3373
9
                                          ARM::VLD3q32Pseudo_UPD };
3374
9
    static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
3375
9
                                          ARM::VLD3q16oddPseudo_UPD,
3376
9
                                          ARM::VLD3q32oddPseudo_UPD };
3377
9
    SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3378
9
    return;
3379
119
  }
3380
119
3381
119
  case ARMISD::VLD4_UPD: {
3382
4
    static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
3383
4
                                         ARM::VLD4d16Pseudo_UPD,
3384
4
                                         ARM::VLD4d32Pseudo_UPD,
3385
4
                                         ARM::VLD1d64QPseudoWB_fixed};
3386
4
    static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3387
4
                                          ARM::VLD4q16Pseudo_UPD,
3388
4
                                          ARM::VLD4q32Pseudo_UPD };
3389
4
    static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
3390
4
                                          ARM::VLD4q16oddPseudo_UPD,
3391
4
                                          ARM::VLD4q32oddPseudo_UPD };
3392
4
    SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3393
4
    return;
3394
119
  }
3395
119
3396
119
  case ARMISD::VLD2LN_UPD: {
3397
6
    static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
3398
6
                                         ARM::VLD2LNd16Pseudo_UPD,
3399
6
                                         ARM::VLD2LNd32Pseudo_UPD };
3400
6
    static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
3401
6
                                         ARM::VLD2LNq32Pseudo_UPD };
3402
6
    SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
3403
6
    return;
3404
119
  }
3405
119
3406
119
  case ARMISD::VLD3LN_UPD: {
3407
3
    static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
3408
3
                                         ARM::VLD3LNd16Pseudo_UPD,
3409
3
                                         ARM::VLD3LNd32Pseudo_UPD };
3410
3
    static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
3411
3
                                         ARM::VLD3LNq32Pseudo_UPD };
3412
3
    SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
3413
3
    return;
3414
119
  }
3415
119
3416
119
  case ARMISD::VLD4LN_UPD: {
3417
3
    static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
3418
3
                                         ARM::VLD4LNd16Pseudo_UPD,
3419
3
                                         ARM::VLD4LNd32Pseudo_UPD };
3420
3
    static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
3421
3
                                         ARM::VLD4LNq32Pseudo_UPD };
3422
3
    SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
3423
3
    return;
3424
119
  }
3425
119
3426
431
  case ARMISD::VST1_UPD: {
3427
431
    static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
3428
431
                                         ARM::VST1d16wb_fixed,
3429
431
                                         ARM::VST1d32wb_fixed,
3430
431
                                         ARM::VST1d64wb_fixed };
3431
431
    static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
3432
431
                                         ARM::VST1q16wb_fixed,
3433
431
                                         ARM::VST1q32wb_fixed,
3434
431
                                         ARM::VST1q64wb_fixed };
3435
431
    SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
3436
431
    return;
3437
119
  }
3438
119
3439
119
  case ARMISD::VST2_UPD: {
3440
5
    static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
3441
5
                                         ARM::VST2d16wb_fixed,
3442
5
                                         ARM::VST2d32wb_fixed,
3443
5
                                         ARM::VST1q64wb_fixed};
3444
5
    static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
3445
5
                                         ARM::VST2q16PseudoWB_fixed,
3446
5
                                         ARM::VST2q32PseudoWB_fixed };
3447
5
    SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
3448
5
    return;
3449
119
  }
3450
119
3451
119
  case ARMISD::VST3_UPD: {
3452
4
    static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
3453
4
                                         ARM::VST3d16Pseudo_UPD,
3454
4
                                         ARM::VST3d32Pseudo_UPD,
3455
4
                                         ARM::VST1d64TPseudoWB_fixed};
3456
4
    static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3457
4
                                          ARM::VST3q16Pseudo_UPD,
3458
4
                                          ARM::VST3q32Pseudo_UPD };
3459
4
    static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
3460
4
                                          ARM::VST3q16oddPseudo_UPD,
3461
4
                                          ARM::VST3q32oddPseudo_UPD };
3462
4
    SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3463
4
    return;
3464
119
  }
3465
119
3466
119
  case ARMISD::VST4_UPD: {
3467
4
    static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
3468
4
                                         ARM::VST4d16Pseudo_UPD,
3469
4
                                         ARM::VST4d32Pseudo_UPD,
3470
4
                                         ARM::VST1d64QPseudoWB_fixed};
3471
4
    static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3472
4
                                          ARM::VST4q16Pseudo_UPD,
3473
4
                                          ARM::VST4q32Pseudo_UPD };
3474
4
    static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
3475
4
                                          ARM::VST4q16oddPseudo_UPD,
3476
4
                                          ARM::VST4q32oddPseudo_UPD };
3477
4
    SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3478
4
    return;
3479
119
  }
3480
119
3481
119
  case ARMISD::VST2LN_UPD: {
3482
1
    static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
3483
1
                                         ARM::VST2LNd16Pseudo_UPD,
3484
1
                                         ARM::VST2LNd32Pseudo_UPD };
3485
1
    static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
3486
1
                                         ARM::VST2LNq32Pseudo_UPD };
3487
1
    SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
3488
1
    return;
3489
119
  }
3490
119
3491
119
  case ARMISD::VST3LN_UPD: {
3492
1
    static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
3493
1
                                         ARM::VST3LNd16Pseudo_UPD,
3494
1
                                         ARM::VST3LNd32Pseudo_UPD };
3495
1
    static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
3496
1
                                         ARM::VST3LNq32Pseudo_UPD };
3497
1
    SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
3498
1
    return;
3499
119
  }
3500
119
3501
119
  case ARMISD::VST4LN_UPD: {
3502
1
    static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
3503
1
                                         ARM::VST4LNd16Pseudo_UPD,
3504
1
                                         ARM::VST4LNd32Pseudo_UPD };
3505
1
    static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
3506
1
                                         ARM::VST4LNq32Pseudo_UPD };
3507
1
    SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3508
1
    return;
3509
119
  }
3510
119
3511
5.45k
  case ISD::INTRINSIC_VOID:
3512
5.45k
  case ISD::INTRINSIC_W_CHAIN: {
3513
5.45k
    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3514
5.45k
    switch (IntNo) {
3515
5.45k
    default:
3516
4.82k
      break;
3517
5.45k
3518
5.45k
    case Intrinsic::arm_mrrc:
3519
4
    case Intrinsic::arm_mrrc2: {
3520
4
      SDLoc dl(N);
3521
4
      SDValue Chain = N->getOperand(0);
3522
4
      unsigned Opc;
3523
4
3524
4
      if (Subtarget->isThumb())
3525
2
        Opc = (IntNo == Intrinsic::arm_mrrc ? 
ARM::t2MRRC1
:
ARM::t2MRRC21
);
3526
2
      else
3527
2
        Opc = (IntNo == Intrinsic::arm_mrrc ? 
ARM::MRRC1
:
ARM::MRRC21
);
3528
4
3529
4
      SmallVector<SDValue, 5> Ops;
3530
4
      Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(), dl)); /* coproc */
3531
4
      Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(), dl)); /* opc */
3532
4
      Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(4))->getZExtValue(), dl)); /* CRm */
3533
4
3534
4
      // The mrrc2 instruction in ARM doesn't allow predicates, the top 4 bits of the encoded
3535
4
      // instruction will always be '1111' but it is possible in assembly language to specify
3536
4
      // AL as a predicate to mrrc2 but it doesn't make any difference to the encoded instruction.
3537
4
      if (Opc != ARM::MRRC2) {
3538
3
        Ops.push_back(getAL(CurDAG, dl));
3539
3
        Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3540
3
      }
3541
4
3542
4
      Ops.push_back(Chain);
3543
4
3544
4
      // Writes to two registers.
3545
4
      const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
3546
4
3547
4
      ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
3548
4
      return;
3549
4
    }
3550
127
    case Intrinsic::arm_ldaexd:
3551
127
    case Intrinsic::arm_ldrexd: {
3552
127
      SDLoc dl(N);
3553
127
      SDValue Chain = N->getOperand(0);
3554
127
      SDValue MemAddr = N->getOperand(2);
3555
127
      bool isThumb = Subtarget->isThumb() && 
Subtarget->hasV8MBaselineOps()58
;
3556
127
3557
127
      bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
3558
127
      unsigned NewOpc = isThumb ? 
(IsAcquire 58
?
ARM::t2LDAEXD15
:
ARM::t2LDREXD43
)
3559
127
                                : 
(IsAcquire 69
?
ARM::LDAEXD15
:
ARM::LDREXD54
);
3560
127
3561
127
      // arm_ldrexd returns a i64 value in {i32, i32}
3562
127
      std::vector<EVT> ResTys;
3563
127
      if (isThumb) {
3564
58
        ResTys.push_back(MVT::i32);
3565
58
        ResTys.push_back(MVT::i32);
3566
58
      } else
3567
69
        ResTys.push_back(MVT::Untyped);
3568
127
      ResTys.push_back(MVT::Other);
3569
127
3570
127
      // Place arguments in the right order.
3571
127
      SDValue Ops[] = {MemAddr, getAL(CurDAG, dl),
3572
127
                       CurDAG->getRegister(0, MVT::i32), Chain};
3573
127
      SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3574
127
      // Transfer memoperands.
3575
127
      MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3576
127
      CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
3577
127
3578
127
      // Remap uses.
3579
127
      SDValue OutChain = isThumb ? 
SDValue(Ld, 2)58
:
SDValue(Ld, 1)69
;
3580
127
      if (!SDValue(N, 0).use_empty()) {
3581
109
        SDValue Result;
3582
109
        if (isThumb)
3583
53
          Result = SDValue(Ld, 0);
3584
56
        else {
3585
56
          SDValue SubRegIdx =
3586
56
            CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
3587
56
          SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3588
56
              dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3589
56
          Result = SDValue(ResNode,0);
3590
56
        }
3591
109
        ReplaceUses(SDValue(N, 0), Result);
3592
109
      }
3593
127
      if (!SDValue(N, 1).use_empty()) {
3594
108
        SDValue Result;
3595
108
        if (isThumb)
3596
53
          Result = SDValue(Ld, 1);
3597
55
        else {
3598
55
          SDValue SubRegIdx =
3599
55
            CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
3600
55
          SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3601
55
              dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3602
55
          Result = SDValue(ResNode,0);
3603
55
        }
3604
108
        ReplaceUses(SDValue(N, 1), Result);
3605
108
      }
3606
127
      ReplaceUses(SDValue(N, 2), OutChain);
3607
127
      CurDAG->RemoveDeadNode(N);
3608
127
      return;
3609
127
    }
3610
127
    case Intrinsic::arm_stlexd:
3611
109
    case Intrinsic::arm_strexd: {
3612
109
      SDLoc dl(N);
3613
109
      SDValue Chain = N->getOperand(0);
3614
109
      SDValue Val0 = N->getOperand(2);
3615
109
      SDValue Val1 = N->getOperand(3);
3616
109
      SDValue MemAddr = N->getOperand(4);
3617
109
3618
109
      // Store exclusive double return a i32 value which is the return status
3619
109
      // of the issued store.
3620
109
      const EVT ResTys[] = {MVT::i32, MVT::Other};
3621
109
3622
109
      bool isThumb = Subtarget->isThumb() && 
Subtarget->hasThumb2()51
;
3623
109
      // Place arguments in the right order.
3624
109
      SmallVector<SDValue, 7> Ops;
3625
109
      if (isThumb) {
3626
51
        Ops.push_back(Val0);
3627
51
        Ops.push_back(Val1);
3628
51
      } else
3629
58
        // arm_strexd uses GPRPair.
3630
58
        Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
3631
109
      Ops.push_back(MemAddr);
3632
109
      Ops.push_back(getAL(CurDAG, dl));
3633
109
      Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3634
109
      Ops.push_back(Chain);
3635
109
3636
109
      bool IsRelease = IntNo == Intrinsic::arm_stlexd;
3637
109
      unsigned NewOpc = isThumb ? 
(IsRelease 51
?
ARM::t2STLEXD13
:
ARM::t2STREXD38
)
3638
109
                                : 
(IsRelease 58
?
ARM::STLEXD13
:
ARM::STREXD45
);
3639
109
3640
109
      SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3641
109
      // Transfer memoperands.
3642
109
      MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3643
109
      CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
3644
109
3645
109
      ReplaceNode(N, St);
3646
109
      return;
3647
109
    }
3648
109
3649
109
    case Intrinsic::arm_neon_vld1: {
3650
68
      static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3651
68
                                           ARM::VLD1d32, ARM::VLD1d64 };
3652
68
      static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3653
68
                                           ARM::VLD1q32, ARM::VLD1q64};
3654
68
      SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
3655
68
      return;
3656
109
    }
3657
109
3658
109
    case Intrinsic::arm_neon_vld1x2: {
3659
8
      static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3660
8
                                           ARM::VLD1q32, ARM::VLD1q64 };
3661
8
      static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
3662
8
                                           ARM::VLD1d16QPseudo,
3663
8
                                           ARM::VLD1d32QPseudo,
3664
8
                                           ARM::VLD1d64QPseudo };
3665
8
      SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3666
8
      return;
3667
109
    }
3668
109
3669
109
    case Intrinsic::arm_neon_vld1x3: {
3670
8
      static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
3671
8
                                           ARM::VLD1d16TPseudo,
3672
8
                                           ARM::VLD1d32TPseudo,
3673
8
                                           ARM::VLD1d64TPseudo };
3674
8
      static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
3675
8
                                            ARM::VLD1q16LowTPseudo_UPD,
3676
8
                                            ARM::VLD1q32LowTPseudo_UPD,
3677
8
                                            ARM::VLD1q64LowTPseudo_UPD };
3678
8
      static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
3679
8
                                            ARM::VLD1q16HighTPseudo,
3680
8
                                            ARM::VLD1q32HighTPseudo,
3681
8
                                            ARM::VLD1q64HighTPseudo };
3682
8
      SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3683
8
      return;
3684
109
    }
3685
109
3686
109
    case Intrinsic::arm_neon_vld1x4: {
3687
8
      static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
3688
8
                                           ARM::VLD1d16QPseudo,
3689
8
                                           ARM::VLD1d32QPseudo,
3690
8
                                           ARM::VLD1d64QPseudo };
3691
8
      static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
3692
8
                                            ARM::VLD1q16LowQPseudo_UPD,
3693
8
                                            ARM::VLD1q32LowQPseudo_UPD,
3694
8
                                            ARM::VLD1q64LowQPseudo_UPD };
3695
8
      static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
3696
8
                                            ARM::VLD1q16HighQPseudo,
3697
8
                                            ARM::VLD1q32HighQPseudo,
3698
8
                                            ARM::VLD1q64HighQPseudo };
3699
8
      SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3700
8
      return;
3701
109
    }
3702
109
3703
109
    case Intrinsic::arm_neon_vld2: {
3704
19
      static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3705
19
                                           ARM::VLD2d32, ARM::VLD1q64 };
3706
19
      static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3707
19
                                           ARM::VLD2q32Pseudo };
3708
19
      SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3709
19
      return;
3710
109
    }
3711
109
3712
109
    case Intrinsic::arm_neon_vld3: {
3713
37
      static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3714
37
                                           ARM::VLD3d16Pseudo,
3715
37
                                           ARM::VLD3d32Pseudo,
3716
37
                                           ARM::VLD1d64TPseudo };
3717
37
      static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3718
37
                                            ARM::VLD3q16Pseudo_UPD,
3719
37
                                            ARM::VLD3q32Pseudo_UPD };
3720
37
      static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3721
37
                                            ARM::VLD3q16oddPseudo,
3722
37
                                            ARM::VLD3q32oddPseudo };
3723
37
      SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3724
37
      return;
3725
109
    }
3726
109
3727
109
    case Intrinsic::arm_neon_vld4: {
3728
11
      static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3729
11
                                           ARM::VLD4d16Pseudo,
3730
11
                                           ARM::VLD4d32Pseudo,
3731
11
                                           ARM::VLD1d64QPseudo };
3732
11
      static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3733
11
                                            ARM::VLD4q16Pseudo_UPD,
3734
11
                                            ARM::VLD4q32Pseudo_UPD };
3735
11
      static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3736
11
                                            ARM::VLD4q16oddPseudo,
3737
11
                                            ARM::VLD4q32oddPseudo };
3738
11
      SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3739
11
      return;
3740
109
    }
3741
109
3742
109
    case Intrinsic::arm_neon_vld2dup: {
3743
8
      static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3744
8
                                           ARM::VLD2DUPd32, ARM::VLD1q64 };
3745
8
      static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
3746
8
                                            ARM::VLD2DUPq16EvenPseudo,
3747
8
                                            ARM::VLD2DUPq32EvenPseudo };
3748
8
      static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
3749
8
                                            ARM::VLD2DUPq16OddPseudo,
3750
8
                                            ARM::VLD2DUPq32OddPseudo };
3751
8
      SelectVLDDup(N, /* IsIntrinsic= */ true, false, 2,
3752
8
                   DOpcodes, QOpcodes0, QOpcodes1);
3753
8
      return;
3754
109
    }
3755
109
3756
109
    case Intrinsic::arm_neon_vld3dup: {
3757
8
      static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
3758
8
                                           ARM::VLD3DUPd16Pseudo,
3759
8
                                           ARM::VLD3DUPd32Pseudo,
3760
8
                                           ARM::VLD1d64TPseudo };
3761
8
      static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
3762
8
                                            ARM::VLD3DUPq16EvenPseudo,
3763
8
                                            ARM::VLD3DUPq32EvenPseudo };
3764
8
      static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
3765
8
                                            ARM::VLD3DUPq16OddPseudo,
3766
8
                                            ARM::VLD3DUPq32OddPseudo };
3767
8
      SelectVLDDup(N, /* IsIntrinsic= */ true, false, 3,
3768
8
                   DOpcodes, QOpcodes0, QOpcodes1);
3769
8
      return;
3770
109
    }
3771
109
3772
109
    case Intrinsic::arm_neon_vld4dup: {
3773
8
      static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
3774
8
                                           ARM::VLD4DUPd16Pseudo,
3775
8
                                           ARM::VLD4DUPd32Pseudo,
3776
8
                                           ARM::VLD1d64QPseudo };
3777
8
      static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
3778
8
                                            ARM::VLD4DUPq16EvenPseudo,
3779
8
                                            ARM::VLD4DUPq32EvenPseudo };
3780
8
      static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
3781
8
                                            ARM::VLD4DUPq16OddPseudo,
3782
8
                                            ARM::VLD4DUPq32OddPseudo };
3783
8
      SelectVLDDup(N, /* IsIntrinsic= */ true, false, 4,
3784
8
                   DOpcodes, QOpcodes0, QOpcodes1);
3785
8
      return;
3786
109
    }
3787
109
3788
109
    case Intrinsic::arm_neon_vld2lane: {
3789
24
      static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3790
24
                                           ARM::VLD2LNd16Pseudo,
3791
24
                                           ARM::VLD2LNd32Pseudo };
3792
24
      static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3793
24
                                           ARM::VLD2LNq32Pseudo };
3794
24
      SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3795
24
      return;
3796
109
    }
3797
109
3798
109
    case Intrinsic::arm_neon_vld3lane: {
3799
21
      static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3800
21
                                           ARM::VLD3LNd16Pseudo,
3801
21
                                           ARM::VLD3LNd32Pseudo };
3802
21
      static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3803
21
                                           ARM::VLD3LNq32Pseudo };
3804
21
      SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3805
21
      return;
3806
109
    }
3807
109
3808
109
    case Intrinsic::arm_neon_vld4lane: {
3809
18
      static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3810
18
                                           ARM::VLD4LNd16Pseudo,
3811
18
                                           ARM::VLD4LNd32Pseudo };
3812
18
      static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3813
18
                                           ARM::VLD4LNq32Pseudo };
3814
18
      SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3815
18
      return;
3816
109
    }
3817
109
3818
109
    case Intrinsic::arm_neon_vst1: {
3819
37
      static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3820
37
                                           ARM::VST1d32, ARM::VST1d64 };
3821
37
      static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3822
37
                                           ARM::VST1q32, ARM::VST1q64 };
3823
37
      SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
3824
37
      return;
3825
109
    }
3826
109
3827
109
    case Intrinsic::arm_neon_vst1x2: {
3828
8
      static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3829
8
                                           ARM::VST1q32, ARM::VST1q64 };
3830
8
      static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
3831
8
                                           ARM::VST1d16QPseudo,
3832
8
                                           ARM::VST1d32QPseudo,
3833
8
                                           ARM::VST1d64QPseudo };
3834
8
      SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3835
8
      return;
3836
109
    }
3837
109
3838
109
    case Intrinsic::arm_neon_vst1x3: {
3839
8
      static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
3840
8
                                           ARM::VST1d16TPseudo,
3841
8
                                           ARM::VST1d32TPseudo,
3842
8
                                           ARM::VST1d64TPseudo };
3843
8
      static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
3844
8
                                            ARM::VST1q16LowTPseudo_UPD,
3845
8
                                            ARM::VST1q32LowTPseudo_UPD,
3846
8
                                            ARM::VST1q64LowTPseudo_UPD };
3847
8
      static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
3848
8
                                            ARM::VST1q16HighTPseudo,
3849
8
                                            ARM::VST1q32HighTPseudo,
3850
8
                                            ARM::VST1q64HighTPseudo };
3851
8
      SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3852
8
      return;
3853
109
    }
3854
109
3855
109
    case Intrinsic::arm_neon_vst1x4: {
3856
8
      static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
3857
8
                                           ARM::VST1d16QPseudo,
3858
8
                                           ARM::VST1d32QPseudo,
3859
8
                                           ARM::VST1d64QPseudo };
3860
8
      static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
3861
8
                                            ARM::VST1q16LowQPseudo_UPD,
3862
8
                                            ARM::VST1q32LowQPseudo_UPD,
3863
8
                                            ARM::VST1q64LowQPseudo_UPD };
3864
8
      static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
3865
8
                                            ARM::VST1q16HighQPseudo,
3866
8
                                            ARM::VST1q32HighQPseudo,
3867
8
                                            ARM::VST1q64HighQPseudo };
3868
8
      SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3869
8
      return;
3870
109
    }
3871
109
3872
109
    case Intrinsic::arm_neon_vst2: {
3873
20
      static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3874
20
                                           ARM::VST2d32, ARM::VST1q64 };
3875
20
      static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3876
20
                                           ARM::VST2q32Pseudo };
3877
20
      SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3878
20
      return;
3879
109
    }
3880
109
3881
109
    case Intrinsic::arm_neon_vst3: {
3882
19
      static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3883
19
                                           ARM::VST3d16Pseudo,
3884
19
                                           ARM::VST3d32Pseudo,
3885
19
                                           ARM::VST1d64TPseudo };
3886
19
      static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3887
19
                                            ARM::VST3q16Pseudo_UPD,
3888
19
                                            ARM::VST3q32Pseudo_UPD };
3889
19
      static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3890
19
                                            ARM::VST3q16oddPseudo,
3891
19
                                            ARM::VST3q32oddPseudo };
3892
19
      SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3893
19
      return;
3894
109
    }
3895
109
3896
109
    case Intrinsic::arm_neon_vst4: {
3897
19
      static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3898
19
                                           ARM::VST4d16Pseudo,
3899
19
                                           ARM::VST4d32Pseudo,
3900
19
                                           ARM::VST1d64QPseudo };
3901
19
      static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3902
19
                                            ARM::VST4q16Pseudo_UPD,
3903
19
                                            ARM::VST4q32Pseudo_UPD };
3904
19
      static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3905
19
                                            ARM::VST4q16oddPseudo,
3906
19
                                            ARM::VST4q32oddPseudo };
3907
19
      SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3908
19
      return;
3909
109
    }
3910
109
3911
109
    case Intrinsic::arm_neon_vst2lane: {
3912
10
      static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3913
10
                                           ARM::VST2LNd16Pseudo,
3914
10
                                           ARM::VST2LNd32Pseudo };
3915
10
      static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3916
10
                                           ARM::VST2LNq32Pseudo };
3917
10
      SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3918
10
      return;
3919
109
    }
3920
109
3921
109
    case Intrinsic::arm_neon_vst3lane: {
3922
10
      static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3923
10
                                           ARM::VST3LNd16Pseudo,
3924
10
                                           ARM::VST3LNd32Pseudo };
3925
10
      static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3926
10
                                           ARM::VST3LNq32Pseudo };
3927
10
      SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3928
10
      return;
3929
109
    }
3930
109
3931
109
    case Intrinsic::arm_neon_vst4lane: {
3932
10
      static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3933
10
                                           ARM::VST4LNd16Pseudo,
3934
10
                                           ARM::VST4LNd32Pseudo };
3935
10
      static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3936
10
                                           ARM::VST4LNq32Pseudo };
3937
10
      SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3938
10
      return;
3939
4.82k
    }
3940
4.82k
    }
3941
4.82k
    break;
3942
4.82k
  }
3943
4.82k
3944
4.82k
  case ISD::ATOMIC_CMP_SWAP:
3945
6
    SelectCMP_SWAP(N);
3946
6
    return;
3947
1.97M
  }
3948
1.97M
3949
1.97M
  SelectCode(N);
3950
1.97M
}
3951
3952
// Inspect a register string of the form
3953
// cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or
3954
// cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string
3955
// and obtain the integer operands from them, adding these operands to the
3956
// provided vector.
3957
static void getIntOperandsFromRegisterString(StringRef RegString,
3958
                                             SelectionDAG *CurDAG,
3959
                                             const SDLoc &DL,
3960
230
                                             std::vector<SDValue> &Ops) {
3961
230
  SmallVector<StringRef, 5> Fields;
3962
230
  RegString.split(Fields, ':');
3963
230
3964
230
  if (Fields.size() > 1) {
3965
8
    bool AllIntFields = true;
3966
8
3967
32
    for (StringRef Field : Fields) {
3968
32
      // Need to trim out leading 'cp' characters and get the integer field.
3969
32
      unsigned IntField;
3970
32
      AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField);
3971
32
      Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32));
3972
32
    }
3973
8
3974
8
    assert(AllIntFields &&
3975
8
            "Unexpected non-integer value in special register string.");
3976
8
  }
3977
230
}
3978
3979
// Maps a Banked Register string to its mask value. The mask value returned is
3980
// for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register
3981
// mask operand, which expresses which register is to be used, e.g. r8, and in
3982
// which mode it is to be used, e.g. usr. Returns -1 to signify that the string
3983
// was invalid.
3984
222
static inline int getBankedRegisterMask(StringRef RegString) {
3985
222
  auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.lower());
3986
222
  if (!TheReg)
3987
222
     return -1;
3988
0
  return TheReg->Encoding;
3989
0
}
3990
3991
// The flags here are common to those allowed for apsr in the A class cores and
3992
// those allowed for the special registers in the M class cores. Returns a
3993
// value representing which flags were present, -1 if invalid.
3994
12
static inline int getMClassFlagsMask(StringRef Flags) {
3995
12
  return StringSwitch<int>(Flags)
3996
12
          .Case("", 0x2) // no flags means nzcvq for psr registers, and 0x2 is
3997
12
                         // correct when flags are not permitted
3998
12
          .Case("g", 0x1)
3999
12
          .Case("nzcvq", 0x2)
4000
12
          .Case("nzcvqg", 0x3)
4001
12
          .Default(-1);
4002
12
}
4003
4004
// Maps MClass special registers string to its value for use in the
4005
// t2MRS_M/t2MSR_M instruction nodes as the SYSm value operand.
4006
// Returns -1 to signify that the string was invalid.
4007
182
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget) {
4008
182
  auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
4009
182
  const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
4010
182
  if (!TheReg || 
!TheReg->hasRequiredFeatures(FeatureBits)181
)
4011
4
    return -1;
4012
178
  return (int)(TheReg->Encoding & 0xFFF); // SYSm value
4013
178
}
4014
4015
24
static int getARClassRegisterMask(StringRef Reg, StringRef Flags) {
4016
24
  // The mask operand contains the special register (R Bit) in bit 4, whether
4017
24
  // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and
4018
24
  // bits 3-0 contains the fields to be accessed in the special register, set by
4019
24
  // the flags provided with the register.
4020
24
  int Mask = 0;
4021
24
  if (Reg == "apsr") {
4022
12
    // The flags permitted for apsr are the same flags that are allowed in
4023
12
    // M class registers. We get the flag value and then shift the flags into
4024
12
    // the correct place to combine with the mask.
4025
12
    Mask = getMClassFlagsMask(Flags);
4026
12
    if (Mask == -1)
4027
0
      return -1;
4028
12
    return Mask << 2;
4029
12
  }
4030
12
4031
12
  if (Reg != "cpsr" && 
Reg != "spsr"7
) {
4032
2
    return -1;
4033
2
  }
4034
10
4035
10
  // This is the same as if the flags were "fc"
4036
10
  if (Flags.empty() || Flags == "all")
4037
0
    return Mask | 0x9;
4038
10
4039
10
  // Inspect the supplied flags string and set the bits in the mask for
4040
10
  // the relevant and valid flags allowed for cpsr and spsr.
4041
16
  
for (char Flag : Flags)10
{
4042
16
    int FlagVal;
4043
16
    switch (Flag) {
4044
16
      case 'c':
4045
4
        FlagVal = 0x1;
4046
4
        break;
4047
16
      case 'x':
4048
4
        FlagVal = 0x2;
4049
4
        break;
4050
16
      case 's':
4051
4
        FlagVal = 0x4;
4052
4
        break;
4053
16
      case 'f':
4054
4
        FlagVal = 0x8;
4055
4
        break;
4056
16
      default:
4057
0
        FlagVal = 0;
4058
16
    }
4059
16
4060
16
    // This avoids allowing strings where the same flag bit appears twice.
4061
16
    if (!FlagVal || (Mask & FlagVal))
4062
0
      return -1;
4063
16
    Mask |= FlagVal;
4064
16
  }
4065
10
4066
10
  // If the register is spsr then we need to set the R bit.
4067
10
  if (Reg == "spsr")
4068
5
    Mask |= 0x10;
4069
10
4070
10
  return Mask;
4071
10
}
4072
4073
// Lower the read_register intrinsic to ARM specific DAG nodes
4074
// using the supplied metadata string to select the instruction node to use
4075
// and the registers/masks to construct as operands for the node.
4076
95
bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){
4077
95
  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
4078
95
  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
4079
95
  bool IsThumb2 = Subtarget->isThumb2();
4080
95
  SDLoc DL(N);
4081
95
4082
95
  std::vector<SDValue> Ops;
4083
95
  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
4084
95
4085
95
  if (!Ops.empty()) {
4086
4
    // If the special register string was constructed of fields (as defined
4087
4
    // in the ACLE) then need to lower to MRC node (32 bit) or
4088
4
    // MRRC node(64 bit), we can make the distinction based on the number of
4089
4
    // operands we have.
4090
4
    unsigned Opcode;
4091
4
    SmallVector<EVT, 3> ResTypes;
4092
4
    if (Ops.size() == 5){
4093
2
      Opcode = IsThumb2 ? 
ARM::t2MRC1
:
ARM::MRC1
;
4094
2
      ResTypes.append({ MVT::i32, MVT::Other });
4095
2
    } else {
4096
2
      assert(Ops.size() == 3 &&
4097
2
              "Invalid number of fields in special register string.");
4098
2
      Opcode = IsThumb2 ? 
ARM::t2MRRC1
:
ARM::MRRC1
;
4099
2
      ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
4100
2
    }
4101
4
4102
4
    Ops.push_back(getAL(CurDAG, DL));
4103
4
    Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4104
4
    Ops.push_back(N->getOperand(0));
4105
4
    ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops));
4106
4
    return true;
4107
4
  }
4108
91
4109
91
  std::string SpecialReg = RegString->getString().lower();
4110
91
4111
91
  int BankedReg = getBankedRegisterMask(SpecialReg);
4112
91
  if (BankedReg != -1) {
4113
0
    Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
4114
0
            getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4115
0
            N->getOperand(0) };
4116
0
    ReplaceNode(
4117
0
        N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
4118
0
                                  DL, MVT::i32, MVT::Other, Ops));
4119
0
    return true;
4120
0
  }
4121
91
4122
91
  // The VFP registers are read by creating SelectionDAG nodes with opcodes
4123
91
  // corresponding to the register that is being read from. So we switch on the
4124
91
  // string to find which opcode we need to use.
4125
91
  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
4126
91
                    .Case("fpscr", ARM::VMRS)
4127
91
                    .Case("fpexc", ARM::VMRS_FPEXC)
4128
91
                    .Case("fpsid", ARM::VMRS_FPSID)
4129
91
                    .Case("mvfr0", ARM::VMRS_MVFR0)
4130
91
                    .Case("mvfr1", ARM::VMRS_MVFR1)
4131
91
                    .Case("mvfr2", ARM::VMRS_MVFR2)
4132
91
                    .Case("fpinst", ARM::VMRS_FPINST)
4133
91
                    .Case("fpinst2", ARM::VMRS_FPINST2)
4134
91
                    .Default(0);
4135
91
4136
91
  // If an opcode was found then we can lower the read to a VFP instruction.
4137
91
  if (Opcode) {
4138
2
    if (!Subtarget->hasVFP2Base())
4139
0
      return false;
4140
2
    if (Opcode == ARM::VMRS_MVFR2 && 
!Subtarget->hasFPARMv8Base()0
)
4141
0
      return false;
4142
2
4143
2
    Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4144
2
            N->getOperand(0) };
4145
2
    ReplaceNode(N,
4146
2
                CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops));
4147
2
    return true;
4148
2
  }
4149
89
4150
89
  // If the target is M Class then need to validate that the register string
4151
89
  // is an acceptable value, so check that a mask can be constructed from the
4152
89
  // string.
4153
89
  if (Subtarget->isMClass()) {
4154
77
    int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
4155
77
    if (SYSmValue == -1)
4156
3
      return false;
4157
74
4158
74
    SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4159
74
                      getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4160
74
                      N->getOperand(0) };
4161
74
    ReplaceNode(
4162
74
        N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops));
4163
74
    return true;
4164
74
  }
4165
12
4166
12
  // Here we know the target is not M Class so we need to check if it is one
4167
12
  // of the remaining possible values which are apsr, cpsr or spsr.
4168
12
  if (SpecialReg == "apsr" || 
SpecialReg == "cpsr"10
) {
4169
4
    Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4170
4
            N->getOperand(0) };
4171
4
    ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? 
ARM::t2MRS_AR0
: ARM::MRS,
4172
4
                                          DL, MVT::i32, MVT::Other, Ops));
4173
4
    return true;
4174
4
  }
4175
8
4176
8
  if (SpecialReg == "spsr") {
4177
1
    Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4178
1
            N->getOperand(0) };
4179
1
    ReplaceNode(
4180
1
        N, CurDAG->getMachineNode(IsThumb2 ? 
ARM::t2MRSsys_AR0
: ARM::MRSsys, DL,
4181
1
                                  MVT::i32, MVT::Other, Ops));
4182
1
    return true;
4183
1
  }
4184
7
4185
7
  return false;
4186
7
}
4187
4188
// Lower the write_register intrinsic to ARM specific DAG nodes
4189
// using the supplied metadata string to select the instruction node to use
4190
// and the registers/masks to use in the nodes
4191
135
bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){
4192
135
  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
4193
135
  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
4194
135
  bool IsThumb2 = Subtarget->isThumb2();
4195
135
  SDLoc DL(N);
4196
135
4197
135
  std::vector<SDValue> Ops;
4198
135
  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
4199
135
4200
135
  if (!Ops.empty()) {
4201
4
    // If the special register string was constructed of fields (as defined
4202
4
    // in the ACLE) then need to lower to MCR node (32 bit) or
4203
4
    // MCRR node(64 bit), we can make the distinction based on the number of
4204
4
    // operands we have.
4205
4
    unsigned Opcode;
4206
4
    if (Ops.size() == 5) {
4207
2
      Opcode = IsThumb2 ? 
ARM::t2MCR1
:
ARM::MCR1
;
4208
2
      Ops.insert(Ops.begin()+2, N->getOperand(2));
4209
2
    } else {
4210
2
      assert(Ops.size() == 3 &&
4211
2
              "Invalid number of fields in special register string.");
4212
2
      Opcode = IsThumb2 ? 
ARM::t2MCRR1
:
ARM::MCRR1
;
4213
2
      SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
4214
2
      Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
4215
2
    }
4216
4
4217
4
    Ops.push_back(getAL(CurDAG, DL));
4218
4
    Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4219
4
    Ops.push_back(N->getOperand(0));
4220
4
4221
4
    ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
4222
4
    return true;
4223
4
  }
4224
131
4225
131
  std::string SpecialReg = RegString->getString().lower();
4226
131
  int BankedReg = getBankedRegisterMask(SpecialReg);
4227
131
  if (BankedReg != -1) {
4228
0
    Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
4229
0
            getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4230
0
            N->getOperand(0) };
4231
0
    ReplaceNode(
4232
0
        N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
4233
0
                                  DL, MVT::Other, Ops));
4234
0
    return true;
4235
0
  }
4236
131
4237
131
  // The VFP registers are written to by creating SelectionDAG nodes with
4238
131
  // opcodes corresponding to the register that is being written. So we switch
4239
131
  // on the string to find which opcode we need to use.
4240
131
  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
4241
131
                    .Case("fpscr", ARM::VMSR)
4242
131
                    .Case("fpexc", ARM::VMSR_FPEXC)
4243
131
                    .Case("fpsid", ARM::VMSR_FPSID)
4244
131
                    .Case("fpinst", ARM::VMSR_FPINST)
4245
131
                    .Case("fpinst2", ARM::VMSR_FPINST2)
4246
131
                    .Default(0);
4247
131
4248
131
  if (Opcode) {
4249
2
    if (!Subtarget->hasVFP2Base())
4250
0
      return false;
4251
2
    Ops = { N->getOperand(2), getAL(CurDAG, DL),
4252
2
            CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
4253
2
    ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
4254
2
    return true;
4255
2
  }
4256
129
4257
129
  std::pair<StringRef, StringRef> Fields;
4258
129
  Fields = StringRef(SpecialReg).rsplit('_');
4259
129
  std::string Reg = Fields.first.str();
4260
129
  StringRef Flags = Fields.second;
4261
129
4262
129
  // If the target was M Class then need to validate the special register value
4263
129
  // and retrieve the mask for use in the instruction node.
4264
129
  if (Subtarget->isMClass()) {
4265
105
    int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
4266
105
    if (SYSmValue == -1)
4267
1
      return false;
4268
104
4269
104
    SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4270
104
                      N->getOperand(2), getAL(CurDAG, DL),
4271
104
                      CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
4272
104
    ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops));
4273
104
    return true;
4274
104
  }
4275
24
4276
24
  // We then check to see if a valid mask can be constructed for one of the
4277
24
  // register string values permitted for the A and R class cores. These values
4278
24
  // are apsr, spsr and cpsr; these are also valid on older cores.
4279
24
  int Mask = getARClassRegisterMask(Reg, Flags);
4280
24
  if (Mask != -1) {
4281
22
    Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
4282
22
            getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4283
22
            N->getOperand(0) };
4284
22
    ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? 
ARM::t2MSR_AR4
:
ARM::MSR18
,
4285
22
                                          DL, MVT::Other, Ops));
4286
22
    return true;
4287
22
  }
4288
2
4289
2
  return false;
4290
2
}
4291
4292
4.95k
bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
4293
4.95k
  std::vector<SDValue> AsmNodeOperands;
4294
4.95k
  unsigned Flag, Kind;
4295
4.95k
  bool Changed = false;
4296
4.95k
  unsigned NumOps = N->getNumOperands();
4297
4.95k
4298
4.95k
  // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
4299
4.95k
  // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
4300
4.95k
  // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
4301
4.95k
  // respectively. Since there is no constraint to explicitly specify a
4302
4.95k
  // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
4303
4.95k
  // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
4304
4.95k
  // them into a GPRPair.
4305
4.95k
4306
4.95k
  SDLoc dl(N);
4307
4.95k
  SDValue Glue = N->getGluedNode() ? 
N->getOperand(NumOps-1)568
4308
4.95k
                                   : 
SDValue(nullptr,0)4.38k
;
4309
4.95k
4310
4.95k
  SmallVector<bool, 8> OpChanged;
4311
4.95k
  // Glue node will be appended late.
4312
34.7k
  for(unsigned i = 0, e = N->getGluedNode() ? 
NumOps - 1568
:
NumOps4.38k
; i < e;
++i29.7k
) {
4313
29.7k
    SDValue op = N->getOperand(i);
4314
29.7k
    AsmNodeOperands.push_back(op);
4315
29.7k
4316
29.7k
    if (i < InlineAsm::Op_FirstOperand)
4317
19.8k
      continue;
4318
9.92k
4319
9.92k
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
4320
5.05k
      Flag = C->getZExtValue();
4321
5.05k
      Kind = InlineAsm::getKind(Flag);
4322
5.05k
    }
4323
4.87k
    else
4324
4.87k
      continue;
4325
5.05k
4326
5.05k
    // Immediate operands to inline asm in the SelectionDAG are modeled with
4327
5.05k
    // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
4328
5.05k
    // the second is a constant with the value of the immediate. If we get here
4329
5.05k
    // and we have a Kind_Imm, skip the next operand, and continue.
4330
5.05k
    if (Kind == InlineAsm::Kind_Imm) {
4331
48
      SDValue op = N->getOperand(++i);
4332
48
      AsmNodeOperands.push_back(op);
4333
48
      continue;
4334
48
    }
4335
5.00k
4336
5.00k
    unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
4337
5.00k
    if (NumRegs)
4338
5.00k
      OpChanged.push_back(false);
4339
5.00k
4340
5.00k
    unsigned DefIdx = 0;
4341
5.00k
    bool IsTiedToChangedOp = false;
4342
5.00k
    // If it's a use that is tied with a previous def, it has no
4343
5.00k
    // reg class constraint.
4344
5.00k
    if (Changed && 
InlineAsm::isUseOperandTiedToDef(Flag, DefIdx)100
)
4345
13
      IsTiedToChangedOp = OpChanged[DefIdx];
4346
5.00k
4347
5.00k
    // Memory operands to inline asm in the SelectionDAG are modeled with two
4348
5.00k
    // operands: a constant of value InlineAsm::Kind_Mem followed by the input
4349
5.00k
    // operand. If we get here and we have a Kind_Mem, skip the next operand (so
4350
5.00k
    // it doesn't get misinterpreted), and continue. We do this here because
4351
5.00k
    // it's important to update the OpChanged array correctly before moving on.
4352
5.00k
    if (Kind == InlineAsm::Kind_Mem) {
4353
25
      SDValue op = N->getOperand(++i);
4354
25
      AsmNodeOperands.push_back(op);
4355
25
      continue;
4356
25
    }
4357
4.97k
4358
4.97k
    if (Kind != InlineAsm::Kind_RegUse && 
Kind != InlineAsm::Kind_RegDef4.19k
4359
4.97k
        && 
Kind != InlineAsm::Kind_RegDefEarlyClobber3.94k
)
4360
3.86k
      continue;
4361
1.10k
4362
1.10k
    unsigned RC;
4363
1.10k
    bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
4364
1.10k
    if ((!IsTiedToChangedOp && 
(1.09k
!HasRC1.09k
||
RC != ARM::GPRRegClassID829
))
4365
1.10k
        || 
NumRegs != 2709
)
4366
994
      continue;
4367
115
4368
115
    assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
4369
115
    SDValue V0 = N->getOperand(i+1);
4370
115
    SDValue V1 = N->getOperand(i+2);
4371
115
    unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
4372
115
    unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
4373
115
    SDValue PairedReg;
4374
115
    MachineRegisterInfo &MRI = MF->getRegInfo();
4375
115
4376
115
    if (Kind == InlineAsm::Kind_RegDef ||
4377
115
        
Kind == InlineAsm::Kind_RegDefEarlyClobber97
) {
4378
56
      // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
4379
56
      // the original GPRs.
4380
56
4381
56
      unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4382
56
      PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4383
56
      SDValue Chain = SDValue(N,0);
4384
56
4385
56
      SDNode *GU = N->getGluedUser();
4386
56
      SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
4387
56
                                               Chain.getValue(1));
4388
56
4389
56
      // Extract values from a GPRPair reg and copy to the original GPR reg.
4390
56
      SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4391
56
                                                    RegCopy);
4392
56
      SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4393
56
                                                    RegCopy);
4394
56
      SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
4395
56
                                        RegCopy.getValue(1));
4396
56
      SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
4397
56
4398
56
      // Update the original glue user.
4399
56
      std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
4400
56
      Ops.push_back(T1.getValue(1));
4401
56
      CurDAG->UpdateNodeOperands(GU, Ops);
4402
56
    }
4403
59
    else {
4404
59
      // For Kind  == InlineAsm::Kind_RegUse, we first copy two GPRs into a
4405
59
      // GPRPair and then pass the GPRPair to the inline asm.
4406
59
      SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
4407
59
4408
59
      // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
4409
59
      SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
4410
59
                                          Chain.getValue(1));
4411
59
      SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
4412
59
                                          T0.getValue(1));
4413
59
      SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
4414
59
4415
59
      // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
4416
59
      // i32 VRs of inline asm with it.
4417
59
      unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4418
59
      PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4419
59
      Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
4420
59
4421
59
      AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
4422
59
      Glue = Chain.getValue(1);
4423
59
    }
4424
115
4425
115
    Changed = true;
4426
115
4427
115
    if(PairedReg.getNode()) {
4428
115
      OpChanged[OpChanged.size() -1 ] = true;
4429
115
      Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
4430
115
      if (IsTiedToChangedOp)
4431
13
        Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
4432
102
      else
4433
102
        Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
4434
115
      // Replace the current flag.
4435
115
      AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
4436
115
          Flag, dl, MVT::i32);
4437
115
      // Add the new register node and skip the original two GPRs.
4438
115
      AsmNodeOperands.push_back(PairedReg);
4439
115
      // Skip the next two GPRs.
4440
115
      i += 2;
4441
115
    }
4442
115
  }
4443
4.95k
4444
4.95k
  if (Glue.getNode())
4445
568
    AsmNodeOperands.push_back(Glue);
4446
4.95k
  if (!Changed)
4447
4.89k
    return false;
4448
59
4449
59
  SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
4450
59
      CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
4451
59
  New->setNodeId(-1);
4452
59
  ReplaceNode(N, New.getNode());
4453
59
  return true;
4454
59
}
4455
4456
4457
bool ARMDAGToDAGISel::
4458
SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
4459
18
                             std::vector<SDValue> &OutOps) {
4460
18
  switch(ConstraintID) {
4461
18
  default:
4462
0
    llvm_unreachable("Unexpected asm memory constraint");
4463
18
  case InlineAsm::Constraint_i:
4464
0
    // FIXME: It seems strange that 'i' is needed here since it's supposed to
4465
0
    //        be an immediate and not a memory constraint.
4466
0
    LLVM_FALLTHROUGH;
4467
18
  case InlineAsm::Constraint_m:
4468
18
  case InlineAsm::Constraint_o:
4469
18
  case InlineAsm::Constraint_Q:
4470
18
  case InlineAsm::Constraint_Um:
4471
18
  case InlineAsm::Constraint_Un:
4472
18
  case InlineAsm::Constraint_Uq:
4473
18
  case InlineAsm::Constraint_Us:
4474
18
  case InlineAsm::Constraint_Ut:
4475
18
  case InlineAsm::Constraint_Uv:
4476
18
  case InlineAsm::Constraint_Uy:
4477
18
    // Require the address to be in a register.  That is safe for all ARM
4478
18
    // variants and it is hard to do anything much smarter without knowing
4479
18
    // how the operand is used.
4480
18
    OutOps.push_back(Op);
4481
18
    return false;
4482
0
  }
4483
0
  return true;
4484
0
}
4485
4486
/// createARMISelDag - This pass converts a legalized DAG into a
4487
/// ARM-specific DAG, ready for instruction scheduling.
4488
///
4489
FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
4490
5.14k
                                     CodeGenOpt::Level OptLevel) {
4491
5.14k
  return new ARMDAGToDAGISel(TM, OptLevel);
4492
5.14k
}