Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains a pass that expands pseudo instructions into target
10
// instructions to allow proper scheduling, if-conversion, and other late
11
// optimizations. This pass should be run after register allocation but before
12
// the post-regalloc scheduling pass.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#include "ARM.h"
17
#include "ARMBaseInstrInfo.h"
18
#include "ARMBaseRegisterInfo.h"
19
#include "ARMConstantPoolValue.h"
20
#include "ARMMachineFunctionInfo.h"
21
#include "ARMSubtarget.h"
22
#include "MCTargetDesc/ARMAddressingModes.h"
23
#include "llvm/CodeGen/LivePhysRegs.h"
24
#include "llvm/CodeGen/MachineFrameInfo.h"
25
#include "llvm/CodeGen/MachineFunctionPass.h"
26
#include "llvm/Support/Debug.h"
27
28
using namespace llvm;
29
30
#define DEBUG_TYPE "arm-pseudo"
31
32
static cl::opt<bool>
33
VerifyARMPseudo("verify-arm-pseudo-expand", cl::Hidden,
34
                cl::desc("Verify machine code after expanding ARM pseudos"));
35
36
31.7k
#define ARM_EXPAND_PSEUDO_NAME "ARM pseudo instruction expansion pass"
37
38
namespace {
39
  class ARMExpandPseudo : public MachineFunctionPass {
40
  public:
41
    static char ID;
42
5.19k
    ARMExpandPseudo() : MachineFunctionPass(ID) {}
43
44
    const ARMBaseInstrInfo *TII;
45
    const TargetRegisterInfo *TRI;
46
    const ARMSubtarget *STI;
47
    ARMFunctionInfo *AFI;
48
49
    bool runOnMachineFunction(MachineFunction &Fn) override;
50
51
5.17k
    MachineFunctionProperties getRequiredProperties() const override {
52
5.17k
      return MachineFunctionProperties().set(
53
5.17k
          MachineFunctionProperties::Property::NoVRegs);
54
5.17k
    }
55
56
31.7k
    StringRef getPassName() const override {
57
31.7k
      return ARM_EXPAND_PSEUDO_NAME;
58
31.7k
    }
59
60
  private:
61
    void TransferImpOps(MachineInstr &OldMI,
62
                        MachineInstrBuilder &UseMI, MachineInstrBuilder &DefMI);
63
    bool ExpandMI(MachineBasicBlock &MBB,
64
                  MachineBasicBlock::iterator MBBI,
65
                  MachineBasicBlock::iterator &NextMBBI);
66
    bool ExpandMBB(MachineBasicBlock &MBB);
67
    void ExpandVLD(MachineBasicBlock::iterator &MBBI);
68
    void ExpandVST(MachineBasicBlock::iterator &MBBI);
69
    void ExpandLaneOp(MachineBasicBlock::iterator &MBBI);
70
    void ExpandVTBL(MachineBasicBlock::iterator &MBBI,
71
                    unsigned Opc, bool IsExt);
72
    void ExpandMOV32BitImm(MachineBasicBlock &MBB,
73
                           MachineBasicBlock::iterator &MBBI);
74
    bool ExpandCMP_SWAP(MachineBasicBlock &MBB,
75
                        MachineBasicBlock::iterator MBBI, unsigned LdrexOp,
76
                        unsigned StrexOp, unsigned UxtOp,
77
                        MachineBasicBlock::iterator &NextMBBI);
78
79
    bool ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
80
                           MachineBasicBlock::iterator MBBI,
81
                           MachineBasicBlock::iterator &NextMBBI);
82
  };
83
  char ARMExpandPseudo::ID = 0;
84
}
85
86
INITIALIZE_PASS(ARMExpandPseudo, DEBUG_TYPE, ARM_EXPAND_PSEUDO_NAME, false,
87
                false)
88
89
/// TransferImpOps - Transfer implicit operands on the pseudo instruction to
90
/// the instructions created from the expansion.
91
void ARMExpandPseudo::TransferImpOps(MachineInstr &OldMI,
92
                                     MachineInstrBuilder &UseMI,
93
30.5k
                                     MachineInstrBuilder &DefMI) {
94
30.5k
  const MCInstrDesc &Desc = OldMI.getDesc();
95
30.5k
  for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands();
96
30.7k
       i != e; 
++i206
) {
97
206
    const MachineOperand &MO = OldMI.getOperand(i);
98
206
    assert(MO.isReg() && MO.getReg());
99
206
    if (MO.isUse())
100
45
      UseMI.add(MO);
101
161
    else
102
161
      DefMI.add(MO);
103
206
  }
104
30.5k
}
105
106
namespace {
107
  // Constants for register spacing in NEON load/store instructions.
108
  // For quad-register load-lane and store-lane pseudo instructors, the
109
  // spacing is initially assumed to be EvenDblSpc, and that is changed to
110
  // OddDblSpc depending on the lane number operand.
111
  enum NEONRegSpacing {
112
    SingleSpc,
113
    SingleLowSpc ,  // Single spacing, low registers, three and four vectors.
114
    SingleHighQSpc, // Single spacing, high registers, four vectors.
115
    SingleHighTSpc, // Single spacing, high registers, three vectors.
116
    EvenDblSpc,
117
    OddDblSpc
118
  };
119
120
  // Entries for NEON load/store information table.  The table is sorted by
121
  // PseudoOpc for fast binary-search lookups.
122
  struct NEONLdStTableEntry {
123
    uint16_t PseudoOpc;
124
    uint16_t RealOpc;
125
    bool IsLoad;
126
    bool isUpdating;
127
    bool hasWritebackOperand;
128
    uint8_t RegSpacing; // One of type NEONRegSpacing
129
    uint8_t NumRegs; // D registers loaded or stored
130
    uint8_t RegElts; // elements per D register; used for lane ops
131
    // FIXME: Temporary flag to denote whether the real instruction takes
132
    // a single register (like the encoding) or all of the registers in
133
    // the list (like the asm syntax and the isel DAG). When all definitions
134
    // are converted to take only the single encoded register, this will
135
    // go away.
136
    bool copyAllListRegs;
137
138
    // Comparison methods for binary search of the table.
139
0
    bool operator<(const NEONLdStTableEntry &TE) const {
140
0
      return PseudoOpc < TE.PseudoOpc;
141
0
    }
142
3.53k
    friend bool operator<(const NEONLdStTableEntry &TE, unsigned PseudoOpc) {
143
3.53k
      return TE.PseudoOpc < PseudoOpc;
144
3.53k
    }
145
    friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned PseudoOpc,
146
0
                                                const NEONLdStTableEntry &TE) {
147
0
      return PseudoOpc < TE.PseudoOpc;
148
0
    }
149
  };
150
}
151
152
static const NEONLdStTableEntry NEONLdStTable[] = {
153
{ ARM::VLD1LNq16Pseudo,     ARM::VLD1LNd16,     true, false, false, EvenDblSpc, 1, 4 ,true},
154
{ ARM::VLD1LNq16Pseudo_UPD, ARM::VLD1LNd16_UPD, true, true, true,  EvenDblSpc, 1, 4 ,true},
155
{ ARM::VLD1LNq32Pseudo,     ARM::VLD1LNd32,     true, false, false, EvenDblSpc, 1, 2 ,true},
156
{ ARM::VLD1LNq32Pseudo_UPD, ARM::VLD1LNd32_UPD, true, true, true,  EvenDblSpc, 1, 2 ,true},
157
{ ARM::VLD1LNq8Pseudo,      ARM::VLD1LNd8,      true, false, false, EvenDblSpc, 1, 8 ,true},
158
{ ARM::VLD1LNq8Pseudo_UPD,  ARM::VLD1LNd8_UPD, true, true, true,  EvenDblSpc, 1, 8 ,true},
159
160
{ ARM::VLD1d16QPseudo,      ARM::VLD1d16Q,     true,  false, false, SingleSpc,  4, 4 ,false},
161
{ ARM::VLD1d16TPseudo,      ARM::VLD1d16T,     true,  false, false, SingleSpc,  3, 4 ,false},
162
{ ARM::VLD1d32QPseudo,      ARM::VLD1d32Q,     true,  false, false, SingleSpc,  4, 2 ,false},
163
{ ARM::VLD1d32TPseudo,      ARM::VLD1d32T,     true,  false, false, SingleSpc,  3, 2 ,false},
164
{ ARM::VLD1d64QPseudo,      ARM::VLD1d64Q,     true,  false, false, SingleSpc,  4, 1 ,false},
165
{ ARM::VLD1d64QPseudoWB_fixed,  ARM::VLD1d64Qwb_fixed,   true,  true, false, SingleSpc,  4, 1 ,false},
166
{ ARM::VLD1d64QPseudoWB_register,  ARM::VLD1d64Qwb_register,   true,  true, true, SingleSpc,  4, 1 ,false},
167
{ ARM::VLD1d64TPseudo,      ARM::VLD1d64T,     true,  false, false, SingleSpc,  3, 1 ,false},
168
{ ARM::VLD1d64TPseudoWB_fixed,  ARM::VLD1d64Twb_fixed,   true,  true, false, SingleSpc,  3, 1 ,false},
169
{ ARM::VLD1d64TPseudoWB_register,  ARM::VLD1d64Twb_register, true, true, true,  SingleSpc,  3, 1 ,false},
170
{ ARM::VLD1d8QPseudo,       ARM::VLD1d8Q,      true,  false, false, SingleSpc,  4, 8 ,false},
171
{ ARM::VLD1d8TPseudo,       ARM::VLD1d8T,      true,  false, false, SingleSpc,  3, 8 ,false},
172
{ ARM::VLD1q16HighQPseudo,  ARM::VLD1d16Q,     true,  false, false, SingleHighQSpc,  4, 4 ,false},
173
{ ARM::VLD1q16HighTPseudo,  ARM::VLD1d16T,     true,  false, false, SingleHighTSpc,  3, 4 ,false},
174
{ ARM::VLD1q16LowQPseudo_UPD,  ARM::VLD1d16Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 4 ,false},
175
{ ARM::VLD1q16LowTPseudo_UPD,  ARM::VLD1d16Twb_fixed,   true,  true, true, SingleLowSpc,  3, 4 ,false},
176
{ ARM::VLD1q32HighQPseudo,  ARM::VLD1d32Q,     true,  false, false, SingleHighQSpc,  4, 2 ,false},
177
{ ARM::VLD1q32HighTPseudo,  ARM::VLD1d32T,     true,  false, false, SingleHighTSpc,  3, 2 ,false},
178
{ ARM::VLD1q32LowQPseudo_UPD,  ARM::VLD1d32Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 2 ,false},
179
{ ARM::VLD1q32LowTPseudo_UPD,  ARM::VLD1d32Twb_fixed,   true,  true, true, SingleLowSpc,  3, 2 ,false},
180
{ ARM::VLD1q64HighQPseudo,  ARM::VLD1d64Q,     true,  false, false, SingleHighQSpc,  4, 1 ,false},
181
{ ARM::VLD1q64HighTPseudo,  ARM::VLD1d64T,     true,  false, false, SingleHighTSpc,  3, 1 ,false},
182
{ ARM::VLD1q64LowQPseudo_UPD,  ARM::VLD1d64Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 1 ,false},
183
{ ARM::VLD1q64LowTPseudo_UPD,  ARM::VLD1d64Twb_fixed,   true,  true, true, SingleLowSpc,  3, 1 ,false},
184
{ ARM::VLD1q8HighQPseudo,   ARM::VLD1d8Q,     true,  false, false, SingleHighQSpc,  4, 8 ,false},
185
{ ARM::VLD1q8HighTPseudo,   ARM::VLD1d8T,     true,  false, false, SingleHighTSpc,  3, 8 ,false},
186
{ ARM::VLD1q8LowQPseudo_UPD,  ARM::VLD1d8Qwb_fixed,   true,  true, true, SingleLowSpc,  4, 8 ,false},
187
{ ARM::VLD1q8LowTPseudo_UPD,  ARM::VLD1d8Twb_fixed,   true,  true, true, SingleLowSpc,  3, 8 ,false},
188
189
{ ARM::VLD2DUPq16EvenPseudo,  ARM::VLD2DUPd16x2,  true, false, false, EvenDblSpc, 2, 4 ,false},
190
{ ARM::VLD2DUPq16OddPseudo,   ARM::VLD2DUPd16x2,  true, false, false, OddDblSpc,  2, 4 ,false},
191
{ ARM::VLD2DUPq32EvenPseudo,  ARM::VLD2DUPd32x2,  true, false, false, EvenDblSpc, 2, 2 ,false},
192
{ ARM::VLD2DUPq32OddPseudo,   ARM::VLD2DUPd32x2,  true, false, false, OddDblSpc,  2, 2 ,false},
193
{ ARM::VLD2DUPq8EvenPseudo,   ARM::VLD2DUPd8x2,   true, false, false, EvenDblSpc, 2, 8 ,false},
194
{ ARM::VLD2DUPq8OddPseudo,    ARM::VLD2DUPd8x2,   true, false, false, OddDblSpc,  2, 8 ,false},
195
196
{ ARM::VLD2LNd16Pseudo,     ARM::VLD2LNd16,     true, false, false, SingleSpc,  2, 4 ,true},
197
{ ARM::VLD2LNd16Pseudo_UPD, ARM::VLD2LNd16_UPD, true, true, true,  SingleSpc,  2, 4 ,true},
198
{ ARM::VLD2LNd32Pseudo,     ARM::VLD2LNd32,     true, false, false, SingleSpc,  2, 2 ,true},
199
{ ARM::VLD2LNd32Pseudo_UPD, ARM::VLD2LNd32_UPD, true, true, true,  SingleSpc,  2, 2 ,true},
200
{ ARM::VLD2LNd8Pseudo,      ARM::VLD2LNd8,      true, false, false, SingleSpc,  2, 8 ,true},
201
{ ARM::VLD2LNd8Pseudo_UPD,  ARM::VLD2LNd8_UPD, true, true, true,  SingleSpc,  2, 8 ,true},
202
{ ARM::VLD2LNq16Pseudo,     ARM::VLD2LNq16,     true, false, false, EvenDblSpc, 2, 4 ,true},
203
{ ARM::VLD2LNq16Pseudo_UPD, ARM::VLD2LNq16_UPD, true, true, true,  EvenDblSpc, 2, 4 ,true},
204
{ ARM::VLD2LNq32Pseudo,     ARM::VLD2LNq32,     true, false, false, EvenDblSpc, 2, 2 ,true},
205
{ ARM::VLD2LNq32Pseudo_UPD, ARM::VLD2LNq32_UPD, true, true, true,  EvenDblSpc, 2, 2 ,true},
206
207
{ ARM::VLD2q16Pseudo,       ARM::VLD2q16,      true,  false, false, SingleSpc,  4, 4 ,false},
208
{ ARM::VLD2q16PseudoWB_fixed,   ARM::VLD2q16wb_fixed, true, true, false,  SingleSpc,  4, 4 ,false},
209
{ ARM::VLD2q16PseudoWB_register,   ARM::VLD2q16wb_register, true, true, true,  SingleSpc,  4, 4 ,false},
210
{ ARM::VLD2q32Pseudo,       ARM::VLD2q32,      true,  false, false, SingleSpc,  4, 2 ,false},
211
{ ARM::VLD2q32PseudoWB_fixed,   ARM::VLD2q32wb_fixed, true, true, false,  SingleSpc,  4, 2 ,false},
212
{ ARM::VLD2q32PseudoWB_register,   ARM::VLD2q32wb_register, true, true, true,  SingleSpc,  4, 2 ,false},
213
{ ARM::VLD2q8Pseudo,        ARM::VLD2q8,       true,  false, false, SingleSpc,  4, 8 ,false},
214
{ ARM::VLD2q8PseudoWB_fixed,    ARM::VLD2q8wb_fixed, true, true, false,  SingleSpc,  4, 8 ,false},
215
{ ARM::VLD2q8PseudoWB_register,    ARM::VLD2q8wb_register, true, true, true,  SingleSpc,  4, 8 ,false},
216
217
{ ARM::VLD3DUPd16Pseudo,     ARM::VLD3DUPd16,     true, false, false, SingleSpc, 3, 4,true},
218
{ ARM::VLD3DUPd16Pseudo_UPD, ARM::VLD3DUPd16_UPD, true, true, true,  SingleSpc, 3, 4,true},
219
{ ARM::VLD3DUPd32Pseudo,     ARM::VLD3DUPd32,     true, false, false, SingleSpc, 3, 2,true},
220
{ ARM::VLD3DUPd32Pseudo_UPD, ARM::VLD3DUPd32_UPD, true, true, true,  SingleSpc, 3, 2,true},
221
{ ARM::VLD3DUPd8Pseudo,      ARM::VLD3DUPd8,      true, false, false, SingleSpc, 3, 8,true},
222
{ ARM::VLD3DUPd8Pseudo_UPD,  ARM::VLD3DUPd8_UPD, true, true, true,  SingleSpc, 3, 8,true},
223
{ ARM::VLD3DUPq16EvenPseudo, ARM::VLD3DUPq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
224
{ ARM::VLD3DUPq16OddPseudo,  ARM::VLD3DUPq16,     true, false, false, OddDblSpc,  3, 4 ,true},
225
{ ARM::VLD3DUPq32EvenPseudo, ARM::VLD3DUPq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
226
{ ARM::VLD3DUPq32OddPseudo,  ARM::VLD3DUPq32,     true, false, false, OddDblSpc,  3, 2 ,true},
227
{ ARM::VLD3DUPq8EvenPseudo,  ARM::VLD3DUPq8,      true, false, false, EvenDblSpc, 3, 8 ,true},
228
{ ARM::VLD3DUPq8OddPseudo,   ARM::VLD3DUPq8,      true, false, false, OddDblSpc,  3, 8 ,true},
229
230
{ ARM::VLD3LNd16Pseudo,     ARM::VLD3LNd16,     true, false, false, SingleSpc,  3, 4 ,true},
231
{ ARM::VLD3LNd16Pseudo_UPD, ARM::VLD3LNd16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
232
{ ARM::VLD3LNd32Pseudo,     ARM::VLD3LNd32,     true, false, false, SingleSpc,  3, 2 ,true},
233
{ ARM::VLD3LNd32Pseudo_UPD, ARM::VLD3LNd32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
234
{ ARM::VLD3LNd8Pseudo,      ARM::VLD3LNd8,      true, false, false, SingleSpc,  3, 8 ,true},
235
{ ARM::VLD3LNd8Pseudo_UPD,  ARM::VLD3LNd8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
236
{ ARM::VLD3LNq16Pseudo,     ARM::VLD3LNq16,     true, false, false, EvenDblSpc, 3, 4 ,true},
237
{ ARM::VLD3LNq16Pseudo_UPD, ARM::VLD3LNq16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
238
{ ARM::VLD3LNq32Pseudo,     ARM::VLD3LNq32,     true, false, false, EvenDblSpc, 3, 2 ,true},
239
{ ARM::VLD3LNq32Pseudo_UPD, ARM::VLD3LNq32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
240
241
{ ARM::VLD3d16Pseudo,       ARM::VLD3d16,      true,  false, false, SingleSpc,  3, 4 ,true},
242
{ ARM::VLD3d16Pseudo_UPD,   ARM::VLD3d16_UPD, true, true, true,  SingleSpc,  3, 4 ,true},
243
{ ARM::VLD3d32Pseudo,       ARM::VLD3d32,      true,  false, false, SingleSpc,  3, 2 ,true},
244
{ ARM::VLD3d32Pseudo_UPD,   ARM::VLD3d32_UPD, true, true, true,  SingleSpc,  3, 2 ,true},
245
{ ARM::VLD3d8Pseudo,        ARM::VLD3d8,       true,  false, false, SingleSpc,  3, 8 ,true},
246
{ ARM::VLD3d8Pseudo_UPD,    ARM::VLD3d8_UPD, true, true, true,  SingleSpc,  3, 8 ,true},
247
248
{ ARM::VLD3q16Pseudo_UPD,    ARM::VLD3q16_UPD, true, true, true,  EvenDblSpc, 3, 4 ,true},
249
{ ARM::VLD3q16oddPseudo,     ARM::VLD3q16,     true,  false, false, OddDblSpc,  3, 4 ,true},
250
{ ARM::VLD3q16oddPseudo_UPD, ARM::VLD3q16_UPD, true, true, true,  OddDblSpc,  3, 4 ,true},
251
{ ARM::VLD3q32Pseudo_UPD,    ARM::VLD3q32_UPD, true, true, true,  EvenDblSpc, 3, 2 ,true},
252
{ ARM::VLD3q32oddPseudo,     ARM::VLD3q32,     true,  false, false, OddDblSpc,  3, 2 ,true},
253
{ ARM::VLD3q32oddPseudo_UPD, ARM::VLD3q32_UPD, true, true, true,  OddDblSpc,  3, 2 ,true},
254
{ ARM::VLD3q8Pseudo_UPD,     ARM::VLD3q8_UPD, true, true, true,  EvenDblSpc, 3, 8 ,true},
255
{ ARM::VLD3q8oddPseudo,      ARM::VLD3q8,      true,  false, false, OddDblSpc,  3, 8 ,true},
256
{ ARM::VLD3q8oddPseudo_UPD,  ARM::VLD3q8_UPD, true, true, true,  OddDblSpc,  3, 8 ,true},
257
258
{ ARM::VLD4DUPd16Pseudo,     ARM::VLD4DUPd16,     true, false, false, SingleSpc, 4, 4,true},
259
{ ARM::VLD4DUPd16Pseudo_UPD, ARM::VLD4DUPd16_UPD, true, true, true,  SingleSpc, 4, 4,true},
260
{ ARM::VLD4DUPd32Pseudo,     ARM::VLD4DUPd32,     true, false, false, SingleSpc, 4, 2,true},
261
{ ARM::VLD4DUPd32Pseudo_UPD, ARM::VLD4DUPd32_UPD, true, true, true,  SingleSpc, 4, 2,true},
262
{ ARM::VLD4DUPd8Pseudo,      ARM::VLD4DUPd8,      true, false, false, SingleSpc, 4, 8,true},
263
{ ARM::VLD4DUPd8Pseudo_UPD,  ARM::VLD4DUPd8_UPD, true, true, true,  SingleSpc, 4, 8,true},
264
{ ARM::VLD4DUPq16EvenPseudo, ARM::VLD4DUPq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
265
{ ARM::VLD4DUPq16OddPseudo,  ARM::VLD4DUPq16,     true, false, false, OddDblSpc,  4, 4 ,true},
266
{ ARM::VLD4DUPq32EvenPseudo, ARM::VLD4DUPq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
267
{ ARM::VLD4DUPq32OddPseudo,  ARM::VLD4DUPq32,     true, false, false, OddDblSpc,  4, 2 ,true},
268
{ ARM::VLD4DUPq8EvenPseudo,  ARM::VLD4DUPq8,      true, false, false, EvenDblSpc, 4, 8 ,true},
269
{ ARM::VLD4DUPq8OddPseudo,   ARM::VLD4DUPq8,      true, false, false, OddDblSpc,  4, 8 ,true},
270
271
{ ARM::VLD4LNd16Pseudo,     ARM::VLD4LNd16,     true, false, false, SingleSpc,  4, 4 ,true},
272
{ ARM::VLD4LNd16Pseudo_UPD, ARM::VLD4LNd16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
273
{ ARM::VLD4LNd32Pseudo,     ARM::VLD4LNd32,     true, false, false, SingleSpc,  4, 2 ,true},
274
{ ARM::VLD4LNd32Pseudo_UPD, ARM::VLD4LNd32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
275
{ ARM::VLD4LNd8Pseudo,      ARM::VLD4LNd8,      true, false, false, SingleSpc,  4, 8 ,true},
276
{ ARM::VLD4LNd8Pseudo_UPD,  ARM::VLD4LNd8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
277
{ ARM::VLD4LNq16Pseudo,     ARM::VLD4LNq16,     true, false, false, EvenDblSpc, 4, 4 ,true},
278
{ ARM::VLD4LNq16Pseudo_UPD, ARM::VLD4LNq16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
279
{ ARM::VLD4LNq32Pseudo,     ARM::VLD4LNq32,     true, false, false, EvenDblSpc, 4, 2 ,true},
280
{ ARM::VLD4LNq32Pseudo_UPD, ARM::VLD4LNq32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
281
282
{ ARM::VLD4d16Pseudo,       ARM::VLD4d16,      true,  false, false, SingleSpc,  4, 4 ,true},
283
{ ARM::VLD4d16Pseudo_UPD,   ARM::VLD4d16_UPD, true, true, true,  SingleSpc,  4, 4 ,true},
284
{ ARM::VLD4d32Pseudo,       ARM::VLD4d32,      true,  false, false, SingleSpc,  4, 2 ,true},
285
{ ARM::VLD4d32Pseudo_UPD,   ARM::VLD4d32_UPD, true, true, true,  SingleSpc,  4, 2 ,true},
286
{ ARM::VLD4d8Pseudo,        ARM::VLD4d8,       true,  false, false, SingleSpc,  4, 8 ,true},
287
{ ARM::VLD4d8Pseudo_UPD,    ARM::VLD4d8_UPD, true, true, true,  SingleSpc,  4, 8 ,true},
288
289
{ ARM::VLD4q16Pseudo_UPD,    ARM::VLD4q16_UPD, true, true, true,  EvenDblSpc, 4, 4 ,true},
290
{ ARM::VLD4q16oddPseudo,     ARM::VLD4q16,     true,  false, false, OddDblSpc,  4, 4 ,true},
291
{ ARM::VLD4q16oddPseudo_UPD, ARM::VLD4q16_UPD, true, true, true,  OddDblSpc,  4, 4 ,true},
292
{ ARM::VLD4q32Pseudo_UPD,    ARM::VLD4q32_UPD, true, true, true,  EvenDblSpc, 4, 2 ,true},
293
{ ARM::VLD4q32oddPseudo,     ARM::VLD4q32,     true,  false, false, OddDblSpc,  4, 2 ,true},
294
{ ARM::VLD4q32oddPseudo_UPD, ARM::VLD4q32_UPD, true, true, true,  OddDblSpc,  4, 2 ,true},
295
{ ARM::VLD4q8Pseudo_UPD,     ARM::VLD4q8_UPD, true, true, true,  EvenDblSpc, 4, 8 ,true},
296
{ ARM::VLD4q8oddPseudo,      ARM::VLD4q8,      true,  false, false, OddDblSpc,  4, 8 ,true},
297
{ ARM::VLD4q8oddPseudo_UPD,  ARM::VLD4q8_UPD, true, true, true,  OddDblSpc,  4, 8 ,true},
298
299
{ ARM::VST1LNq16Pseudo,     ARM::VST1LNd16,    false, false, false, EvenDblSpc, 1, 4 ,true},
300
{ ARM::VST1LNq16Pseudo_UPD, ARM::VST1LNd16_UPD, false, true, true,  EvenDblSpc, 1, 4 ,true},
301
{ ARM::VST1LNq32Pseudo,     ARM::VST1LNd32,    false, false, false, EvenDblSpc, 1, 2 ,true},
302
{ ARM::VST1LNq32Pseudo_UPD, ARM::VST1LNd32_UPD, false, true, true,  EvenDblSpc, 1, 2 ,true},
303
{ ARM::VST1LNq8Pseudo,      ARM::VST1LNd8,     false, false, false, EvenDblSpc, 1, 8 ,true},
304
{ ARM::VST1LNq8Pseudo_UPD,  ARM::VST1LNd8_UPD, false, true, true,  EvenDblSpc, 1, 8 ,true},
305
306
{ ARM::VST1d16QPseudo,      ARM::VST1d16Q,     false, false, false, SingleSpc,  4, 4 ,false},
307
{ ARM::VST1d16TPseudo,      ARM::VST1d16T,     false, false, false, SingleSpc,  3, 4 ,false},
308
{ ARM::VST1d32QPseudo,      ARM::VST1d32Q,     false, false, false, SingleSpc,  4, 2 ,false},
309
{ ARM::VST1d32TPseudo,      ARM::VST1d32T,     false, false, false, SingleSpc,  3, 2 ,false},
310
{ ARM::VST1d64QPseudo,      ARM::VST1d64Q,     false, false, false, SingleSpc,  4, 1 ,false},
311
{ ARM::VST1d64QPseudoWB_fixed,  ARM::VST1d64Qwb_fixed, false, true, false,  SingleSpc,  4, 1 ,false},
312
{ ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true,  SingleSpc,  4, 1 ,false},
313
{ ARM::VST1d64TPseudo,      ARM::VST1d64T,     false, false, false, SingleSpc,  3, 1 ,false},
314
{ ARM::VST1d64TPseudoWB_fixed,  ARM::VST1d64Twb_fixed, false, true, false,  SingleSpc,  3, 1 ,false},
315
{ ARM::VST1d64TPseudoWB_register,  ARM::VST1d64Twb_register, false, true, true,  SingleSpc,  3, 1 ,false},
316
{ ARM::VST1d8QPseudo,       ARM::VST1d8Q,      false, false, false, SingleSpc,  4, 8 ,false},
317
{ ARM::VST1d8TPseudo,       ARM::VST1d8T,      false, false, false, SingleSpc,  3, 8 ,false},
318
{ ARM::VST1q16HighQPseudo,  ARM::VST1d16Q,      false, false, false, SingleHighQSpc,   4, 4 ,false},
319
{ ARM::VST1q16HighTPseudo,  ARM::VST1d16T,      false, false, false, SingleHighTSpc,   3, 4 ,false},
320
{ ARM::VST1q16LowQPseudo_UPD,   ARM::VST1d16Qwb_fixed,  false, true, true, SingleLowSpc,   4, 4 ,false},
321
{ ARM::VST1q16LowTPseudo_UPD,   ARM::VST1d16Twb_fixed,  false, true, true, SingleLowSpc,   3, 4 ,false},
322
{ ARM::VST1q32HighQPseudo,  ARM::VST1d32Q,      false, false, false, SingleHighQSpc,   4, 2 ,false},
323
{ ARM::VST1q32HighTPseudo,  ARM::VST1d32T,      false, false, false, SingleHighTSpc,   3, 2 ,false},
324
{ ARM::VST1q32LowQPseudo_UPD,   ARM::VST1d32Qwb_fixed,  false, true, true, SingleLowSpc,   4, 2 ,false},
325
{ ARM::VST1q32LowTPseudo_UPD,   ARM::VST1d32Twb_fixed,  false, true, true, SingleLowSpc,   3, 2 ,false},
326
{ ARM::VST1q64HighQPseudo,  ARM::VST1d64Q,      false, false, false, SingleHighQSpc,   4, 1 ,false},
327
{ ARM::VST1q64HighTPseudo,  ARM::VST1d64T,      false, false, false, SingleHighTSpc,   3, 1 ,false},
328
{ ARM::VST1q64LowQPseudo_UPD,   ARM::VST1d64Qwb_fixed,  false, true, true, SingleLowSpc,   4, 1 ,false},
329
{ ARM::VST1q64LowTPseudo_UPD,   ARM::VST1d64Twb_fixed,  false, true, true, SingleLowSpc,   3, 1 ,false},
330
{ ARM::VST1q8HighQPseudo,   ARM::VST1d8Q,      false, false, false, SingleHighQSpc,   4, 8 ,false},
331
{ ARM::VST1q8HighTPseudo,   ARM::VST1d8T,      false, false, false, SingleHighTSpc,   3, 8 ,false},
332
{ ARM::VST1q8LowQPseudo_UPD,   ARM::VST1d8Qwb_fixed,  false, true, true, SingleLowSpc,   4, 8 ,false},
333
{ ARM::VST1q8LowTPseudo_UPD,   ARM::VST1d8Twb_fixed,  false, true, true, SingleLowSpc,   3, 8 ,false},
334
335
{ ARM::VST2LNd16Pseudo,     ARM::VST2LNd16,     false, false, false, SingleSpc, 2, 4 ,true},
336
{ ARM::VST2LNd16Pseudo_UPD, ARM::VST2LNd16_UPD, false, true, true,  SingleSpc, 2, 4 ,true},
337
{ ARM::VST2LNd32Pseudo,     ARM::VST2LNd32,     false, false, false, SingleSpc, 2, 2 ,true},
338
{ ARM::VST2LNd32Pseudo_UPD, ARM::VST2LNd32_UPD, false, true, true,  SingleSpc, 2, 2 ,true},
339
{ ARM::VST2LNd8Pseudo,      ARM::VST2LNd8,      false, false, false, SingleSpc, 2, 8 ,true},
340
{ ARM::VST2LNd8Pseudo_UPD,  ARM::VST2LNd8_UPD, false, true, true,  SingleSpc, 2, 8 ,true},
341
{ ARM::VST2LNq16Pseudo,     ARM::VST2LNq16,     false, false, false, EvenDblSpc, 2, 4,true},
342
{ ARM::VST2LNq16Pseudo_UPD, ARM::VST2LNq16_UPD, false, true, true,  EvenDblSpc, 2, 4,true},
343
{ ARM::VST2LNq32Pseudo,     ARM::VST2LNq32,     false, false, false, EvenDblSpc, 2, 2,true},
344
{ ARM::VST2LNq32Pseudo_UPD, ARM::VST2LNq32_UPD, false, true, true,  EvenDblSpc, 2, 2,true},
345
346
{ ARM::VST2q16Pseudo,       ARM::VST2q16,      false, false, false, SingleSpc,  4, 4 ,false},
347
{ ARM::VST2q16PseudoWB_fixed,   ARM::VST2q16wb_fixed, false, true, false,  SingleSpc,  4, 4 ,false},
348
{ ARM::VST2q16PseudoWB_register,   ARM::VST2q16wb_register, false, true, true,  SingleSpc,  4, 4 ,false},
349
{ ARM::VST2q32Pseudo,       ARM::VST2q32,      false, false, false, SingleSpc,  4, 2 ,false},
350
{ ARM::VST2q32PseudoWB_fixed,   ARM::VST2q32wb_fixed, false, true, false,  SingleSpc,  4, 2 ,false},
351
{ ARM::VST2q32PseudoWB_register,   ARM::VST2q32wb_register, false, true, true,  SingleSpc,  4, 2 ,false},
352
{ ARM::VST2q8Pseudo,        ARM::VST2q8,       false, false, false, SingleSpc,  4, 8 ,false},
353
{ ARM::VST2q8PseudoWB_fixed,    ARM::VST2q8wb_fixed, false, true, false,  SingleSpc,  4, 8 ,false},
354
{ ARM::VST2q8PseudoWB_register,    ARM::VST2q8wb_register, false, true, true,  SingleSpc,  4, 8 ,false},
355
356
{ ARM::VST3LNd16Pseudo,     ARM::VST3LNd16,     false, false, false, SingleSpc, 3, 4 ,true},
357
{ ARM::VST3LNd16Pseudo_UPD, ARM::VST3LNd16_UPD, false, true, true,  SingleSpc, 3, 4 ,true},
358
{ ARM::VST3LNd32Pseudo,     ARM::VST3LNd32,     false, false, false, SingleSpc, 3, 2 ,true},
359
{ ARM::VST3LNd32Pseudo_UPD, ARM::VST3LNd32_UPD, false, true, true,  SingleSpc, 3, 2 ,true},
360
{ ARM::VST3LNd8Pseudo,      ARM::VST3LNd8,      false, false, false, SingleSpc, 3, 8 ,true},
361
{ ARM::VST3LNd8Pseudo_UPD,  ARM::VST3LNd8_UPD, false, true, true,  SingleSpc, 3, 8 ,true},
362
{ ARM::VST3LNq16Pseudo,     ARM::VST3LNq16,     false, false, false, EvenDblSpc, 3, 4,true},
363
{ ARM::VST3LNq16Pseudo_UPD, ARM::VST3LNq16_UPD, false, true, true,  EvenDblSpc, 3, 4,true},
364
{ ARM::VST3LNq32Pseudo,     ARM::VST3LNq32,     false, false, false, EvenDblSpc, 3, 2,true},
365
{ ARM::VST3LNq32Pseudo_UPD, ARM::VST3LNq32_UPD, false, true, true,  EvenDblSpc, 3, 2,true},
366
367
{ ARM::VST3d16Pseudo,       ARM::VST3d16,      false, false, false, SingleSpc,  3, 4 ,true},
368
{ ARM::VST3d16Pseudo_UPD,   ARM::VST3d16_UPD, false, true, true,  SingleSpc,  3, 4 ,true},
369
{ ARM::VST3d32Pseudo,       ARM::VST3d32,      false, false, false, SingleSpc,  3, 2 ,true},
370
{ ARM::VST3d32Pseudo_UPD,   ARM::VST3d32_UPD, false, true, true,  SingleSpc,  3, 2 ,true},
371
{ ARM::VST3d8Pseudo,        ARM::VST3d8,       false, false, false, SingleSpc,  3, 8 ,true},
372
{ ARM::VST3d8Pseudo_UPD,    ARM::VST3d8_UPD, false, true, true,  SingleSpc,  3, 8 ,true},
373
374
{ ARM::VST3q16Pseudo_UPD,    ARM::VST3q16_UPD, false, true, true,  EvenDblSpc, 3, 4 ,true},
375
{ ARM::VST3q16oddPseudo,     ARM::VST3q16,     false, false, false, OddDblSpc,  3, 4 ,true},
376
{ ARM::VST3q16oddPseudo_UPD, ARM::VST3q16_UPD, false, true, true,  OddDblSpc,  3, 4 ,true},
377
{ ARM::VST3q32Pseudo_UPD,    ARM::VST3q32_UPD, false, true, true,  EvenDblSpc, 3, 2 ,true},
378
{ ARM::VST3q32oddPseudo,     ARM::VST3q32,     false, false, false, OddDblSpc,  3, 2 ,true},
379
{ ARM::VST3q32oddPseudo_UPD, ARM::VST3q32_UPD, false, true, true,  OddDblSpc,  3, 2 ,true},
380
{ ARM::VST3q8Pseudo_UPD,     ARM::VST3q8_UPD, false, true, true,  EvenDblSpc, 3, 8 ,true},
381
{ ARM::VST3q8oddPseudo,      ARM::VST3q8,      false, false, false, OddDblSpc,  3, 8 ,true},
382
{ ARM::VST3q8oddPseudo_UPD,  ARM::VST3q8_UPD, false, true, true,  OddDblSpc,  3, 8 ,true},
383
384
{ ARM::VST4LNd16Pseudo,     ARM::VST4LNd16,     false, false, false, SingleSpc, 4, 4 ,true},
385
{ ARM::VST4LNd16Pseudo_UPD, ARM::VST4LNd16_UPD, false, true, true,  SingleSpc, 4, 4 ,true},
386
{ ARM::VST4LNd32Pseudo,     ARM::VST4LNd32,     false, false, false, SingleSpc, 4, 2 ,true},
387
{ ARM::VST4LNd32Pseudo_UPD, ARM::VST4LNd32_UPD, false, true, true,  SingleSpc, 4, 2 ,true},
388
{ ARM::VST4LNd8Pseudo,      ARM::VST4LNd8,      false, false, false, SingleSpc, 4, 8 ,true},
389
{ ARM::VST4LNd8Pseudo_UPD,  ARM::VST4LNd8_UPD, false, true, true,  SingleSpc, 4, 8 ,true},
390
{ ARM::VST4LNq16Pseudo,     ARM::VST4LNq16,     false, false, false, EvenDblSpc, 4, 4,true},
391
{ ARM::VST4LNq16Pseudo_UPD, ARM::VST4LNq16_UPD, false, true, true,  EvenDblSpc, 4, 4,true},
392
{ ARM::VST4LNq32Pseudo,     ARM::VST4LNq32,     false, false, false, EvenDblSpc, 4, 2,true},
393
{ ARM::VST4LNq32Pseudo_UPD, ARM::VST4LNq32_UPD, false, true, true,  EvenDblSpc, 4, 2,true},
394
395
{ ARM::VST4d16Pseudo,       ARM::VST4d16,      false, false, false, SingleSpc,  4, 4 ,true},
396
{ ARM::VST4d16Pseudo_UPD,   ARM::VST4d16_UPD, false, true, true,  SingleSpc,  4, 4 ,true},
397
{ ARM::VST4d32Pseudo,       ARM::VST4d32,      false, false, false, SingleSpc,  4, 2 ,true},
398
{ ARM::VST4d32Pseudo_UPD,   ARM::VST4d32_UPD, false, true, true,  SingleSpc,  4, 2 ,true},
399
{ ARM::VST4d8Pseudo,        ARM::VST4d8,       false, false, false, SingleSpc,  4, 8 ,true},
400
{ ARM::VST4d8Pseudo_UPD,    ARM::VST4d8_UPD, false, true, true,  SingleSpc,  4, 8 ,true},
401
402
{ ARM::VST4q16Pseudo_UPD,    ARM::VST4q16_UPD, false, true, true,  EvenDblSpc, 4, 4 ,true},
403
{ ARM::VST4q16oddPseudo,     ARM::VST4q16,     false, false, false, OddDblSpc,  4, 4 ,true},
404
{ ARM::VST4q16oddPseudo_UPD, ARM::VST4q16_UPD, false, true, true,  OddDblSpc,  4, 4 ,true},
405
{ ARM::VST4q32Pseudo_UPD,    ARM::VST4q32_UPD, false, true, true,  EvenDblSpc, 4, 2 ,true},
406
{ ARM::VST4q32oddPseudo,     ARM::VST4q32,     false, false, false, OddDblSpc,  4, 2 ,true},
407
{ ARM::VST4q32oddPseudo_UPD, ARM::VST4q32_UPD, false, true, true,  OddDblSpc,  4, 2 ,true},
408
{ ARM::VST4q8Pseudo_UPD,     ARM::VST4q8_UPD, false, true, true,  EvenDblSpc, 4, 8 ,true},
409
{ ARM::VST4q8oddPseudo,      ARM::VST4q8,      false, false, false, OddDblSpc,  4, 8 ,true},
410
{ ARM::VST4q8oddPseudo_UPD,  ARM::VST4q8_UPD, false, true, true,  OddDblSpc,  4, 8 ,true}
411
};
412
413
/// LookupNEONLdSt - Search the NEONLdStTable for information about a NEON
414
/// load or store pseudo instruction.
415
444
static const NEONLdStTableEntry *LookupNEONLdSt(unsigned Opcode) {
416
#ifndef NDEBUG
417
  // Make sure the table is sorted.
418
  static std::atomic<bool> TableChecked(false);
419
  if (!TableChecked.load(std::memory_order_relaxed)) {
420
    assert(std::is_sorted(std::begin(NEONLdStTable), std::end(NEONLdStTable)) &&
421
           "NEONLdStTable is not sorted!");
422
    TableChecked.store(true, std::memory_order_relaxed);
423
  }
424
#endif
425
426
444
  auto I = llvm::lower_bound(NEONLdStTable, Opcode);
427
444
  if (I != std::end(NEONLdStTable) && I->PseudoOpc == Opcode)
428
444
    return I;
429
0
  return nullptr;
430
0
}
431
432
/// GetDSubRegs - Get 4 D subregisters of a Q, QQ, or QQQQ register,
433
/// corresponding to the specified register spacing.  Not all of the results
434
/// are necessarily valid, e.g., a Q register only has 2 D subregisters.
435
static void GetDSubRegs(unsigned Reg, NEONRegSpacing RegSpc,
436
                        const TargetRegisterInfo *TRI, unsigned &D0,
437
443
                        unsigned &D1, unsigned &D2, unsigned &D3) {
438
443
  if (RegSpc == SingleSpc || 
RegSpc == SingleLowSpc241
) {
439
218
    D0 = TRI->getSubReg(Reg, ARM::dsub_0);
440
218
    D1 = TRI->getSubReg(Reg, ARM::dsub_1);
441
218
    D2 = TRI->getSubReg(Reg, ARM::dsub_2);
442
218
    D3 = TRI->getSubReg(Reg, ARM::dsub_3);
443
225
  } else if (RegSpc == SingleHighQSpc) {
444
8
    D0 = TRI->getSubReg(Reg, ARM::dsub_4);
445
8
    D1 = TRI->getSubReg(Reg, ARM::dsub_5);
446
8
    D2 = TRI->getSubReg(Reg, ARM::dsub_6);
447
8
    D3 = TRI->getSubReg(Reg, ARM::dsub_7);
448
217
  } else if (RegSpc == SingleHighTSpc) {
449
8
    D0 = TRI->getSubReg(Reg, ARM::dsub_3);
450
8
    D1 = TRI->getSubReg(Reg, ARM::dsub_4);
451
8
    D2 = TRI->getSubReg(Reg, ARM::dsub_5);
452
8
    D3 = TRI->getSubReg(Reg, ARM::dsub_6);
453
209
  } else if (RegSpc == EvenDblSpc) {
454
107
    D0 = TRI->getSubReg(Reg, ARM::dsub_0);
455
107
    D1 = TRI->getSubReg(Reg, ARM::dsub_2);
456
107
    D2 = TRI->getSubReg(Reg, ARM::dsub_4);
457
107
    D3 = TRI->getSubReg(Reg, ARM::dsub_6);
458
107
  } else {
459
102
    assert(RegSpc == OddDblSpc && "unknown register spacing");
460
102
    D0 = TRI->getSubReg(Reg, ARM::dsub_1);
461
102
    D1 = TRI->getSubReg(Reg, ARM::dsub_3);
462
102
    D2 = TRI->getSubReg(Reg, ARM::dsub_5);
463
102
    D3 = TRI->getSubReg(Reg, ARM::dsub_7);
464
102
  }
465
443
}
466
467
/// ExpandVLD - Translate VLD pseudo instructions with Q, QQ or QQQQ register
468
/// operands to real VLD instructions with D register operands.
469
148
void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
470
148
  MachineInstr &MI = *MBBI;
471
148
  MachineBasicBlock &MBB = *MI.getParent();
472
148
  LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
473
148
474
148
  const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
475
148
  assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");
476
148
  NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
477
148
  unsigned NumRegs = TableEntry->NumRegs;
478
148
479
148
  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
480
148
                                    TII->get(TableEntry->RealOpc));
481
148
  unsigned OpIdx = 0;
482
148
483
148
  bool DstIsDead = MI.getOperand(OpIdx).isDead();
484
148
  unsigned DstReg = MI.getOperand(OpIdx++).getReg();
485
148
  if(TableEntry->RealOpc == ARM::VLD2DUPd8x2 ||
486
148
     
TableEntry->RealOpc == ARM::VLD2DUPd16x2146
||
487
148
     
TableEntry->RealOpc == ARM::VLD2DUPd32x2144
) {
488
6
    unsigned SubRegIndex;
489
6
    if (RegSpc == EvenDblSpc) {
490
3
      SubRegIndex = ARM::dsub_0;
491
3
    } else {
492
3
      assert(RegSpc == OddDblSpc && "Unexpected spacing!");
493
3
      SubRegIndex = ARM::dsub_1;
494
3
    }
495
6
    unsigned SubReg = TRI->getSubReg(DstReg, SubRegIndex);
496
6
    unsigned DstRegPair = TRI->getMatchingSuperReg(SubReg, ARM::dsub_0,
497
6
                                                   &ARM::DPairSpcRegClass);
498
6
    MIB.addReg(DstRegPair, RegState::Define | getDeadRegState(DstIsDead));
499
142
  } else {
500
142
    unsigned D0, D1, D2, D3;
501
142
    GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
502
142
    MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
503
142
    if (NumRegs > 1 && TableEntry->copyAllListRegs)
504
87
      MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
505
142
    if (NumRegs > 2 && TableEntry->copyAllListRegs)
506
87
      MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
507
142
    if (NumRegs > 3 && 
TableEntry->copyAllListRegs64
)
508
28
      MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
509
142
  }
510
148
511
148
  if (TableEntry->isUpdating)
512
47
    MIB.add(MI.getOperand(OpIdx++));
513
148
514
148
  // Copy the addrmode6 operands.
515
148
  MIB.add(MI.getOperand(OpIdx++));
516
148
  MIB.add(MI.getOperand(OpIdx++));
517
148
518
148
  // Copy the am6offset operand.
519
148
  if (TableEntry->hasWritebackOperand) {
520
39
    // TODO: The writing-back pseudo instructions we translate here are all
521
39
    // defined to take am6offset nodes that are capable to represent both fixed
522
39
    // and register forms. Some real instructions, however, do not rely on
523
39
    // am6offset and have separate definitions for such forms. When this is the
524
39
    // case, fixed forms do not take any offset nodes, so here we skip them for
525
39
    // such instructions. Once all real and pseudo writing-back instructions are
526
39
    // rewritten without use of am6offset nodes, this code will go away.
527
39
    const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
528
39
    if (TableEntry->RealOpc == ARM::VLD1d8Qwb_fixed ||
529
39
        
TableEntry->RealOpc == ARM::VLD1d16Qwb_fixed38
||
530
39
        
TableEntry->RealOpc == ARM::VLD1d32Qwb_fixed37
||
531
39
        
TableEntry->RealOpc == ARM::VLD1d64Qwb_fixed36
||
532
39
        
TableEntry->RealOpc == ARM::VLD1d8Twb_fixed35
||
533
39
        
TableEntry->RealOpc == ARM::VLD1d16Twb_fixed34
||
534
39
        
TableEntry->RealOpc == ARM::VLD1d32Twb_fixed33
||
535
39
        
TableEntry->RealOpc == ARM::VLD1d64Twb_fixed32
) {
536
8
      assert(AM6Offset.getReg() == 0 &&
537
8
             "A fixed writing-back pseudo instruction provides an offset "
538
8
             "register!");
539
31
    } else {
540
31
      MIB.add(AM6Offset);
541
31
    }
542
39
  }
543
148
544
148
  // For an instruction writing double-spaced subregs, the pseudo instruction
545
148
  // has an extra operand that is a use of the super-register.  Record the
546
148
  // operand index and skip over it.
547
148
  unsigned SrcOpIdx = 0;
548
148
  if(TableEntry->RealOpc != ARM::VLD2DUPd8x2 &&
549
148
     
TableEntry->RealOpc != ARM::VLD2DUPd16x2146
&&
550
148
     
TableEntry->RealOpc != ARM::VLD2DUPd32x2144
) {
551
142
    if (RegSpc == EvenDblSpc || 
RegSpc == OddDblSpc116
||
552
142
        
RegSpc == SingleLowSpc91
||
RegSpc == SingleHighQSpc83
||
553
142
        
RegSpc == SingleHighTSpc79
)
554
67
      SrcOpIdx = OpIdx++;
555
142
  }
556
148
557
148
  // Copy the predicate operands.
558
148
  MIB.add(MI.getOperand(OpIdx++));
559
148
  MIB.add(MI.getOperand(OpIdx++));
560
148
561
148
  // Copy the super-register source operand used for double-spaced subregs over
562
148
  // to the new instruction as an implicit operand.
563
148
  if (SrcOpIdx != 0) {
564
67
    MachineOperand MO = MI.getOperand(SrcOpIdx);
565
67
    MO.setImplicit(true);
566
67
    MIB.add(MO);
567
67
  }
568
148
  // Add an implicit def for the super-register.
569
148
  MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
570
148
  TransferImpOps(MI, MIB, MIB);
571
148
572
148
  // Transfer memoperands.
573
148
  MIB.cloneMemRefs(MI);
574
148
  MI.eraseFromParent();
575
148
  LLVM_DEBUG(dbgs() << "To:        "; MIB.getInstr()->dump(););
576
148
}
577
578
/// ExpandVST - Translate VST pseudo instructions with Q, QQ or QQQQ register
579
/// operands to real VST instructions with D register operands.
580
107
void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
581
107
  MachineInstr &MI = *MBBI;
582
107
  MachineBasicBlock &MBB = *MI.getParent();
583
107
  LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
584
107
585
107
  const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
586
107
  assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");
587
107
  NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
588
107
  unsigned NumRegs = TableEntry->NumRegs;
589
107
590
107
  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
591
107
                                    TII->get(TableEntry->RealOpc));
592
107
  unsigned OpIdx = 0;
593
107
  if (TableEntry->isUpdating)
594
36
    MIB.add(MI.getOperand(OpIdx++));
595
107
596
107
  // Copy the addrmode6 operands.
597
107
  MIB.add(MI.getOperand(OpIdx++));
598
107
  MIB.add(MI.getOperand(OpIdx++));
599
107
600
107
  if (TableEntry->hasWritebackOperand) {
601
32
    // TODO: The writing-back pseudo instructions we translate here are all
602
32
    // defined to take am6offset nodes that are capable to represent both fixed
603
32
    // and register forms. Some real instructions, however, do not rely on
604
32
    // am6offset and have separate definitions for such forms. When this is the
605
32
    // case, fixed forms do not take any offset nodes, so here we skip them for
606
32
    // such instructions. Once all real and pseudo writing-back instructions are
607
32
    // rewritten without use of am6offset nodes, this code will go away.
608
32
    const MachineOperand &AM6Offset = MI.getOperand(OpIdx++);
609
32
    if (TableEntry->RealOpc == ARM::VST1d8Qwb_fixed ||
610
32
        
TableEntry->RealOpc == ARM::VST1d16Qwb_fixed31
||
611
32
        
TableEntry->RealOpc == ARM::VST1d32Qwb_fixed30
||
612
32
        
TableEntry->RealOpc == ARM::VST1d64Qwb_fixed29
||
613
32
        
TableEntry->RealOpc == ARM::VST1d8Twb_fixed28
||
614
32
        
TableEntry->RealOpc == ARM::VST1d16Twb_fixed27
||
615
32
        
TableEntry->RealOpc == ARM::VST1d32Twb_fixed26
||
616
32
        
TableEntry->RealOpc == ARM::VST1d64Twb_fixed25
) {
617
8
      assert(AM6Offset.getReg() == 0 &&
618
8
             "A fixed writing-back pseudo instruction provides an offset "
619
8
             "register!");
620
24
    } else {
621
24
      MIB.add(AM6Offset);
622
24
    }
623
32
  }
624
107
625
107
  bool SrcIsKill = MI.getOperand(OpIdx).isKill();
626
107
  bool SrcIsUndef = MI.getOperand(OpIdx).isUndef();
627
107
  unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
628
107
  unsigned D0, D1, D2, D3;
629
107
  GetDSubRegs(SrcReg, RegSpc, TRI, D0, D1, D2, D3);
630
107
  MIB.addReg(D0, getUndefRegState(SrcIsUndef));
631
107
  if (NumRegs > 1 && TableEntry->copyAllListRegs)
632
57
    MIB.addReg(D1, getUndefRegState(SrcIsUndef));
633
107
  if (NumRegs > 2 && TableEntry->copyAllListRegs)
634
57
    MIB.addReg(D2, getUndefRegState(SrcIsUndef));
635
107
  if (NumRegs > 3 && 
TableEntry->copyAllListRegs67
)
636
32
    MIB.addReg(D3, getUndefRegState(SrcIsUndef));
637
107
638
107
  // Copy the predicate operands.
639
107
  MIB.add(MI.getOperand(OpIdx++));
640
107
  MIB.add(MI.getOperand(OpIdx++));
641
107
642
107
  if (SrcIsKill && 
!SrcIsUndef72
) // Add an implicit kill for the super-reg.
643
72
    MIB->addRegisterKilled(SrcReg, TRI, true);
644
35
  else if (!SrcIsUndef)
645
34
    MIB.addReg(SrcReg, RegState::Implicit); // Add implicit uses for src reg.
646
107
  TransferImpOps(MI, MIB, MIB);
647
107
648
107
  // Transfer memoperands.
649
107
  MIB.cloneMemRefs(MI);
650
107
  MI.eraseFromParent();
651
107
  LLVM_DEBUG(dbgs() << "To:        "; MIB.getInstr()->dump(););
652
107
}
653
654
/// ExpandLaneOp - Translate VLD*LN and VST*LN instructions with Q, QQ or QQQQ
655
/// register operands to real instructions with D register operands.
656
189
void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
657
189
  MachineInstr &MI = *MBBI;
658
189
  MachineBasicBlock &MBB = *MI.getParent();
659
189
  LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
660
189
661
189
  const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
662
189
  assert(TableEntry && "NEONLdStTable lookup failed");
663
189
  NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
664
189
  unsigned NumRegs = TableEntry->NumRegs;
665
189
  unsigned RegElts = TableEntry->RegElts;
666
189
667
189
  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
668
189
                                    TII->get(TableEntry->RealOpc));
669
189
  unsigned OpIdx = 0;
670
189
  // The lane operand is always the 3rd from last operand, before the 2
671
189
  // predicate operands.
672
189
  unsigned Lane = MI.getOperand(MI.getDesc().getNumOperands() - 3).getImm();
673
189
674
189
  // Adjust the lane and spacing as needed for Q registers.
675
189
  assert(RegSpc != OddDblSpc && "unexpected register spacing for VLD/VST-lane");
676
189
  if (RegSpc == EvenDblSpc && 
Lane >= RegElts122
) {
677
59
    RegSpc = OddDblSpc;
678
59
    Lane -= RegElts;
679
59
  }
680
189
  assert(Lane < RegElts && "out of range lane for VLD/VST-lane");
681
189
682
189
  unsigned D0 = 0, D1 = 0, D2 = 0, D3 = 0;
683
189
  unsigned DstReg = 0;
684
189
  bool DstIsDead = false;
685
189
  if (TableEntry->IsLoad) {
686
96
    DstIsDead = MI.getOperand(OpIdx).isDead();
687
96
    DstReg = MI.getOperand(OpIdx++).getReg();
688
96
    GetDSubRegs(DstReg, RegSpc, TRI, D0, D1, D2, D3);
689
96
    MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead));
690
96
    if (NumRegs > 1)
691
75
      MIB.addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
692
96
    if (NumRegs > 2)
693
45
      MIB.addReg(D2, RegState::Define | getDeadRegState(DstIsDead));
694
96
    if (NumRegs > 3)
695
21
      MIB.addReg(D3, RegState::Define | getDeadRegState(DstIsDead));
696
96
  }
697
189
698
189
  if (TableEntry->isUpdating)
699
15
    MIB.add(MI.getOperand(OpIdx++));
700
189
701
189
  // Copy the addrmode6 operands.
702
189
  MIB.add(MI.getOperand(OpIdx++));
703
189
  MIB.add(MI.getOperand(OpIdx++));
704
189
  // Copy the am6offset operand.
705
189
  if (TableEntry->hasWritebackOperand)
706
15
    MIB.add(MI.getOperand(OpIdx++));
707
189
708
189
  // Grab the super-register source.
709
189
  MachineOperand MO = MI.getOperand(OpIdx++);
710
189
  if (!TableEntry->IsLoad)
711
93
    GetDSubRegs(MO.getReg(), RegSpc, TRI, D0, D1, D2, D3);
712
189
713
189
  // Add the subregs as sources of the new instruction.
714
189
  unsigned SrcFlags = (getUndefRegState(MO.isUndef()) |
715
189
                       getKillRegState(MO.isKill()));
716
189
  MIB.addReg(D0, SrcFlags);
717
189
  if (NumRegs > 1)
718
108
    MIB.addReg(D1, SrcFlags);
719
189
  if (NumRegs > 2)
720
67
    MIB.addReg(D2, SrcFlags);
721
189
  if (NumRegs > 3)
722
32
    MIB.addReg(D3, SrcFlags);
723
189
724
189
  // Add the lane number operand.
725
189
  MIB.addImm(Lane);
726
189
  OpIdx += 1;
727
189
728
189
  // Copy the predicate operands.
729
189
  MIB.add(MI.getOperand(OpIdx++));
730
189
  MIB.add(MI.getOperand(OpIdx++));
731
189
732
189
  // Copy the super-register source to be an implicit source.
733
189
  MO.setImplicit(true);
734
189
  MIB.add(MO);
735
189
  if (TableEntry->IsLoad)
736
96
    // Add an implicit def for the super-register.
737
96
    MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
738
189
  TransferImpOps(MI, MIB, MIB);
739
189
  // Transfer memoperands.
740
189
  MIB.cloneMemRefs(MI);
741
189
  MI.eraseFromParent();
742
189
}
743
744
/// ExpandVTBL - Translate VTBL and VTBX pseudo instructions with Q or QQ
745
/// register operands to real instructions with D register operands.
746
void ARMExpandPseudo::ExpandVTBL(MachineBasicBlock::iterator &MBBI,
747
5
                                 unsigned Opc, bool IsExt) {
748
5
  MachineInstr &MI = *MBBI;
749
5
  MachineBasicBlock &MBB = *MI.getParent();
750
5
  LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
751
5
752
5
  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc));
753
5
  unsigned OpIdx = 0;
754
5
755
5
  // Transfer the destination register operand.
756
5
  MIB.add(MI.getOperand(OpIdx++));
757
5
  if (IsExt) {
758
3
    MachineOperand VdSrc(MI.getOperand(OpIdx++));
759
3
    MIB.add(VdSrc);
760
3
  }
761
5
762
5
  bool SrcIsKill = MI.getOperand(OpIdx).isKill();
763
5
  unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
764
5
  unsigned D0, D1, D2, D3;
765
5
  GetDSubRegs(SrcReg, SingleSpc, TRI, D0, D1, D2, D3);
766
5
  MIB.addReg(D0);
767
5
768
5
  // Copy the other source register operand.
769
5
  MachineOperand VmSrc(MI.getOperand(OpIdx++));
770
5
  MIB.add(VmSrc);
771
5
772
5
  // Copy the predicate operands.
773
5
  MIB.add(MI.getOperand(OpIdx++));
774
5
  MIB.add(MI.getOperand(OpIdx++));
775
5
776
5
  // Add an implicit kill and use for the super-reg.
777
5
  MIB.addReg(SrcReg, RegState::Implicit | getKillRegState(SrcIsKill));
778
5
  TransferImpOps(MI, MIB, MIB);
779
5
  MI.eraseFromParent();
780
5
  LLVM_DEBUG(dbgs() << "To:        "; MIB.getInstr()->dump(););
781
5
}
782
783
67
static bool IsAnAddressOperand(const MachineOperand &MO) {
784
67
  // This check is overly conservative.  Unless we are certain that the machine
785
67
  // operand is not a symbol reference, we return that it is a symbol reference.
786
67
  // This is important as the load pair may not be split up Windows.
787
67
  switch (MO.getType()) {
788
67
  case MachineOperand::MO_Register:
789
0
  case MachineOperand::MO_Immediate:
790
0
  case MachineOperand::MO_CImmediate:
791
0
  case MachineOperand::MO_FPImmediate:
792
0
    return false;
793
0
  case MachineOperand::MO_MachineBasicBlock:
794
0
    return true;
795
0
  case MachineOperand::MO_FrameIndex:
796
0
    return false;
797
67
  case MachineOperand::MO_ConstantPoolIndex:
798
67
  case MachineOperand::MO_TargetIndex:
799
67
  case MachineOperand::MO_JumpTableIndex:
800
67
  case MachineOperand::MO_ExternalSymbol:
801
67
  case MachineOperand::MO_GlobalAddress:
802
67
  case MachineOperand::MO_BlockAddress:
803
67
    return true;
804
67
  case MachineOperand::MO_RegisterMask:
805
0
  case MachineOperand::MO_RegisterLiveOut:
806
0
    return false;
807
0
  case MachineOperand::MO_Metadata:
808
0
  case MachineOperand::MO_MCSymbol:
809
0
    return true;
810
0
  case MachineOperand::MO_CFIIndex:
811
0
    return false;
812
0
  case MachineOperand::MO_IntrinsicID:
813
0
  case MachineOperand::MO_Predicate:
814
0
    llvm_unreachable("should not exist post-isel");
815
0
  }
816
0
  llvm_unreachable("unhandled machine operand type");
817
0
}
818
819
5.54k
static MachineOperand makeImplicit(const MachineOperand &MO) {
820
5.54k
  MachineOperand NewMO = MO;
821
5.54k
  NewMO.setImplicit();
822
5.54k
  return NewMO;
823
5.54k
}
824
825
void ARMExpandPseudo::ExpandMOV32BitImm(MachineBasicBlock &MBB,
826
2.76k
                                        MachineBasicBlock::iterator &MBBI) {
827
2.76k
  MachineInstr &MI = *MBBI;
828
2.76k
  unsigned Opcode = MI.getOpcode();
829
2.76k
  unsigned PredReg = 0;
830
2.76k
  ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
831
2.76k
  unsigned DstReg = MI.getOperand(0).getReg();
832
2.76k
  bool DstIsDead = MI.getOperand(0).isDead();
833
2.76k
  bool isCC = Opcode == ARM::MOVCCi32imm || 
Opcode == ARM::t2MOVCCi32imm2.74k
;
834
2.76k
  const MachineOperand &MO = MI.getOperand(isCC ? 
236
:
12.72k
);
835
2.76k
  bool RequiresBundling = STI->isTargetWindows() && 
IsAnAddressOperand(MO)67
;
836
2.76k
  MachineInstrBuilder LO16, HI16;
837
2.76k
  LLVM_DEBUG(dbgs() << "Expanding: "; MI.dump());
838
2.76k
839
2.76k
  if (!STI->hasV6T2Ops() &&
840
2.76k
      
(206
Opcode == ARM::MOVi32imm206
||
Opcode == ARM::MOVCCi32imm23
)) {
841
183
    // FIXME Windows CE supports older ARM CPUs
842
183
    assert(!STI->isTargetWindows() && "Windows on ARM requires ARMv7+");
843
183
844
183
    // Expand into a movi + orr.
845
183
    LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVi), DstReg);
846
183
    HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::ORRri))
847
183
      .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
848
183
      .addReg(DstReg);
849
183
850
183
    assert (MO.isImm() && "MOVi32imm w/ non-immediate source operand!");
851
183
    unsigned ImmVal = (unsigned)MO.getImm();
852
183
    unsigned SOImmValV1 = ARM_AM::getSOImmTwoPartFirst(ImmVal);
853
183
    unsigned SOImmValV2 = ARM_AM::getSOImmTwoPartSecond(ImmVal);
854
183
    LO16 = LO16.addImm(SOImmValV1);
855
183
    HI16 = HI16.addImm(SOImmValV2);
856
183
    LO16.cloneMemRefs(MI);
857
183
    HI16.cloneMemRefs(MI);
858
183
    LO16.addImm(Pred).addReg(PredReg).add(condCodeOp());
859
183
    HI16.addImm(Pred).addReg(PredReg).add(condCodeOp());
860
183
    if (isCC)
861
0
      LO16.add(makeImplicit(MI.getOperand(1)));
862
183
    TransferImpOps(MI, LO16, HI16);
863
183
    MI.eraseFromParent();
864
183
    return;
865
183
  }
866
2.58k
867
2.58k
  unsigned LO16Opc = 0;
868
2.58k
  unsigned HI16Opc = 0;
869
2.58k
  if (Opcode == ARM::t2MOVi32imm || 
Opcode == ARM::t2MOVCCi32imm893
) {
870
1.70k
    LO16Opc = ARM::t2MOVi16;
871
1.70k
    HI16Opc = ARM::t2MOVTi16;
872
1.70k
  } else {
873
876
    LO16Opc = ARM::MOVi16;
874
876
    HI16Opc = ARM::MOVTi16;
875
876
  }
876
2.58k
877
2.58k
  LO16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LO16Opc), DstReg);
878
2.58k
  HI16 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc))
879
2.58k
    .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
880
2.58k
    .addReg(DstReg);
881
2.58k
882
2.58k
  switch (MO.getType()) {
883
2.58k
  case MachineOperand::MO_Immediate: {
884
1.47k
    unsigned Imm = MO.getImm();
885
1.47k
    unsigned Lo16 = Imm & 0xffff;
886
1.47k
    unsigned Hi16 = (Imm >> 16) & 0xffff;
887
1.47k
    LO16 = LO16.addImm(Lo16);
888
1.47k
    HI16 = HI16.addImm(Hi16);
889
1.47k
    break;
890
2.58k
  }
891
2.58k
  case MachineOperand::MO_ExternalSymbol: {
892
10
    const char *ES = MO.getSymbolName();
893
10
    unsigned TF = MO.getTargetFlags();
894
10
    LO16 = LO16.addExternalSymbol(ES, TF | ARMII::MO_LO16);
895
10
    HI16 = HI16.addExternalSymbol(ES, TF | ARMII::MO_HI16);
896
10
    break;
897
2.58k
  }
898
2.58k
  default: {
899
1.09k
    const GlobalValue *GV = MO.getGlobal();
900
1.09k
    unsigned TF = MO.getTargetFlags();
901
1.09k
    LO16 = LO16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_LO16);
902
1.09k
    HI16 = HI16.addGlobalAddress(GV, MO.getOffset(), TF | ARMII::MO_HI16);
903
1.09k
    break;
904
2.58k
  }
905
2.58k
  }
906
2.58k
907
2.58k
  LO16.cloneMemRefs(MI);
908
2.58k
  HI16.cloneMemRefs(MI);
909
2.58k
  LO16.addImm(Pred).addReg(PredReg);
910
2.58k
  HI16.addImm(Pred).addReg(PredReg);
911
2.58k
912
2.58k
  if (RequiresBundling)
913
67
    finalizeBundle(MBB, LO16->getIterator(), MBBI->getIterator());
914
2.58k
915
2.58k
  if (isCC)
916
36
    LO16.add(makeImplicit(MI.getOperand(1)));
917
2.58k
  TransferImpOps(MI, LO16, HI16);
918
2.58k
  MI.eraseFromParent();
919
2.58k
  LLVM_DEBUG(dbgs() << "To:        "; LO16.getInstr()->dump(););
920
2.58k
  LLVM_DEBUG(dbgs() << "And:       "; HI16.getInstr()->dump(););
921
2.58k
}
922
923
/// Expand a CMP_SWAP pseudo-inst to an ldrex/strex loop as simply as
924
/// possible. This only gets used at -O0 so we don't care about efficiency of
925
/// the generated code.
926
bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,
927
                                     MachineBasicBlock::iterator MBBI,
928
                                     unsigned LdrexOp, unsigned StrexOp,
929
                                     unsigned UxtOp,
930
6
                                     MachineBasicBlock::iterator &NextMBBI) {
931
6
  bool IsThumb = STI->isThumb();
932
6
  MachineInstr &MI = *MBBI;
933
6
  DebugLoc DL = MI.getDebugLoc();
934
6
  const MachineOperand &Dest = MI.getOperand(0);
935
6
  unsigned TempReg = MI.getOperand(1).getReg();
936
6
  // Duplicating undef operands into 2 instructions does not guarantee the same
937
6
  // value on both; However undef should be replaced by xzr anyway.
938
6
  assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
939
6
  unsigned AddrReg = MI.getOperand(2).getReg();
940
6
  unsigned DesiredReg = MI.getOperand(3).getReg();
941
6
  unsigned NewReg = MI.getOperand(4).getReg();
942
6
943
6
  MachineFunction *MF = MBB.getParent();
944
6
  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
945
6
  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
946
6
  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
947
6
948
6
  MF->insert(++MBB.getIterator(), LoadCmpBB);
949
6
  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
950
6
  MF->insert(++StoreBB->getIterator(), DoneBB);
951
6
952
6
  if (UxtOp) {
953
4
    MachineInstrBuilder MIB =
954
4
        BuildMI(MBB, MBBI, DL, TII->get(UxtOp), DesiredReg)
955
4
            .addReg(DesiredReg, RegState::Kill);
956
4
    if (!IsThumb)
957
2
      MIB.addImm(0);
958
4
    MIB.add(predOps(ARMCC::AL));
959
4
  }
960
6
961
6
  // .Lloadcmp:
962
6
  //     ldrex rDest, [rAddr]
963
6
  //     cmp rDest, rDesired
964
6
  //     bne .Ldone
965
6
966
6
  MachineInstrBuilder MIB;
967
6
  MIB = BuildMI(LoadCmpBB, DL, TII->get(LdrexOp), Dest.getReg());
968
6
  MIB.addReg(AddrReg);
969
6
  if (LdrexOp == ARM::t2LDREX)
970
1
    MIB.addImm(0); // a 32-bit Thumb ldrex (only) allows an offset.
971
6
  MIB.add(predOps(ARMCC::AL));
972
6
973
6
  unsigned CMPrr = IsThumb ? 
ARM::tCMPhir3
:
ARM::CMPrr3
;
974
6
  BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
975
6
      .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
976
6
      .addReg(DesiredReg)
977
6
      .add(predOps(ARMCC::AL));
978
6
  unsigned Bcc = IsThumb ? 
ARM::tBcc3
:
ARM::Bcc3
;
979
6
  BuildMI(LoadCmpBB, DL, TII->get(Bcc))
980
6
      .addMBB(DoneBB)
981
6
      .addImm(ARMCC::NE)
982
6
      .addReg(ARM::CPSR, RegState::Kill);
983
6
  LoadCmpBB->addSuccessor(DoneBB);
984
6
  LoadCmpBB->addSuccessor(StoreBB);
985
6
986
6
  // .Lstore:
987
6
  //     strex rTempReg, rNew, [rAddr]
988
6
  //     cmp rTempReg, #0
989
6
  //     bne .Lloadcmp
990
6
  MIB = BuildMI(StoreBB, DL, TII->get(StrexOp), TempReg)
991
6
    .addReg(NewReg)
992
6
    .addReg(AddrReg);
993
6
  if (StrexOp == ARM::t2STREX)
994
1
    MIB.addImm(0); // a 32-bit Thumb strex (only) allows an offset.
995
6
  MIB.add(predOps(ARMCC::AL));
996
6
997
6
  unsigned CMPri = IsThumb ? 
ARM::t2CMPri3
:
ARM::CMPri3
;
998
6
  BuildMI(StoreBB, DL, TII->get(CMPri))
999
6
      .addReg(TempReg, RegState::Kill)
1000
6
      .addImm(0)
1001
6
      .add(predOps(ARMCC::AL));
1002
6
  BuildMI(StoreBB, DL, TII->get(Bcc))
1003
6
      .addMBB(LoadCmpBB)
1004
6
      .addImm(ARMCC::NE)
1005
6
      .addReg(ARM::CPSR, RegState::Kill);
1006
6
  StoreBB->addSuccessor(LoadCmpBB);
1007
6
  StoreBB->addSuccessor(DoneBB);
1008
6
1009
6
  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
1010
6
  DoneBB->transferSuccessors(&MBB);
1011
6
1012
6
  MBB.addSuccessor(LoadCmpBB);
1013
6
1014
6
  NextMBBI = MBB.end();
1015
6
  MI.eraseFromParent();
1016
6
1017
6
  // Recompute livein lists.
1018
6
  LivePhysRegs LiveRegs;
1019
6
  computeAndAddLiveIns(LiveRegs, *DoneBB);
1020
6
  computeAndAddLiveIns(LiveRegs, *StoreBB);
1021
6
  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
1022
6
  // Do an extra pass around the loop to get loop carried registers right.
1023
6
  StoreBB->clearLiveIns();
1024
6
  computeAndAddLiveIns(LiveRegs, *StoreBB);
1025
6
  LoadCmpBB->clearLiveIns();
1026
6
  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
1027
6
1028
6
  return true;
1029
6
}
1030
1031
/// ARM's ldrexd/strexd take a consecutive register pair (represented as a
1032
/// single GPRPair register), Thumb's take two separate registers so we need to
1033
/// extract the subregs from the pair.
1034
static void addExclusiveRegPair(MachineInstrBuilder &MIB, MachineOperand &Reg,
1035
                                unsigned Flags, bool IsThumb,
1036
16
                                const TargetRegisterInfo *TRI) {
1037
16
  if (IsThumb) {
1038
6
    unsigned RegLo = TRI->getSubReg(Reg.getReg(), ARM::gsub_0);
1039
6
    unsigned RegHi = TRI->getSubReg(Reg.getReg(), ARM::gsub_1);
1040
6
    MIB.addReg(RegLo, Flags);
1041
6
    MIB.addReg(RegHi, Flags);
1042
6
  } else
1043
10
    MIB.addReg(Reg.getReg(), Flags);
1044
16
}
1045
1046
/// Expand a 64-bit CMP_SWAP to an ldrexd/strexd loop.
1047
bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
1048
                                        MachineBasicBlock::iterator MBBI,
1049
8
                                        MachineBasicBlock::iterator &NextMBBI) {
1050
8
  bool IsThumb = STI->isThumb();
1051
8
  MachineInstr &MI = *MBBI;
1052
8
  DebugLoc DL = MI.getDebugLoc();
1053
8
  MachineOperand &Dest = MI.getOperand(0);
1054
8
  unsigned TempReg = MI.getOperand(1).getReg();
1055
8
  // Duplicating undef operands into 2 instructions does not guarantee the same
1056
8
  // value on both; However undef should be replaced by xzr anyway.
1057
8
  assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
1058
8
  unsigned AddrReg = MI.getOperand(2).getReg();
1059
8
  unsigned DesiredReg = MI.getOperand(3).getReg();
1060
8
  MachineOperand New = MI.getOperand(4);
1061
8
  New.setIsKill(false);
1062
8
1063
8
  unsigned DestLo = TRI->getSubReg(Dest.getReg(), ARM::gsub_0);
1064
8
  unsigned DestHi = TRI->getSubReg(Dest.getReg(), ARM::gsub_1);
1065
8
  unsigned DesiredLo = TRI->getSubReg(DesiredReg, ARM::gsub_0);
1066
8
  unsigned DesiredHi = TRI->getSubReg(DesiredReg, ARM::gsub_1);
1067
8
1068
8
  MachineFunction *MF = MBB.getParent();
1069
8
  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
1070
8
  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
1071
8
  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
1072
8
1073
8
  MF->insert(++MBB.getIterator(), LoadCmpBB);
1074
8
  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
1075
8
  MF->insert(++StoreBB->getIterator(), DoneBB);
1076
8
1077
8
  // .Lloadcmp:
1078
8
  //     ldrexd rDestLo, rDestHi, [rAddr]
1079
8
  //     cmp rDestLo, rDesiredLo
1080
8
  //     sbcs dead rTempReg, rDestHi, rDesiredHi
1081
8
  //     bne .Ldone
1082
8
  unsigned LDREXD = IsThumb ? 
ARM::t2LDREXD3
:
ARM::LDREXD5
;
1083
8
  MachineInstrBuilder MIB;
1084
8
  MIB = BuildMI(LoadCmpBB, DL, TII->get(LDREXD));
1085
8
  addExclusiveRegPair(MIB, Dest, RegState::Define, IsThumb, TRI);
1086
8
  MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
1087
8
1088
8
  unsigned CMPrr = IsThumb ? 
ARM::tCMPhir3
:
ARM::CMPrr5
;
1089
8
  BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
1090
8
      .addReg(DestLo, getKillRegState(Dest.isDead()))
1091
8
      .addReg(DesiredLo)
1092
8
      .add(predOps(ARMCC::AL));
1093
8
1094
8
  BuildMI(LoadCmpBB, DL, TII->get(CMPrr))
1095
8
      .addReg(DestHi, getKillRegState(Dest.isDead()))
1096
8
      .addReg(DesiredHi)
1097
8
      .addImm(ARMCC::EQ).addReg(ARM::CPSR, RegState::Kill);
1098
8
1099
8
  unsigned Bcc = IsThumb ? 
ARM::tBcc3
:
ARM::Bcc5
;
1100
8
  BuildMI(LoadCmpBB, DL, TII->get(Bcc))
1101
8
      .addMBB(DoneBB)
1102
8
      .addImm(ARMCC::NE)
1103
8
      .addReg(ARM::CPSR, RegState::Kill);
1104
8
  LoadCmpBB->addSuccessor(DoneBB);
1105
8
  LoadCmpBB->addSuccessor(StoreBB);
1106
8
1107
8
  // .Lstore:
1108
8
  //     strexd rTempReg, rNewLo, rNewHi, [rAddr]
1109
8
  //     cmp rTempReg, #0
1110
8
  //     bne .Lloadcmp
1111
8
  unsigned STREXD = IsThumb ? 
ARM::t2STREXD3
:
ARM::STREXD5
;
1112
8
  MIB = BuildMI(StoreBB, DL, TII->get(STREXD), TempReg);
1113
8
  unsigned Flags = getKillRegState(New.isDead());
1114
8
  addExclusiveRegPair(MIB, New, Flags, IsThumb, TRI);
1115
8
  MIB.addReg(AddrReg).add(predOps(ARMCC::AL));
1116
8
1117
8
  unsigned CMPri = IsThumb ? 
ARM::t2CMPri3
:
ARM::CMPri5
;
1118
8
  BuildMI(StoreBB, DL, TII->get(CMPri))
1119
8
      .addReg(TempReg, RegState::Kill)
1120
8
      .addImm(0)
1121
8
      .add(predOps(ARMCC::AL));
1122
8
  BuildMI(StoreBB, DL, TII->get(Bcc))
1123
8
      .addMBB(LoadCmpBB)
1124
8
      .addImm(ARMCC::NE)
1125
8
      .addReg(ARM::CPSR, RegState::Kill);
1126
8
  StoreBB->addSuccessor(LoadCmpBB);
1127
8
  StoreBB->addSuccessor(DoneBB);
1128
8
1129
8
  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
1130
8
  DoneBB->transferSuccessors(&MBB);
1131
8
1132
8
  MBB.addSuccessor(LoadCmpBB);
1133
8
1134
8
  NextMBBI = MBB.end();
1135
8
  MI.eraseFromParent();
1136
8
1137
8
  // Recompute livein lists.
1138
8
  LivePhysRegs LiveRegs;
1139
8
  computeAndAddLiveIns(LiveRegs, *DoneBB);
1140
8
  computeAndAddLiveIns(LiveRegs, *StoreBB);
1141
8
  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
1142
8
  // Do an extra pass around the loop to get loop carried registers right.
1143
8
  StoreBB->clearLiveIns();
1144
8
  computeAndAddLiveIns(LiveRegs, *StoreBB);
1145
8
  LoadCmpBB->clearLiveIns();
1146
8
  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
1147
8
1148
8
  return true;
1149
8
}
1150
1151
1152
bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
1153
                               MachineBasicBlock::iterator MBBI,
1154
762k
                               MachineBasicBlock::iterator &NextMBBI) {
1155
762k
  MachineInstr &MI = *MBBI;
1156
762k
  unsigned Opcode = MI.getOpcode();
1157
762k
  switch (Opcode) {
1158
762k
    default:
1159
721k
      return false;
1160
762k
1161
762k
    case ARM::TCRETURNdi:
1162
3.00k
    case ARM::TCRETURNri: {
1163
3.00k
      MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
1164
3.00k
      assert(MBBI->isReturn() &&
1165
3.00k
             "Can only insert epilog into returning blocks");
1166
3.00k
      unsigned RetOpcode = MBBI->getOpcode();
1167
3.00k
      DebugLoc dl = MBBI->getDebugLoc();
1168
3.00k
      const ARMBaseInstrInfo &TII = *static_cast<const ARMBaseInstrInfo *>(
1169
3.00k
          MBB.getParent()->getSubtarget().getInstrInfo());
1170
3.00k
1171
3.00k
      // Tail call return: adjust the stack pointer and jump to callee.
1172
3.00k
      MBBI = MBB.getLastNonDebugInstr();
1173
3.00k
      MachineOperand &JumpTarget = MBBI->getOperand(0);
1174
3.00k
1175
3.00k
      // Jump to label or value in register.
1176
3.00k
      if (RetOpcode == ARM::TCRETURNdi) {
1177
2.88k
        unsigned TCOpcode =
1178
2.88k
            STI->isThumb()
1179
2.88k
                ? 
(STI->isTargetMachO() 2.68k
?
ARM::tTAILJMPd2.33k
:
ARM::tTAILJMPdND342
)
1180
2.88k
                : 
ARM::TAILJMPd204
;
1181
2.88k
        MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
1182
2.88k
        if (JumpTarget.isGlobal())
1183
2.62k
          MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
1184
2.62k
                               JumpTarget.getTargetFlags());
1185
264
        else {
1186
264
          assert(JumpTarget.isSymbol());
1187
264
          MIB.addExternalSymbol(JumpTarget.getSymbolName(),
1188
264
                                JumpTarget.getTargetFlags());
1189
264
        }
1190
2.88k
1191
2.88k
        // Add the default predicate in Thumb mode.
1192
2.88k
        if (STI->isThumb())
1193
2.68k
          MIB.add(predOps(ARMCC::AL));
1194
2.88k
      } else 
if (120
RetOpcode == ARM::TCRETURNri120
) {
1195
120
        unsigned Opcode =
1196
120
          STI->isThumb() ? 
ARM::tTAILJMPr110
1197
120
                         : 
(STI->hasV4TOps() 10
?
ARM::TAILJMPr7
:
ARM::TAILJMPr43
);
1198
120
        BuildMI(MBB, MBBI, dl,
1199
120
                TII.get(Opcode))
1200
120
            .addReg(JumpTarget.getReg(), RegState::Kill);
1201
120
      }
1202
3.00k
1203
3.00k
      auto NewMI = std::prev(MBBI);
1204
11.7k
      for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; 
++i8.71k
)
1205
8.71k
        NewMI->addOperand(MBBI->getOperand(i));
1206
3.00k
1207
3.00k
      // Delete the pseudo instruction TCRETURN.
1208
3.00k
      MBB.erase(MBBI);
1209
3.00k
      MBBI = NewMI;
1210
3.00k
      return true;
1211
3.00k
    }
1212
3.00k
    case ARM::VMOVScc:
1213
469
    case ARM::VMOVDcc: {
1214
469
      unsigned newOpc = Opcode == ARM::VMOVScc ? 
ARM::VMOVS297
:
ARM::VMOVD172
;
1215
469
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(newOpc),
1216
469
              MI.getOperand(1).getReg())
1217
469
          .add(MI.getOperand(2))
1218
469
          .addImm(MI.getOperand(3).getImm()) // 'pred'
1219
469
          .add(MI.getOperand(4))
1220
469
          .add(makeImplicit(MI.getOperand(1)));
1221
469
1222
469
      MI.eraseFromParent();
1223
469
      return true;
1224
469
    }
1225
2.28k
    case ARM::t2MOVCCr:
1226
2.28k
    case ARM::MOVCCr: {
1227
2.28k
      unsigned Opc = AFI->isThumbFunction() ? 
ARM::t2MOVr2.11k
:
ARM::MOVr174
;
1228
2.28k
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
1229
2.28k
              MI.getOperand(1).getReg())
1230
2.28k
          .add(MI.getOperand(2))
1231
2.28k
          .addImm(MI.getOperand(3).getImm()) // 'pred'
1232
2.28k
          .add(MI.getOperand(4))
1233
2.28k
          .add(condCodeOp()) // 's' bit
1234
2.28k
          .add(makeImplicit(MI.getOperand(1)));
1235
2.28k
1236
2.28k
      MI.eraseFromParent();
1237
2.28k
      return true;
1238
2.28k
    }
1239
2.28k
    case ARM::MOVCCsi: {
1240
2
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
1241
2
              (MI.getOperand(1).getReg()))
1242
2
          .add(MI.getOperand(2))
1243
2
          .addImm(MI.getOperand(3).getImm())
1244
2
          .addImm(MI.getOperand(4).getImm()) // 'pred'
1245
2
          .add(MI.getOperand(5))
1246
2
          .add(condCodeOp()) // 's' bit
1247
2
          .add(makeImplicit(MI.getOperand(1)));
1248
2
1249
2
      MI.eraseFromParent();
1250
2
      return true;
1251
2.28k
    }
1252
2.28k
    case ARM::MOVCCsr: {
1253
19
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsr),
1254
19
              (MI.getOperand(1).getReg()))
1255
19
          .add(MI.getOperand(2))
1256
19
          .add(MI.getOperand(3))
1257
19
          .addImm(MI.getOperand(4).getImm())
1258
19
          .addImm(MI.getOperand(5).getImm()) // 'pred'
1259
19
          .add(MI.getOperand(6))
1260
19
          .add(condCodeOp()) // 's' bit
1261
19
          .add(makeImplicit(MI.getOperand(1)));
1262
19
1263
19
      MI.eraseFromParent();
1264
19
      return true;
1265
2.28k
    }
1266
2.28k
    case ARM::t2MOVCCi16:
1267
138
    case ARM::MOVCCi16: {
1268
138
      unsigned NewOpc = AFI->isThumbFunction() ? 
ARM::t2MOVi1620
:
ARM::MOVi16118
;
1269
138
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
1270
138
              MI.getOperand(1).getReg())
1271
138
          .addImm(MI.getOperand(2).getImm())
1272
138
          .addImm(MI.getOperand(3).getImm()) // 'pred'
1273
138
          .add(MI.getOperand(4))
1274
138
          .add(makeImplicit(MI.getOperand(1)));
1275
138
      MI.eraseFromParent();
1276
138
      return true;
1277
138
    }
1278
2.48k
    case ARM::t2MOVCCi:
1279
2.48k
    case ARM::MOVCCi: {
1280
2.48k
      unsigned Opc = AFI->isThumbFunction() ? 
ARM::t2MOVi2.17k
:
ARM::MOVi308
;
1281
2.48k
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
1282
2.48k
              MI.getOperand(1).getReg())
1283
2.48k
          .addImm(MI.getOperand(2).getImm())
1284
2.48k
          .addImm(MI.getOperand(3).getImm()) // 'pred'
1285
2.48k
          .add(MI.getOperand(4))
1286
2.48k
          .add(condCodeOp()) // 's' bit
1287
2.48k
          .add(makeImplicit(MI.getOperand(1)));
1288
2.48k
1289
2.48k
      MI.eraseFromParent();
1290
2.48k
      return true;
1291
2.48k
    }
1292
2.48k
    case ARM::t2MVNCCi:
1293
84
    case ARM::MVNCCi: {
1294
84
      unsigned Opc = AFI->isThumbFunction() ? 
ARM::t2MVNi35
:
ARM::MVNi49
;
1295
84
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opc),
1296
84
              MI.getOperand(1).getReg())
1297
84
          .addImm(MI.getOperand(2).getImm())
1298
84
          .addImm(MI.getOperand(3).getImm()) // 'pred'
1299
84
          .add(MI.getOperand(4))
1300
84
          .add(condCodeOp()) // 's' bit
1301
84
          .add(makeImplicit(MI.getOperand(1)));
1302
84
1303
84
      MI.eraseFromParent();
1304
84
      return true;
1305
84
    }
1306
84
    case ARM::t2MOVCClsl:
1307
28
    case ARM::t2MOVCClsr:
1308
28
    case ARM::t2MOVCCasr:
1309
28
    case ARM::t2MOVCCror: {
1310
28
      unsigned NewOpc;
1311
28
      switch (Opcode) {
1312
28
      
case ARM::t2MOVCClsl: NewOpc = ARM::t2LSLri; break11
;
1313
28
      
case ARM::t2MOVCClsr: NewOpc = ARM::t2LSRri; break13
;
1314
28
      
case ARM::t2MOVCCasr: NewOpc = ARM::t2ASRri; break3
;
1315
28
      
case ARM::t2MOVCCror: NewOpc = ARM::t2RORri; break1
;
1316
28
      
default: 0
llvm_unreachable0
("unexpeced conditional move");
1317
28
      }
1318
28
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc),
1319
28
              MI.getOperand(1).getReg())
1320
28
          .add(MI.getOperand(2))
1321
28
          .addImm(MI.getOperand(3).getImm())
1322
28
          .addImm(MI.getOperand(4).getImm()) // 'pred'
1323
28
          .add(MI.getOperand(5))
1324
28
          .add(condCodeOp()) // 's' bit
1325
28
          .add(makeImplicit(MI.getOperand(1)));
1326
28
      MI.eraseFromParent();
1327
28
      return true;
1328
28
    }
1329
32
    case ARM::Int_eh_sjlj_dispatchsetup: {
1330
32
      MachineFunction &MF = *MI.getParent()->getParent();
1331
32
      const ARMBaseInstrInfo *AII =
1332
32
        static_cast<const ARMBaseInstrInfo*>(TII);
1333
32
      const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
1334
32
      // For functions using a base pointer, we rematerialize it (via the frame
1335
32
      // pointer) here since eh.sjlj.setjmp and eh.sjlj.longjmp don't do it
1336
32
      // for us. Otherwise, expand to nothing.
1337
32
      if (RI.hasBasePointer(MF)) {
1338
0
        int32_t NumBytes = AFI->getFramePtrSpillOffset();
1339
0
        unsigned FramePtr = RI.getFrameRegister(MF);
1340
0
        assert(MF.getSubtarget().getFrameLowering()->hasFP(MF) &&
1341
0
               "base pointer without frame pointer?");
1342
0
1343
0
        if (AFI->isThumb2Function()) {
1344
0
          emitT2RegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
1345
0
                                 FramePtr, -NumBytes, ARMCC::AL, 0, *TII);
1346
0
        } else if (AFI->isThumbFunction()) {
1347
0
          emitThumbRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
1348
0
                                    FramePtr, -NumBytes, *TII, RI);
1349
0
        } else {
1350
0
          emitARMRegPlusImmediate(MBB, MBBI, MI.getDebugLoc(), ARM::R6,
1351
0
                                  FramePtr, -NumBytes, ARMCC::AL, 0,
1352
0
                                  *TII);
1353
0
        }
1354
0
        // If there's dynamic realignment, adjust for it.
1355
0
        if (RI.needsStackRealignment(MF)) {
1356
0
          MachineFrameInfo &MFI = MF.getFrameInfo();
1357
0
          unsigned MaxAlign = MFI.getMaxAlignment();
1358
0
          assert (!AFI->isThumb1OnlyFunction());
1359
0
          // Emit bic r6, r6, MaxAlign
1360
0
          assert(MaxAlign <= 256 && "The BIC instruction cannot encode "
1361
0
                                    "immediates larger than 256 with all lower "
1362
0
                                    "bits set.");
1363
0
          unsigned bicOpc = AFI->isThumbFunction() ?
1364
0
            ARM::t2BICri : ARM::BICri;
1365
0
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(bicOpc), ARM::R6)
1366
0
              .addReg(ARM::R6, RegState::Kill)
1367
0
              .addImm(MaxAlign - 1)
1368
0
              .add(predOps(ARMCC::AL))
1369
0
              .add(condCodeOp());
1370
0
        }
1371
0
1372
0
      }
1373
32
      MI.eraseFromParent();
1374
32
      return true;
1375
28
    }
1376
28
1377
28
    case ARM::MOVsrl_flag:
1378
2
    case ARM::MOVsra_flag: {
1379
2
      // These are just fancy MOVs instructions.
1380
2
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
1381
2
              MI.getOperand(0).getReg())
1382
2
          .add(MI.getOperand(1))
1383
2
          .addImm(ARM_AM::getSORegOpc(
1384
2
              (Opcode == ARM::MOVsrl_flag ? ARM_AM::lsr : 
ARM_AM::asr0
), 1))
1385
2
          .add(predOps(ARMCC::AL))
1386
2
          .addReg(ARM::CPSR, RegState::Define);
1387
2
      MI.eraseFromParent();
1388
2
      return true;
1389
2
    }
1390
2
    case ARM::RRX: {
1391
2
      // This encodes as "MOVs Rd, Rm, rrx
1392
2
      MachineInstrBuilder MIB =
1393
2
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::MOVsi),
1394
2
                  MI.getOperand(0).getReg())
1395
2
              .add(MI.getOperand(1))
1396
2
              .addImm(ARM_AM::getSORegOpc(ARM_AM::rrx, 0))
1397
2
              .add(predOps(ARMCC::AL))
1398
2
              .add(condCodeOp());
1399
2
      TransferImpOps(MI, MIB, MIB);
1400
2
      MI.eraseFromParent();
1401
2
      return true;
1402
2
    }
1403
39
    case ARM::tTPsoft:
1404
39
    case ARM::TPsoft: {
1405
39
      const bool Thumb = Opcode == ARM::tTPsoft;
1406
39
1407
39
      MachineInstrBuilder MIB;
1408
39
      if (STI->genLongCalls()) {
1409
2
        MachineFunction *MF = MBB.getParent();
1410
2
        MachineConstantPool *MCP = MF->getConstantPool();
1411
2
        unsigned PCLabelID = AFI->createPICLabelUId();
1412
2
        MachineConstantPoolValue *CPV =
1413
2
            ARMConstantPoolSymbol::Create(MF->getFunction().getContext(),
1414
2
                                          "__aeabi_read_tp", PCLabelID, 0);
1415
2
        unsigned Reg = MI.getOperand(0).getReg();
1416
2
        MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
1417
2
                      TII->get(Thumb ? 
ARM::tLDRpci1
:
ARM::LDRi121
), Reg)
1418
2
                  .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
1419
2
        if (!Thumb)
1420
1
          MIB.addImm(0);
1421
2
        MIB.add(predOps(ARMCC::AL));
1422
2
1423
2
        MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
1424
2
                      TII->get(Thumb ? 
ARM::tBLXr1
:
ARM::BLX1
));
1425
2
        if (Thumb)
1426
1
          MIB.add(predOps(ARMCC::AL));
1427
2
        MIB.addReg(Reg, RegState::Kill);
1428
37
      } else {
1429
37
        MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
1430
37
                      TII->get(Thumb ? 
ARM::tBL11
:
ARM::BL26
));
1431
37
        if (Thumb)
1432
11
          MIB.add(predOps(ARMCC::AL));
1433
37
        MIB.addExternalSymbol("__aeabi_read_tp", 0);
1434
37
      }
1435
39
1436
39
      MIB.cloneMemRefs(MI);
1437
39
      TransferImpOps(MI, MIB, MIB);
1438
39
      MI.eraseFromParent();
1439
39
      return true;
1440
39
    }
1441
45
    case ARM::tLDRpci_pic:
1442
45
    case ARM::t2LDRpci_pic: {
1443
45
      unsigned NewLdOpc = (Opcode == ARM::tLDRpci_pic)
1444
45
        ? 
ARM::tLDRpci14
:
ARM::t2LDRpci31
;
1445
45
      unsigned DstReg = MI.getOperand(0).getReg();
1446
45
      bool DstIsDead = MI.getOperand(0).isDead();
1447
45
      MachineInstrBuilder MIB1 =
1448
45
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewLdOpc), DstReg)
1449
45
              .add(MI.getOperand(1))
1450
45
              .add(predOps(ARMCC::AL));
1451
45
      MIB1.cloneMemRefs(MI);
1452
45
      MachineInstrBuilder MIB2 =
1453
45
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::tPICADD))
1454
45
              .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
1455
45
              .addReg(DstReg)
1456
45
              .add(MI.getOperand(2));
1457
45
      TransferImpOps(MI, MIB1, MIB2);
1458
45
      MI.eraseFromParent();
1459
45
      return true;
1460
45
    }
1461
45
1462
1.22k
    case ARM::LDRLIT_ga_abs:
1463
1.22k
    case ARM::LDRLIT_ga_pcrel:
1464
1.22k
    case ARM::LDRLIT_ga_pcrel_ldr:
1465
1.22k
    case ARM::tLDRLIT_ga_abs:
1466
1.22k
    case ARM::tLDRLIT_ga_pcrel: {
1467
1.22k
      unsigned DstReg = MI.getOperand(0).getReg();
1468
1.22k
      bool DstIsDead = MI.getOperand(0).isDead();
1469
1.22k
      const MachineOperand &MO1 = MI.getOperand(1);
1470
1.22k
      auto Flags = MO1.getTargetFlags();
1471
1.22k
      const GlobalValue *GV = MO1.getGlobal();
1472
1.22k
      bool IsARM =
1473
1.22k
          Opcode != ARM::tLDRLIT_ga_pcrel && 
Opcode != ARM::tLDRLIT_ga_abs475
;
1474
1.22k
      bool IsPIC =
1475
1.22k
          Opcode != ARM::LDRLIT_ga_abs && 
Opcode != ARM::tLDRLIT_ga_abs1.20k
;
1476
1.22k
      unsigned LDRLITOpc = IsARM ? 
ARM::LDRi12204
:
ARM::tLDRpci1.02k
;
1477
1.22k
      unsigned PICAddOpc =
1478
1.22k
          IsARM
1479
1.22k
              ? 
(Opcode == ARM::LDRLIT_ga_pcrel_ldr 204
?
ARM::PICLDR102
:
ARM::PICADD102
)
1480
1.22k
              : 
ARM::tPICADD1.02k
;
1481
1.22k
1482
1.22k
      // We need a new const-pool entry to load from.
1483
1.22k
      MachineConstantPool *MCP = MBB.getParent()->getConstantPool();
1484
1.22k
      unsigned ARMPCLabelIndex = 0;
1485
1.22k
      MachineConstantPoolValue *CPV;
1486
1.22k
1487
1.22k
      if (IsPIC) {
1488
937
        unsigned PCAdj = IsARM ? 
8186
:
4751
;
1489
937
        auto Modifier = (Flags & ARMII::MO_GOT)
1490
937
                            ? 
ARMCP::GOT_PREL66
1491
937
                            : 
ARMCP::no_modifier871
;
1492
937
        ARMPCLabelIndex = AFI->createPICLabelUId();
1493
937
        CPV = ARMConstantPoolConstant::Create(
1494
937
            GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj, Modifier,
1495
937
            /*AddCurrentAddr*/ Modifier == ARMCP::GOT_PREL);
1496
937
      } else
1497
289
        CPV = ARMConstantPoolConstant::Create(GV, ARMCP::no_modifier);
1498
1.22k
1499
1.22k
      MachineInstrBuilder MIB =
1500
1.22k
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(LDRLITOpc), DstReg)
1501
1.22k
            .addConstantPoolIndex(MCP->getConstantPoolIndex(CPV, 4));
1502
1.22k
      if (IsARM)
1503
204
        MIB.addImm(0);
1504
1.22k
      MIB.add(predOps(ARMCC::AL));
1505
1.22k
1506
1.22k
      if (IsPIC) {
1507
937
        MachineInstrBuilder MIB =
1508
937
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(PICAddOpc))
1509
937
            .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
1510
937
            .addReg(DstReg)
1511
937
            .addImm(ARMPCLabelIndex);
1512
937
1513
937
        if (IsARM)
1514
186
          MIB.add(predOps(ARMCC::AL));
1515
937
      }
1516
1.22k
1517
1.22k
      MI.eraseFromParent();
1518
1.22k
      return true;
1519
1.22k
    }
1520
27.1k
    case ARM::MOV_ga_pcrel:
1521
27.1k
    case ARM::MOV_ga_pcrel_ldr:
1522
27.1k
    case ARM::t2MOV_ga_pcrel: {
1523
27.1k
      // Expand into movw + movw. Also "add pc" / ldr [pc] in PIC mode.
1524
27.1k
      unsigned LabelId = AFI->createPICLabelUId();
1525
27.1k
      unsigned DstReg = MI.getOperand(0).getReg();
1526
27.1k
      bool DstIsDead = MI.getOperand(0).isDead();
1527
27.1k
      const MachineOperand &MO1 = MI.getOperand(1);
1528
27.1k
      const GlobalValue *GV = MO1.getGlobal();
1529
27.1k
      unsigned TF = MO1.getTargetFlags();
1530
27.1k
      bool isARM = Opcode != ARM::t2MOV_ga_pcrel;
1531
27.1k
      unsigned LO16Opc = isARM ? 
ARM::MOVi16_ga_pcrel225
:
ARM::t2MOVi16_ga_pcrel26.9k
;
1532
27.1k
      unsigned HI16Opc = isARM ? 
ARM::MOVTi16_ga_pcrel225
:
ARM::t2MOVTi16_ga_pcrel26.9k
;
1533
27.1k
      unsigned LO16TF = TF | ARMII::MO_LO16;
1534
27.1k
      unsigned HI16TF = TF | ARMII::MO_HI16;
1535
27.1k
      unsigned PICAddOpc = isARM
1536
27.1k
        ? 
(Opcode == ARM::MOV_ga_pcrel_ldr 225
?
ARM::PICLDR136
:
ARM::PICADD89
)
1537
27.1k
        : 
ARM::tPICADD26.9k
;
1538
27.1k
      MachineInstrBuilder MIB1 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
1539
27.1k
                                         TII->get(LO16Opc), DstReg)
1540
27.1k
        .addGlobalAddress(GV, MO1.getOffset(), TF | LO16TF)
1541
27.1k
        .addImm(LabelId);
1542
27.1k
1543
27.1k
      BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(HI16Opc), DstReg)
1544
27.1k
        .addReg(DstReg)
1545
27.1k
        .addGlobalAddress(GV, MO1.getOffset(), TF | HI16TF)
1546
27.1k
        .addImm(LabelId);
1547
27.1k
1548
27.1k
      MachineInstrBuilder MIB3 = BuildMI(MBB, MBBI, MI.getDebugLoc(),
1549
27.1k
                                         TII->get(PICAddOpc))
1550
27.1k
        .addReg(DstReg, RegState::Define | getDeadRegState(DstIsDead))
1551
27.1k
        .addReg(DstReg).addImm(LabelId);
1552
27.1k
      if (isARM) {
1553
225
        MIB3.add(predOps(ARMCC::AL));
1554
225
        if (Opcode == ARM::MOV_ga_pcrel_ldr)
1555
136
          MIB3.cloneMemRefs(MI);
1556
225
      }
1557
27.1k
      TransferImpOps(MI, MIB1, MIB3);
1558
27.1k
      MI.eraseFromParent();
1559
27.1k
      return true;
1560
27.1k
    }
1561
27.1k
1562
27.1k
    case ARM::MOVi32imm:
1563
2.76k
    case ARM::MOVCCi32imm:
1564
2.76k
    case ARM::t2MOVi32imm:
1565
2.76k
    case ARM::t2MOVCCi32imm:
1566
2.76k
      ExpandMOV32BitImm(MBB, MBBI);
1567
2.76k
      return true;
1568
2.76k
1569
2.76k
    case ARM::SUBS_PC_LR: {
1570
6
      MachineInstrBuilder MIB =
1571
6
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(ARM::SUBri), ARM::PC)
1572
6
              .addReg(ARM::LR)
1573
6
              .add(MI.getOperand(0))
1574
6
              .add(MI.getOperand(1))
1575
6
              .add(MI.getOperand(2))
1576
6
              .addReg(ARM::CPSR, RegState::Undef);
1577
6
      TransferImpOps(MI, MIB, MIB);
1578
6
      MI.eraseFromParent();
1579
6
      return true;
1580
2.76k
    }
1581
2.76k
    case ARM::VLDMQIA: {
1582
2
      unsigned NewOpc = ARM::VLDMDIA;
1583
2
      MachineInstrBuilder MIB =
1584
2
        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
1585
2
      unsigned OpIdx = 0;
1586
2
1587
2
      // Grab the Q register destination.
1588
2
      bool DstIsDead = MI.getOperand(OpIdx).isDead();
1589
2
      unsigned DstReg = MI.getOperand(OpIdx++).getReg();
1590
2
1591
2
      // Copy the source register.
1592
2
      MIB.add(MI.getOperand(OpIdx++));
1593
2
1594
2
      // Copy the predicate operands.
1595
2
      MIB.add(MI.getOperand(OpIdx++));
1596
2
      MIB.add(MI.getOperand(OpIdx++));
1597
2
1598
2
      // Add the destination operands (D subregs).
1599
2
      unsigned D0 = TRI->getSubReg(DstReg, ARM::dsub_0);
1600
2
      unsigned D1 = TRI->getSubReg(DstReg, ARM::dsub_1);
1601
2
      MIB.addReg(D0, RegState::Define | getDeadRegState(DstIsDead))
1602
2
        .addReg(D1, RegState::Define | getDeadRegState(DstIsDead));
1603
2
1604
2
      // Add an implicit def for the super-register.
1605
2
      MIB.addReg(DstReg, RegState::ImplicitDefine | getDeadRegState(DstIsDead));
1606
2
      TransferImpOps(MI, MIB, MIB);
1607
2
      MIB.cloneMemRefs(MI);
1608
2
      MI.eraseFromParent();
1609
2
      return true;
1610
2.76k
    }
1611
2.76k
1612
2.76k
    case ARM::VSTMQIA: {
1613
2
      unsigned NewOpc = ARM::VSTMDIA;
1614
2
      MachineInstrBuilder MIB =
1615
2
        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(NewOpc));
1616
2
      unsigned OpIdx = 0;
1617
2
1618
2
      // Grab the Q register source.
1619
2
      bool SrcIsKill = MI.getOperand(OpIdx).isKill();
1620
2
      unsigned SrcReg = MI.getOperand(OpIdx++).getReg();
1621
2
1622
2
      // Copy the destination register.
1623
2
      MachineOperand Dst(MI.getOperand(OpIdx++));
1624
2
      MIB.add(Dst);
1625
2
1626
2
      // Copy the predicate operands.
1627
2
      MIB.add(MI.getOperand(OpIdx++));
1628
2
      MIB.add(MI.getOperand(OpIdx++));
1629
2
1630
2
      // Add the source operands (D subregs).
1631
2
      unsigned D0 = TRI->getSubReg(SrcReg, ARM::dsub_0);
1632
2
      unsigned D1 = TRI->getSubReg(SrcReg, ARM::dsub_1);
1633
2
      MIB.addReg(D0, SrcIsKill ? RegState::Kill : 
00
)
1634
2
         .addReg(D1, SrcIsKill ? RegState::Kill : 
00
);
1635
2
1636
2
      if (SrcIsKill)      // Add an implicit kill for the Q register.
1637
2
        MIB->addRegisterKilled(SrcReg, TRI, true);
1638
2
1639
2
      TransferImpOps(MI, MIB, MIB);
1640
2
      MIB.cloneMemRefs(MI);
1641
2
      MI.eraseFromParent();
1642
2
      return true;
1643
2.76k
    }
1644
2.76k
1645
2.76k
    case ARM::VLD2q8Pseudo:
1646
148
    case ARM::VLD2q16Pseudo:
1647
148
    case ARM::VLD2q32Pseudo:
1648
148
    case ARM::VLD2q8PseudoWB_fixed:
1649
148
    case ARM::VLD2q16PseudoWB_fixed:
1650
148
    case ARM::VLD2q32PseudoWB_fixed:
1651
148
    case ARM::VLD2q8PseudoWB_register:
1652
148
    case ARM::VLD2q16PseudoWB_register:
1653
148
    case ARM::VLD2q32PseudoWB_register:
1654
148
    case ARM::VLD3d8Pseudo:
1655
148
    case ARM::VLD3d16Pseudo:
1656
148
    case ARM::VLD3d32Pseudo:
1657
148
    case ARM::VLD1d8TPseudo:
1658
148
    case ARM::VLD1d16TPseudo:
1659
148
    case ARM::VLD1d32TPseudo:
1660
148
    case ARM::VLD1d64TPseudo:
1661
148
    case ARM::VLD1d64TPseudoWB_fixed:
1662
148
    case ARM::VLD1d64TPseudoWB_register:
1663
148
    case ARM::VLD3d8Pseudo_UPD:
1664
148
    case ARM::VLD3d16Pseudo_UPD:
1665
148
    case ARM::VLD3d32Pseudo_UPD:
1666
148
    case ARM::VLD3q8Pseudo_UPD:
1667
148
    case ARM::VLD3q16Pseudo_UPD:
1668
148
    case ARM::VLD3q32Pseudo_UPD:
1669
148
    case ARM::VLD3q8oddPseudo:
1670
148
    case ARM::VLD3q16oddPseudo:
1671
148
    case ARM::VLD3q32oddPseudo:
1672
148
    case ARM::VLD3q8oddPseudo_UPD:
1673
148
    case ARM::VLD3q16oddPseudo_UPD:
1674
148
    case ARM::VLD3q32oddPseudo_UPD:
1675
148
    case ARM::VLD4d8Pseudo:
1676
148
    case ARM::VLD4d16Pseudo:
1677
148
    case ARM::VLD4d32Pseudo:
1678
148
    case ARM::VLD1d8QPseudo:
1679
148
    case ARM::VLD1d16QPseudo:
1680
148
    case ARM::VLD1d32QPseudo:
1681
148
    case ARM::VLD1d64QPseudo:
1682
148
    case ARM::VLD1d64QPseudoWB_fixed:
1683
148
    case ARM::VLD1d64QPseudoWB_register:
1684
148
    case ARM::VLD1q8HighQPseudo:
1685
148
    case ARM::VLD1q8LowQPseudo_UPD:
1686
148
    case ARM::VLD1q8HighTPseudo:
1687
148
    case ARM::VLD1q8LowTPseudo_UPD:
1688
148
    case ARM::VLD1q16HighQPseudo:
1689
148
    case ARM::VLD1q16LowQPseudo_UPD:
1690
148
    case ARM::VLD1q16HighTPseudo:
1691
148
    case ARM::VLD1q16LowTPseudo_UPD:
1692
148
    case ARM::VLD1q32HighQPseudo:
1693
148
    case ARM::VLD1q32LowQPseudo_UPD:
1694
148
    case ARM::VLD1q32HighTPseudo:
1695
148
    case ARM::VLD1q32LowTPseudo_UPD:
1696
148
    case ARM::VLD1q64HighQPseudo:
1697
148
    case ARM::VLD1q64LowQPseudo_UPD:
1698
148
    case ARM::VLD1q64HighTPseudo:
1699
148
    case ARM::VLD1q64LowTPseudo_UPD:
1700
148
    case ARM::VLD4d8Pseudo_UPD:
1701
148
    case ARM::VLD4d16Pseudo_UPD:
1702
148
    case ARM::VLD4d32Pseudo_UPD:
1703
148
    case ARM::VLD4q8Pseudo_UPD:
1704
148
    case ARM::VLD4q16Pseudo_UPD:
1705
148
    case ARM::VLD4q32Pseudo_UPD:
1706
148
    case ARM::VLD4q8oddPseudo:
1707
148
    case ARM::VLD4q16oddPseudo:
1708
148
    case ARM::VLD4q32oddPseudo:
1709
148
    case ARM::VLD4q8oddPseudo_UPD:
1710
148
    case ARM::VLD4q16oddPseudo_UPD:
1711
148
    case ARM::VLD4q32oddPseudo_UPD:
1712
148
    case ARM::VLD3DUPd8Pseudo:
1713
148
    case ARM::VLD3DUPd16Pseudo:
1714
148
    case ARM::VLD3DUPd32Pseudo:
1715
148
    case ARM::VLD3DUPd8Pseudo_UPD:
1716
148
    case ARM::VLD3DUPd16Pseudo_UPD:
1717
148
    case ARM::VLD3DUPd32Pseudo_UPD:
1718
148
    case ARM::VLD4DUPd8Pseudo:
1719
148
    case ARM::VLD4DUPd16Pseudo:
1720
148
    case ARM::VLD4DUPd32Pseudo:
1721
148
    case ARM::VLD4DUPd8Pseudo_UPD:
1722
148
    case ARM::VLD4DUPd16Pseudo_UPD:
1723
148
    case ARM::VLD4DUPd32Pseudo_UPD:
1724
148
    case ARM::VLD2DUPq8EvenPseudo:
1725
148
    case ARM::VLD2DUPq8OddPseudo:
1726
148
    case ARM::VLD2DUPq16EvenPseudo:
1727
148
    case ARM::VLD2DUPq16OddPseudo:
1728
148
    case ARM::VLD2DUPq32EvenPseudo:
1729
148
    case ARM::VLD2DUPq32OddPseudo:
1730
148
    case ARM::VLD3DUPq8EvenPseudo:
1731
148
    case ARM::VLD3DUPq8OddPseudo:
1732
148
    case ARM::VLD3DUPq16EvenPseudo:
1733
148
    case ARM::VLD3DUPq16OddPseudo:
1734
148
    case ARM::VLD3DUPq32EvenPseudo:
1735
148
    case ARM::VLD3DUPq32OddPseudo:
1736
148
    case ARM::VLD4DUPq8EvenPseudo:
1737
148
    case ARM::VLD4DUPq8OddPseudo:
1738
148
    case ARM::VLD4DUPq16EvenPseudo:
1739
148
    case ARM::VLD4DUPq16OddPseudo:
1740
148
    case ARM::VLD4DUPq32EvenPseudo:
1741
148
    case ARM::VLD4DUPq32OddPseudo:
1742
148
      ExpandVLD(MBBI);
1743
148
      return true;
1744
148
1745
148
    case ARM::VST2q8Pseudo:
1746
107
    case ARM::VST2q16Pseudo:
1747
107
    case ARM::VST2q32Pseudo:
1748
107
    case ARM::VST2q8PseudoWB_fixed:
1749
107
    case ARM::VST2q16PseudoWB_fixed:
1750
107
    case ARM::VST2q32PseudoWB_fixed:
1751
107
    case ARM::VST2q8PseudoWB_register:
1752
107
    case ARM::VST2q16PseudoWB_register:
1753
107
    case ARM::VST2q32PseudoWB_register:
1754
107
    case ARM::VST3d8Pseudo:
1755
107
    case ARM::VST3d16Pseudo:
1756
107
    case ARM::VST3d32Pseudo:
1757
107
    case ARM::VST1d8TPseudo:
1758
107
    case ARM::VST1d16TPseudo:
1759
107
    case ARM::VST1d32TPseudo:
1760
107
    case ARM::VST1d64TPseudo:
1761
107
    case ARM::VST3d8Pseudo_UPD:
1762
107
    case ARM::VST3d16Pseudo_UPD:
1763
107
    case ARM::VST3d32Pseudo_UPD:
1764
107
    case ARM::VST1d64TPseudoWB_fixed:
1765
107
    case ARM::VST1d64TPseudoWB_register:
1766
107
    case ARM::VST3q8Pseudo_UPD:
1767
107
    case ARM::VST3q16Pseudo_UPD:
1768
107
    case ARM::VST3q32Pseudo_UPD:
1769
107
    case ARM::VST3q8oddPseudo:
1770
107
    case ARM::VST3q16oddPseudo:
1771
107
    case ARM::VST3q32oddPseudo:
1772
107
    case ARM::VST3q8oddPseudo_UPD:
1773
107
    case ARM::VST3q16oddPseudo_UPD:
1774
107
    case ARM::VST3q32oddPseudo_UPD:
1775
107
    case ARM::VST4d8Pseudo:
1776
107
    case ARM::VST4d16Pseudo:
1777
107
    case ARM::VST4d32Pseudo:
1778
107
    case ARM::VST1d8QPseudo:
1779
107
    case ARM::VST1d16QPseudo:
1780
107
    case ARM::VST1d32QPseudo:
1781
107
    case ARM::VST1d64QPseudo:
1782
107
    case ARM::VST4d8Pseudo_UPD:
1783
107
    case ARM::VST4d16Pseudo_UPD:
1784
107
    case ARM::VST4d32Pseudo_UPD:
1785
107
    case ARM::VST1d64QPseudoWB_fixed:
1786
107
    case ARM::VST1d64QPseudoWB_register:
1787
107
    case ARM::VST1q8HighQPseudo:
1788
107
    case ARM::VST1q8LowQPseudo_UPD:
1789
107
    case ARM::VST1q8HighTPseudo:
1790
107
    case ARM::VST1q8LowTPseudo_UPD:
1791
107
    case ARM::VST1q16HighQPseudo:
1792
107
    case ARM::VST1q16LowQPseudo_UPD:
1793
107
    case ARM::VST1q16HighTPseudo:
1794
107
    case ARM::VST1q16LowTPseudo_UPD:
1795
107
    case ARM::VST1q32HighQPseudo:
1796
107
    case ARM::VST1q32LowQPseudo_UPD:
1797
107
    case ARM::VST1q32HighTPseudo:
1798
107
    case ARM::VST1q32LowTPseudo_UPD:
1799
107
    case ARM::VST1q64HighQPseudo:
1800
107
    case ARM::VST1q64LowQPseudo_UPD:
1801
107
    case ARM::VST1q64HighTPseudo:
1802
107
    case ARM::VST1q64LowTPseudo_UPD:
1803
107
    case ARM::VST4q8Pseudo_UPD:
1804
107
    case ARM::VST4q16Pseudo_UPD:
1805
107
    case ARM::VST4q32Pseudo_UPD:
1806
107
    case ARM::VST4q8oddPseudo:
1807
107
    case ARM::VST4q16oddPseudo:
1808
107
    case ARM::VST4q32oddPseudo:
1809
107
    case ARM::VST4q8oddPseudo_UPD:
1810
107
    case ARM::VST4q16oddPseudo_UPD:
1811
107
    case ARM::VST4q32oddPseudo_UPD:
1812
107
      ExpandVST(MBBI);
1813
107
      return true;
1814
107
1815
189
    case ARM::VLD1LNq8Pseudo:
1816
189
    case ARM::VLD1LNq16Pseudo:
1817
189
    case ARM::VLD1LNq32Pseudo:
1818
189
    case ARM::VLD1LNq8Pseudo_UPD:
1819
189
    case ARM::VLD1LNq16Pseudo_UPD:
1820
189
    case ARM::VLD1LNq32Pseudo_UPD:
1821
189
    case ARM::VLD2LNd8Pseudo:
1822
189
    case ARM::VLD2LNd16Pseudo:
1823
189
    case ARM::VLD2LNd32Pseudo:
1824
189
    case ARM::VLD2LNq16Pseudo:
1825
189
    case ARM::VLD2LNq32Pseudo:
1826
189
    case ARM::VLD2LNd8Pseudo_UPD:
1827
189
    case ARM::VLD2LNd16Pseudo_UPD:
1828
189
    case ARM::VLD2LNd32Pseudo_UPD:
1829
189
    case ARM::VLD2LNq16Pseudo_UPD:
1830
189
    case ARM::VLD2LNq32Pseudo_UPD:
1831
189
    case ARM::VLD3LNd8Pseudo:
1832
189
    case ARM::VLD3LNd16Pseudo:
1833
189
    case ARM::VLD3LNd32Pseudo:
1834
189
    case ARM::VLD3LNq16Pseudo:
1835
189
    case ARM::VLD3LNq32Pseudo:
1836
189
    case ARM::VLD3LNd8Pseudo_UPD:
1837
189
    case ARM::VLD3LNd16Pseudo_UPD:
1838
189
    case ARM::VLD3LNd32Pseudo_UPD:
1839
189
    case ARM::VLD3LNq16Pseudo_UPD:
1840
189
    case ARM::VLD3LNq32Pseudo_UPD:
1841
189
    case ARM::VLD4LNd8Pseudo:
1842
189
    case ARM::VLD4LNd16Pseudo:
1843
189
    case ARM::VLD4LNd32Pseudo:
1844
189
    case ARM::VLD4LNq16Pseudo:
1845
189
    case ARM::VLD4LNq32Pseudo:
1846
189
    case ARM::VLD4LNd8Pseudo_UPD:
1847
189
    case ARM::VLD4LNd16Pseudo_UPD:
1848
189
    case ARM::VLD4LNd32Pseudo_UPD:
1849
189
    case ARM::VLD4LNq16Pseudo_UPD:
1850
189
    case ARM::VLD4LNq32Pseudo_UPD:
1851
189
    case ARM::VST1LNq8Pseudo:
1852
189
    case ARM::VST1LNq16Pseudo:
1853
189
    case ARM::VST1LNq32Pseudo:
1854
189
    case ARM::VST1LNq8Pseudo_UPD:
1855
189
    case ARM::VST1LNq16Pseudo_UPD:
1856
189
    case ARM::VST1LNq32Pseudo_UPD:
1857
189
    case ARM::VST2LNd8Pseudo:
1858
189
    case ARM::VST2LNd16Pseudo:
1859
189
    case ARM::VST2LNd32Pseudo:
1860
189
    case ARM::VST2LNq16Pseudo:
1861
189
    case ARM::VST2LNq32Pseudo:
1862
189
    case ARM::VST2LNd8Pseudo_UPD:
1863
189
    case ARM::VST2LNd16Pseudo_UPD:
1864
189
    case ARM::VST2LNd32Pseudo_UPD:
1865
189
    case ARM::VST2LNq16Pseudo_UPD:
1866
189
    case ARM::VST2LNq32Pseudo_UPD:
1867
189
    case ARM::VST3LNd8Pseudo:
1868
189
    case ARM::VST3LNd16Pseudo:
1869
189
    case ARM::VST3LNd32Pseudo:
1870
189
    case ARM::VST3LNq16Pseudo:
1871
189
    case ARM::VST3LNq32Pseudo:
1872
189
    case ARM::VST3LNd8Pseudo_UPD:
1873
189
    case ARM::VST3LNd16Pseudo_UPD:
1874
189
    case ARM::VST3LNd32Pseudo_UPD:
1875
189
    case ARM::VST3LNq16Pseudo_UPD:
1876
189
    case ARM::VST3LNq32Pseudo_UPD:
1877
189
    case ARM::VST4LNd8Pseudo:
1878
189
    case ARM::VST4LNd16Pseudo:
1879
189
    case ARM::VST4LNd32Pseudo:
1880
189
    case ARM::VST4LNq16Pseudo:
1881
189
    case ARM::VST4LNq32Pseudo:
1882
189
    case ARM::VST4LNd8Pseudo_UPD:
1883
189
    case ARM::VST4LNd16Pseudo_UPD:
1884
189
    case ARM::VST4LNd32Pseudo_UPD:
1885
189
    case ARM::VST4LNq16Pseudo_UPD:
1886
189
    case ARM::VST4LNq32Pseudo_UPD:
1887
189
      ExpandLaneOp(MBBI);
1888
189
      return true;
1889
189
1890
189
    
case ARM::VTBL3Pseudo: ExpandVTBL(MBBI, ARM::VTBL3, false); return true1
;
1891
189
    
case ARM::VTBL4Pseudo: ExpandVTBL(MBBI, ARM::VTBL4, false); return true1
;
1892
189
    
case ARM::VTBX3Pseudo: ExpandVTBL(MBBI, ARM::VTBX3, true); return true1
;
1893
189
    
case ARM::VTBX4Pseudo: ExpandVTBL(MBBI, ARM::VTBX4, true); return true2
;
1894
189
1895
189
    case ARM::CMP_SWAP_8:
1896
2
      if (STI->isThumb())
1897
1
        return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXB, ARM::t2STREXB,
1898
1
                              ARM::tUXTB, NextMBBI);
1899
1
      else
1900
1
        return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXB, ARM::STREXB,
1901
1
                              ARM::UXTB, NextMBBI);
1902
2
    case ARM::CMP_SWAP_16:
1903
2
      if (STI->isThumb())
1904
1
        return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREXH, ARM::t2STREXH,
1905
1
                              ARM::tUXTH, NextMBBI);
1906
1
      else
1907
1
        return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREXH, ARM::STREXH,
1908
1
                              ARM::UXTH, NextMBBI);
1909
2
    case ARM::CMP_SWAP_32:
1910
2
      if (STI->isThumb())
1911
1
        return ExpandCMP_SWAP(MBB, MBBI, ARM::t2LDREX, ARM::t2STREX, 0,
1912
1
                              NextMBBI);
1913
1
      else
1914
1
        return ExpandCMP_SWAP(MBB, MBBI, ARM::LDREX, ARM::STREX, 0, NextMBBI);
1915
0
1916
8
    case ARM::CMP_SWAP_64:
1917
8
      return ExpandCMP_SWAP_64(MBB, MBBI, NextMBBI);
1918
762k
  }
1919
762k
}
1920
1921
112k
bool ARMExpandPseudo::ExpandMBB(MachineBasicBlock &MBB) {
1922
112k
  bool Modified = false;
1923
112k
1924
112k
  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
1925
875k
  while (MBBI != E) {
1926
762k
    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
1927
762k
    Modified |= ExpandMI(MBB, MBBI, NMBBI);
1928
762k
    MBBI = NMBBI;
1929
762k
  }
1930
112k
1931
112k
  return Modified;
1932
112k
}
1933
1934
26.5k
bool ARMExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
1935
26.5k
  STI = &static_cast<const ARMSubtarget &>(MF.getSubtarget());
1936
26.5k
  TII = STI->getInstrInfo();
1937
26.5k
  TRI = STI->getRegisterInfo();
1938
26.5k
  AFI = MF.getInfo<ARMFunctionInfo>();
1939
26.5k
1940
26.5k
  LLVM_DEBUG(dbgs() << "********** ARM EXPAND PSEUDO INSTRUCTIONS **********\n"
1941
26.5k
                    << "********** Function: " << MF.getName() << '\n');
1942
26.5k
1943
26.5k
  bool Modified = false;
1944
26.5k
  for (MachineBasicBlock &MBB : MF)
1945
112k
    Modified |= ExpandMBB(MBB);
1946
26.5k
  if (VerifyARMPseudo)
1947
4
    MF.verify(this, "After expanding ARM pseudo instructions.");
1948
26.5k
1949
26.5k
  LLVM_DEBUG(dbgs() << "***************************************************\n");
1950
26.5k
  return Modified;
1951
26.5k
}
1952
1953
/// createARMExpandPseudoPass - returns an instance of the pseudo instruction
1954
/// expansion pass.
1955
5.19k
FunctionPass *llvm::createARMExpandPseudoPass() {
1956
5.19k
  return new ARMExpandPseudo();
1957
5.19k
}