Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains a pass that expands pseudo instructions into target
10
// instructions to allow proper scheduling and other late optimizations.  This
11
// pass should be run after register allocation but before the post-regalloc
12
// scheduling pass.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#include "AArch64ExpandImm.h"
17
#include "AArch64InstrInfo.h"
18
#include "AArch64MachineFunctionInfo.h"
19
#include "AArch64Subtarget.h"
20
#include "MCTargetDesc/AArch64AddressingModes.h"
21
#include "Utils/AArch64BaseInfo.h"
22
#include "llvm/ADT/DenseMap.h"
23
#include "llvm/ADT/Triple.h"
24
#include "llvm/CodeGen/LivePhysRegs.h"
25
#include "llvm/CodeGen/MachineBasicBlock.h"
26
#include "llvm/CodeGen/MachineFunction.h"
27
#include "llvm/CodeGen/MachineFunctionPass.h"
28
#include "llvm/CodeGen/MachineInstr.h"
29
#include "llvm/CodeGen/MachineInstrBuilder.h"
30
#include "llvm/CodeGen/MachineOperand.h"
31
#include "llvm/CodeGen/TargetSubtargetInfo.h"
32
#include "llvm/IR/DebugLoc.h"
33
#include "llvm/MC/MCInstrDesc.h"
34
#include "llvm/Pass.h"
35
#include "llvm/Support/CodeGen.h"
36
#include "llvm/Support/MathExtras.h"
37
#include "llvm/Target/TargetMachine.h"
38
#include <cassert>
39
#include <cstdint>
40
#include <iterator>
41
#include <limits>
42
#include <utility>
43
44
using namespace llvm;
45
46
266k
#define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
47
48
namespace {
49
50
class AArch64ExpandPseudo : public MachineFunctionPass {
51
public:
52
  const AArch64InstrInfo *TII;
53
54
  static char ID;
55
56
8.78k
  AArch64ExpandPseudo() : MachineFunctionPass(ID) {
57
8.78k
    initializeAArch64ExpandPseudoPass(*PassRegistry::getPassRegistry());
58
8.78k
  }
59
60
  bool runOnMachineFunction(MachineFunction &Fn) override;
61
62
266k
  StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }
63
64
private:
65
  bool expandMBB(MachineBasicBlock &MBB);
66
  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
67
                MachineBasicBlock::iterator &NextMBBI);
68
  bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
69
                    unsigned BitSize);
70
71
  bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
72
                      unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
73
                      unsigned ExtendImm, unsigned ZeroReg,
74
                      MachineBasicBlock::iterator &NextMBBI);
75
  bool expandCMP_SWAP_128(MachineBasicBlock &MBB,
76
                          MachineBasicBlock::iterator MBBI,
77
                          MachineBasicBlock::iterator &NextMBBI);
78
  bool expandSetTagLoop(MachineBasicBlock &MBB,
79
                        MachineBasicBlock::iterator MBBI,
80
                        MachineBasicBlock::iterator &NextMBBI);
81
};
82
83
} // end anonymous namespace
84
85
char AArch64ExpandPseudo::ID = 0;
86
87
INITIALIZE_PASS(AArch64ExpandPseudo, "aarch64-expand-pseudo",
88
                AARCH64_EXPAND_PSEUDO_NAME, false, false)
89
90
/// Transfer implicit operands on the pseudo instruction to the
91
/// instructions created from the expansion.
92
static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,
93
4.91M
                           MachineInstrBuilder &DefMI) {
94
4.91M
  const MCInstrDesc &Desc = OldMI.getDesc();
95
5.93M
  for (unsigned i = Desc.getNumOperands(), e = OldMI.getNumOperands(); i != e;
96
4.91M
       
++i1.02M
) {
97
1.02M
    const MachineOperand &MO = OldMI.getOperand(i);
98
1.02M
    assert(MO.isReg() && MO.getReg());
99
1.02M
    if (MO.isUse())
100
438k
      UseMI.add(MO);
101
583k
    else
102
583k
      DefMI.add(MO);
103
1.02M
  }
104
4.91M
}
105
106
/// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more
107
/// real move-immediate instructions to synthesize the immediate.
108
bool AArch64ExpandPseudo::expandMOVImm(MachineBasicBlock &MBB,
109
                                       MachineBasicBlock::iterator MBBI,
110
1.30M
                                       unsigned BitSize) {
111
1.30M
  MachineInstr &MI = *MBBI;
112
1.30M
  unsigned DstReg = MI.getOperand(0).getReg();
113
1.30M
  uint64_t Imm = MI.getOperand(1).getImm();
114
1.30M
115
1.30M
  if (DstReg == AArch64::XZR || 
DstReg == AArch64::WZR1.30M
) {
116
2
    // Useless def, and we don't want to risk creating an invalid ORR (which
117
2
    // would really write to sp).
118
2
    MI.eraseFromParent();
119
2
    return true;
120
2
  }
121
1.30M
122
1.30M
  SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
123
1.30M
  AArch64_IMM::expandMOVImm(Imm, BitSize, Insn);
124
1.30M
  assert(Insn.size() != 0);
125
1.30M
126
1.30M
  SmallVector<MachineInstrBuilder, 4> MIBS;
127
2.68M
  for (auto I = Insn.begin(), E = Insn.end(); I != E; 
++I1.37M
) {
128
1.37M
    bool LastItem = std::next(I) == E;
129
1.37M
    switch (I->Opcode)
130
1.37M
    {
131
1.37M
    
default: 0
llvm_unreachable0
("unhandled!");
break0
;
132
1.37M
133
1.37M
    case AArch64::ORRWri:
134
9.11k
    case AArch64::ORRXri:
135
9.11k
      MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
136
9.11k
        .add(MI.getOperand(0))
137
9.11k
        .addReg(BitSize == 32 ? 
AArch64::WZR551
:
AArch64::XZR8.56k
)
138
9.11k
        .addImm(I->Op2));
139
9.11k
      break;
140
1.29M
    case AArch64::MOVNWi:
141
1.29M
    case AArch64::MOVNXi:
142
1.29M
    case AArch64::MOVZWi:
143
1.29M
    case AArch64::MOVZXi: {
144
1.29M
      bool DstIsDead = MI.getOperand(0).isDead();
145
1.29M
      MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
146
1.29M
        .addReg(DstReg, RegState::Define |
147
1.29M
                getDeadRegState(DstIsDead && 
LastItem18.7k
))
148
1.29M
        .addImm(I->Op1)
149
1.29M
        .addImm(I->Op2));
150
1.29M
      } break;
151
1.29M
    case AArch64::MOVKWi:
152
74.4k
    case AArch64::MOVKXi: {
153
74.4k
      unsigned DstReg = MI.getOperand(0).getReg();
154
74.4k
      bool DstIsDead = MI.getOperand(0).isDead();
155
74.4k
      MIBS.push_back(BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(I->Opcode))
156
74.4k
        .addReg(DstReg,
157
74.4k
                RegState::Define |
158
74.4k
                getDeadRegState(DstIsDead && 
LastItem5.37k
))
159
74.4k
        .addReg(DstReg)
160
74.4k
        .addImm(I->Op1)
161
74.4k
        .addImm(I->Op2));
162
74.4k
      } break;
163
1.37M
    }
164
1.37M
  }
165
1.30M
  transferImpOps(MI, MIBS.front(), MIBS.back());
166
1.30M
  MI.eraseFromParent();
167
1.30M
  return true;
168
1.30M
}
169
170
bool AArch64ExpandPseudo::expandCMP_SWAP(
171
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp,
172
    unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
173
7
    MachineBasicBlock::iterator &NextMBBI) {
174
7
  MachineInstr &MI = *MBBI;
175
7
  DebugLoc DL = MI.getDebugLoc();
176
7
  const MachineOperand &Dest = MI.getOperand(0);
177
7
  unsigned StatusReg = MI.getOperand(1).getReg();
178
7
  bool StatusDead = MI.getOperand(1).isDead();
179
7
  // Duplicating undef operands into 2 instructions does not guarantee the same
180
7
  // value on both; However undef should be replaced by xzr anyway.
181
7
  assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
182
7
  unsigned AddrReg = MI.getOperand(2).getReg();
183
7
  unsigned DesiredReg = MI.getOperand(3).getReg();
184
7
  unsigned NewReg = MI.getOperand(4).getReg();
185
7
186
7
  MachineFunction *MF = MBB.getParent();
187
7
  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
188
7
  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
189
7
  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
190
7
191
7
  MF->insert(++MBB.getIterator(), LoadCmpBB);
192
7
  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
193
7
  MF->insert(++StoreBB->getIterator(), DoneBB);
194
7
195
7
  // .Lloadcmp:
196
7
  //     mov wStatus, 0
197
7
  //     ldaxr xDest, [xAddr]
198
7
  //     cmp xDest, xDesired
199
7
  //     b.ne .Ldone
200
7
  if (!StatusDead)
201
7
    BuildMI(LoadCmpBB, DL, TII->get(AArch64::MOVZWi), StatusReg)
202
7
      .addImm(0).addImm(0);
203
7
  BuildMI(LoadCmpBB, DL, TII->get(LdarOp), Dest.getReg())
204
7
      .addReg(AddrReg);
205
7
  BuildMI(LoadCmpBB, DL, TII->get(CmpOp), ZeroReg)
206
7
      .addReg(Dest.getReg(), getKillRegState(Dest.isDead()))
207
7
      .addReg(DesiredReg)
208
7
      .addImm(ExtendImm);
209
7
  BuildMI(LoadCmpBB, DL, TII->get(AArch64::Bcc))
210
7
      .addImm(AArch64CC::NE)
211
7
      .addMBB(DoneBB)
212
7
      .addReg(AArch64::NZCV, RegState::Implicit | RegState::Kill);
213
7
  LoadCmpBB->addSuccessor(DoneBB);
214
7
  LoadCmpBB->addSuccessor(StoreBB);
215
7
216
7
  // .Lstore:
217
7
  //     stlxr wStatus, xNew, [xAddr]
218
7
  //     cbnz wStatus, .Lloadcmp
219
7
  BuildMI(StoreBB, DL, TII->get(StlrOp), StatusReg)
220
7
      .addReg(NewReg)
221
7
      .addReg(AddrReg);
222
7
  BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
223
7
      .addReg(StatusReg, getKillRegState(StatusDead))
224
7
      .addMBB(LoadCmpBB);
225
7
  StoreBB->addSuccessor(LoadCmpBB);
226
7
  StoreBB->addSuccessor(DoneBB);
227
7
228
7
  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
229
7
  DoneBB->transferSuccessors(&MBB);
230
7
231
7
  MBB.addSuccessor(LoadCmpBB);
232
7
233
7
  NextMBBI = MBB.end();
234
7
  MI.eraseFromParent();
235
7
236
7
  // Recompute livein lists.
237
7
  LivePhysRegs LiveRegs;
238
7
  computeAndAddLiveIns(LiveRegs, *DoneBB);
239
7
  computeAndAddLiveIns(LiveRegs, *StoreBB);
240
7
  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
241
7
  // Do an extra pass around the loop to get loop carried registers right.
242
7
  StoreBB->clearLiveIns();
243
7
  computeAndAddLiveIns(LiveRegs, *StoreBB);
244
7
  LoadCmpBB->clearLiveIns();
245
7
  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
246
7
247
7
  return true;
248
7
}
249
250
bool AArch64ExpandPseudo::expandCMP_SWAP_128(
251
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
252
2
    MachineBasicBlock::iterator &NextMBBI) {
253
2
  MachineInstr &MI = *MBBI;
254
2
  DebugLoc DL = MI.getDebugLoc();
255
2
  MachineOperand &DestLo = MI.getOperand(0);
256
2
  MachineOperand &DestHi = MI.getOperand(1);
257
2
  unsigned StatusReg = MI.getOperand(2).getReg();
258
2
  bool StatusDead = MI.getOperand(2).isDead();
259
2
  // Duplicating undef operands into 2 instructions does not guarantee the same
260
2
  // value on both; However undef should be replaced by xzr anyway.
261
2
  assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
262
2
  unsigned AddrReg = MI.getOperand(3).getReg();
263
2
  unsigned DesiredLoReg = MI.getOperand(4).getReg();
264
2
  unsigned DesiredHiReg = MI.getOperand(5).getReg();
265
2
  unsigned NewLoReg = MI.getOperand(6).getReg();
266
2
  unsigned NewHiReg = MI.getOperand(7).getReg();
267
2
268
2
  MachineFunction *MF = MBB.getParent();
269
2
  auto LoadCmpBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
270
2
  auto StoreBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
271
2
  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
272
2
273
2
  MF->insert(++MBB.getIterator(), LoadCmpBB);
274
2
  MF->insert(++LoadCmpBB->getIterator(), StoreBB);
275
2
  MF->insert(++StoreBB->getIterator(), DoneBB);
276
2
277
2
  // .Lloadcmp:
278
2
  //     ldaxp xDestLo, xDestHi, [xAddr]
279
2
  //     cmp xDestLo, xDesiredLo
280
2
  //     sbcs xDestHi, xDesiredHi
281
2
  //     b.ne .Ldone
282
2
  BuildMI(LoadCmpBB, DL, TII->get(AArch64::LDAXPX))
283
2
      .addReg(DestLo.getReg(), RegState::Define)
284
2
      .addReg(DestHi.getReg(), RegState::Define)
285
2
      .addReg(AddrReg);
286
2
  BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
287
2
      .addReg(DestLo.getReg(), getKillRegState(DestLo.isDead()))
288
2
      .addReg(DesiredLoReg)
289
2
      .addImm(0);
290
2
  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
291
2
    .addUse(AArch64::WZR)
292
2
    .addUse(AArch64::WZR)
293
2
    .addImm(AArch64CC::EQ);
294
2
  BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR)
295
2
      .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead()))
296
2
      .addReg(DesiredHiReg)
297
2
      .addImm(0);
298
2
  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg)
299
2
      .addUse(StatusReg, RegState::Kill)
300
2
      .addUse(StatusReg, RegState::Kill)
301
2
      .addImm(AArch64CC::EQ);
302
2
  BuildMI(LoadCmpBB, DL, TII->get(AArch64::CBNZW))
303
2
      .addUse(StatusReg, getKillRegState(StatusDead))
304
2
      .addMBB(DoneBB);
305
2
  LoadCmpBB->addSuccessor(DoneBB);
306
2
  LoadCmpBB->addSuccessor(StoreBB);
307
2
308
2
  // .Lstore:
309
2
  //     stlxp wStatus, xNewLo, xNewHi, [xAddr]
310
2
  //     cbnz wStatus, .Lloadcmp
311
2
  BuildMI(StoreBB, DL, TII->get(AArch64::STLXPX), StatusReg)
312
2
      .addReg(NewLoReg)
313
2
      .addReg(NewHiReg)
314
2
      .addReg(AddrReg);
315
2
  BuildMI(StoreBB, DL, TII->get(AArch64::CBNZW))
316
2
      .addReg(StatusReg, getKillRegState(StatusDead))
317
2
      .addMBB(LoadCmpBB);
318
2
  StoreBB->addSuccessor(LoadCmpBB);
319
2
  StoreBB->addSuccessor(DoneBB);
320
2
321
2
  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
322
2
  DoneBB->transferSuccessors(&MBB);
323
2
324
2
  MBB.addSuccessor(LoadCmpBB);
325
2
326
2
  NextMBBI = MBB.end();
327
2
  MI.eraseFromParent();
328
2
329
2
  // Recompute liveness bottom up.
330
2
  LivePhysRegs LiveRegs;
331
2
  computeAndAddLiveIns(LiveRegs, *DoneBB);
332
2
  computeAndAddLiveIns(LiveRegs, *StoreBB);
333
2
  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
334
2
  // Do an extra pass in the loop to get the loop carried dependencies right.
335
2
  StoreBB->clearLiveIns();
336
2
  computeAndAddLiveIns(LiveRegs, *StoreBB);
337
2
  LoadCmpBB->clearLiveIns();
338
2
  computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
339
2
340
2
  return true;
341
2
}
342
343
bool AArch64ExpandPseudo::expandSetTagLoop(
344
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
345
4
    MachineBasicBlock::iterator &NextMBBI) {
346
4
  MachineInstr &MI = *MBBI;
347
4
  DebugLoc DL = MI.getDebugLoc();
348
4
  Register SizeReg = MI.getOperand(2).getReg();
349
4
  Register AddressReg = MI.getOperand(3).getReg();
350
4
351
4
  MachineFunction *MF = MBB.getParent();
352
4
353
4
  bool ZeroData = MI.getOpcode() == AArch64::STZGloop;
354
4
  const unsigned OpCode =
355
4
      ZeroData ? 
AArch64::STZ2GPostIndex1
:
AArch64::ST2GPostIndex3
;
356
4
357
4
  auto LoopBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
358
4
  auto DoneBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
359
4
360
4
  MF->insert(++MBB.getIterator(), LoopBB);
361
4
  MF->insert(++LoopBB->getIterator(), DoneBB);
362
4
363
4
  BuildMI(LoopBB, DL, TII->get(OpCode))
364
4
      .addDef(AddressReg)
365
4
      .addReg(AddressReg)
366
4
      .addReg(AddressReg)
367
4
      .addImm(2)
368
4
      .cloneMemRefs(MI)
369
4
      .setMIFlags(MI.getFlags());
370
4
  BuildMI(LoopBB, DL, TII->get(AArch64::SUBXri))
371
4
      .addDef(SizeReg)
372
4
      .addReg(SizeReg)
373
4
      .addImm(16 * 2)
374
4
      .addImm(0);
375
4
  BuildMI(LoopBB, DL, TII->get(AArch64::CBNZX)).addUse(SizeReg).addMBB(LoopBB);
376
4
377
4
  LoopBB->addSuccessor(LoopBB);
378
4
  LoopBB->addSuccessor(DoneBB);
379
4
380
4
  DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
381
4
  DoneBB->transferSuccessors(&MBB);
382
4
383
4
  MBB.addSuccessor(LoopBB);
384
4
385
4
  NextMBBI = MBB.end();
386
4
  MI.eraseFromParent();
387
4
  // Recompute liveness bottom up.
388
4
  LivePhysRegs LiveRegs;
389
4
  computeAndAddLiveIns(LiveRegs, *DoneBB);
390
4
  computeAndAddLiveIns(LiveRegs, *LoopBB);
391
4
  // Do an extra pass in the loop to get the loop carried dependencies right.
392
4
  // FIXME: is this necessary?
393
4
  LoopBB->clearLiveIns();
394
4
  computeAndAddLiveIns(LiveRegs, *LoopBB);
395
4
  DoneBB->clearLiveIns();
396
4
  computeAndAddLiveIns(LiveRegs, *DoneBB);
397
4
398
4
  return true;
399
4
}
400
401
/// If MBBI references a pseudo instruction that should be expanded here,
402
/// do the expansion and return true.  Otherwise return false.
403
bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
404
                                   MachineBasicBlock::iterator MBBI,
405
14.1M
                                   MachineBasicBlock::iterator &NextMBBI) {
406
14.1M
  MachineInstr &MI = *MBBI;
407
14.1M
  unsigned Opcode = MI.getOpcode();
408
14.1M
  switch (Opcode) {
409
14.1M
  default:
410
9.22M
    break;
411
14.1M
412
14.1M
  case AArch64::ADDWrr:
413
2.73M
  case AArch64::SUBWrr:
414
2.73M
  case AArch64::ADDXrr:
415
2.73M
  case AArch64::SUBXrr:
416
2.73M
  case AArch64::ADDSWrr:
417
2.73M
  case AArch64::SUBSWrr:
418
2.73M
  case AArch64::ADDSXrr:
419
2.73M
  case AArch64::SUBSXrr:
420
2.73M
  case AArch64::ANDWrr:
421
2.73M
  case AArch64::ANDXrr:
422
2.73M
  case AArch64::BICWrr:
423
2.73M
  case AArch64::BICXrr:
424
2.73M
  case AArch64::ANDSWrr:
425
2.73M
  case AArch64::ANDSXrr:
426
2.73M
  case AArch64::BICSWrr:
427
2.73M
  case AArch64::BICSXrr:
428
2.73M
  case AArch64::EONWrr:
429
2.73M
  case AArch64::EONXrr:
430
2.73M
  case AArch64::EORWrr:
431
2.73M
  case AArch64::EORXrr:
432
2.73M
  case AArch64::ORNWrr:
433
2.73M
  case AArch64::ORNXrr:
434
2.73M
  case AArch64::ORRWrr:
435
2.73M
  case AArch64::ORRXrr: {
436
2.73M
    unsigned Opcode;
437
2.73M
    switch (MI.getOpcode()) {
438
2.73M
    default:
439
0
      return false;
440
2.73M
    
case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break73.4k
;
441
2.73M
    
case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break9.88k
;
442
2.73M
    
case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break327k
;
443
2.73M
    
case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break11.4k
;
444
2.73M
    
case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break6.79k
;
445
2.73M
    
case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break152k
;
446
2.73M
    
case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break17.6k
;
447
2.73M
    
case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break197k
;
448
2.73M
    
case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break68.4k
;
449
2.73M
    
case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break20.7k
;
450
2.73M
    
case AArch64::BICWrr: Opcode = AArch64::BICWrs; break1.72k
;
451
2.73M
    
case AArch64::BICXrr: Opcode = AArch64::BICXrs; break290
;
452
2.73M
    
case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break6.41k
;
453
2.73M
    
case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break3.44k
;
454
2.73M
    
case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break39
;
455
2.73M
    
case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break9
;
456
2.73M
    
case AArch64::EONWrr: Opcode = AArch64::EONWrs; break10
;
457
2.73M
    
case AArch64::EONXrr: Opcode = AArch64::EONXrs; break4
;
458
2.73M
    
case AArch64::EORWrr: Opcode = AArch64::EORWrs; break3.44k
;
459
2.73M
    
case AArch64::EORXrr: Opcode = AArch64::EORXrs; break2.27k
;
460
2.73M
    
case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break9.00k
;
461
2.73M
    
case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break705
;
462
2.73M
    
case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break35.8k
;
463
2.73M
    
case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break1.78M
;
464
2.73M
    }
465
2.73M
    MachineInstrBuilder MIB1 =
466
2.73M
        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(Opcode),
467
2.73M
                MI.getOperand(0).getReg())
468
2.73M
            .add(MI.getOperand(1))
469
2.73M
            .add(MI.getOperand(2))
470
2.73M
            .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0));
471
2.73M
    transferImpOps(MI, MIB1, MIB1);
472
2.73M
    MI.eraseFromParent();
473
2.73M
    return true;
474
2.73M
  }
475
2.73M
476
2.73M
  case AArch64::LOADgot: {
477
158k
    MachineFunction *MF = MBB.getParent();
478
158k
    unsigned DstReg = MI.getOperand(0).getReg();
479
158k
    const MachineOperand &MO1 = MI.getOperand(1);
480
158k
    unsigned Flags = MO1.getTargetFlags();
481
158k
482
158k
    if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {
483
30
      // Tiny codemodel expand to LDR
484
30
      MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
485
30
                                        TII->get(AArch64::LDRXl), DstReg);
486
30
487
30
      if (MO1.isGlobal()) {
488
30
        MIB.addGlobalAddress(MO1.getGlobal(), 0, Flags);
489
30
      } else 
if (0
MO1.isSymbol()0
) {
490
0
        MIB.addExternalSymbol(MO1.getSymbolName(), Flags);
491
0
      } else {
492
0
        assert(MO1.isCPI() &&
493
0
               "Only expect globals, externalsymbols, or constant pools");
494
0
        MIB.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(), Flags);
495
0
      }
496
158k
    } else {
497
158k
      // Small codemodel expand into ADRP + LDR.
498
158k
      MachineInstrBuilder MIB1 =
499
158k
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg);
500
158k
      MachineInstrBuilder MIB2 =
501
158k
          BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRXui))
502
158k
              .add(MI.getOperand(0))
503
158k
              .addReg(DstReg);
504
158k
505
158k
      if (MO1.isGlobal()) {
506
158k
        MIB1.addGlobalAddress(MO1.getGlobal(), 0, Flags | AArch64II::MO_PAGE);
507
158k
        MIB2.addGlobalAddress(MO1.getGlobal(), 0,
508
158k
                              Flags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
509
158k
      } else 
if (0
MO1.isSymbol()0
) {
510
0
        MIB1.addExternalSymbol(MO1.getSymbolName(), Flags | AArch64II::MO_PAGE);
511
0
        MIB2.addExternalSymbol(MO1.getSymbolName(), Flags |
512
0
                                                        AArch64II::MO_PAGEOFF |
513
0
                                                        AArch64II::MO_NC);
514
0
      } else {
515
0
        assert(MO1.isCPI() &&
516
0
               "Only expect globals, externalsymbols, or constant pools");
517
0
        MIB1.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
518
0
                                  Flags | AArch64II::MO_PAGE);
519
0
        MIB2.addConstantPoolIndex(MO1.getIndex(), MO1.getOffset(),
520
0
                                  Flags | AArch64II::MO_PAGEOFF |
521
0
                                      AArch64II::MO_NC);
522
0
      }
523
158k
524
158k
      transferImpOps(MI, MIB1, MIB2);
525
158k
    }
526
158k
    MI.eraseFromParent();
527
158k
    return true;
528
2.73M
  }
529
2.73M
530
2.73M
  case AArch64::MOVaddr:
531
447k
  case AArch64::MOVaddrJT:
532
447k
  case AArch64::MOVaddrCP:
533
447k
  case AArch64::MOVaddrBA:
534
447k
  case AArch64::MOVaddrTLS:
535
447k
  case AArch64::MOVaddrEXT: {
536
447k
    // Expand into ADRP + ADD.
537
447k
    unsigned DstReg = MI.getOperand(0).getReg();
538
447k
    MachineInstrBuilder MIB1 =
539
447k
        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADRP), DstReg)
540
447k
            .add(MI.getOperand(1));
541
447k
542
447k
    MachineInstrBuilder MIB2 =
543
447k
        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
544
447k
            .add(MI.getOperand(0))
545
447k
            .addReg(DstReg)
546
447k
            .add(MI.getOperand(2))
547
447k
            .addImm(0);
548
447k
549
447k
    transferImpOps(MI, MIB1, MIB2);
550
447k
    MI.eraseFromParent();
551
447k
    return true;
552
447k
  }
553
447k
  case AArch64::ADDlowTLS:
554
1
    // Produce a plain ADD
555
1
    BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDXri))
556
1
        .add(MI.getOperand(0))
557
1
        .add(MI.getOperand(1))
558
1
        .add(MI.getOperand(2))
559
1
        .addImm(0);
560
1
    MI.eraseFromParent();
561
1
    return true;
562
447k
563
447k
  case AArch64::MOVbaseTLS: {
564
53
    unsigned DstReg = MI.getOperand(0).getReg();
565
53
    auto SysReg = AArch64SysReg::TPIDR_EL0;
566
53
    MachineFunction *MF = MBB.getParent();
567
53
    if (MF->getTarget().getTargetTriple().isOSFuchsia() &&
568
53
        
MF->getTarget().getCodeModel() == CodeModel::Kernel4
)
569
2
      SysReg = AArch64SysReg::TPIDR_EL1;
570
51
    else if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())
571
1
      SysReg = AArch64SysReg::TPIDR_EL3;
572
50
    else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())
573
1
      SysReg = AArch64SysReg::TPIDR_EL2;
574
49
    else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())
575
1
      SysReg = AArch64SysReg::TPIDR_EL1;
576
53
    BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::MRS), DstReg)
577
53
        .addImm(SysReg);
578
53
    MI.eraseFromParent();
579
53
    return true;
580
447k
  }
581
447k
582
746k
  case AArch64::MOVi32imm:
583
746k
    return expandMOVImm(MBB, MBBI, 32);
584
558k
  case AArch64::MOVi64imm:
585
558k
    return expandMOVImm(MBB, MBBI, 64);
586
447k
  case AArch64::RET_ReallyLR: {
587
266k
    // Hiding the LR use with RET_ReallyLR may lead to extra kills in the
588
266k
    // function and missing live-ins. We are fine in practice because callee
589
266k
    // saved register handling ensures the register value is restored before
590
266k
    // RET, but we need the undef flag here to appease the MachineVerifier
591
266k
    // liveness checks.
592
266k
    MachineInstrBuilder MIB =
593
266k
        BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET))
594
266k
          .addReg(AArch64::LR, RegState::Undef);
595
266k
    transferImpOps(MI, MIB, MIB);
596
266k
    MI.eraseFromParent();
597
266k
    return true;
598
447k
  }
599
447k
  case AArch64::CMP_SWAP_8:
600
1
    return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRB, AArch64::STLXRB,
601
1
                          AArch64::SUBSWrx,
602
1
                          AArch64_AM::getArithExtendImm(AArch64_AM::UXTB, 0),
603
1
                          AArch64::WZR, NextMBBI);
604
447k
  case AArch64::CMP_SWAP_16:
605
1
    return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRH, AArch64::STLXRH,
606
1
                          AArch64::SUBSWrx,
607
1
                          AArch64_AM::getArithExtendImm(AArch64_AM::UXTH, 0),
608
1
                          AArch64::WZR, NextMBBI);
609
447k
  case AArch64::CMP_SWAP_32:
610
3
    return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW,
611
3
                          AArch64::SUBSWrs,
612
3
                          AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
613
3
                          AArch64::WZR, NextMBBI);
614
447k
  case AArch64::CMP_SWAP_64:
615
2
    return expandCMP_SWAP(MBB, MBBI,
616
2
                          AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,
617
2
                          AArch64_AM::getShifterImm(AArch64_AM::LSL, 0),
618
2
                          AArch64::XZR, NextMBBI);
619
447k
  case AArch64::CMP_SWAP_128:
620
2
    return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
621
447k
622
447k
  case AArch64::AESMCrrTied:
623
198
  case AArch64::AESIMCrrTied: {
624
198
    MachineInstrBuilder MIB =
625
198
    BuildMI(MBB, MBBI, MI.getDebugLoc(),
626
198
            TII->get(Opcode == AArch64::AESMCrrTied ? 
AArch64::AESMCrr110
:
627
198
                                                      
AArch64::AESIMCrr88
))
628
198
      .add(MI.getOperand(0))
629
198
      .add(MI.getOperand(1));
630
198
    transferImpOps(MI, MIB, MIB);
631
198
    MI.eraseFromParent();
632
198
    return true;
633
198
   }
634
198
   case AArch64::IRGstack: {
635
6
     MachineFunction &MF = *MBB.getParent();
636
6
     const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
637
6
     const AArch64FrameLowering *TFI =
638
6
         MF.getSubtarget<AArch64Subtarget>().getFrameLowering();
639
6
640
6
     // IRG does not allow immediate offset. getTaggedBasePointerOffset should
641
6
     // almost always point to SP-after-prologue; if not, emit a longer
642
6
     // instruction sequence.
643
6
     int BaseOffset = -AFI->getTaggedBasePointerOffset();
644
6
     unsigned FrameReg;
645
6
     int FrameRegOffset = TFI->resolveFrameOffsetReference(
646
6
         MF, BaseOffset, false /*isFixed*/, FrameReg, /*PreferFP=*/false,
647
6
         /*ForSimm=*/true);
648
6
     Register SrcReg = FrameReg;
649
6
     if (FrameRegOffset != 0) {
650
1
       // Use output register as temporary.
651
1
       SrcReg = MI.getOperand(0).getReg();
652
1
       emitFrameOffset(MBB, &MI, MI.getDebugLoc(), SrcReg, FrameReg,
653
1
                       FrameRegOffset, TII);
654
1
     }
655
6
     BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::IRG))
656
6
         .add(MI.getOperand(0))
657
6
         .addUse(SrcReg)
658
6
         .add(MI.getOperand(2));
659
6
     MI.eraseFromParent();
660
6
     return true;
661
198
   }
662
198
   case AArch64::TAGPstack: {
663
6
     BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ADDG))
664
6
         .add(MI.getOperand(0))
665
6
         .add(MI.getOperand(1))
666
6
         .add(MI.getOperand(2))
667
6
         .add(MI.getOperand(4));
668
6
     MI.eraseFromParent();
669
6
     return true;
670
198
   }
671
198
   case AArch64::STGloop:
672
4
   case AArch64::STZGloop:
673
4
     return expandSetTagLoop(MBB, MBBI, NextMBBI);
674
9.22M
  }
675
9.22M
  return false;
676
9.22M
}
677
678
/// Iterate over the instructions in basic block MBB and expand any
679
/// pseudo instructions.  Return true if anything was modified.
680
2.09M
bool AArch64ExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
681
2.09M
  bool Modified = false;
682
2.09M
683
2.09M
  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
684
16.2M
  while (MBBI != E) {
685
14.1M
    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
686
14.1M
    Modified |= expandMI(MBB, MBBI, NMBBI);
687
14.1M
    MBBI = NMBBI;
688
14.1M
  }
689
2.09M
690
2.09M
  return Modified;
691
2.09M
}
692
693
257k
bool AArch64ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
694
257k
  TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
695
257k
696
257k
  bool Modified = false;
697
257k
  for (auto &MBB : MF)
698
2.09M
    Modified |= expandMBB(MBB);
699
257k
  return Modified;
700
257k
}
701
702
/// Returns an instance of the pseudo instruction expansion pass.
703
8.78k
FunctionPass *llvm::createAArch64ExpandPseudoPass() {
704
8.78k
  return new AArch64ExpandPseudo();
705
8.78k
}