Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- RISCVExpandPseudoInsts.cpp - Expand pseudo instructions -----------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains a pass that expands pseudo instructions into target
10
// instructions. This pass should be run after register allocation but before
11
// the post-regalloc scheduling pass.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#include "RISCV.h"
16
#include "RISCVInstrInfo.h"
17
#include "RISCVTargetMachine.h"
18
19
#include "llvm/CodeGen/LivePhysRegs.h"
20
#include "llvm/CodeGen/MachineFunctionPass.h"
21
#include "llvm/CodeGen/MachineInstrBuilder.h"
22
23
using namespace llvm;
24
25
3.68k
#define RISCV_EXPAND_PSEUDO_NAME "RISCV pseudo instruction expansion pass"
26
27
namespace {
28
29
class RISCVExpandPseudo : public MachineFunctionPass {
30
public:
31
  const RISCVInstrInfo *TII;
32
  static char ID;
33
34
329
  RISCVExpandPseudo() : MachineFunctionPass(ID) {
35
329
    initializeRISCVExpandPseudoPass(*PassRegistry::getPassRegistry());
36
329
  }
37
38
  bool runOnMachineFunction(MachineFunction &MF) override;
39
40
3.68k
  StringRef getPassName() const override { return RISCV_EXPAND_PSEUDO_NAME; }
41
42
private:
43
  bool expandMBB(MachineBasicBlock &MBB);
44
  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
45
                MachineBasicBlock::iterator &NextMBBI);
46
  bool expandAtomicBinOp(MachineBasicBlock &MBB,
47
                         MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
48
                         bool IsMasked, int Width,
49
                         MachineBasicBlock::iterator &NextMBBI);
50
  bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
51
                            MachineBasicBlock::iterator MBBI,
52
                            AtomicRMWInst::BinOp, bool IsMasked, int Width,
53
                            MachineBasicBlock::iterator &NextMBBI);
54
  bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
55
                           MachineBasicBlock::iterator MBBI, bool IsMasked,
56
                           int Width, MachineBasicBlock::iterator &NextMBBI);
57
  bool expandAuipcInstPair(MachineBasicBlock &MBB,
58
                           MachineBasicBlock::iterator MBBI,
59
                           MachineBasicBlock::iterator &NextMBBI,
60
                           unsigned FlagsHi, unsigned SecondOpcode);
61
  bool expandLoadLocalAddress(MachineBasicBlock &MBB,
62
                              MachineBasicBlock::iterator MBBI,
63
                              MachineBasicBlock::iterator &NextMBBI);
64
  bool expandLoadAddress(MachineBasicBlock &MBB,
65
                         MachineBasicBlock::iterator MBBI,
66
                         MachineBasicBlock::iterator &NextMBBI);
67
  bool expandLoadTLSIEAddress(MachineBasicBlock &MBB,
68
                              MachineBasicBlock::iterator MBBI,
69
                              MachineBasicBlock::iterator &NextMBBI);
70
  bool expandLoadTLSGDAddress(MachineBasicBlock &MBB,
71
                              MachineBasicBlock::iterator MBBI,
72
                              MachineBasicBlock::iterator &NextMBBI);
73
};
74
75
char RISCVExpandPseudo::ID = 0;
76
77
3.35k
bool RISCVExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
78
3.35k
  TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());
79
3.35k
  bool Modified = false;
80
3.35k
  for (auto &MBB : MF)
81
5.45k
    Modified |= expandMBB(MBB);
82
3.35k
  return Modified;
83
3.35k
}
84
85
5.45k
bool RISCVExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
86
5.45k
  bool Modified = false;
87
5.45k
88
5.45k
  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
89
55.3k
  while (MBBI != E) {
90
49.8k
    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
91
49.8k
    Modified |= expandMI(MBB, MBBI, NMBBI);
92
49.8k
    MBBI = NMBBI;
93
49.8k
  }
94
5.45k
95
5.45k
  return Modified;
96
5.45k
}
97
98
bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
99
                                 MachineBasicBlock::iterator MBBI,
100
49.8k
                                 MachineBasicBlock::iterator &NextMBBI) {
101
49.8k
  switch (MBBI->getOpcode()) {
102
49.8k
  case RISCV::PseudoAtomicLoadNand32:
103
10
    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
104
10
                             NextMBBI);
105
49.8k
  case RISCV::PseudoAtomicLoadNand64:
106
5
    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
107
5
                             NextMBBI);
108
49.8k
  case RISCV::PseudoMaskedAtomicSwap32:
109
20
    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
110
20
                             NextMBBI);
111
49.8k
  case RISCV::PseudoMaskedAtomicLoadAdd32:
112
20
    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
113
49.8k
  case RISCV::PseudoMaskedAtomicLoadSub32:
114
20
    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
115
49.8k
  case RISCV::PseudoMaskedAtomicLoadNand32:
116
20
    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
117
20
                             NextMBBI);
118
49.8k
  case RISCV::PseudoMaskedAtomicLoadMax32:
119
20
    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
120
20
                                NextMBBI);
121
49.8k
  case RISCV::PseudoMaskedAtomicLoadMin32:
122
20
    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
123
20
                                NextMBBI);
124
49.8k
  case RISCV::PseudoMaskedAtomicLoadUMax32:
125
20
    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
126
20
                                NextMBBI);
127
49.8k
  case RISCV::PseudoMaskedAtomicLoadUMin32:
128
20
    return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
129
20
                                NextMBBI);
130
49.8k
  case RISCV::PseudoCmpXchg32:
131
21
    return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
132
49.8k
  case RISCV::PseudoCmpXchg64:
133
10
    return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
134
49.8k
  case RISCV::PseudoMaskedCmpXchg32:
135
40
    return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
136
49.8k
  case RISCV::PseudoLLA:
137
6
    return expandLoadLocalAddress(MBB, MBBI, NextMBBI);
138
49.8k
  case RISCV::PseudoLA:
139
2
    return expandLoadAddress(MBB, MBBI, NextMBBI);
140
49.8k
  case RISCV::PseudoLA_TLS_IE:
141
2
    return expandLoadTLSIEAddress(MBB, MBBI, NextMBBI);
142
49.8k
  case RISCV::PseudoLA_TLS_GD:
143
4
    return expandLoadTLSGDAddress(MBB, MBBI, NextMBBI);
144
49.6k
  }
145
49.6k
146
49.6k
  return false;
147
49.6k
}
148
149
231
static unsigned getLRForRMW32(AtomicOrdering Ordering) {
150
231
  switch (Ordering) {
151
231
  default:
152
0
    llvm_unreachable("Unexpected AtomicOrdering");
153
231
  case AtomicOrdering::Monotonic:
154
40
    return RISCV::LR_W;
155
231
  case AtomicOrdering::Acquire:
156
46
    return RISCV::LR_W_AQ;
157
231
  case AtomicOrdering::Release:
158
46
    return RISCV::LR_W;
159
231
  case AtomicOrdering::AcquireRelease:
160
46
    return RISCV::LR_W_AQ;
161
231
  case AtomicOrdering::SequentiallyConsistent:
162
53
    return RISCV::LR_W_AQ_RL;
163
231
  }
164
231
}
165
166
231
static unsigned getSCForRMW32(AtomicOrdering Ordering) {
167
231
  switch (Ordering) {
168
231
  default:
169
0
    llvm_unreachable("Unexpected AtomicOrdering");
170
231
  case AtomicOrdering::Monotonic:
171
40
    return RISCV::SC_W;
172
231
  case AtomicOrdering::Acquire:
173
46
    return RISCV::SC_W;
174
231
  case AtomicOrdering::Release:
175
46
    return RISCV::SC_W_RL;
176
231
  case AtomicOrdering::AcquireRelease:
177
46
    return RISCV::SC_W_RL;
178
231
  case AtomicOrdering::SequentiallyConsistent:
179
53
    return RISCV::SC_W_AQ_RL;
180
231
  }
181
231
}
182
183
15
static unsigned getLRForRMW64(AtomicOrdering Ordering) {
184
15
  switch (Ordering) {
185
15
  default:
186
0
    llvm_unreachable("Unexpected AtomicOrdering");
187
15
  case AtomicOrdering::Monotonic:
188
2
    return RISCV::LR_D;
189
15
  case AtomicOrdering::Acquire:
190
3
    return RISCV::LR_D_AQ;
191
15
  case AtomicOrdering::Release:
192
3
    return RISCV::LR_D;
193
15
  case AtomicOrdering::AcquireRelease:
194
3
    return RISCV::LR_D_AQ;
195
15
  case AtomicOrdering::SequentiallyConsistent:
196
4
    return RISCV::LR_D_AQ_RL;
197
15
  }
198
15
}
199
200
15
static unsigned getSCForRMW64(AtomicOrdering Ordering) {
201
15
  switch (Ordering) {
202
15
  default:
203
0
    llvm_unreachable("Unexpected AtomicOrdering");
204
15
  case AtomicOrdering::Monotonic:
205
2
    return RISCV::SC_D;
206
15
  case AtomicOrdering::Acquire:
207
3
    return RISCV::SC_D;
208
15
  case AtomicOrdering::Release:
209
3
    return RISCV::SC_D_RL;
210
15
  case AtomicOrdering::AcquireRelease:
211
3
    return RISCV::SC_D_RL;
212
15
  case AtomicOrdering::SequentiallyConsistent:
213
4
    return RISCV::SC_D_AQ_RL;
214
15
  }
215
15
}
216
217
86
static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
218
86
  if (Width == 32)
219
71
    return getLRForRMW32(Ordering);
220
15
  if (Width == 64)
221
15
    return getLRForRMW64(Ordering);
222
0
  llvm_unreachable("Unexpected LR width\n");
223
0
}
224
225
86
static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
226
86
  if (Width == 32)
227
71
    return getSCForRMW32(Ordering);
228
15
  if (Width == 64)
229
15
    return getSCForRMW64(Ordering);
230
0
  llvm_unreachable("Unexpected SC width\n");
231
0
}
232
233
static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
234
                                   DebugLoc DL, MachineBasicBlock *ThisMBB,
235
                                   MachineBasicBlock *LoopMBB,
236
                                   MachineBasicBlock *DoneMBB,
237
15
                                   AtomicRMWInst::BinOp BinOp, int Width) {
238
15
  unsigned DestReg = MI.getOperand(0).getReg();
239
15
  unsigned ScratchReg = MI.getOperand(1).getReg();
240
15
  unsigned AddrReg = MI.getOperand(2).getReg();
241
15
  unsigned IncrReg = MI.getOperand(3).getReg();
242
15
  AtomicOrdering Ordering =
243
15
      static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
244
15
245
15
  // .loop:
246
15
  //   lr.[w|d] dest, (addr)
247
15
  //   binop scratch, dest, val
248
15
  //   sc.[w|d] scratch, scratch, (addr)
249
15
  //   bnez scratch, loop
250
15
  BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
251
15
      .addReg(AddrReg);
252
15
  switch (BinOp) {
253
15
  default:
254
0
    llvm_unreachable("Unexpected AtomicRMW BinOp");
255
15
  case AtomicRMWInst::Nand:
256
15
    BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
257
15
        .addReg(DestReg)
258
15
        .addReg(IncrReg);
259
15
    BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
260
15
        .addReg(ScratchReg)
261
15
        .addImm(-1);
262
15
    break;
263
15
  }
264
15
  BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
265
15
      .addReg(AddrReg)
266
15
      .addReg(ScratchReg);
267
15
  BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
268
15
      .addReg(ScratchReg)
269
15
      .addReg(RISCV::X0)
270
15
      .addMBB(LoopMBB);
271
15
}
272
273
static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
274
                              MachineBasicBlock *MBB, unsigned DestReg,
275
                              unsigned OldValReg, unsigned NewValReg,
276
200
                              unsigned MaskReg, unsigned ScratchReg) {
277
200
  assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
278
200
  assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
279
200
  assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
280
200
281
200
  // We select bits from newval and oldval using:
282
200
  // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
283
200
  // r = oldval ^ ((oldval ^ newval) & masktargetdata);
284
200
  BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
285
200
      .addReg(OldValReg)
286
200
      .addReg(NewValReg);
287
200
  BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
288
200
      .addReg(ScratchReg)
289
200
      .addReg(MaskReg);
290
200
  BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
291
200
      .addReg(OldValReg)
292
200
      .addReg(ScratchReg);
293
200
}
294
295
static void doMaskedAtomicBinOpExpansion(
296
    const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
297
    MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
298
80
    MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
299
80
  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
300
80
  unsigned DestReg = MI.getOperand(0).getReg();
301
80
  unsigned ScratchReg = MI.getOperand(1).getReg();
302
80
  unsigned AddrReg = MI.getOperand(2).getReg();
303
80
  unsigned IncrReg = MI.getOperand(3).getReg();
304
80
  unsigned MaskReg = MI.getOperand(4).getReg();
305
80
  AtomicOrdering Ordering =
306
80
      static_cast<AtomicOrdering>(MI.getOperand(5).getImm());
307
80
308
80
  // .loop:
309
80
  //   lr.w destreg, (alignedaddr)
310
80
  //   binop scratch, destreg, incr
311
80
  //   xor scratch, destreg, scratch
312
80
  //   and scratch, scratch, masktargetdata
313
80
  //   xor scratch, destreg, scratch
314
80
  //   sc.w scratch, scratch, (alignedaddr)
315
80
  //   bnez scratch, loop
316
80
  BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
317
80
      .addReg(AddrReg);
318
80
  switch (BinOp) {
319
80
  default:
320
0
    llvm_unreachable("Unexpected AtomicRMW BinOp");
321
80
  case AtomicRMWInst::Xchg:
322
20
    BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
323
20
        .addReg(RISCV::X0)
324
20
        .addReg(IncrReg);
325
20
    break;
326
80
  case AtomicRMWInst::Add:
327
20
    BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
328
20
        .addReg(DestReg)
329
20
        .addReg(IncrReg);
330
20
    break;
331
80
  case AtomicRMWInst::Sub:
332
20
    BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
333
20
        .addReg(DestReg)
334
20
        .addReg(IncrReg);
335
20
    break;
336
80
  case AtomicRMWInst::Nand:
337
20
    BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
338
20
        .addReg(DestReg)
339
20
        .addReg(IncrReg);
340
20
    BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
341
20
        .addReg(ScratchReg)
342
20
        .addImm(-1);
343
20
    break;
344
80
  }
345
80
346
80
  insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
347
80
                    ScratchReg);
348
80
349
80
  BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
350
80
      .addReg(AddrReg)
351
80
      .addReg(ScratchReg);
352
80
  BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
353
80
      .addReg(ScratchReg)
354
80
      .addReg(RISCV::X0)
355
80
      .addMBB(LoopMBB);
356
80
}
357
358
bool RISCVExpandPseudo::expandAtomicBinOp(
359
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
360
    AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
361
95
    MachineBasicBlock::iterator &NextMBBI) {
362
95
  MachineInstr &MI = *MBBI;
363
95
  DebugLoc DL = MI.getDebugLoc();
364
95
365
95
  MachineFunction *MF = MBB.getParent();
366
95
  auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
367
95
  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
368
95
369
95
  // Insert new MBBs.
370
95
  MF->insert(++MBB.getIterator(), LoopMBB);
371
95
  MF->insert(++LoopMBB->getIterator(), DoneMBB);
372
95
373
95
  // Set up successors and transfer remaining instructions to DoneMBB.
374
95
  LoopMBB->addSuccessor(LoopMBB);
375
95
  LoopMBB->addSuccessor(DoneMBB);
376
95
  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
377
95
  DoneMBB->transferSuccessors(&MBB);
378
95
  MBB.addSuccessor(LoopMBB);
379
95
380
95
  if (!IsMasked)
381
15
    doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
382
80
  else
383
80
    doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
384
80
                                 Width);
385
95
386
95
  NextMBBI = MBB.end();
387
95
  MI.eraseFromParent();
388
95
389
95
  LivePhysRegs LiveRegs;
390
95
  computeAndAddLiveIns(LiveRegs, *LoopMBB);
391
95
  computeAndAddLiveIns(LiveRegs, *DoneMBB);
392
95
393
95
  return true;
394
95
}
395
396
static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
397
                       MachineBasicBlock *MBB, unsigned ValReg,
398
40
                       unsigned ShamtReg) {
399
40
  BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
400
40
      .addReg(ValReg)
401
40
      .addReg(ShamtReg);
402
40
  BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
403
40
      .addReg(ValReg)
404
40
      .addReg(ShamtReg);
405
40
}
406
407
bool RISCVExpandPseudo::expandAtomicMinMaxOp(
408
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
409
    AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
410
80
    MachineBasicBlock::iterator &NextMBBI) {
411
80
  assert(IsMasked == true &&
412
80
         "Should only need to expand masked atomic max/min");
413
80
  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
414
80
415
80
  MachineInstr &MI = *MBBI;
416
80
  DebugLoc DL = MI.getDebugLoc();
417
80
  MachineFunction *MF = MBB.getParent();
418
80
  auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
419
80
  auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
420
80
  auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
421
80
  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
422
80
423
80
  // Insert new MBBs.
424
80
  MF->insert(++MBB.getIterator(), LoopHeadMBB);
425
80
  MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
426
80
  MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
427
80
  MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
428
80
429
80
  // Set up successors and transfer remaining instructions to DoneMBB.
430
80
  LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
431
80
  LoopHeadMBB->addSuccessor(LoopTailMBB);
432
80
  LoopIfBodyMBB->addSuccessor(LoopTailMBB);
433
80
  LoopTailMBB->addSuccessor(LoopHeadMBB);
434
80
  LoopTailMBB->addSuccessor(DoneMBB);
435
80
  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
436
80
  DoneMBB->transferSuccessors(&MBB);
437
80
  MBB.addSuccessor(LoopHeadMBB);
438
80
439
80
  unsigned DestReg = MI.getOperand(0).getReg();
440
80
  unsigned Scratch1Reg = MI.getOperand(1).getReg();
441
80
  unsigned Scratch2Reg = MI.getOperand(2).getReg();
442
80
  unsigned AddrReg = MI.getOperand(3).getReg();
443
80
  unsigned IncrReg = MI.getOperand(4).getReg();
444
80
  unsigned MaskReg = MI.getOperand(5).getReg();
445
80
  bool IsSigned = BinOp == AtomicRMWInst::Min || 
BinOp == AtomicRMWInst::Max60
;
446
80
  AtomicOrdering Ordering =
447
80
      static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 
740
:
640
).getImm());
448
80
449
80
  //
450
80
  // .loophead:
451
80
  //   lr.w destreg, (alignedaddr)
452
80
  //   and scratch2, destreg, mask
453
80
  //   mv scratch1, destreg
454
80
  //   [sext scratch2 if signed min/max]
455
80
  //   ifnochangeneeded scratch2, incr, .looptail
456
80
  BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
457
80
      .addReg(AddrReg);
458
80
  BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
459
80
      .addReg(DestReg)
460
80
      .addReg(MaskReg);
461
80
  BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
462
80
      .addReg(DestReg)
463
80
      .addImm(0);
464
80
465
80
  switch (BinOp) {
466
80
  default:
467
0
    llvm_unreachable("Unexpected AtomicRMW BinOp");
468
80
  case AtomicRMWInst::Max: {
469
20
    insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
470
20
    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
471
20
        .addReg(Scratch2Reg)
472
20
        .addReg(IncrReg)
473
20
        .addMBB(LoopTailMBB);
474
20
    break;
475
80
  }
476
80
  case AtomicRMWInst::Min: {
477
20
    insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
478
20
    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
479
20
        .addReg(IncrReg)
480
20
        .addReg(Scratch2Reg)
481
20
        .addMBB(LoopTailMBB);
482
20
    break;
483
80
  }
484
80
  case AtomicRMWInst::UMax:
485
20
    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
486
20
        .addReg(Scratch2Reg)
487
20
        .addReg(IncrReg)
488
20
        .addMBB(LoopTailMBB);
489
20
    break;
490
80
  case AtomicRMWInst::UMin:
491
20
    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
492
20
        .addReg(IncrReg)
493
20
        .addReg(Scratch2Reg)
494
20
        .addMBB(LoopTailMBB);
495
20
    break;
496
80
  }
497
80
498
80
  // .loopifbody:
499
80
  //   xor scratch1, destreg, incr
500
80
  //   and scratch1, scratch1, mask
501
80
  //   xor scratch1, destreg, scratch1
502
80
  insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
503
80
                    MaskReg, Scratch1Reg);
504
80
505
80
  // .looptail:
506
80
  //   sc.w scratch1, scratch1, (addr)
507
80
  //   bnez scratch1, loop
508
80
  BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
509
80
      .addReg(AddrReg)
510
80
      .addReg(Scratch1Reg);
511
80
  BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
512
80
      .addReg(Scratch1Reg)
513
80
      .addReg(RISCV::X0)
514
80
      .addMBB(LoopHeadMBB);
515
80
516
80
  NextMBBI = MBB.end();
517
80
  MI.eraseFromParent();
518
80
519
80
  LivePhysRegs LiveRegs;
520
80
  computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
521
80
  computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
522
80
  computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
523
80
  computeAndAddLiveIns(LiveRegs, *DoneMBB);
524
80
525
80
  return true;
526
80
}
527
528
bool RISCVExpandPseudo::expandAtomicCmpXchg(
529
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
530
71
    int Width, MachineBasicBlock::iterator &NextMBBI) {
531
71
  MachineInstr &MI = *MBBI;
532
71
  DebugLoc DL = MI.getDebugLoc();
533
71
  MachineFunction *MF = MBB.getParent();
534
71
  auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
535
71
  auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
536
71
  auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
537
71
538
71
  // Insert new MBBs.
539
71
  MF->insert(++MBB.getIterator(), LoopHeadMBB);
540
71
  MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
541
71
  MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
542
71
543
71
  // Set up successors and transfer remaining instructions to DoneMBB.
544
71
  LoopHeadMBB->addSuccessor(LoopTailMBB);
545
71
  LoopHeadMBB->addSuccessor(DoneMBB);
546
71
  LoopTailMBB->addSuccessor(DoneMBB);
547
71
  LoopTailMBB->addSuccessor(LoopHeadMBB);
548
71
  DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
549
71
  DoneMBB->transferSuccessors(&MBB);
550
71
  MBB.addSuccessor(LoopHeadMBB);
551
71
552
71
  unsigned DestReg = MI.getOperand(0).getReg();
553
71
  unsigned ScratchReg = MI.getOperand(1).getReg();
554
71
  unsigned AddrReg = MI.getOperand(2).getReg();
555
71
  unsigned CmpValReg = MI.getOperand(3).getReg();
556
71
  unsigned NewValReg = MI.getOperand(4).getReg();
557
71
  AtomicOrdering Ordering =
558
71
      static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 
640
:
531
).getImm());
559
71
560
71
  if (!IsMasked) {
561
31
    // .loophead:
562
31
    //   lr.[w|d] dest, (addr)
563
31
    //   bne dest, cmpval, done
564
31
    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
565
31
        .addReg(AddrReg);
566
31
    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
567
31
        .addReg(DestReg)
568
31
        .addReg(CmpValReg)
569
31
        .addMBB(DoneMBB);
570
31
    // .looptail:
571
31
    //   sc.[w|d] scratch, newval, (addr)
572
31
    //   bnez scratch, loophead
573
31
    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
574
31
        .addReg(AddrReg)
575
31
        .addReg(NewValReg);
576
31
    BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
577
31
        .addReg(ScratchReg)
578
31
        .addReg(RISCV::X0)
579
31
        .addMBB(LoopHeadMBB);
580
40
  } else {
581
40
    // .loophead:
582
40
    //   lr.w dest, (addr)
583
40
    //   and scratch, dest, mask
584
40
    //   bne scratch, cmpval, done
585
40
    unsigned MaskReg = MI.getOperand(5).getReg();
586
40
    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
587
40
        .addReg(AddrReg);
588
40
    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
589
40
        .addReg(DestReg)
590
40
        .addReg(MaskReg);
591
40
    BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
592
40
        .addReg(ScratchReg)
593
40
        .addReg(CmpValReg)
594
40
        .addMBB(DoneMBB);
595
40
596
40
    // .looptail:
597
40
    //   xor scratch, dest, newval
598
40
    //   and scratch, scratch, mask
599
40
    //   xor scratch, dest, scratch
600
40
    //   sc.w scratch, scratch, (adrr)
601
40
    //   bnez scratch, loophead
602
40
    insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
603
40
                      MaskReg, ScratchReg);
604
40
    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
605
40
        .addReg(AddrReg)
606
40
        .addReg(ScratchReg);
607
40
    BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
608
40
        .addReg(ScratchReg)
609
40
        .addReg(RISCV::X0)
610
40
        .addMBB(LoopHeadMBB);
611
40
  }
612
71
613
71
  NextMBBI = MBB.end();
614
71
  MI.eraseFromParent();
615
71
616
71
  LivePhysRegs LiveRegs;
617
71
  computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
618
71
  computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
619
71
  computeAndAddLiveIns(LiveRegs, *DoneMBB);
620
71
621
71
  return true;
622
71
}
623
624
bool RISCVExpandPseudo::expandAuipcInstPair(
625
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
626
    MachineBasicBlock::iterator &NextMBBI, unsigned FlagsHi,
627
14
    unsigned SecondOpcode) {
628
14
  MachineFunction *MF = MBB.getParent();
629
14
  MachineInstr &MI = *MBBI;
630
14
  DebugLoc DL = MI.getDebugLoc();
631
14
632
14
  unsigned DestReg = MI.getOperand(0).getReg();
633
14
  const MachineOperand &Symbol = MI.getOperand(1);
634
14
635
14
  MachineBasicBlock *NewMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
636
14
637
14
  // Tell AsmPrinter that we unconditionally want the symbol of this label to be
638
14
  // emitted.
639
14
  NewMBB->setLabelMustBeEmitted();
640
14
641
14
  MF->insert(++MBB.getIterator(), NewMBB);
642
14
643
14
  BuildMI(NewMBB, DL, TII->get(RISCV::AUIPC), DestReg)
644
14
      .addDisp(Symbol, 0, FlagsHi);
645
14
  BuildMI(NewMBB, DL, TII->get(SecondOpcode), DestReg)
646
14
      .addReg(DestReg)
647
14
      .addMBB(NewMBB, RISCVII::MO_PCREL_LO);
648
14
649
14
  // Move all the rest of the instructions to NewMBB.
650
14
  NewMBB->splice(NewMBB->end(), &MBB, std::next(MBBI), MBB.end());
651
14
  // Update machine-CFG edges.
652
14
  NewMBB->transferSuccessorsAndUpdatePHIs(&MBB);
653
14
  // Make the original basic block fall-through to the new.
654
14
  MBB.addSuccessor(NewMBB);
655
14
656
14
  // Make sure live-ins are correctly attached to this new basic block.
657
14
  LivePhysRegs LiveRegs;
658
14
  computeAndAddLiveIns(LiveRegs, *NewMBB);
659
14
660
14
  NextMBBI = MBB.end();
661
14
  MI.eraseFromParent();
662
14
  return true;
663
14
}
664
665
bool RISCVExpandPseudo::expandLoadLocalAddress(
666
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
667
6
    MachineBasicBlock::iterator &NextMBBI) {
668
6
  return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_PCREL_HI,
669
6
                             RISCV::ADDI);
670
6
}
671
672
bool RISCVExpandPseudo::expandLoadAddress(
673
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
674
2
    MachineBasicBlock::iterator &NextMBBI) {
675
2
  MachineFunction *MF = MBB.getParent();
676
2
677
2
  unsigned SecondOpcode;
678
2
  unsigned FlagsHi;
679
2
  if (MF->getTarget().isPositionIndependent()) {
680
2
    const auto &STI = MF->getSubtarget<RISCVSubtarget>();
681
2
    SecondOpcode = STI.is64Bit() ? 
RISCV::LD1
:
RISCV::LW1
;
682
2
    FlagsHi = RISCVII::MO_GOT_HI;
683
2
  } else {
684
0
    SecondOpcode = RISCV::ADDI;
685
0
    FlagsHi = RISCVII::MO_PCREL_HI;
686
0
  }
687
2
  return expandAuipcInstPair(MBB, MBBI, NextMBBI, FlagsHi, SecondOpcode);
688
2
}
689
690
bool RISCVExpandPseudo::expandLoadTLSIEAddress(
691
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
692
2
    MachineBasicBlock::iterator &NextMBBI) {
693
2
  MachineFunction *MF = MBB.getParent();
694
2
695
2
  const auto &STI = MF->getSubtarget<RISCVSubtarget>();
696
2
  unsigned SecondOpcode = STI.is64Bit() ? 
RISCV::LD1
:
RISCV::LW1
;
697
2
  return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GOT_HI,
698
2
                             SecondOpcode);
699
2
}
700
701
bool RISCVExpandPseudo::expandLoadTLSGDAddress(
702
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
703
4
    MachineBasicBlock::iterator &NextMBBI) {
704
4
  return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GD_HI,
705
4
                             RISCV::ADDI);
706
4
}
707
708
} // end of anonymous namespace
709
710
INITIALIZE_PASS(RISCVExpandPseudo, "riscv-expand-pseudo",
711
                RISCV_EXPAND_PSEUDO_NAME, false, false)
712
namespace llvm {
713
714
329
FunctionPass *createRISCVExpandPseudoPass() { return new RISCVExpandPseudo(); }
715
716
} // end of namespace llvm