/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===-- RISCVExpandPseudoInsts.cpp - Expand pseudo instructions -----------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This file contains a pass that expands pseudo instructions into target |
10 | | // instructions. This pass should be run after register allocation but before |
11 | | // the post-regalloc scheduling pass. |
12 | | // |
13 | | //===----------------------------------------------------------------------===// |
14 | | |
15 | | #include "RISCV.h" |
16 | | #include "RISCVInstrInfo.h" |
17 | | #include "RISCVTargetMachine.h" |
18 | | |
19 | | #include "llvm/CodeGen/LivePhysRegs.h" |
20 | | #include "llvm/CodeGen/MachineFunctionPass.h" |
21 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
22 | | |
23 | | using namespace llvm; |
24 | | |
25 | 3.68k | #define RISCV_EXPAND_PSEUDO_NAME "RISCV pseudo instruction expansion pass" |
26 | | |
27 | | namespace { |
28 | | |
29 | | class RISCVExpandPseudo : public MachineFunctionPass { |
30 | | public: |
31 | | const RISCVInstrInfo *TII; |
32 | | static char ID; |
33 | | |
34 | 329 | RISCVExpandPseudo() : MachineFunctionPass(ID) { |
35 | 329 | initializeRISCVExpandPseudoPass(*PassRegistry::getPassRegistry()); |
36 | 329 | } |
37 | | |
38 | | bool runOnMachineFunction(MachineFunction &MF) override; |
39 | | |
40 | 3.68k | StringRef getPassName() const override { return RISCV_EXPAND_PSEUDO_NAME; } |
41 | | |
42 | | private: |
43 | | bool expandMBB(MachineBasicBlock &MBB); |
44 | | bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
45 | | MachineBasicBlock::iterator &NextMBBI); |
46 | | bool expandAtomicBinOp(MachineBasicBlock &MBB, |
47 | | MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp, |
48 | | bool IsMasked, int Width, |
49 | | MachineBasicBlock::iterator &NextMBBI); |
50 | | bool expandAtomicMinMaxOp(MachineBasicBlock &MBB, |
51 | | MachineBasicBlock::iterator MBBI, |
52 | | AtomicRMWInst::BinOp, bool IsMasked, int Width, |
53 | | MachineBasicBlock::iterator &NextMBBI); |
54 | | bool expandAtomicCmpXchg(MachineBasicBlock &MBB, |
55 | | MachineBasicBlock::iterator MBBI, bool IsMasked, |
56 | | int Width, MachineBasicBlock::iterator &NextMBBI); |
57 | | bool expandAuipcInstPair(MachineBasicBlock &MBB, |
58 | | MachineBasicBlock::iterator MBBI, |
59 | | MachineBasicBlock::iterator &NextMBBI, |
60 | | unsigned FlagsHi, unsigned SecondOpcode); |
61 | | bool expandLoadLocalAddress(MachineBasicBlock &MBB, |
62 | | MachineBasicBlock::iterator MBBI, |
63 | | MachineBasicBlock::iterator &NextMBBI); |
64 | | bool expandLoadAddress(MachineBasicBlock &MBB, |
65 | | MachineBasicBlock::iterator MBBI, |
66 | | MachineBasicBlock::iterator &NextMBBI); |
67 | | bool expandLoadTLSIEAddress(MachineBasicBlock &MBB, |
68 | | MachineBasicBlock::iterator MBBI, |
69 | | MachineBasicBlock::iterator &NextMBBI); |
70 | | bool expandLoadTLSGDAddress(MachineBasicBlock &MBB, |
71 | | MachineBasicBlock::iterator MBBI, |
72 | | MachineBasicBlock::iterator &NextMBBI); |
73 | | }; |
74 | | |
75 | | char RISCVExpandPseudo::ID = 0; |
76 | | |
77 | 3.35k | bool RISCVExpandPseudo::runOnMachineFunction(MachineFunction &MF) { |
78 | 3.35k | TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo()); |
79 | 3.35k | bool Modified = false; |
80 | 3.35k | for (auto &MBB : MF) |
81 | 5.45k | Modified |= expandMBB(MBB); |
82 | 3.35k | return Modified; |
83 | 3.35k | } |
84 | | |
85 | 5.45k | bool RISCVExpandPseudo::expandMBB(MachineBasicBlock &MBB) { |
86 | 5.45k | bool Modified = false; |
87 | 5.45k | |
88 | 5.45k | MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); |
89 | 55.3k | while (MBBI != E) { |
90 | 49.8k | MachineBasicBlock::iterator NMBBI = std::next(MBBI); |
91 | 49.8k | Modified |= expandMI(MBB, MBBI, NMBBI); |
92 | 49.8k | MBBI = NMBBI; |
93 | 49.8k | } |
94 | 5.45k | |
95 | 5.45k | return Modified; |
96 | 5.45k | } |
97 | | |
98 | | bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB, |
99 | | MachineBasicBlock::iterator MBBI, |
100 | 49.8k | MachineBasicBlock::iterator &NextMBBI) { |
101 | 49.8k | switch (MBBI->getOpcode()) { |
102 | 49.8k | case RISCV::PseudoAtomicLoadNand32: |
103 | 10 | return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32, |
104 | 10 | NextMBBI); |
105 | 49.8k | case RISCV::PseudoAtomicLoadNand64: |
106 | 5 | return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64, |
107 | 5 | NextMBBI); |
108 | 49.8k | case RISCV::PseudoMaskedAtomicSwap32: |
109 | 20 | return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32, |
110 | 20 | NextMBBI); |
111 | 49.8k | case RISCV::PseudoMaskedAtomicLoadAdd32: |
112 | 20 | return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI); |
113 | 49.8k | case RISCV::PseudoMaskedAtomicLoadSub32: |
114 | 20 | return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI); |
115 | 49.8k | case RISCV::PseudoMaskedAtomicLoadNand32: |
116 | 20 | return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32, |
117 | 20 | NextMBBI); |
118 | 49.8k | case RISCV::PseudoMaskedAtomicLoadMax32: |
119 | 20 | return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32, |
120 | 20 | NextMBBI); |
121 | 49.8k | case RISCV::PseudoMaskedAtomicLoadMin32: |
122 | 20 | return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32, |
123 | 20 | NextMBBI); |
124 | 49.8k | case RISCV::PseudoMaskedAtomicLoadUMax32: |
125 | 20 | return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32, |
126 | 20 | NextMBBI); |
127 | 49.8k | case RISCV::PseudoMaskedAtomicLoadUMin32: |
128 | 20 | return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32, |
129 | 20 | NextMBBI); |
130 | 49.8k | case RISCV::PseudoCmpXchg32: |
131 | 21 | return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI); |
132 | 49.8k | case RISCV::PseudoCmpXchg64: |
133 | 10 | return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI); |
134 | 49.8k | case RISCV::PseudoMaskedCmpXchg32: |
135 | 40 | return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI); |
136 | 49.8k | case RISCV::PseudoLLA: |
137 | 6 | return expandLoadLocalAddress(MBB, MBBI, NextMBBI); |
138 | 49.8k | case RISCV::PseudoLA: |
139 | 2 | return expandLoadAddress(MBB, MBBI, NextMBBI); |
140 | 49.8k | case RISCV::PseudoLA_TLS_IE: |
141 | 2 | return expandLoadTLSIEAddress(MBB, MBBI, NextMBBI); |
142 | 49.8k | case RISCV::PseudoLA_TLS_GD: |
143 | 4 | return expandLoadTLSGDAddress(MBB, MBBI, NextMBBI); |
144 | 49.6k | } |
145 | 49.6k | |
146 | 49.6k | return false; |
147 | 49.6k | } |
148 | | |
149 | 231 | static unsigned getLRForRMW32(AtomicOrdering Ordering) { |
150 | 231 | switch (Ordering) { |
151 | 231 | default: |
152 | 0 | llvm_unreachable("Unexpected AtomicOrdering"); |
153 | 231 | case AtomicOrdering::Monotonic: |
154 | 40 | return RISCV::LR_W; |
155 | 231 | case AtomicOrdering::Acquire: |
156 | 46 | return RISCV::LR_W_AQ; |
157 | 231 | case AtomicOrdering::Release: |
158 | 46 | return RISCV::LR_W; |
159 | 231 | case AtomicOrdering::AcquireRelease: |
160 | 46 | return RISCV::LR_W_AQ; |
161 | 231 | case AtomicOrdering::SequentiallyConsistent: |
162 | 53 | return RISCV::LR_W_AQ_RL; |
163 | 231 | } |
164 | 231 | } |
165 | | |
166 | 231 | static unsigned getSCForRMW32(AtomicOrdering Ordering) { |
167 | 231 | switch (Ordering) { |
168 | 231 | default: |
169 | 0 | llvm_unreachable("Unexpected AtomicOrdering"); |
170 | 231 | case AtomicOrdering::Monotonic: |
171 | 40 | return RISCV::SC_W; |
172 | 231 | case AtomicOrdering::Acquire: |
173 | 46 | return RISCV::SC_W; |
174 | 231 | case AtomicOrdering::Release: |
175 | 46 | return RISCV::SC_W_RL; |
176 | 231 | case AtomicOrdering::AcquireRelease: |
177 | 46 | return RISCV::SC_W_RL; |
178 | 231 | case AtomicOrdering::SequentiallyConsistent: |
179 | 53 | return RISCV::SC_W_AQ_RL; |
180 | 231 | } |
181 | 231 | } |
182 | | |
183 | 15 | static unsigned getLRForRMW64(AtomicOrdering Ordering) { |
184 | 15 | switch (Ordering) { |
185 | 15 | default: |
186 | 0 | llvm_unreachable("Unexpected AtomicOrdering"); |
187 | 15 | case AtomicOrdering::Monotonic: |
188 | 2 | return RISCV::LR_D; |
189 | 15 | case AtomicOrdering::Acquire: |
190 | 3 | return RISCV::LR_D_AQ; |
191 | 15 | case AtomicOrdering::Release: |
192 | 3 | return RISCV::LR_D; |
193 | 15 | case AtomicOrdering::AcquireRelease: |
194 | 3 | return RISCV::LR_D_AQ; |
195 | 15 | case AtomicOrdering::SequentiallyConsistent: |
196 | 4 | return RISCV::LR_D_AQ_RL; |
197 | 15 | } |
198 | 15 | } |
199 | | |
200 | 15 | static unsigned getSCForRMW64(AtomicOrdering Ordering) { |
201 | 15 | switch (Ordering) { |
202 | 15 | default: |
203 | 0 | llvm_unreachable("Unexpected AtomicOrdering"); |
204 | 15 | case AtomicOrdering::Monotonic: |
205 | 2 | return RISCV::SC_D; |
206 | 15 | case AtomicOrdering::Acquire: |
207 | 3 | return RISCV::SC_D; |
208 | 15 | case AtomicOrdering::Release: |
209 | 3 | return RISCV::SC_D_RL; |
210 | 15 | case AtomicOrdering::AcquireRelease: |
211 | 3 | return RISCV::SC_D_RL; |
212 | 15 | case AtomicOrdering::SequentiallyConsistent: |
213 | 4 | return RISCV::SC_D_AQ_RL; |
214 | 15 | } |
215 | 15 | } |
216 | | |
217 | 86 | static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) { |
218 | 86 | if (Width == 32) |
219 | 71 | return getLRForRMW32(Ordering); |
220 | 15 | if (Width == 64) |
221 | 15 | return getLRForRMW64(Ordering); |
222 | 0 | llvm_unreachable("Unexpected LR width\n"); |
223 | 0 | } |
224 | | |
225 | 86 | static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) { |
226 | 86 | if (Width == 32) |
227 | 71 | return getSCForRMW32(Ordering); |
228 | 15 | if (Width == 64) |
229 | 15 | return getSCForRMW64(Ordering); |
230 | 0 | llvm_unreachable("Unexpected SC width\n"); |
231 | 0 | } |
232 | | |
233 | | static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI, |
234 | | DebugLoc DL, MachineBasicBlock *ThisMBB, |
235 | | MachineBasicBlock *LoopMBB, |
236 | | MachineBasicBlock *DoneMBB, |
237 | 15 | AtomicRMWInst::BinOp BinOp, int Width) { |
238 | 15 | unsigned DestReg = MI.getOperand(0).getReg(); |
239 | 15 | unsigned ScratchReg = MI.getOperand(1).getReg(); |
240 | 15 | unsigned AddrReg = MI.getOperand(2).getReg(); |
241 | 15 | unsigned IncrReg = MI.getOperand(3).getReg(); |
242 | 15 | AtomicOrdering Ordering = |
243 | 15 | static_cast<AtomicOrdering>(MI.getOperand(4).getImm()); |
244 | 15 | |
245 | 15 | // .loop: |
246 | 15 | // lr.[w|d] dest, (addr) |
247 | 15 | // binop scratch, dest, val |
248 | 15 | // sc.[w|d] scratch, scratch, (addr) |
249 | 15 | // bnez scratch, loop |
250 | 15 | BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg) |
251 | 15 | .addReg(AddrReg); |
252 | 15 | switch (BinOp) { |
253 | 15 | default: |
254 | 0 | llvm_unreachable("Unexpected AtomicRMW BinOp"); |
255 | 15 | case AtomicRMWInst::Nand: |
256 | 15 | BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg) |
257 | 15 | .addReg(DestReg) |
258 | 15 | .addReg(IncrReg); |
259 | 15 | BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg) |
260 | 15 | .addReg(ScratchReg) |
261 | 15 | .addImm(-1); |
262 | 15 | break; |
263 | 15 | } |
264 | 15 | BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg) |
265 | 15 | .addReg(AddrReg) |
266 | 15 | .addReg(ScratchReg); |
267 | 15 | BuildMI(LoopMBB, DL, TII->get(RISCV::BNE)) |
268 | 15 | .addReg(ScratchReg) |
269 | 15 | .addReg(RISCV::X0) |
270 | 15 | .addMBB(LoopMBB); |
271 | 15 | } |
272 | | |
273 | | static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL, |
274 | | MachineBasicBlock *MBB, unsigned DestReg, |
275 | | unsigned OldValReg, unsigned NewValReg, |
276 | 200 | unsigned MaskReg, unsigned ScratchReg) { |
277 | 200 | assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique"); |
278 | 200 | assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique"); |
279 | 200 | assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique"); |
280 | 200 | |
281 | 200 | // We select bits from newval and oldval using: |
282 | 200 | // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge |
283 | 200 | // r = oldval ^ ((oldval ^ newval) & masktargetdata); |
284 | 200 | BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg) |
285 | 200 | .addReg(OldValReg) |
286 | 200 | .addReg(NewValReg); |
287 | 200 | BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg) |
288 | 200 | .addReg(ScratchReg) |
289 | 200 | .addReg(MaskReg); |
290 | 200 | BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg) |
291 | 200 | .addReg(OldValReg) |
292 | 200 | .addReg(ScratchReg); |
293 | 200 | } |
294 | | |
295 | | static void doMaskedAtomicBinOpExpansion( |
296 | | const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL, |
297 | | MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB, |
298 | 80 | MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) { |
299 | 80 | assert(Width == 32 && "Should never need to expand masked 64-bit operations"); |
300 | 80 | unsigned DestReg = MI.getOperand(0).getReg(); |
301 | 80 | unsigned ScratchReg = MI.getOperand(1).getReg(); |
302 | 80 | unsigned AddrReg = MI.getOperand(2).getReg(); |
303 | 80 | unsigned IncrReg = MI.getOperand(3).getReg(); |
304 | 80 | unsigned MaskReg = MI.getOperand(4).getReg(); |
305 | 80 | AtomicOrdering Ordering = |
306 | 80 | static_cast<AtomicOrdering>(MI.getOperand(5).getImm()); |
307 | 80 | |
308 | 80 | // .loop: |
309 | 80 | // lr.w destreg, (alignedaddr) |
310 | 80 | // binop scratch, destreg, incr |
311 | 80 | // xor scratch, destreg, scratch |
312 | 80 | // and scratch, scratch, masktargetdata |
313 | 80 | // xor scratch, destreg, scratch |
314 | 80 | // sc.w scratch, scratch, (alignedaddr) |
315 | 80 | // bnez scratch, loop |
316 | 80 | BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg) |
317 | 80 | .addReg(AddrReg); |
318 | 80 | switch (BinOp) { |
319 | 80 | default: |
320 | 0 | llvm_unreachable("Unexpected AtomicRMW BinOp"); |
321 | 80 | case AtomicRMWInst::Xchg: |
322 | 20 | BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg) |
323 | 20 | .addReg(RISCV::X0) |
324 | 20 | .addReg(IncrReg); |
325 | 20 | break; |
326 | 80 | case AtomicRMWInst::Add: |
327 | 20 | BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg) |
328 | 20 | .addReg(DestReg) |
329 | 20 | .addReg(IncrReg); |
330 | 20 | break; |
331 | 80 | case AtomicRMWInst::Sub: |
332 | 20 | BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg) |
333 | 20 | .addReg(DestReg) |
334 | 20 | .addReg(IncrReg); |
335 | 20 | break; |
336 | 80 | case AtomicRMWInst::Nand: |
337 | 20 | BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg) |
338 | 20 | .addReg(DestReg) |
339 | 20 | .addReg(IncrReg); |
340 | 20 | BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg) |
341 | 20 | .addReg(ScratchReg) |
342 | 20 | .addImm(-1); |
343 | 20 | break; |
344 | 80 | } |
345 | 80 | |
346 | 80 | insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg, |
347 | 80 | ScratchReg); |
348 | 80 | |
349 | 80 | BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg) |
350 | 80 | .addReg(AddrReg) |
351 | 80 | .addReg(ScratchReg); |
352 | 80 | BuildMI(LoopMBB, DL, TII->get(RISCV::BNE)) |
353 | 80 | .addReg(ScratchReg) |
354 | 80 | .addReg(RISCV::X0) |
355 | 80 | .addMBB(LoopMBB); |
356 | 80 | } |
357 | | |
358 | | bool RISCVExpandPseudo::expandAtomicBinOp( |
359 | | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
360 | | AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width, |
361 | 95 | MachineBasicBlock::iterator &NextMBBI) { |
362 | 95 | MachineInstr &MI = *MBBI; |
363 | 95 | DebugLoc DL = MI.getDebugLoc(); |
364 | 95 | |
365 | 95 | MachineFunction *MF = MBB.getParent(); |
366 | 95 | auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
367 | 95 | auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
368 | 95 | |
369 | 95 | // Insert new MBBs. |
370 | 95 | MF->insert(++MBB.getIterator(), LoopMBB); |
371 | 95 | MF->insert(++LoopMBB->getIterator(), DoneMBB); |
372 | 95 | |
373 | 95 | // Set up successors and transfer remaining instructions to DoneMBB. |
374 | 95 | LoopMBB->addSuccessor(LoopMBB); |
375 | 95 | LoopMBB->addSuccessor(DoneMBB); |
376 | 95 | DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end()); |
377 | 95 | DoneMBB->transferSuccessors(&MBB); |
378 | 95 | MBB.addSuccessor(LoopMBB); |
379 | 95 | |
380 | 95 | if (!IsMasked) |
381 | 15 | doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width); |
382 | 80 | else |
383 | 80 | doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, |
384 | 80 | Width); |
385 | 95 | |
386 | 95 | NextMBBI = MBB.end(); |
387 | 95 | MI.eraseFromParent(); |
388 | 95 | |
389 | 95 | LivePhysRegs LiveRegs; |
390 | 95 | computeAndAddLiveIns(LiveRegs, *LoopMBB); |
391 | 95 | computeAndAddLiveIns(LiveRegs, *DoneMBB); |
392 | 95 | |
393 | 95 | return true; |
394 | 95 | } |
395 | | |
396 | | static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL, |
397 | | MachineBasicBlock *MBB, unsigned ValReg, |
398 | 40 | unsigned ShamtReg) { |
399 | 40 | BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg) |
400 | 40 | .addReg(ValReg) |
401 | 40 | .addReg(ShamtReg); |
402 | 40 | BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg) |
403 | 40 | .addReg(ValReg) |
404 | 40 | .addReg(ShamtReg); |
405 | 40 | } |
406 | | |
407 | | bool RISCVExpandPseudo::expandAtomicMinMaxOp( |
408 | | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
409 | | AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width, |
410 | 80 | MachineBasicBlock::iterator &NextMBBI) { |
411 | 80 | assert(IsMasked == true && |
412 | 80 | "Should only need to expand masked atomic max/min"); |
413 | 80 | assert(Width == 32 && "Should never need to expand masked 64-bit operations"); |
414 | 80 | |
415 | 80 | MachineInstr &MI = *MBBI; |
416 | 80 | DebugLoc DL = MI.getDebugLoc(); |
417 | 80 | MachineFunction *MF = MBB.getParent(); |
418 | 80 | auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
419 | 80 | auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
420 | 80 | auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
421 | 80 | auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
422 | 80 | |
423 | 80 | // Insert new MBBs. |
424 | 80 | MF->insert(++MBB.getIterator(), LoopHeadMBB); |
425 | 80 | MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB); |
426 | 80 | MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB); |
427 | 80 | MF->insert(++LoopTailMBB->getIterator(), DoneMBB); |
428 | 80 | |
429 | 80 | // Set up successors and transfer remaining instructions to DoneMBB. |
430 | 80 | LoopHeadMBB->addSuccessor(LoopIfBodyMBB); |
431 | 80 | LoopHeadMBB->addSuccessor(LoopTailMBB); |
432 | 80 | LoopIfBodyMBB->addSuccessor(LoopTailMBB); |
433 | 80 | LoopTailMBB->addSuccessor(LoopHeadMBB); |
434 | 80 | LoopTailMBB->addSuccessor(DoneMBB); |
435 | 80 | DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end()); |
436 | 80 | DoneMBB->transferSuccessors(&MBB); |
437 | 80 | MBB.addSuccessor(LoopHeadMBB); |
438 | 80 | |
439 | 80 | unsigned DestReg = MI.getOperand(0).getReg(); |
440 | 80 | unsigned Scratch1Reg = MI.getOperand(1).getReg(); |
441 | 80 | unsigned Scratch2Reg = MI.getOperand(2).getReg(); |
442 | 80 | unsigned AddrReg = MI.getOperand(3).getReg(); |
443 | 80 | unsigned IncrReg = MI.getOperand(4).getReg(); |
444 | 80 | unsigned MaskReg = MI.getOperand(5).getReg(); |
445 | 80 | bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max60 ; |
446 | 80 | AtomicOrdering Ordering = |
447 | 80 | static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 740 : 640 ).getImm()); |
448 | 80 | |
449 | 80 | // |
450 | 80 | // .loophead: |
451 | 80 | // lr.w destreg, (alignedaddr) |
452 | 80 | // and scratch2, destreg, mask |
453 | 80 | // mv scratch1, destreg |
454 | 80 | // [sext scratch2 if signed min/max] |
455 | 80 | // ifnochangeneeded scratch2, incr, .looptail |
456 | 80 | BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg) |
457 | 80 | .addReg(AddrReg); |
458 | 80 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg) |
459 | 80 | .addReg(DestReg) |
460 | 80 | .addReg(MaskReg); |
461 | 80 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg) |
462 | 80 | .addReg(DestReg) |
463 | 80 | .addImm(0); |
464 | 80 | |
465 | 80 | switch (BinOp) { |
466 | 80 | default: |
467 | 0 | llvm_unreachable("Unexpected AtomicRMW BinOp"); |
468 | 80 | case AtomicRMWInst::Max: { |
469 | 20 | insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg()); |
470 | 20 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE)) |
471 | 20 | .addReg(Scratch2Reg) |
472 | 20 | .addReg(IncrReg) |
473 | 20 | .addMBB(LoopTailMBB); |
474 | 20 | break; |
475 | 80 | } |
476 | 80 | case AtomicRMWInst::Min: { |
477 | 20 | insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg()); |
478 | 20 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE)) |
479 | 20 | .addReg(IncrReg) |
480 | 20 | .addReg(Scratch2Reg) |
481 | 20 | .addMBB(LoopTailMBB); |
482 | 20 | break; |
483 | 80 | } |
484 | 80 | case AtomicRMWInst::UMax: |
485 | 20 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU)) |
486 | 20 | .addReg(Scratch2Reg) |
487 | 20 | .addReg(IncrReg) |
488 | 20 | .addMBB(LoopTailMBB); |
489 | 20 | break; |
490 | 80 | case AtomicRMWInst::UMin: |
491 | 20 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU)) |
492 | 20 | .addReg(IncrReg) |
493 | 20 | .addReg(Scratch2Reg) |
494 | 20 | .addMBB(LoopTailMBB); |
495 | 20 | break; |
496 | 80 | } |
497 | 80 | |
498 | 80 | // .loopifbody: |
499 | 80 | // xor scratch1, destreg, incr |
500 | 80 | // and scratch1, scratch1, mask |
501 | 80 | // xor scratch1, destreg, scratch1 |
502 | 80 | insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg, |
503 | 80 | MaskReg, Scratch1Reg); |
504 | 80 | |
505 | 80 | // .looptail: |
506 | 80 | // sc.w scratch1, scratch1, (addr) |
507 | 80 | // bnez scratch1, loop |
508 | 80 | BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg) |
509 | 80 | .addReg(AddrReg) |
510 | 80 | .addReg(Scratch1Reg); |
511 | 80 | BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE)) |
512 | 80 | .addReg(Scratch1Reg) |
513 | 80 | .addReg(RISCV::X0) |
514 | 80 | .addMBB(LoopHeadMBB); |
515 | 80 | |
516 | 80 | NextMBBI = MBB.end(); |
517 | 80 | MI.eraseFromParent(); |
518 | 80 | |
519 | 80 | LivePhysRegs LiveRegs; |
520 | 80 | computeAndAddLiveIns(LiveRegs, *LoopHeadMBB); |
521 | 80 | computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB); |
522 | 80 | computeAndAddLiveIns(LiveRegs, *LoopTailMBB); |
523 | 80 | computeAndAddLiveIns(LiveRegs, *DoneMBB); |
524 | 80 | |
525 | 80 | return true; |
526 | 80 | } |
527 | | |
528 | | bool RISCVExpandPseudo::expandAtomicCmpXchg( |
529 | | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked, |
530 | 71 | int Width, MachineBasicBlock::iterator &NextMBBI) { |
531 | 71 | MachineInstr &MI = *MBBI; |
532 | 71 | DebugLoc DL = MI.getDebugLoc(); |
533 | 71 | MachineFunction *MF = MBB.getParent(); |
534 | 71 | auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
535 | 71 | auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
536 | 71 | auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
537 | 71 | |
538 | 71 | // Insert new MBBs. |
539 | 71 | MF->insert(++MBB.getIterator(), LoopHeadMBB); |
540 | 71 | MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB); |
541 | 71 | MF->insert(++LoopTailMBB->getIterator(), DoneMBB); |
542 | 71 | |
543 | 71 | // Set up successors and transfer remaining instructions to DoneMBB. |
544 | 71 | LoopHeadMBB->addSuccessor(LoopTailMBB); |
545 | 71 | LoopHeadMBB->addSuccessor(DoneMBB); |
546 | 71 | LoopTailMBB->addSuccessor(DoneMBB); |
547 | 71 | LoopTailMBB->addSuccessor(LoopHeadMBB); |
548 | 71 | DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end()); |
549 | 71 | DoneMBB->transferSuccessors(&MBB); |
550 | 71 | MBB.addSuccessor(LoopHeadMBB); |
551 | 71 | |
552 | 71 | unsigned DestReg = MI.getOperand(0).getReg(); |
553 | 71 | unsigned ScratchReg = MI.getOperand(1).getReg(); |
554 | 71 | unsigned AddrReg = MI.getOperand(2).getReg(); |
555 | 71 | unsigned CmpValReg = MI.getOperand(3).getReg(); |
556 | 71 | unsigned NewValReg = MI.getOperand(4).getReg(); |
557 | 71 | AtomicOrdering Ordering = |
558 | 71 | static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 640 : 531 ).getImm()); |
559 | 71 | |
560 | 71 | if (!IsMasked) { |
561 | 31 | // .loophead: |
562 | 31 | // lr.[w|d] dest, (addr) |
563 | 31 | // bne dest, cmpval, done |
564 | 31 | BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg) |
565 | 31 | .addReg(AddrReg); |
566 | 31 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE)) |
567 | 31 | .addReg(DestReg) |
568 | 31 | .addReg(CmpValReg) |
569 | 31 | .addMBB(DoneMBB); |
570 | 31 | // .looptail: |
571 | 31 | // sc.[w|d] scratch, newval, (addr) |
572 | 31 | // bnez scratch, loophead |
573 | 31 | BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg) |
574 | 31 | .addReg(AddrReg) |
575 | 31 | .addReg(NewValReg); |
576 | 31 | BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE)) |
577 | 31 | .addReg(ScratchReg) |
578 | 31 | .addReg(RISCV::X0) |
579 | 31 | .addMBB(LoopHeadMBB); |
580 | 40 | } else { |
581 | 40 | // .loophead: |
582 | 40 | // lr.w dest, (addr) |
583 | 40 | // and scratch, dest, mask |
584 | 40 | // bne scratch, cmpval, done |
585 | 40 | unsigned MaskReg = MI.getOperand(5).getReg(); |
586 | 40 | BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg) |
587 | 40 | .addReg(AddrReg); |
588 | 40 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg) |
589 | 40 | .addReg(DestReg) |
590 | 40 | .addReg(MaskReg); |
591 | 40 | BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE)) |
592 | 40 | .addReg(ScratchReg) |
593 | 40 | .addReg(CmpValReg) |
594 | 40 | .addMBB(DoneMBB); |
595 | 40 | |
596 | 40 | // .looptail: |
597 | 40 | // xor scratch, dest, newval |
598 | 40 | // and scratch, scratch, mask |
599 | 40 | // xor scratch, dest, scratch |
600 | 40 | // sc.w scratch, scratch, (adrr) |
601 | 40 | // bnez scratch, loophead |
602 | 40 | insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg, |
603 | 40 | MaskReg, ScratchReg); |
604 | 40 | BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg) |
605 | 40 | .addReg(AddrReg) |
606 | 40 | .addReg(ScratchReg); |
607 | 40 | BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE)) |
608 | 40 | .addReg(ScratchReg) |
609 | 40 | .addReg(RISCV::X0) |
610 | 40 | .addMBB(LoopHeadMBB); |
611 | 40 | } |
612 | 71 | |
613 | 71 | NextMBBI = MBB.end(); |
614 | 71 | MI.eraseFromParent(); |
615 | 71 | |
616 | 71 | LivePhysRegs LiveRegs; |
617 | 71 | computeAndAddLiveIns(LiveRegs, *LoopHeadMBB); |
618 | 71 | computeAndAddLiveIns(LiveRegs, *LoopTailMBB); |
619 | 71 | computeAndAddLiveIns(LiveRegs, *DoneMBB); |
620 | 71 | |
621 | 71 | return true; |
622 | 71 | } |
623 | | |
624 | | bool RISCVExpandPseudo::expandAuipcInstPair( |
625 | | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
626 | | MachineBasicBlock::iterator &NextMBBI, unsigned FlagsHi, |
627 | 14 | unsigned SecondOpcode) { |
628 | 14 | MachineFunction *MF = MBB.getParent(); |
629 | 14 | MachineInstr &MI = *MBBI; |
630 | 14 | DebugLoc DL = MI.getDebugLoc(); |
631 | 14 | |
632 | 14 | unsigned DestReg = MI.getOperand(0).getReg(); |
633 | 14 | const MachineOperand &Symbol = MI.getOperand(1); |
634 | 14 | |
635 | 14 | MachineBasicBlock *NewMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock()); |
636 | 14 | |
637 | 14 | // Tell AsmPrinter that we unconditionally want the symbol of this label to be |
638 | 14 | // emitted. |
639 | 14 | NewMBB->setLabelMustBeEmitted(); |
640 | 14 | |
641 | 14 | MF->insert(++MBB.getIterator(), NewMBB); |
642 | 14 | |
643 | 14 | BuildMI(NewMBB, DL, TII->get(RISCV::AUIPC), DestReg) |
644 | 14 | .addDisp(Symbol, 0, FlagsHi); |
645 | 14 | BuildMI(NewMBB, DL, TII->get(SecondOpcode), DestReg) |
646 | 14 | .addReg(DestReg) |
647 | 14 | .addMBB(NewMBB, RISCVII::MO_PCREL_LO); |
648 | 14 | |
649 | 14 | // Move all the rest of the instructions to NewMBB. |
650 | 14 | NewMBB->splice(NewMBB->end(), &MBB, std::next(MBBI), MBB.end()); |
651 | 14 | // Update machine-CFG edges. |
652 | 14 | NewMBB->transferSuccessorsAndUpdatePHIs(&MBB); |
653 | 14 | // Make the original basic block fall-through to the new. |
654 | 14 | MBB.addSuccessor(NewMBB); |
655 | 14 | |
656 | 14 | // Make sure live-ins are correctly attached to this new basic block. |
657 | 14 | LivePhysRegs LiveRegs; |
658 | 14 | computeAndAddLiveIns(LiveRegs, *NewMBB); |
659 | 14 | |
660 | 14 | NextMBBI = MBB.end(); |
661 | 14 | MI.eraseFromParent(); |
662 | 14 | return true; |
663 | 14 | } |
664 | | |
665 | | bool RISCVExpandPseudo::expandLoadLocalAddress( |
666 | | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
667 | 6 | MachineBasicBlock::iterator &NextMBBI) { |
668 | 6 | return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_PCREL_HI, |
669 | 6 | RISCV::ADDI); |
670 | 6 | } |
671 | | |
672 | | bool RISCVExpandPseudo::expandLoadAddress( |
673 | | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
674 | 2 | MachineBasicBlock::iterator &NextMBBI) { |
675 | 2 | MachineFunction *MF = MBB.getParent(); |
676 | 2 | |
677 | 2 | unsigned SecondOpcode; |
678 | 2 | unsigned FlagsHi; |
679 | 2 | if (MF->getTarget().isPositionIndependent()) { |
680 | 2 | const auto &STI = MF->getSubtarget<RISCVSubtarget>(); |
681 | 2 | SecondOpcode = STI.is64Bit() ? RISCV::LD1 : RISCV::LW1 ; |
682 | 2 | FlagsHi = RISCVII::MO_GOT_HI; |
683 | 2 | } else { |
684 | 0 | SecondOpcode = RISCV::ADDI; |
685 | 0 | FlagsHi = RISCVII::MO_PCREL_HI; |
686 | 0 | } |
687 | 2 | return expandAuipcInstPair(MBB, MBBI, NextMBBI, FlagsHi, SecondOpcode); |
688 | 2 | } |
689 | | |
690 | | bool RISCVExpandPseudo::expandLoadTLSIEAddress( |
691 | | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
692 | 2 | MachineBasicBlock::iterator &NextMBBI) { |
693 | 2 | MachineFunction *MF = MBB.getParent(); |
694 | 2 | |
695 | 2 | const auto &STI = MF->getSubtarget<RISCVSubtarget>(); |
696 | 2 | unsigned SecondOpcode = STI.is64Bit() ? RISCV::LD1 : RISCV::LW1 ; |
697 | 2 | return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GOT_HI, |
698 | 2 | SecondOpcode); |
699 | 2 | } |
700 | | |
701 | | bool RISCVExpandPseudo::expandLoadTLSGDAddress( |
702 | | MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
703 | 4 | MachineBasicBlock::iterator &NextMBBI) { |
704 | 4 | return expandAuipcInstPair(MBB, MBBI, NextMBBI, RISCVII::MO_TLS_GD_HI, |
705 | 4 | RISCV::ADDI); |
706 | 4 | } |
707 | | |
708 | | } // end of anonymous namespace |
709 | | |
710 | | INITIALIZE_PASS(RISCVExpandPseudo, "riscv-expand-pseudo", |
711 | | RISCV_EXPAND_PSEUDO_NAME, false, false) |
712 | | namespace llvm { |
713 | | |
714 | 329 | FunctionPass *createRISCVExpandPseudoPass() { return new RISCVExpandPseudo(); } |
715 | | |
716 | | } // end of namespace llvm |