/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/Mips/MipsExpandPseudo.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This file contains a pass that expands pseudo instructions into target |
10 | | // instructions to allow proper scheduling, if-conversion, and other late |
11 | | // optimizations. This pass should be run after register allocation but before |
12 | | // the post-regalloc scheduling pass. |
13 | | // |
14 | | // This is currently only used for expanding atomic pseudos after register |
15 | | // allocation. We do this to avoid the fast register allocator introducing |
16 | | // spills between ll and sc. These stores cause some MIPS implementations to |
17 | | // abort the atomic RMW sequence. |
18 | | // |
19 | | //===----------------------------------------------------------------------===// |
20 | | |
21 | | #include "Mips.h" |
22 | | #include "MipsInstrInfo.h" |
23 | | #include "MipsSubtarget.h" |
24 | | #include "llvm/CodeGen/LivePhysRegs.h" |
25 | | #include "llvm/CodeGen/MachineFunctionPass.h" |
26 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
27 | | |
28 | | using namespace llvm; |
29 | | |
30 | | #define DEBUG_TYPE "mips-pseudo" |
31 | | |
32 | | namespace { |
33 | | class MipsExpandPseudo : public MachineFunctionPass { |
34 | | public: |
35 | | static char ID; |
36 | 2.09k | MipsExpandPseudo() : MachineFunctionPass(ID) {} |
37 | | |
38 | | const MipsInstrInfo *TII; |
39 | | const MipsSubtarget *STI; |
40 | | |
41 | | bool runOnMachineFunction(MachineFunction &Fn) override; |
42 | | |
43 | 2.01k | MachineFunctionProperties getRequiredProperties() const override { |
44 | 2.01k | return MachineFunctionProperties().set( |
45 | 2.01k | MachineFunctionProperties::Property::NoVRegs); |
46 | 2.01k | } |
47 | | |
48 | 15.2k | StringRef getPassName() const override { |
49 | 15.2k | return "Mips pseudo instruction expansion pass"; |
50 | 15.2k | } |
51 | | |
52 | | private: |
53 | | bool expandAtomicCmpSwap(MachineBasicBlock &MBB, |
54 | | MachineBasicBlock::iterator MBBI, |
55 | | MachineBasicBlock::iterator &NextMBBI); |
56 | | bool expandAtomicCmpSwapSubword(MachineBasicBlock &MBB, |
57 | | MachineBasicBlock::iterator MBBI, |
58 | | MachineBasicBlock::iterator &NextMBBI); |
59 | | |
60 | | bool expandAtomicBinOp(MachineBasicBlock &BB, |
61 | | MachineBasicBlock::iterator I, |
62 | | MachineBasicBlock::iterator &NMBBI, unsigned Size); |
63 | | bool expandAtomicBinOpSubword(MachineBasicBlock &BB, |
64 | | MachineBasicBlock::iterator I, |
65 | | MachineBasicBlock::iterator &NMBBI); |
66 | | |
67 | | bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, |
68 | | MachineBasicBlock::iterator &NMBB); |
69 | | bool expandMBB(MachineBasicBlock &MBB); |
70 | | }; |
71 | | char MipsExpandPseudo::ID = 0; |
72 | | } |
73 | | |
74 | | bool MipsExpandPseudo::expandAtomicCmpSwapSubword( |
75 | | MachineBasicBlock &BB, MachineBasicBlock::iterator I, |
76 | 46 | MachineBasicBlock::iterator &NMBBI) { |
77 | 46 | |
78 | 46 | MachineFunction *MF = BB.getParent(); |
79 | 46 | |
80 | 46 | const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); |
81 | 46 | DebugLoc DL = I->getDebugLoc(); |
82 | 46 | unsigned LL, SC; |
83 | 46 | |
84 | 46 | unsigned ZERO = Mips::ZERO; |
85 | 46 | unsigned BNE = Mips::BNE; |
86 | 46 | unsigned BEQ = Mips::BEQ; |
87 | 46 | unsigned SEOp = |
88 | 46 | I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB31 : Mips::SEH15 ; |
89 | 46 | |
90 | 46 | if (STI->inMicroMipsMode()) { |
91 | 4 | LL = STI->hasMips32r6() ? Mips::LL_MMR60 : Mips::LL_MM; |
92 | 4 | SC = STI->hasMips32r6() ? Mips::SC_MMR60 : Mips::SC_MM; |
93 | 4 | BNE = STI->hasMips32r6() ? Mips::BNEC_MMR60 : Mips::BNE_MM; |
94 | 4 | BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR60 : Mips::BEQ_MM; |
95 | 42 | } else { |
96 | 42 | LL = STI->hasMips32r6() ? (ArePtrs64bit 12 ? Mips::LL64_R66 : Mips::LL_R66 ) |
97 | 42 | : (ArePtrs64bit 30 ? Mips::LL649 : Mips::LL21 ); |
98 | 42 | SC = STI->hasMips32r6() ? (ArePtrs64bit 12 ? Mips::SC64_R66 : Mips::SC_R66 ) |
99 | 42 | : (ArePtrs64bit 30 ? Mips::SC649 : Mips::SC21 ); |
100 | 42 | } |
101 | 46 | |
102 | 46 | unsigned Dest = I->getOperand(0).getReg(); |
103 | 46 | unsigned Ptr = I->getOperand(1).getReg(); |
104 | 46 | unsigned Mask = I->getOperand(2).getReg(); |
105 | 46 | unsigned ShiftCmpVal = I->getOperand(3).getReg(); |
106 | 46 | unsigned Mask2 = I->getOperand(4).getReg(); |
107 | 46 | unsigned ShiftNewVal = I->getOperand(5).getReg(); |
108 | 46 | unsigned ShiftAmnt = I->getOperand(6).getReg(); |
109 | 46 | unsigned Scratch = I->getOperand(7).getReg(); |
110 | 46 | unsigned Scratch2 = I->getOperand(8).getReg(); |
111 | 46 | |
112 | 46 | // insert new blocks after the current block |
113 | 46 | const BasicBlock *LLVM_BB = BB.getBasicBlock(); |
114 | 46 | MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); |
115 | 46 | MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); |
116 | 46 | MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
117 | 46 | MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
118 | 46 | MachineFunction::iterator It = ++BB.getIterator(); |
119 | 46 | MF->insert(It, loop1MBB); |
120 | 46 | MF->insert(It, loop2MBB); |
121 | 46 | MF->insert(It, sinkMBB); |
122 | 46 | MF->insert(It, exitMBB); |
123 | 46 | |
124 | 46 | // Transfer the remainder of BB and its successor edges to exitMBB. |
125 | 46 | exitMBB->splice(exitMBB->begin(), &BB, |
126 | 46 | std::next(MachineBasicBlock::iterator(I)), BB.end()); |
127 | 46 | exitMBB->transferSuccessorsAndUpdatePHIs(&BB); |
128 | 46 | |
129 | 46 | // thisMBB: |
130 | 46 | // ... |
131 | 46 | // fallthrough --> loop1MBB |
132 | 46 | BB.addSuccessor(loop1MBB, BranchProbability::getOne()); |
133 | 46 | loop1MBB->addSuccessor(sinkMBB); |
134 | 46 | loop1MBB->addSuccessor(loop2MBB); |
135 | 46 | loop1MBB->normalizeSuccProbs(); |
136 | 46 | loop2MBB->addSuccessor(loop1MBB); |
137 | 46 | loop2MBB->addSuccessor(sinkMBB); |
138 | 46 | loop2MBB->normalizeSuccProbs(); |
139 | 46 | sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); |
140 | 46 | |
141 | 46 | // loop1MBB: |
142 | 46 | // ll dest, 0(ptr) |
143 | 46 | // and Mask', dest, Mask |
144 | 46 | // bne Mask', ShiftCmpVal, exitMBB |
145 | 46 | BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0); |
146 | 46 | BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2) |
147 | 46 | .addReg(Scratch) |
148 | 46 | .addReg(Mask); |
149 | 46 | BuildMI(loop1MBB, DL, TII->get(BNE)) |
150 | 46 | .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB); |
151 | 46 | |
152 | 46 | // loop2MBB: |
153 | 46 | // and dest, dest, mask2 |
154 | 46 | // or dest, dest, ShiftNewVal |
155 | 46 | // sc dest, dest, 0(ptr) |
156 | 46 | // beq dest, $0, loop1MBB |
157 | 46 | BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch) |
158 | 46 | .addReg(Scratch, RegState::Kill) |
159 | 46 | .addReg(Mask2); |
160 | 46 | BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch) |
161 | 46 | .addReg(Scratch, RegState::Kill) |
162 | 46 | .addReg(ShiftNewVal); |
163 | 46 | BuildMI(loop2MBB, DL, TII->get(SC), Scratch) |
164 | 46 | .addReg(Scratch, RegState::Kill) |
165 | 46 | .addReg(Ptr) |
166 | 46 | .addImm(0); |
167 | 46 | BuildMI(loop2MBB, DL, TII->get(BEQ)) |
168 | 46 | .addReg(Scratch, RegState::Kill) |
169 | 46 | .addReg(ZERO) |
170 | 46 | .addMBB(loop1MBB); |
171 | 46 | |
172 | 46 | // sinkMBB: |
173 | 46 | // srl srlres, Mask', shiftamt |
174 | 46 | // sign_extend dest,srlres |
175 | 46 | BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) |
176 | 46 | .addReg(Scratch2) |
177 | 46 | .addReg(ShiftAmnt); |
178 | 46 | if (STI->hasMips32r2()) { |
179 | 21 | BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); |
180 | 25 | } else { |
181 | 25 | const unsigned ShiftImm = |
182 | 25 | I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 168 : 2417 ; |
183 | 25 | BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) |
184 | 25 | .addReg(Dest, RegState::Kill) |
185 | 25 | .addImm(ShiftImm); |
186 | 25 | BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) |
187 | 25 | .addReg(Dest, RegState::Kill) |
188 | 25 | .addImm(ShiftImm); |
189 | 25 | } |
190 | 46 | |
191 | 46 | LivePhysRegs LiveRegs; |
192 | 46 | computeAndAddLiveIns(LiveRegs, *loop1MBB); |
193 | 46 | computeAndAddLiveIns(LiveRegs, *loop2MBB); |
194 | 46 | computeAndAddLiveIns(LiveRegs, *sinkMBB); |
195 | 46 | computeAndAddLiveIns(LiveRegs, *exitMBB); |
196 | 46 | |
197 | 46 | NMBBI = BB.end(); |
198 | 46 | I->eraseFromParent(); |
199 | 46 | return true; |
200 | 46 | } |
201 | | |
202 | | bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB, |
203 | | MachineBasicBlock::iterator I, |
204 | 42 | MachineBasicBlock::iterator &NMBBI) { |
205 | 42 | |
206 | 42 | const unsigned Size = |
207 | 42 | I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 433 : 89 ; |
208 | 42 | MachineFunction *MF = BB.getParent(); |
209 | 42 | |
210 | 42 | const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); |
211 | 42 | DebugLoc DL = I->getDebugLoc(); |
212 | 42 | |
213 | 42 | unsigned LL, SC, ZERO, BNE, BEQ, MOVE; |
214 | 42 | |
215 | 42 | if (Size == 4) { |
216 | 33 | if (STI->inMicroMipsMode()) { |
217 | 2 | LL = STI->hasMips32r6() ? Mips::LL_MMR60 : Mips::LL_MM; |
218 | 2 | SC = STI->hasMips32r6() ? Mips::SC_MMR60 : Mips::SC_MM; |
219 | 2 | BNE = STI->hasMips32r6() ? Mips::BNEC_MMR60 : Mips::BNE_MM; |
220 | 2 | BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR60 : Mips::BEQ_MM; |
221 | 31 | } else { |
222 | 31 | LL = STI->hasMips32r6() |
223 | 31 | ? (ArePtrs64bit 8 ? Mips::LL64_R64 : Mips::LL_R64 ) |
224 | 31 | : (ArePtrs64bit 23 ? Mips::LL647 : Mips::LL16 ); |
225 | 31 | SC = STI->hasMips32r6() |
226 | 31 | ? (ArePtrs64bit 8 ? Mips::SC64_R64 : Mips::SC_R64 ) |
227 | 31 | : (ArePtrs64bit 23 ? Mips::SC647 : Mips::SC16 ); |
228 | 31 | BNE = Mips::BNE; |
229 | 31 | BEQ = Mips::BEQ; |
230 | 31 | } |
231 | 33 | |
232 | 33 | ZERO = Mips::ZERO; |
233 | 33 | MOVE = Mips::OR; |
234 | 33 | } else { |
235 | 9 | LL = STI->hasMips64r6() ? Mips::LLD_R62 : Mips::LLD7 ; |
236 | 9 | SC = STI->hasMips64r6() ? Mips::SCD_R62 : Mips::SCD7 ; |
237 | 9 | ZERO = Mips::ZERO_64; |
238 | 9 | BNE = Mips::BNE64; |
239 | 9 | BEQ = Mips::BEQ64; |
240 | 9 | MOVE = Mips::OR64; |
241 | 9 | } |
242 | 42 | |
243 | 42 | unsigned Dest = I->getOperand(0).getReg(); |
244 | 42 | unsigned Ptr = I->getOperand(1).getReg(); |
245 | 42 | unsigned OldVal = I->getOperand(2).getReg(); |
246 | 42 | unsigned NewVal = I->getOperand(3).getReg(); |
247 | 42 | unsigned Scratch = I->getOperand(4).getReg(); |
248 | 42 | |
249 | 42 | // insert new blocks after the current block |
250 | 42 | const BasicBlock *LLVM_BB = BB.getBasicBlock(); |
251 | 42 | MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); |
252 | 42 | MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); |
253 | 42 | MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
254 | 42 | MachineFunction::iterator It = ++BB.getIterator(); |
255 | 42 | MF->insert(It, loop1MBB); |
256 | 42 | MF->insert(It, loop2MBB); |
257 | 42 | MF->insert(It, exitMBB); |
258 | 42 | |
259 | 42 | // Transfer the remainder of BB and its successor edges to exitMBB. |
260 | 42 | exitMBB->splice(exitMBB->begin(), &BB, |
261 | 42 | std::next(MachineBasicBlock::iterator(I)), BB.end()); |
262 | 42 | exitMBB->transferSuccessorsAndUpdatePHIs(&BB); |
263 | 42 | |
264 | 42 | // thisMBB: |
265 | 42 | // ... |
266 | 42 | // fallthrough --> loop1MBB |
267 | 42 | BB.addSuccessor(loop1MBB, BranchProbability::getOne()); |
268 | 42 | loop1MBB->addSuccessor(exitMBB); |
269 | 42 | loop1MBB->addSuccessor(loop2MBB); |
270 | 42 | loop1MBB->normalizeSuccProbs(); |
271 | 42 | loop2MBB->addSuccessor(loop1MBB); |
272 | 42 | loop2MBB->addSuccessor(exitMBB); |
273 | 42 | loop2MBB->normalizeSuccProbs(); |
274 | 42 | |
275 | 42 | // loop1MBB: |
276 | 42 | // ll dest, 0(ptr) |
277 | 42 | // bne dest, oldval, exitMBB |
278 | 42 | BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0); |
279 | 42 | BuildMI(loop1MBB, DL, TII->get(BNE)) |
280 | 42 | .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB); |
281 | 42 | |
282 | 42 | // loop2MBB: |
283 | 42 | // move scratch, NewVal |
284 | 42 | // sc Scratch, Scratch, 0(ptr) |
285 | 42 | // beq Scratch, $0, loop1MBB |
286 | 42 | BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO); |
287 | 42 | BuildMI(loop2MBB, DL, TII->get(SC), Scratch) |
288 | 42 | .addReg(Scratch).addReg(Ptr).addImm(0); |
289 | 42 | BuildMI(loop2MBB, DL, TII->get(BEQ)) |
290 | 42 | .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB); |
291 | 42 | |
292 | 42 | LivePhysRegs LiveRegs; |
293 | 42 | computeAndAddLiveIns(LiveRegs, *loop1MBB); |
294 | 42 | computeAndAddLiveIns(LiveRegs, *loop2MBB); |
295 | 42 | computeAndAddLiveIns(LiveRegs, *exitMBB); |
296 | 42 | |
297 | 42 | NMBBI = BB.end(); |
298 | 42 | I->eraseFromParent(); |
299 | 42 | return true; |
300 | 42 | } |
301 | | |
302 | | bool MipsExpandPseudo::expandAtomicBinOpSubword( |
303 | | MachineBasicBlock &BB, MachineBasicBlock::iterator I, |
304 | 76 | MachineBasicBlock::iterator &NMBBI) { |
305 | 76 | |
306 | 76 | MachineFunction *MF = BB.getParent(); |
307 | 76 | |
308 | 76 | const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); |
309 | 76 | DebugLoc DL = I->getDebugLoc(); |
310 | 76 | |
311 | 76 | unsigned LL, SC; |
312 | 76 | unsigned BEQ = Mips::BEQ; |
313 | 76 | unsigned SEOp = Mips::SEH; |
314 | 76 | |
315 | 76 | if (STI->inMicroMipsMode()) { |
316 | 6 | LL = STI->hasMips32r6() ? Mips::LL_MMR60 : Mips::LL_MM; |
317 | 6 | SC = STI->hasMips32r6() ? Mips::SC_MMR60 : Mips::SC_MM; |
318 | 6 | BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR60 : Mips::BEQ_MM; |
319 | 70 | } else { |
320 | 70 | LL = STI->hasMips32r6() ? (ArePtrs64bit 20 ? Mips::LL64_R610 : Mips::LL_R610 ) |
321 | 70 | : (ArePtrs64bit 50 ? Mips::LL6415 : Mips::LL35 ); |
322 | 70 | SC = STI->hasMips32r6() ? (ArePtrs64bit 20 ? Mips::SC64_R610 : Mips::SC_R610 ) |
323 | 70 | : (ArePtrs64bit 50 ? Mips::SC6415 : Mips::SC35 ); |
324 | 70 | } |
325 | 76 | |
326 | 76 | bool IsSwap = false; |
327 | 76 | bool IsNand = false; |
328 | 76 | |
329 | 76 | unsigned Opcode = 0; |
330 | 76 | switch (I->getOpcode()) { |
331 | 76 | case Mips::ATOMIC_LOAD_NAND_I8_POSTRA: |
332 | 15 | SEOp = Mips::SEB; |
333 | 15 | LLVM_FALLTHROUGH; |
334 | 15 | case Mips::ATOMIC_LOAD_NAND_I16_POSTRA: |
335 | 15 | IsNand = true; |
336 | 15 | break; |
337 | 15 | case Mips::ATOMIC_SWAP_I8_POSTRA: |
338 | 15 | SEOp = Mips::SEB; |
339 | 15 | LLVM_FALLTHROUGH; |
340 | 15 | case Mips::ATOMIC_SWAP_I16_POSTRA: |
341 | 15 | IsSwap = true; |
342 | 15 | break; |
343 | 16 | case Mips::ATOMIC_LOAD_ADD_I8_POSTRA: |
344 | 16 | SEOp = Mips::SEB; |
345 | 16 | LLVM_FALLTHROUGH; |
346 | 31 | case Mips::ATOMIC_LOAD_ADD_I16_POSTRA: |
347 | 31 | Opcode = Mips::ADDu; |
348 | 31 | break; |
349 | 16 | case Mips::ATOMIC_LOAD_SUB_I8_POSTRA: |
350 | 15 | SEOp = Mips::SEB; |
351 | 15 | LLVM_FALLTHROUGH; |
352 | 15 | case Mips::ATOMIC_LOAD_SUB_I16_POSTRA: |
353 | 15 | Opcode = Mips::SUBu; |
354 | 15 | break; |
355 | 15 | case Mips::ATOMIC_LOAD_AND_I8_POSTRA: |
356 | 0 | SEOp = Mips::SEB; |
357 | 0 | LLVM_FALLTHROUGH; |
358 | 0 | case Mips::ATOMIC_LOAD_AND_I16_POSTRA: |
359 | 0 | Opcode = Mips::AND; |
360 | 0 | break; |
361 | 0 | case Mips::ATOMIC_LOAD_OR_I8_POSTRA: |
362 | 0 | SEOp = Mips::SEB; |
363 | 0 | LLVM_FALLTHROUGH; |
364 | 0 | case Mips::ATOMIC_LOAD_OR_I16_POSTRA: |
365 | 0 | Opcode = Mips::OR; |
366 | 0 | break; |
367 | 0 | case Mips::ATOMIC_LOAD_XOR_I8_POSTRA: |
368 | 0 | SEOp = Mips::SEB; |
369 | 0 | LLVM_FALLTHROUGH; |
370 | 0 | case Mips::ATOMIC_LOAD_XOR_I16_POSTRA: |
371 | 0 | Opcode = Mips::XOR; |
372 | 0 | break; |
373 | 0 | default: |
374 | 0 | llvm_unreachable("Unknown subword atomic pseudo for expansion!"); |
375 | 76 | } |
376 | 76 | |
377 | 76 | unsigned Dest = I->getOperand(0).getReg(); |
378 | 76 | unsigned Ptr = I->getOperand(1).getReg(); |
379 | 76 | unsigned Incr = I->getOperand(2).getReg(); |
380 | 76 | unsigned Mask = I->getOperand(3).getReg(); |
381 | 76 | unsigned Mask2 = I->getOperand(4).getReg(); |
382 | 76 | unsigned ShiftAmnt = I->getOperand(5).getReg(); |
383 | 76 | unsigned OldVal = I->getOperand(6).getReg(); |
384 | 76 | unsigned BinOpRes = I->getOperand(7).getReg(); |
385 | 76 | unsigned StoreVal = I->getOperand(8).getReg(); |
386 | 76 | |
387 | 76 | const BasicBlock *LLVM_BB = BB.getBasicBlock(); |
388 | 76 | MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
389 | 76 | MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
390 | 76 | MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
391 | 76 | MachineFunction::iterator It = ++BB.getIterator(); |
392 | 76 | MF->insert(It, loopMBB); |
393 | 76 | MF->insert(It, sinkMBB); |
394 | 76 | MF->insert(It, exitMBB); |
395 | 76 | |
396 | 76 | exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); |
397 | 76 | exitMBB->transferSuccessorsAndUpdatePHIs(&BB); |
398 | 76 | |
399 | 76 | BB.addSuccessor(loopMBB, BranchProbability::getOne()); |
400 | 76 | loopMBB->addSuccessor(sinkMBB); |
401 | 76 | loopMBB->addSuccessor(loopMBB); |
402 | 76 | loopMBB->normalizeSuccProbs(); |
403 | 76 | |
404 | 76 | BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); |
405 | 76 | if (IsNand) { |
406 | 15 | // and andres, oldval, incr2 |
407 | 15 | // nor binopres, $0, andres |
408 | 15 | // and newval, binopres, mask |
409 | 15 | BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) |
410 | 15 | .addReg(OldVal) |
411 | 15 | .addReg(Incr); |
412 | 15 | BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes) |
413 | 15 | .addReg(Mips::ZERO) |
414 | 15 | .addReg(BinOpRes); |
415 | 15 | BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) |
416 | 15 | .addReg(BinOpRes) |
417 | 15 | .addReg(Mask); |
418 | 61 | } else if (!IsSwap) { |
419 | 46 | // <binop> binopres, oldval, incr2 |
420 | 46 | // and newval, binopres, mask |
421 | 46 | BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes) |
422 | 46 | .addReg(OldVal) |
423 | 46 | .addReg(Incr); |
424 | 46 | BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) |
425 | 46 | .addReg(BinOpRes) |
426 | 46 | .addReg(Mask); |
427 | 46 | } else { // atomic.swap |
428 | 15 | // and newval, incr2, mask |
429 | 15 | BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) |
430 | 15 | .addReg(Incr) |
431 | 15 | .addReg(Mask); |
432 | 15 | } |
433 | 76 | |
434 | 76 | // and StoreVal, OlddVal, Mask2 |
435 | 76 | // or StoreVal, StoreVal, BinOpRes |
436 | 76 | // StoreVal<tied1> = sc StoreVal, 0(Ptr) |
437 | 76 | // beq StoreVal, zero, loopMBB |
438 | 76 | BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal) |
439 | 76 | .addReg(OldVal).addReg(Mask2); |
440 | 76 | BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal) |
441 | 76 | .addReg(StoreVal).addReg(BinOpRes); |
442 | 76 | BuildMI(loopMBB, DL, TII->get(SC), StoreVal) |
443 | 76 | .addReg(StoreVal).addReg(Ptr).addImm(0); |
444 | 76 | BuildMI(loopMBB, DL, TII->get(BEQ)) |
445 | 76 | .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB); |
446 | 76 | |
447 | 76 | // sinkMBB: |
448 | 76 | // and maskedoldval1,oldval,mask |
449 | 76 | // srl srlres,maskedoldval1,shiftamt |
450 | 76 | // sign_extend dest,srlres |
451 | 76 | |
452 | 76 | sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); |
453 | 76 | |
454 | 76 | BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest) |
455 | 76 | .addReg(OldVal).addReg(Mask); |
456 | 76 | BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) |
457 | 76 | .addReg(Dest).addReg(ShiftAmnt); |
458 | 76 | |
459 | 76 | if (STI->hasMips32r2()) { |
460 | 35 | BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); |
461 | 41 | } else { |
462 | 41 | const unsigned ShiftImm = SEOp == Mips::SEH ? 168 : 2433 ; |
463 | 41 | BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) |
464 | 41 | .addReg(Dest, RegState::Kill) |
465 | 41 | .addImm(ShiftImm); |
466 | 41 | BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) |
467 | 41 | .addReg(Dest, RegState::Kill) |
468 | 41 | .addImm(ShiftImm); |
469 | 41 | } |
470 | 76 | |
471 | 76 | LivePhysRegs LiveRegs; |
472 | 76 | computeAndAddLiveIns(LiveRegs, *loopMBB); |
473 | 76 | computeAndAddLiveIns(LiveRegs, *sinkMBB); |
474 | 76 | computeAndAddLiveIns(LiveRegs, *exitMBB); |
475 | 76 | |
476 | 76 | NMBBI = BB.end(); |
477 | 76 | I->eraseFromParent(); |
478 | 76 | |
479 | 76 | return true; |
480 | 76 | } |
481 | | |
482 | | bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, |
483 | | MachineBasicBlock::iterator I, |
484 | | MachineBasicBlock::iterator &NMBBI, |
485 | 202 | unsigned Size) { |
486 | 202 | MachineFunction *MF = BB.getParent(); |
487 | 202 | |
488 | 202 | const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); |
489 | 202 | DebugLoc DL = I->getDebugLoc(); |
490 | 202 | |
491 | 202 | unsigned LL, SC, ZERO, BEQ; |
492 | 202 | |
493 | 202 | if (Size == 4) { |
494 | 138 | if (STI->inMicroMipsMode()) { |
495 | 10 | LL = STI->hasMips32r6() ? Mips::LL_MMR60 : Mips::LL_MM; |
496 | 10 | SC = STI->hasMips32r6() ? Mips::SC_MMR60 : Mips::SC_MM; |
497 | 10 | BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR60 : Mips::BEQ_MM; |
498 | 128 | } else { |
499 | 128 | LL = STI->hasMips32r6() |
500 | 128 | ? (ArePtrs64bit 38 ? Mips::LL64_R619 : Mips::LL_R619 ) |
501 | 128 | : (ArePtrs64bit 90 ? Mips::LL6427 : Mips::LL63 ); |
502 | 128 | SC = STI->hasMips32r6() |
503 | 128 | ? (ArePtrs64bit 38 ? Mips::SC64_R619 : Mips::SC_R619 ) |
504 | 128 | : (ArePtrs64bit 90 ? Mips::SC6427 : Mips::SC63 ); |
505 | 128 | BEQ = Mips::BEQ; |
506 | 128 | } |
507 | 138 | |
508 | 138 | ZERO = Mips::ZERO; |
509 | 138 | } else { |
510 | 64 | LL = STI->hasMips64r6() ? Mips::LLD_R615 : Mips::LLD49 ; |
511 | 64 | SC = STI->hasMips64r6() ? Mips::SCD_R615 : Mips::SCD49 ; |
512 | 64 | ZERO = Mips::ZERO_64; |
513 | 64 | BEQ = Mips::BEQ64; |
514 | 64 | } |
515 | 202 | |
516 | 202 | unsigned OldVal = I->getOperand(0).getReg(); |
517 | 202 | unsigned Ptr = I->getOperand(1).getReg(); |
518 | 202 | unsigned Incr = I->getOperand(2).getReg(); |
519 | 202 | unsigned Scratch = I->getOperand(3).getReg(); |
520 | 202 | |
521 | 202 | unsigned Opcode = 0; |
522 | 202 | unsigned OR = 0; |
523 | 202 | unsigned AND = 0; |
524 | 202 | unsigned NOR = 0; |
525 | 202 | bool IsNand = false; |
526 | 202 | switch (I->getOpcode()) { |
527 | 202 | case Mips::ATOMIC_LOAD_ADD_I32_POSTRA: |
528 | 48 | Opcode = Mips::ADDu; |
529 | 48 | break; |
530 | 202 | case Mips::ATOMIC_LOAD_SUB_I32_POSTRA: |
531 | 15 | Opcode = Mips::SUBu; |
532 | 15 | break; |
533 | 202 | case Mips::ATOMIC_LOAD_AND_I32_POSTRA: |
534 | 15 | Opcode = Mips::AND; |
535 | 15 | break; |
536 | 202 | case Mips::ATOMIC_LOAD_OR_I32_POSTRA: |
537 | 15 | Opcode = Mips::OR; |
538 | 15 | break; |
539 | 202 | case Mips::ATOMIC_LOAD_XOR_I32_POSTRA: |
540 | 15 | Opcode = Mips::XOR; |
541 | 15 | break; |
542 | 202 | case Mips::ATOMIC_LOAD_NAND_I32_POSTRA: |
543 | 15 | IsNand = true; |
544 | 15 | AND = Mips::AND; |
545 | 15 | NOR = Mips::NOR; |
546 | 15 | break; |
547 | 202 | case Mips::ATOMIC_SWAP_I32_POSTRA: |
548 | 15 | OR = Mips::OR; |
549 | 15 | break; |
550 | 202 | case Mips::ATOMIC_LOAD_ADD_I64_POSTRA: |
551 | 10 | Opcode = Mips::DADDu; |
552 | 10 | break; |
553 | 202 | case Mips::ATOMIC_LOAD_SUB_I64_POSTRA: |
554 | 9 | Opcode = Mips::DSUBu; |
555 | 9 | break; |
556 | 202 | case Mips::ATOMIC_LOAD_AND_I64_POSTRA: |
557 | 9 | Opcode = Mips::AND64; |
558 | 9 | break; |
559 | 202 | case Mips::ATOMIC_LOAD_OR_I64_POSTRA: |
560 | 9 | Opcode = Mips::OR64; |
561 | 9 | break; |
562 | 202 | case Mips::ATOMIC_LOAD_XOR_I64_POSTRA: |
563 | 9 | Opcode = Mips::XOR64; |
564 | 9 | break; |
565 | 202 | case Mips::ATOMIC_LOAD_NAND_I64_POSTRA: |
566 | 9 | IsNand = true; |
567 | 9 | AND = Mips::AND64; |
568 | 9 | NOR = Mips::NOR64; |
569 | 9 | break; |
570 | 202 | case Mips::ATOMIC_SWAP_I64_POSTRA: |
571 | 9 | OR = Mips::OR64; |
572 | 9 | break; |
573 | 202 | default: |
574 | 0 | llvm_unreachable("Unknown pseudo atomic!"); |
575 | 202 | } |
576 | 202 | |
577 | 202 | const BasicBlock *LLVM_BB = BB.getBasicBlock(); |
578 | 202 | MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
579 | 202 | MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); |
580 | 202 | MachineFunction::iterator It = ++BB.getIterator(); |
581 | 202 | MF->insert(It, loopMBB); |
582 | 202 | MF->insert(It, exitMBB); |
583 | 202 | |
584 | 202 | exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); |
585 | 202 | exitMBB->transferSuccessorsAndUpdatePHIs(&BB); |
586 | 202 | |
587 | 202 | BB.addSuccessor(loopMBB, BranchProbability::getOne()); |
588 | 202 | loopMBB->addSuccessor(exitMBB); |
589 | 202 | loopMBB->addSuccessor(loopMBB); |
590 | 202 | loopMBB->normalizeSuccProbs(); |
591 | 202 | |
592 | 202 | BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); |
593 | 202 | assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!"); |
594 | 202 | assert((OldVal != Incr) && "Clobbered the wrong reg!"); |
595 | 202 | if (Opcode) { |
596 | 154 | BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr); |
597 | 154 | } else if (48 IsNand48 ) { |
598 | 24 | assert(AND && NOR && |
599 | 24 | "Unknown nand instruction for atomic pseudo expansion"); |
600 | 24 | BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr); |
601 | 24 | BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch); |
602 | 24 | } else { |
603 | 24 | assert(OR && "Unknown instruction for atomic pseudo expansion!"); |
604 | 24 | BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO); |
605 | 24 | } |
606 | 202 | |
607 | 202 | BuildMI(loopMBB, DL, TII->get(SC), Scratch).addReg(Scratch).addReg(Ptr).addImm(0); |
608 | 202 | BuildMI(loopMBB, DL, TII->get(BEQ)).addReg(Scratch).addReg(ZERO).addMBB(loopMBB); |
609 | 202 | |
610 | 202 | NMBBI = BB.end(); |
611 | 202 | I->eraseFromParent(); |
612 | 202 | |
613 | 202 | LivePhysRegs LiveRegs; |
614 | 202 | computeAndAddLiveIns(LiveRegs, *loopMBB); |
615 | 202 | computeAndAddLiveIns(LiveRegs, *exitMBB); |
616 | 202 | |
617 | 202 | return true; |
618 | 202 | } |
619 | | |
620 | | bool MipsExpandPseudo::expandMI(MachineBasicBlock &MBB, |
621 | | MachineBasicBlock::iterator MBBI, |
622 | 128k | MachineBasicBlock::iterator &NMBB) { |
623 | 128k | |
624 | 128k | bool Modified = false; |
625 | 128k | |
626 | 128k | switch (MBBI->getOpcode()) { |
627 | 128k | case Mips::ATOMIC_CMP_SWAP_I32_POSTRA: |
628 | 42 | case Mips::ATOMIC_CMP_SWAP_I64_POSTRA: |
629 | 42 | return expandAtomicCmpSwap(MBB, MBBI, NMBB); |
630 | 46 | case Mips::ATOMIC_CMP_SWAP_I8_POSTRA: |
631 | 46 | case Mips::ATOMIC_CMP_SWAP_I16_POSTRA: |
632 | 46 | return expandAtomicCmpSwapSubword(MBB, MBBI, NMBB); |
633 | 76 | case Mips::ATOMIC_SWAP_I8_POSTRA: |
634 | 76 | case Mips::ATOMIC_SWAP_I16_POSTRA: |
635 | 76 | case Mips::ATOMIC_LOAD_NAND_I8_POSTRA: |
636 | 76 | case Mips::ATOMIC_LOAD_NAND_I16_POSTRA: |
637 | 76 | case Mips::ATOMIC_LOAD_ADD_I8_POSTRA: |
638 | 76 | case Mips::ATOMIC_LOAD_ADD_I16_POSTRA: |
639 | 76 | case Mips::ATOMIC_LOAD_SUB_I8_POSTRA: |
640 | 76 | case Mips::ATOMIC_LOAD_SUB_I16_POSTRA: |
641 | 76 | case Mips::ATOMIC_LOAD_AND_I8_POSTRA: |
642 | 76 | case Mips::ATOMIC_LOAD_AND_I16_POSTRA: |
643 | 76 | case Mips::ATOMIC_LOAD_OR_I8_POSTRA: |
644 | 76 | case Mips::ATOMIC_LOAD_OR_I16_POSTRA: |
645 | 76 | case Mips::ATOMIC_LOAD_XOR_I8_POSTRA: |
646 | 76 | case Mips::ATOMIC_LOAD_XOR_I16_POSTRA: |
647 | 76 | return expandAtomicBinOpSubword(MBB, MBBI, NMBB); |
648 | 138 | case Mips::ATOMIC_LOAD_ADD_I32_POSTRA: |
649 | 138 | case Mips::ATOMIC_LOAD_SUB_I32_POSTRA: |
650 | 138 | case Mips::ATOMIC_LOAD_AND_I32_POSTRA: |
651 | 138 | case Mips::ATOMIC_LOAD_OR_I32_POSTRA: |
652 | 138 | case Mips::ATOMIC_LOAD_XOR_I32_POSTRA: |
653 | 138 | case Mips::ATOMIC_LOAD_NAND_I32_POSTRA: |
654 | 138 | case Mips::ATOMIC_SWAP_I32_POSTRA: |
655 | 138 | return expandAtomicBinOp(MBB, MBBI, NMBB, 4); |
656 | 138 | case Mips::ATOMIC_LOAD_ADD_I64_POSTRA: |
657 | 64 | case Mips::ATOMIC_LOAD_SUB_I64_POSTRA: |
658 | 64 | case Mips::ATOMIC_LOAD_AND_I64_POSTRA: |
659 | 64 | case Mips::ATOMIC_LOAD_OR_I64_POSTRA: |
660 | 64 | case Mips::ATOMIC_LOAD_XOR_I64_POSTRA: |
661 | 64 | case Mips::ATOMIC_LOAD_NAND_I64_POSTRA: |
662 | 64 | case Mips::ATOMIC_SWAP_I64_POSTRA: |
663 | 64 | return expandAtomicBinOp(MBB, MBBI, NMBB, 8); |
664 | 128k | default: |
665 | 128k | return Modified; |
666 | 128k | } |
667 | 128k | } |
668 | | |
669 | 16.3k | bool MipsExpandPseudo::expandMBB(MachineBasicBlock &MBB) { |
670 | 16.3k | bool Modified = false; |
671 | 16.3k | |
672 | 16.3k | MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); |
673 | 145k | while (MBBI != E) { |
674 | 128k | MachineBasicBlock::iterator NMBBI = std::next(MBBI); |
675 | 128k | Modified |= expandMI(MBB, MBBI, NMBBI); |
676 | 128k | MBBI = NMBBI; |
677 | 128k | } |
678 | 16.3k | |
679 | 16.3k | return Modified; |
680 | 16.3k | } |
681 | | |
682 | 13.2k | bool MipsExpandPseudo::runOnMachineFunction(MachineFunction &MF) { |
683 | 13.2k | STI = &static_cast<const MipsSubtarget &>(MF.getSubtarget()); |
684 | 13.2k | TII = STI->getInstrInfo(); |
685 | 13.2k | |
686 | 13.2k | bool Modified = false; |
687 | 29.5k | for (MachineFunction::iterator MFI = MF.begin(), E = MF.end(); MFI != E; |
688 | 16.3k | ++MFI) |
689 | 16.3k | Modified |= expandMBB(*MFI); |
690 | 13.2k | |
691 | 13.2k | if (Modified) |
692 | 366 | MF.RenumberBlocks(); |
693 | 13.2k | |
694 | 13.2k | return Modified; |
695 | 13.2k | } |
696 | | |
697 | | /// createMipsExpandPseudoPass - returns an instance of the pseudo instruction |
698 | | /// expansion pass. |
699 | 2.09k | FunctionPass *llvm::createMipsExpandPseudoPass() { |
700 | 2.09k | return new MipsExpandPseudo(); |
701 | 2.09k | } |