/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- HexagonSplitDouble.cpp ---------------------------------------------===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | |
10 | | #define DEBUG_TYPE "hsdr" |
11 | | |
12 | | #include "HexagonInstrInfo.h" |
13 | | #include "HexagonRegisterInfo.h" |
14 | | #include "HexagonSubtarget.h" |
15 | | #include "llvm/ADT/BitVector.h" |
16 | | #include "llvm/ADT/STLExtras.h" |
17 | | #include "llvm/ADT/SmallVector.h" |
18 | | #include "llvm/ADT/StringRef.h" |
19 | | #include "llvm/CodeGen/MachineBasicBlock.h" |
20 | | #include "llvm/CodeGen/MachineFunction.h" |
21 | | #include "llvm/CodeGen/MachineFunctionPass.h" |
22 | | #include "llvm/CodeGen/MachineInstr.h" |
23 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
24 | | #include "llvm/CodeGen/MachineLoopInfo.h" |
25 | | #include "llvm/CodeGen/MachineMemOperand.h" |
26 | | #include "llvm/CodeGen/MachineOperand.h" |
27 | | #include "llvm/CodeGen/MachineRegisterInfo.h" |
28 | | #include "llvm/IR/DebugLoc.h" |
29 | | #include "llvm/Pass.h" |
30 | | #include "llvm/Support/CommandLine.h" |
31 | | #include "llvm/Support/Compiler.h" |
32 | | #include "llvm/Support/Debug.h" |
33 | | #include "llvm/Support/ErrorHandling.h" |
34 | | #include "llvm/Support/raw_ostream.h" |
35 | | #include "llvm/Target/TargetRegisterInfo.h" |
36 | | #include <algorithm> |
37 | | #include <cassert> |
38 | | #include <cstdint> |
39 | | #include <limits> |
40 | | #include <map> |
41 | | #include <set> |
42 | | #include <utility> |
43 | | #include <vector> |
44 | | |
45 | | using namespace llvm; |
46 | | |
47 | | namespace llvm { |
48 | | |
49 | | FunctionPass *createHexagonSplitDoubleRegs(); |
50 | | void initializeHexagonSplitDoubleRegsPass(PassRegistry&); |
51 | | |
52 | | } // end namespace llvm |
53 | | |
54 | | static cl::opt<int> MaxHSDR("max-hsdr", cl::Hidden, cl::init(-1), |
55 | | cl::desc("Maximum number of split partitions")); |
56 | | static cl::opt<bool> MemRefsFixed("hsdr-no-mem", cl::Hidden, cl::init(true), |
57 | | cl::desc("Do not split loads or stores")); |
58 | | |
59 | | namespace { |
60 | | |
61 | | class HexagonSplitDoubleRegs : public MachineFunctionPass { |
62 | | public: |
63 | | static char ID; |
64 | | |
65 | 392 | HexagonSplitDoubleRegs() : MachineFunctionPass(ID) { |
66 | 392 | initializeHexagonSplitDoubleRegsPass(*PassRegistry::getPassRegistry()); |
67 | 392 | } |
68 | | |
69 | 0 | StringRef getPassName() const override { |
70 | 0 | return "Hexagon Split Double Registers"; |
71 | 0 | } |
72 | | |
73 | 388 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
74 | 388 | AU.addRequired<MachineLoopInfo>(); |
75 | 388 | AU.addPreserved<MachineLoopInfo>(); |
76 | 388 | MachineFunctionPass::getAnalysisUsage(AU); |
77 | 388 | } |
78 | | |
79 | | bool runOnMachineFunction(MachineFunction &MF) override; |
80 | | |
81 | | private: |
82 | | static const TargetRegisterClass *const DoubleRC; |
83 | | |
84 | | const HexagonRegisterInfo *TRI = nullptr; |
85 | | const HexagonInstrInfo *TII = nullptr; |
86 | | const MachineLoopInfo *MLI; |
87 | | MachineRegisterInfo *MRI; |
88 | | |
89 | | using USet = std::set<unsigned>; |
90 | | using UUSetMap = std::map<unsigned, USet>; |
91 | | using UUPair = std::pair<unsigned, unsigned>; |
92 | | using UUPairMap = std::map<unsigned, UUPair>; |
93 | | using LoopRegMap = std::map<const MachineLoop *, USet>; |
94 | | |
95 | | bool isInduction(unsigned Reg, LoopRegMap &IRM) const; |
96 | | bool isVolatileInstr(const MachineInstr *MI) const; |
97 | | bool isFixedInstr(const MachineInstr *MI) const; |
98 | | void partitionRegisters(UUSetMap &P2Rs); |
99 | | int32_t profit(const MachineInstr *MI) const; |
100 | | bool isProfitable(const USet &Part, LoopRegMap &IRM) const; |
101 | | |
102 | | void collectIndRegsForLoop(const MachineLoop *L, USet &Rs); |
103 | | void collectIndRegs(LoopRegMap &IRM); |
104 | | |
105 | | void createHalfInstr(unsigned Opc, MachineInstr *MI, |
106 | | const UUPairMap &PairMap, unsigned SubR); |
107 | | void splitMemRef(MachineInstr *MI, const UUPairMap &PairMap); |
108 | | void splitImmediate(MachineInstr *MI, const UUPairMap &PairMap); |
109 | | void splitCombine(MachineInstr *MI, const UUPairMap &PairMap); |
110 | | void splitExt(MachineInstr *MI, const UUPairMap &PairMap); |
111 | | void splitShift(MachineInstr *MI, const UUPairMap &PairMap); |
112 | | void splitAslOr(MachineInstr *MI, const UUPairMap &PairMap); |
113 | | bool splitInstr(MachineInstr *MI, const UUPairMap &PairMap); |
114 | | void replaceSubregUses(MachineInstr *MI, const UUPairMap &PairMap); |
115 | | void collapseRegPairs(MachineInstr *MI, const UUPairMap &PairMap); |
116 | | bool splitPartition(const USet &Part); |
117 | | |
118 | | static int Counter; |
119 | | |
120 | | static void dump_partition(raw_ostream&, const USet&, |
121 | | const TargetRegisterInfo&); |
122 | | }; |
123 | | |
124 | | } // end anonymous namespace |
125 | | |
126 | | char HexagonSplitDoubleRegs::ID; |
127 | | int HexagonSplitDoubleRegs::Counter = 0; |
128 | | const TargetRegisterClass *const HexagonSplitDoubleRegs::DoubleRC = |
129 | | &Hexagon::DoubleRegsRegClass; |
130 | | |
131 | | INITIALIZE_PASS(HexagonSplitDoubleRegs, "hexagon-split-double", |
132 | | "Hexagon Split Double Registers", false, false) |
133 | | |
134 | | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
135 | | LLVM_DUMP_METHOD void HexagonSplitDoubleRegs::dump_partition(raw_ostream &os, |
136 | | const USet &Part, const TargetRegisterInfo &TRI) { |
137 | | dbgs() << '{'; |
138 | | for (auto I : Part) |
139 | | dbgs() << ' ' << PrintReg(I, &TRI); |
140 | | dbgs() << " }"; |
141 | | } |
142 | | #endif |
143 | | |
144 | 284 | bool HexagonSplitDoubleRegs::isInduction(unsigned Reg, LoopRegMap &IRM) const { |
145 | 0 | for (auto I : IRM) { |
146 | 0 | const USet &Rs = I.second; |
147 | 0 | if (Rs.find(Reg) != Rs.end()) |
148 | 0 | return true; |
149 | 284 | } |
150 | 284 | return false; |
151 | 284 | } |
152 | | |
153 | 0 | bool HexagonSplitDoubleRegs::isVolatileInstr(const MachineInstr *MI) const { |
154 | 0 | for (auto &I : MI->memoperands()) |
155 | 0 | if (0 I->isVolatile()0 ) |
156 | 0 | return true; |
157 | 0 | return false; |
158 | 0 | } |
159 | | |
160 | 1.98k | bool HexagonSplitDoubleRegs::isFixedInstr(const MachineInstr *MI) const { |
161 | 1.98k | if (MI->mayLoad() || 1.98k MI->mayStore()1.86k ) |
162 | 234 | if (234 MemRefsFixed || 234 isVolatileInstr(MI)0 ) |
163 | 234 | return true; |
164 | 1.74k | if (1.74k MI->isDebugValue()1.74k ) |
165 | 0 | return false; |
166 | 1.74k | |
167 | 1.74k | unsigned Opc = MI->getOpcode(); |
168 | 1.74k | switch (Opc) { |
169 | 719 | default: |
170 | 719 | return true; |
171 | 1.74k | |
172 | 513 | case TargetOpcode::PHI: |
173 | 513 | case TargetOpcode::COPY: |
174 | 513 | break; |
175 | 513 | |
176 | 0 | case Hexagon::L2_loadrd_io: |
177 | 0 | // Not handling stack stores (only reg-based addresses). |
178 | 0 | if (MI->getOperand(1).isReg()) |
179 | 0 | break; |
180 | 0 | return true; |
181 | 0 | case Hexagon::S2_storerd_io: |
182 | 0 | // Not handling stack stores (only reg-based addresses). |
183 | 0 | if (MI->getOperand(0).isReg()) |
184 | 0 | break; |
185 | 0 | return true; |
186 | 515 | case Hexagon::L2_loadrd_pi: |
187 | 515 | case Hexagon::S2_storerd_pi: |
188 | 515 | |
189 | 515 | case Hexagon::A2_tfrpi: |
190 | 515 | case Hexagon::A2_combineii: |
191 | 515 | case Hexagon::A4_combineir: |
192 | 515 | case Hexagon::A4_combineii: |
193 | 515 | case Hexagon::A4_combineri: |
194 | 515 | case Hexagon::A2_combinew: |
195 | 515 | case Hexagon::CONST64: |
196 | 515 | |
197 | 515 | case Hexagon::A2_sxtw: |
198 | 515 | |
199 | 515 | case Hexagon::A2_andp: |
200 | 515 | case Hexagon::A2_orp: |
201 | 515 | case Hexagon::A2_xorp: |
202 | 515 | case Hexagon::S2_asl_i_p_or: |
203 | 515 | case Hexagon::S2_asl_i_p: |
204 | 515 | case Hexagon::S2_asr_i_p: |
205 | 515 | case Hexagon::S2_lsr_i_p: |
206 | 515 | break; |
207 | 1.02k | } |
208 | 1.02k | |
209 | 1.02k | for (auto &Op : MI->operands()) 1.02k { |
210 | 2.73k | if (!Op.isReg()) |
211 | 597 | continue; |
212 | 2.14k | unsigned R = Op.getReg(); |
213 | 2.14k | if (!TargetRegisterInfo::isVirtualRegister(R)) |
214 | 165 | return true; |
215 | 863 | } |
216 | 863 | return false; |
217 | 863 | } |
218 | | |
219 | 839 | void HexagonSplitDoubleRegs::partitionRegisters(UUSetMap &P2Rs) { |
220 | 839 | using UUMap = std::map<unsigned, unsigned>; |
221 | 839 | using UVect = std::vector<unsigned>; |
222 | 839 | |
223 | 839 | unsigned NumRegs = MRI->getNumVirtRegs(); |
224 | 839 | BitVector DoubleRegs(NumRegs); |
225 | 9.76k | for (unsigned i = 0; i < NumRegs9.76k ; ++i8.92k ) { |
226 | 8.92k | unsigned R = TargetRegisterInfo::index2VirtReg(i); |
227 | 8.92k | if (MRI->getRegClass(R) == DoubleRC) |
228 | 822 | DoubleRegs.set(i); |
229 | 8.92k | } |
230 | 839 | |
231 | 839 | BitVector FixedRegs(NumRegs); |
232 | 1.66k | for (int x = DoubleRegs.find_first(); x >= 01.66k ; x = DoubleRegs.find_next(x)822 ) { |
233 | 822 | unsigned R = TargetRegisterInfo::index2VirtReg(x); |
234 | 822 | MachineInstr *DefI = MRI->getVRegDef(R); |
235 | 822 | // In some cases a register may exist, but never be defined or used. |
236 | 822 | // It should never appear anywhere, but mark it as "fixed", just to be |
237 | 822 | // safe. |
238 | 822 | if (!DefI || 822 isFixedInstr(DefI)819 ) |
239 | 538 | FixedRegs.set(x); |
240 | 822 | } |
241 | 839 | |
242 | 839 | UUSetMap AssocMap; |
243 | 1.66k | for (int x = DoubleRegs.find_first(); x >= 01.66k ; x = DoubleRegs.find_next(x)822 ) { |
244 | 822 | if (FixedRegs[x]) |
245 | 538 | continue; |
246 | 284 | unsigned R = TargetRegisterInfo::index2VirtReg(x); |
247 | 284 | DEBUG(dbgs() << PrintReg(R, TRI) << " ~~"); |
248 | 284 | USet &Asc = AssocMap[R]; |
249 | 284 | for (auto U = MRI->use_nodbg_begin(R), Z = MRI->use_nodbg_end(); |
250 | 641 | U != Z641 ; ++U357 ) { |
251 | 357 | MachineOperand &Op = *U; |
252 | 357 | MachineInstr *UseI = Op.getParent(); |
253 | 357 | if (isFixedInstr(UseI)) |
254 | 212 | continue; |
255 | 580 | for (unsigned i = 0, n = UseI->getNumOperands(); 145 i < n580 ; ++i435 ) { |
256 | 435 | MachineOperand &MO = UseI->getOperand(i); |
257 | 435 | // Skip non-registers or registers with subregisters. |
258 | 435 | if (&MO == &Op || 435 !MO.isReg()290 || MO.getSubReg()204 ) |
259 | 231 | continue; |
260 | 204 | unsigned T = MO.getReg(); |
261 | 204 | if (!TargetRegisterInfo::isVirtualRegister(T)204 ) { |
262 | 0 | FixedRegs.set(x); |
263 | 0 | continue; |
264 | 0 | } |
265 | 204 | if (204 MRI->getRegClass(T) != DoubleRC204 ) |
266 | 89 | continue; |
267 | 115 | unsigned u = TargetRegisterInfo::virtReg2Index(T); |
268 | 115 | if (FixedRegs[u]) |
269 | 25 | continue; |
270 | 90 | DEBUG90 (dbgs() << ' ' << PrintReg(T, TRI)); |
271 | 90 | Asc.insert(T); |
272 | 90 | // Make it symmetric. |
273 | 90 | AssocMap[T].insert(R); |
274 | 90 | } |
275 | 357 | } |
276 | 284 | DEBUG(dbgs() << '\n'); |
277 | 822 | } |
278 | 839 | |
279 | 839 | UUMap R2P; |
280 | 839 | unsigned NextP = 1; |
281 | 839 | USet Visited; |
282 | 1.66k | for (int x = DoubleRegs.find_first(); x >= 01.66k ; x = DoubleRegs.find_next(x)822 ) { |
283 | 822 | unsigned R = TargetRegisterInfo::index2VirtReg(x); |
284 | 822 | if (Visited.count(R)) |
285 | 54 | continue; |
286 | 768 | // Create a new partition for R. |
287 | 768 | unsigned ThisP = FixedRegs[x] ? 768 0538 : NextP++230 ; |
288 | 768 | UVect WorkQ; |
289 | 768 | WorkQ.push_back(R); |
290 | 1.67k | for (unsigned i = 0; i < WorkQ.size()1.67k ; ++i910 ) { |
291 | 910 | unsigned T = WorkQ[i]; |
292 | 910 | if (Visited.count(T)) |
293 | 88 | continue; |
294 | 822 | R2P[T] = ThisP; |
295 | 822 | Visited.insert(T); |
296 | 822 | // Add all registers associated with T. |
297 | 822 | USet &Asc = AssocMap[T]; |
298 | 964 | for (USet::iterator J = Asc.begin(), F = Asc.end(); J != F964 ; ++J142 ) |
299 | 142 | WorkQ.push_back(*J); |
300 | 910 | } |
301 | 822 | } |
302 | 839 | |
303 | 839 | for (auto I : R2P) |
304 | 822 | P2Rs[I.second].insert(I.first); |
305 | 839 | } |
306 | | |
307 | 89 | static inline int32_t profitImm(unsigned Lo, unsigned Hi) { |
308 | 89 | int32_t P = 0; |
309 | 89 | bool LoZ1 = false, HiZ1 = false; |
310 | 89 | if (Lo == 0 || 89 Lo == 0xFFFFFFFF55 ) |
311 | 37 | P += 10, LoZ1 = true; |
312 | 89 | if (Hi == 0 || 89 Hi == 0xFFFFFFFF42 ) |
313 | 52 | P += 10, HiZ1 = true; |
314 | 89 | if (!LoZ1 && 89 !HiZ152 && Lo == Hi31 ) |
315 | 5 | P += 3; |
316 | 89 | return P; |
317 | 89 | } |
318 | | |
319 | 429 | int32_t HexagonSplitDoubleRegs::profit(const MachineInstr *MI) const { |
320 | 429 | unsigned ImmX = 0; |
321 | 429 | unsigned Opc = MI->getOpcode(); |
322 | 429 | switch (Opc) { |
323 | 51 | case TargetOpcode::PHI: |
324 | 51 | for (const auto &Op : MI->operands()) |
325 | 51 | if (51 !Op.getSubReg()51 ) |
326 | 51 | return 0; |
327 | 0 | return 10; |
328 | 91 | case TargetOpcode::COPY: |
329 | 91 | if (MI->getOperand(1).getSubReg() != 0) |
330 | 89 | return 10; |
331 | 2 | return 0; |
332 | 2 | |
333 | 0 | case Hexagon::L2_loadrd_io: |
334 | 0 | case Hexagon::S2_storerd_io: |
335 | 0 | return -1; |
336 | 0 | case Hexagon::L2_loadrd_pi: |
337 | 0 | case Hexagon::S2_storerd_pi: |
338 | 0 | return 2; |
339 | 0 |
|
340 | 69 | case Hexagon::A2_tfrpi: |
341 | 69 | case Hexagon::CONST64: { |
342 | 69 | uint64_t D = MI->getOperand(1).getImm(); |
343 | 69 | unsigned Lo = D & 0xFFFFFFFFULL; |
344 | 69 | unsigned Hi = D >> 32; |
345 | 69 | return profitImm(Lo, Hi); |
346 | 69 | } |
347 | 20 | case Hexagon::A2_combineii: |
348 | 20 | case Hexagon::A4_combineii: |
349 | 20 | return profitImm(MI->getOperand(1).getImm(), |
350 | 20 | MI->getOperand(2).getImm()); |
351 | 0 | case Hexagon::A4_combineri: |
352 | 0 | ImmX++; |
353 | 0 | // Fall through into A4_combineir. |
354 | 0 | LLVM_FALLTHROUGH; |
355 | 51 | case Hexagon::A4_combineir: { |
356 | 51 | ImmX++; |
357 | 51 | int64_t V = MI->getOperand(ImmX).getImm(); |
358 | 51 | if (V == 0 || 51 V == -11 ) |
359 | 50 | return 10; |
360 | 1 | // Fall through into A2_combinew. |
361 | 1 | LLVM_FALLTHROUGH1 ; |
362 | 1 | } |
363 | 50 | case Hexagon::A2_combinew: |
364 | 50 | return 2; |
365 | 1 | |
366 | 5 | case Hexagon::A2_sxtw: |
367 | 5 | return 3; |
368 | 1 | |
369 | 31 | case Hexagon::A2_andp: |
370 | 31 | case Hexagon::A2_orp: |
371 | 31 | case Hexagon::A2_xorp: |
372 | 31 | return 1; |
373 | 31 | |
374 | 6 | case Hexagon::S2_asl_i_p_or: { |
375 | 6 | unsigned S = MI->getOperand(3).getImm(); |
376 | 6 | if (S == 0 || 6 S == 326 ) |
377 | 0 | return 10; |
378 | 6 | return -1; |
379 | 6 | } |
380 | 56 | case Hexagon::S2_asl_i_p: |
381 | 56 | case Hexagon::S2_asr_i_p: |
382 | 56 | case Hexagon::S2_lsr_i_p: |
383 | 56 | unsigned S = MI->getOperand(2).getImm(); |
384 | 56 | if (S == 0 || 56 S == 3256 ) |
385 | 25 | return 10; |
386 | 31 | if (31 S == 1631 ) |
387 | 6 | return 5; |
388 | 25 | if (25 S == 4825 ) |
389 | 6 | return 7; |
390 | 19 | return -10; |
391 | 0 | } |
392 | 0 |
|
393 | 0 | return 0; |
394 | 0 | } |
395 | | |
396 | | bool HexagonSplitDoubleRegs::isProfitable(const USet &Part, LoopRegMap &IRM) |
397 | 230 | const { |
398 | 230 | unsigned FixedNum = 0, LoopPhiNum = 0; |
399 | 230 | int32_t TotalP = 0; |
400 | 230 | |
401 | 284 | for (unsigned DR : Part) { |
402 | 284 | MachineInstr *DefI = MRI->getVRegDef(DR); |
403 | 284 | int32_t P = profit(DefI); |
404 | 284 | if (P == std::numeric_limits<int>::min()) |
405 | 0 | return false; |
406 | 284 | TotalP += P; |
407 | 284 | // Reduce the profitability of splitting induction registers. |
408 | 284 | if (isInduction(DR, IRM)) |
409 | 0 | TotalP -= 30; |
410 | 284 | |
411 | 284 | for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end(); |
412 | 641 | U != W641 ; ++U357 ) { |
413 | 357 | MachineInstr *UseI = U->getParent(); |
414 | 357 | if (isFixedInstr(UseI)357 ) { |
415 | 212 | FixedNum++; |
416 | 212 | // Calculate the cost of generating REG_SEQUENCE instructions. |
417 | 714 | for (auto &Op : UseI->operands()) { |
418 | 714 | if (Op.isReg() && 714 Part.count(Op.getReg())602 ) |
419 | 212 | if (212 Op.getSubReg()212 ) |
420 | 0 | TotalP -= 2; |
421 | 714 | } |
422 | 212 | continue; |
423 | 212 | } |
424 | 145 | // If a register from this partition is used in a fixed instruction, |
425 | 145 | // and there is also a register in this partition that is used in |
426 | 145 | // a loop phi node, then decrease the splitting profit as this can |
427 | 145 | // confuse the modulo scheduler. |
428 | 145 | if (145 UseI->isPHI()145 ) { |
429 | 23 | const MachineBasicBlock *PB = UseI->getParent(); |
430 | 23 | const MachineLoop *L = MLI->getLoopFor(PB); |
431 | 23 | if (L && 23 L->getHeader() == PB13 ) |
432 | 11 | LoopPhiNum++; |
433 | 23 | } |
434 | 145 | // Splittable instruction. |
435 | 145 | int32_t P = profit(UseI); |
436 | 145 | if (P == std::numeric_limits<int>::min()) |
437 | 0 | return false; |
438 | 145 | TotalP += P; |
439 | 145 | } |
440 | 284 | } |
441 | 230 | |
442 | 230 | if (230 FixedNum > 0 && 230 LoopPhiNum > 0175 ) |
443 | 9 | TotalP -= 20*LoopPhiNum; |
444 | 230 | |
445 | 230 | DEBUG(dbgs() << "Partition profit: " << TotalP << '\n'); |
446 | 230 | return TotalP > 0; |
447 | 230 | } |
448 | | |
449 | | void HexagonSplitDoubleRegs::collectIndRegsForLoop(const MachineLoop *L, |
450 | 206 | USet &Rs) { |
451 | 206 | const MachineBasicBlock *HB = L->getHeader(); |
452 | 206 | const MachineBasicBlock *LB = L->getLoopLatch(); |
453 | 206 | if (!HB || 206 !LB206 ) |
454 | 2 | return; |
455 | 204 | |
456 | 204 | // Examine the latch branch. Expect it to be a conditional branch to |
457 | 204 | // the header (either "br-cond header" or "br-cond exit; br header"). |
458 | 204 | MachineBasicBlock *TB = nullptr, *FB = nullptr; |
459 | 204 | MachineBasicBlock *TmpLB = const_cast<MachineBasicBlock*>(LB); |
460 | 204 | SmallVector<MachineOperand,2> Cond; |
461 | 204 | bool BadLB = TII->analyzeBranch(*TmpLB, TB, FB, Cond, false); |
462 | 204 | // Only analyzable conditional branches. HII::analyzeBranch will put |
463 | 204 | // the branch opcode as the first element of Cond, and the predicate |
464 | 204 | // operand as the second. |
465 | 204 | if (BadLB || 204 Cond.size() != 2204 ) |
466 | 23 | return; |
467 | 181 | // Only simple jump-conditional (with or without negation). |
468 | 181 | if (181 !TII->PredOpcodeHasJMP_c(Cond[0].getImm())181 ) |
469 | 0 | return; |
470 | 181 | // Must go to the header. |
471 | 181 | if (181 TB != HB && 181 FB != HB15 ) |
472 | 0 | return; |
473 | 181 | assert(Cond[1].isReg() && "Unexpected Cond vector from analyzeBranch"); |
474 | 181 | // Expect a predicate register. |
475 | 181 | unsigned PR = Cond[1].getReg(); |
476 | 181 | assert(MRI->getRegClass(PR) == &Hexagon::PredRegsRegClass); |
477 | 181 | |
478 | 181 | // Get the registers on which the loop controlling compare instruction |
479 | 181 | // depends. |
480 | 181 | unsigned CmpR1 = 0, CmpR2 = 0; |
481 | 181 | const MachineInstr *CmpI = MRI->getVRegDef(PR); |
482 | 199 | while (CmpI->getOpcode() == Hexagon::C2_not) |
483 | 18 | CmpI = MRI->getVRegDef(CmpI->getOperand(1).getReg()); |
484 | 181 | |
485 | 181 | int Mask = 0, Val = 0; |
486 | 181 | bool OkCI = TII->analyzeCompare(*CmpI, CmpR1, CmpR2, Mask, Val); |
487 | 181 | if (!OkCI) |
488 | 2 | return; |
489 | 179 | // Eliminate non-double input registers. |
490 | 179 | if (179 CmpR1 && 179 MRI->getRegClass(CmpR1) != DoubleRC179 ) |
491 | 179 | CmpR1 = 0; |
492 | 179 | if (CmpR2 && 179 MRI->getRegClass(CmpR2) != DoubleRC59 ) |
493 | 59 | CmpR2 = 0; |
494 | 179 | if (!CmpR1 && 179 !CmpR2179 ) |
495 | 179 | return; |
496 | 0 |
|
497 | 0 | // Now examine the top of the loop: the phi nodes that could poten- |
498 | 0 | // tially define loop induction registers. The registers defined by |
499 | 0 | // such a phi node would be used in a 64-bit add, which then would |
500 | 0 | // be used in the loop compare instruction. |
501 | 0 |
|
502 | 0 | // Get the set of all double registers defined by phi nodes in the |
503 | 0 | // loop header. |
504 | 0 | using UVect = std::vector<unsigned>; |
505 | 0 |
|
506 | 0 | UVect DP; |
507 | 0 | for (auto &MI : *HB) { |
508 | 0 | if (!MI.isPHI()) |
509 | 0 | break; |
510 | 0 | const MachineOperand &MD = MI.getOperand(0); |
511 | 0 | unsigned R = MD.getReg(); |
512 | 0 | if (MRI->getRegClass(R) == DoubleRC) |
513 | 0 | DP.push_back(R); |
514 | 0 | } |
515 | 0 | if (DP.empty()) |
516 | 0 | return; |
517 | 0 |
|
518 | 0 | auto NoIndOp = [this, CmpR1, CmpR2] (unsigned R) -> bool 0 { |
519 | 0 | for (auto I = MRI->use_nodbg_begin(R), E = MRI->use_nodbg_end(); |
520 | 0 | I != E0 ; ++I0 ) { |
521 | 0 | const MachineInstr *UseI = I->getParent(); |
522 | 0 | if (UseI->getOpcode() != Hexagon::A2_addp) |
523 | 0 | continue; |
524 | 0 | // Get the output from the add. If it is one of the inputs to the |
525 | 0 | // loop-controlling compare instruction, then R is likely an induc- |
526 | 0 | // tion register. |
527 | 0 | unsigned T = UseI->getOperand(0).getReg(); |
528 | 0 | if (T == CmpR1 || 0 T == CmpR20 ) |
529 | 0 | return false; |
530 | 0 | } |
531 | 0 | return true; |
532 | 0 | }; |
533 | 0 | UVect::iterator End = llvm::remove_if(DP, NoIndOp); |
534 | 0 | Rs.insert(DP.begin(), End); |
535 | 0 | Rs.insert(CmpR1); |
536 | 0 | Rs.insert(CmpR2); |
537 | 0 |
|
538 | 0 | DEBUG({ |
539 | 206 | dbgs() << "For loop at BB#" << HB->getNumber() << " ind regs: "; |
540 | 206 | dump_partition(dbgs(), Rs, *TRI); |
541 | 206 | dbgs() << '\n'; |
542 | 206 | }); |
543 | 206 | } |
544 | | |
545 | 839 | void HexagonSplitDoubleRegs::collectIndRegs(LoopRegMap &IRM) { |
546 | 839 | using LoopVector = std::vector<MachineLoop *>; |
547 | 839 | |
548 | 839 | LoopVector WorkQ; |
549 | 839 | |
550 | 839 | for (auto I : *MLI) |
551 | 189 | WorkQ.push_back(I); |
552 | 1.04k | for (unsigned i = 0; i < WorkQ.size()1.04k ; ++i206 ) { |
553 | 206 | for (auto I : *WorkQ[i]) |
554 | 17 | WorkQ.push_back(I); |
555 | 206 | } |
556 | 839 | |
557 | 839 | USet Rs; |
558 | 1.04k | for (unsigned i = 0, n = WorkQ.size(); i < n1.04k ; ++i206 ) { |
559 | 206 | MachineLoop *L = WorkQ[i]; |
560 | 206 | Rs.clear(); |
561 | 206 | collectIndRegsForLoop(L, Rs); |
562 | 206 | if (!Rs.empty()) |
563 | 0 | IRM.insert(std::make_pair(L, Rs)); |
564 | 206 | } |
565 | 839 | } |
566 | | |
567 | | void HexagonSplitDoubleRegs::createHalfInstr(unsigned Opc, MachineInstr *MI, |
568 | 54 | const UUPairMap &PairMap, unsigned SubR) { |
569 | 54 | MachineBasicBlock &B = *MI->getParent(); |
570 | 54 | DebugLoc DL = MI->getDebugLoc(); |
571 | 54 | MachineInstr *NewI = BuildMI(B, MI, DL, TII->get(Opc)); |
572 | 54 | |
573 | 210 | for (auto &Op : MI->operands()) { |
574 | 210 | if (!Op.isReg()210 ) { |
575 | 44 | NewI->addOperand(Op); |
576 | 44 | continue; |
577 | 44 | } |
578 | 166 | // For register operands, set the subregister. |
579 | 166 | unsigned R = Op.getReg(); |
580 | 166 | unsigned SR = Op.getSubReg(); |
581 | 166 | bool isVirtReg = TargetRegisterInfo::isVirtualRegister(R); |
582 | 166 | bool isKill = Op.isKill(); |
583 | 166 | if (isVirtReg && 166 MRI->getRegClass(R) == DoubleRC166 ) { |
584 | 166 | isKill = false; |
585 | 166 | UUPairMap::const_iterator F = PairMap.find(R); |
586 | 166 | if (F == PairMap.end()166 ) { |
587 | 74 | SR = SubR; |
588 | 166 | } else { |
589 | 92 | const UUPair &P = F->second; |
590 | 92 | R = (SubR == Hexagon::isub_lo) ? P.first46 : P.second46 ; |
591 | 92 | SR = 0; |
592 | 92 | } |
593 | 166 | } |
594 | 210 | auto CO = MachineOperand::CreateReg(R, Op.isDef(), Op.isImplicit(), isKill, |
595 | 210 | Op.isDead(), Op.isUndef(), Op.isEarlyClobber(), SR, Op.isDebug(), |
596 | 210 | Op.isInternalRead()); |
597 | 210 | NewI->addOperand(CO); |
598 | 210 | } |
599 | 54 | } |
600 | | |
601 | | void HexagonSplitDoubleRegs::splitMemRef(MachineInstr *MI, |
602 | 0 | const UUPairMap &PairMap) { |
603 | 0 | bool Load = MI->mayLoad(); |
604 | 0 | unsigned OrigOpc = MI->getOpcode(); |
605 | 0 | bool PostInc = (OrigOpc == Hexagon::L2_loadrd_pi || |
606 | 0 | OrigOpc == Hexagon::S2_storerd_pi); |
607 | 0 | MachineInstr *LowI, *HighI; |
608 | 0 | MachineBasicBlock &B = *MI->getParent(); |
609 | 0 | DebugLoc DL = MI->getDebugLoc(); |
610 | 0 |
|
611 | 0 | // Index of the base-address-register operand. |
612 | 0 | unsigned AdrX = PostInc ? (Load ? 0 20 : 10 ) |
613 | 0 | : (Load ? 0 10 : 00 ); |
614 | 0 | MachineOperand &AdrOp = MI->getOperand(AdrX); |
615 | 0 | unsigned RSA = getRegState(AdrOp); |
616 | 0 | MachineOperand &ValOp = Load ? MI->getOperand(0) |
617 | 0 | : (PostInc ? 0 MI->getOperand(3)0 |
618 | 0 | : MI->getOperand(2)); |
619 | 0 | UUPairMap::const_iterator F = PairMap.find(ValOp.getReg()); |
620 | 0 | assert(F != PairMap.end()); |
621 | 0 |
|
622 | 0 | if (Load0 ) { |
623 | 0 | const UUPair &P = F->second; |
624 | 0 | int64_t Off = PostInc ? 00 : MI->getOperand(2).getImm()0 ; |
625 | 0 | LowI = BuildMI(B, MI, DL, TII->get(Hexagon::L2_loadri_io), P.first) |
626 | 0 | .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg()) |
627 | 0 | .addImm(Off); |
628 | 0 | HighI = BuildMI(B, MI, DL, TII->get(Hexagon::L2_loadri_io), P.second) |
629 | 0 | .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg()) |
630 | 0 | .addImm(Off+4); |
631 | 0 | } else { |
632 | 0 | const UUPair &P = F->second; |
633 | 0 | int64_t Off = PostInc ? 00 : MI->getOperand(1).getImm()0 ; |
634 | 0 | LowI = BuildMI(B, MI, DL, TII->get(Hexagon::S2_storeri_io)) |
635 | 0 | .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg()) |
636 | 0 | .addImm(Off) |
637 | 0 | .addReg(P.first); |
638 | 0 | HighI = BuildMI(B, MI, DL, TII->get(Hexagon::S2_storeri_io)) |
639 | 0 | .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg()) |
640 | 0 | .addImm(Off+4) |
641 | 0 | .addReg(P.second); |
642 | 0 | } |
643 | 0 |
|
644 | 0 | if (PostInc0 ) { |
645 | 0 | // Create the increment of the address register. |
646 | 0 | int64_t Inc = Load ? MI->getOperand(3).getImm() |
647 | 0 | : MI->getOperand(2).getImm(); |
648 | 0 | MachineOperand &UpdOp = Load ? MI->getOperand(1)0 : MI->getOperand(0)0 ; |
649 | 0 | const TargetRegisterClass *RC = MRI->getRegClass(UpdOp.getReg()); |
650 | 0 | unsigned NewR = MRI->createVirtualRegister(RC); |
651 | 0 | assert(!UpdOp.getSubReg() && "Def operand with subreg"); |
652 | 0 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_addi), NewR) |
653 | 0 | .addReg(AdrOp.getReg(), RSA) |
654 | 0 | .addImm(Inc); |
655 | 0 | MRI->replaceRegWith(UpdOp.getReg(), NewR); |
656 | 0 | // The original instruction will be deleted later. |
657 | 0 | } |
658 | 0 |
|
659 | 0 | // Generate a new pair of memory-operands. |
660 | 0 | MachineFunction &MF = *B.getParent(); |
661 | 0 | for (auto &MO : MI->memoperands()) { |
662 | 0 | const MachinePointerInfo &Ptr = MO->getPointerInfo(); |
663 | 0 | MachineMemOperand::Flags F = MO->getFlags(); |
664 | 0 | int A = MO->getAlignment(); |
665 | 0 |
|
666 | 0 | auto *Tmp1 = MF.getMachineMemOperand(Ptr, F, 4/*size*/, A); |
667 | 0 | LowI->addMemOperand(MF, Tmp1); |
668 | 0 | auto *Tmp2 = MF.getMachineMemOperand(Ptr, F, 4/*size*/, std::min(A, 4)); |
669 | 0 | HighI->addMemOperand(MF, Tmp2); |
670 | 0 | } |
671 | 0 | } |
672 | | |
673 | | void HexagonSplitDoubleRegs::splitImmediate(MachineInstr *MI, |
674 | 50 | const UUPairMap &PairMap) { |
675 | 50 | MachineOperand &Op0 = MI->getOperand(0); |
676 | 50 | MachineOperand &Op1 = MI->getOperand(1); |
677 | 50 | assert(Op0.isReg() && Op1.isImm()); |
678 | 50 | uint64_t V = Op1.getImm(); |
679 | 50 | |
680 | 50 | MachineBasicBlock &B = *MI->getParent(); |
681 | 50 | DebugLoc DL = MI->getDebugLoc(); |
682 | 50 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
683 | 50 | assert(F != PairMap.end()); |
684 | 50 | const UUPair &P = F->second; |
685 | 50 | |
686 | 50 | // The operand to A2_tfrsi can only have 32 significant bits. Immediate |
687 | 50 | // values in MachineOperand are stored as 64-bit integers, and so the |
688 | 50 | // value -1 may be represented either as 64-bit -1, or 4294967295. Both |
689 | 50 | // will have the 32 higher bits truncated in the end, but -1 will remain |
690 | 50 | // as -1, while the latter may appear to be a large unsigned value |
691 | 50 | // requiring a constant extender. The casting to int32_t will select the |
692 | 50 | // former representation. (The same reasoning applies to all 32-bit |
693 | 50 | // values.) |
694 | 50 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.first) |
695 | 50 | .addImm(int32_t(V & 0xFFFFFFFFULL)); |
696 | 50 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.second) |
697 | 50 | .addImm(int32_t(V >> 32)); |
698 | 50 | } |
699 | | |
700 | | void HexagonSplitDoubleRegs::splitCombine(MachineInstr *MI, |
701 | 95 | const UUPairMap &PairMap) { |
702 | 95 | MachineOperand &Op0 = MI->getOperand(0); |
703 | 95 | MachineOperand &Op1 = MI->getOperand(1); |
704 | 95 | MachineOperand &Op2 = MI->getOperand(2); |
705 | 95 | assert(Op0.isReg()); |
706 | 95 | |
707 | 95 | MachineBasicBlock &B = *MI->getParent(); |
708 | 95 | DebugLoc DL = MI->getDebugLoc(); |
709 | 95 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
710 | 95 | assert(F != PairMap.end()); |
711 | 95 | const UUPair &P = F->second; |
712 | 95 | |
713 | 95 | if (Op1.isImm()95 ) { |
714 | 56 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.second) |
715 | 56 | .addImm(Op1.getImm()); |
716 | 95 | } else if (39 Op1.isReg()39 ) { |
717 | 39 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.second) |
718 | 39 | .addReg(Op1.getReg(), getRegState(Op1), Op1.getSubReg()); |
719 | 39 | } else |
720 | 0 | llvm_unreachable("Unexpected operand"); |
721 | 95 | |
722 | 95 | if (95 Op2.isImm()95 ) { |
723 | 5 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.first) |
724 | 5 | .addImm(Op2.getImm()); |
725 | 95 | } else if (90 Op2.isReg()90 ) { |
726 | 90 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.first) |
727 | 90 | .addReg(Op2.getReg(), getRegState(Op2), Op2.getSubReg()); |
728 | 90 | } else |
729 | 0 | llvm_unreachable("Unexpected operand"); |
730 | 95 | } |
731 | | |
732 | | void HexagonSplitDoubleRegs::splitExt(MachineInstr *MI, |
733 | 5 | const UUPairMap &PairMap) { |
734 | 5 | MachineOperand &Op0 = MI->getOperand(0); |
735 | 5 | MachineOperand &Op1 = MI->getOperand(1); |
736 | 5 | assert(Op0.isReg() && Op1.isReg()); |
737 | 5 | |
738 | 5 | MachineBasicBlock &B = *MI->getParent(); |
739 | 5 | DebugLoc DL = MI->getDebugLoc(); |
740 | 5 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
741 | 5 | assert(F != PairMap.end()); |
742 | 5 | const UUPair &P = F->second; |
743 | 5 | unsigned RS = getRegState(Op1); |
744 | 5 | |
745 | 5 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.first) |
746 | 5 | .addReg(Op1.getReg(), RS & ~RegState::Kill, Op1.getSubReg()); |
747 | 5 | BuildMI(B, MI, DL, TII->get(Hexagon::S2_asr_i_r), P.second) |
748 | 5 | .addReg(Op1.getReg(), RS, Op1.getSubReg()) |
749 | 5 | .addImm(31); |
750 | 5 | } |
751 | | |
752 | | void HexagonSplitDoubleRegs::splitShift(MachineInstr *MI, |
753 | 29 | const UUPairMap &PairMap) { |
754 | 29 | using namespace Hexagon; |
755 | 29 | |
756 | 29 | MachineOperand &Op0 = MI->getOperand(0); |
757 | 29 | MachineOperand &Op1 = MI->getOperand(1); |
758 | 29 | MachineOperand &Op2 = MI->getOperand(2); |
759 | 29 | assert(Op0.isReg() && Op1.isReg() && Op2.isImm()); |
760 | 29 | int64_t Sh64 = Op2.getImm(); |
761 | 29 | assert(Sh64 >= 0 && Sh64 < 64); |
762 | 29 | unsigned S = Sh64; |
763 | 29 | |
764 | 29 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
765 | 29 | assert(F != PairMap.end()); |
766 | 29 | const UUPair &P = F->second; |
767 | 29 | unsigned LoR = P.first; |
768 | 29 | unsigned HiR = P.second; |
769 | 29 | |
770 | 29 | unsigned Opc = MI->getOpcode(); |
771 | 2 | bool Right = (Opc == S2_lsr_i_p || Opc == S2_asr_i_p); |
772 | 29 | bool Left = !Right; |
773 | 29 | bool Signed = (Opc == S2_asr_i_p); |
774 | 29 | |
775 | 29 | MachineBasicBlock &B = *MI->getParent(); |
776 | 29 | DebugLoc DL = MI->getDebugLoc(); |
777 | 29 | unsigned RS = getRegState(Op1); |
778 | 1 | unsigned ShiftOpc = Left ? S2_asl_i_r |
779 | 28 | : (Signed ? 28 S2_asr_i_r1 : S2_lsr_i_r27 ); |
780 | 29 | unsigned LoSR = isub_lo; |
781 | 29 | unsigned HiSR = isub_hi; |
782 | 29 | |
783 | 29 | if (S == 029 ) { |
784 | 0 | // No shift, subregister copy. |
785 | 0 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR) |
786 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR); |
787 | 0 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), HiR) |
788 | 0 | .addReg(Op1.getReg(), RS, HiSR); |
789 | 29 | } else if (29 S < 3229 ) { |
790 | 4 | const TargetRegisterClass *IntRC = &IntRegsRegClass; |
791 | 4 | unsigned TmpR = MRI->createVirtualRegister(IntRC); |
792 | 4 | // Expansion: |
793 | 4 | // Shift left: DR = shl R, #s |
794 | 4 | // LoR = shl R.lo, #s |
795 | 4 | // TmpR = extractu R.lo, #s, #32-s |
796 | 4 | // HiR = or (TmpR, asl(R.hi, #s)) |
797 | 4 | // Shift right: DR = shr R, #s |
798 | 4 | // HiR = shr R.hi, #s |
799 | 4 | // TmpR = shr R.lo, #s |
800 | 4 | // LoR = insert TmpR, R.hi, #s, #32-s |
801 | 4 | |
802 | 4 | // Shift left: |
803 | 4 | // LoR = shl R.lo, #s |
804 | 4 | // Shift right: |
805 | 4 | // TmpR = shr R.lo, #s |
806 | 4 | |
807 | 4 | // Make a special case for A2_aslh and A2_asrh (they are predicable as |
808 | 4 | // opposed to S2_asl_i_r/S2_asr_i_r). |
809 | 4 | if (S == 16 && 4 Left4 ) |
810 | 0 | BuildMI(B, MI, DL, TII->get(A2_aslh), LoR) |
811 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR); |
812 | 4 | else if (4 S == 16 && 4 Signed4 ) |
813 | 0 | BuildMI(B, MI, DL, TII->get(A2_asrh), TmpR) |
814 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR); |
815 | 4 | else |
816 | 4 | BuildMI(B, MI, DL, TII->get(ShiftOpc), (Left ? 4 LoR0 : TmpR4 )) |
817 | 4 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR) |
818 | 4 | .addImm(S); |
819 | 4 | |
820 | 4 | if (Left4 ) { |
821 | 0 | // TmpR = extractu R.lo, #s, #32-s |
822 | 0 | BuildMI(B, MI, DL, TII->get(S2_extractu), TmpR) |
823 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR) |
824 | 0 | .addImm(S) |
825 | 0 | .addImm(32-S); |
826 | 0 | // HiR = or (TmpR, asl(R.hi, #s)) |
827 | 0 | BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR) |
828 | 0 | .addReg(TmpR) |
829 | 0 | .addReg(Op1.getReg(), RS, HiSR) |
830 | 0 | .addImm(S); |
831 | 4 | } else { |
832 | 4 | // HiR = shr R.hi, #s |
833 | 4 | BuildMI(B, MI, DL, TII->get(ShiftOpc), HiR) |
834 | 4 | .addReg(Op1.getReg(), RS & ~RegState::Kill, HiSR) |
835 | 4 | .addImm(S); |
836 | 4 | // LoR = insert TmpR, R.hi, #s, #32-s |
837 | 4 | BuildMI(B, MI, DL, TII->get(S2_insert), LoR) |
838 | 4 | .addReg(TmpR) |
839 | 4 | .addReg(Op1.getReg(), RS, HiSR) |
840 | 4 | .addImm(S) |
841 | 4 | .addImm(32-S); |
842 | 4 | } |
843 | 29 | } else if (25 S == 3225 ) { |
844 | 20 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), (Left ? HiR1 : LoR19 )) |
845 | 20 | .addReg(Op1.getReg(), RS & ~RegState::Kill, (Left ? LoSR1 : HiSR19 )); |
846 | 20 | if (!Signed) |
847 | 19 | BuildMI(B, MI, DL, TII->get(A2_tfrsi), (Left ? 19 LoR1 : HiR18 )) |
848 | 19 | .addImm(0); |
849 | 20 | else // Must be right shift. |
850 | 1 | BuildMI(B, MI, DL, TII->get(S2_asr_i_r), HiR) |
851 | 1 | .addReg(Op1.getReg(), RS, HiSR) |
852 | 1 | .addImm(31); |
853 | 25 | } else if (5 S < 645 ) { |
854 | 5 | S -= 32; |
855 | 5 | if (S == 16 && 5 Left4 ) |
856 | 0 | BuildMI(B, MI, DL, TII->get(A2_aslh), HiR) |
857 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR); |
858 | 5 | else if (5 S == 16 && 5 Signed4 ) |
859 | 0 | BuildMI(B, MI, DL, TII->get(A2_asrh), LoR) |
860 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, HiSR); |
861 | 5 | else |
862 | 5 | BuildMI(B, MI, DL, TII->get(ShiftOpc), (Left ? 5 HiR0 : LoR5 )) |
863 | 5 | .addReg(Op1.getReg(), RS & ~RegState::Kill, (Left ? LoSR0 : HiSR5 )) |
864 | 5 | .addImm(S); |
865 | 5 | |
866 | 5 | if (Signed) |
867 | 0 | BuildMI(B, MI, DL, TII->get(S2_asr_i_r), HiR) |
868 | 0 | .addReg(Op1.getReg(), RS, HiSR) |
869 | 0 | .addImm(31); |
870 | 5 | else |
871 | 5 | BuildMI(B, MI, DL, TII->get(A2_tfrsi), (Left ? 5 LoR0 : HiR5 )) |
872 | 5 | .addImm(0); |
873 | 29 | } |
874 | 29 | } |
875 | | |
876 | | void HexagonSplitDoubleRegs::splitAslOr(MachineInstr *MI, |
877 | 0 | const UUPairMap &PairMap) { |
878 | 0 | using namespace Hexagon; |
879 | 0 |
|
880 | 0 | MachineOperand &Op0 = MI->getOperand(0); |
881 | 0 | MachineOperand &Op1 = MI->getOperand(1); |
882 | 0 | MachineOperand &Op2 = MI->getOperand(2); |
883 | 0 | MachineOperand &Op3 = MI->getOperand(3); |
884 | 0 | assert(Op0.isReg() && Op1.isReg() && Op2.isReg() && Op3.isImm()); |
885 | 0 | int64_t Sh64 = Op3.getImm(); |
886 | 0 | assert(Sh64 >= 0 && Sh64 < 64); |
887 | 0 | unsigned S = Sh64; |
888 | 0 |
|
889 | 0 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
890 | 0 | assert(F != PairMap.end()); |
891 | 0 | const UUPair &P = F->second; |
892 | 0 | unsigned LoR = P.first; |
893 | 0 | unsigned HiR = P.second; |
894 | 0 |
|
895 | 0 | MachineBasicBlock &B = *MI->getParent(); |
896 | 0 | DebugLoc DL = MI->getDebugLoc(); |
897 | 0 | unsigned RS1 = getRegState(Op1); |
898 | 0 | unsigned RS2 = getRegState(Op2); |
899 | 0 | const TargetRegisterClass *IntRC = &IntRegsRegClass; |
900 | 0 |
|
901 | 0 | unsigned LoSR = isub_lo; |
902 | 0 | unsigned HiSR = isub_hi; |
903 | 0 |
|
904 | 0 | // Op0 = S2_asl_i_p_or Op1, Op2, Op3 |
905 | 0 | // means: Op0 = or (Op1, asl(Op2, Op3)) |
906 | 0 |
|
907 | 0 | // Expansion of |
908 | 0 | // DR = or (R1, asl(R2, #s)) |
909 | 0 | // |
910 | 0 | // LoR = or (R1.lo, asl(R2.lo, #s)) |
911 | 0 | // Tmp1 = extractu R2.lo, #s, #32-s |
912 | 0 | // Tmp2 = or R1.hi, Tmp1 |
913 | 0 | // HiR = or (Tmp2, asl(R2.hi, #s)) |
914 | 0 |
|
915 | 0 | if (S == 00 ) { |
916 | 0 | // DR = or (R1, asl(R2, #0)) |
917 | 0 | // -> or (R1, R2) |
918 | 0 | // i.e. LoR = or R1.lo, R2.lo |
919 | 0 | // HiR = or R1.hi, R2.hi |
920 | 0 | BuildMI(B, MI, DL, TII->get(A2_or), LoR) |
921 | 0 | .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR) |
922 | 0 | .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR); |
923 | 0 | BuildMI(B, MI, DL, TII->get(A2_or), HiR) |
924 | 0 | .addReg(Op1.getReg(), RS1, HiSR) |
925 | 0 | .addReg(Op2.getReg(), RS2, HiSR); |
926 | 0 | } else if (0 S < 320 ) { |
927 | 0 | BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), LoR) |
928 | 0 | .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR) |
929 | 0 | .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR) |
930 | 0 | .addImm(S); |
931 | 0 | unsigned TmpR1 = MRI->createVirtualRegister(IntRC); |
932 | 0 | BuildMI(B, MI, DL, TII->get(S2_extractu), TmpR1) |
933 | 0 | .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR) |
934 | 0 | .addImm(S) |
935 | 0 | .addImm(32-S); |
936 | 0 | unsigned TmpR2 = MRI->createVirtualRegister(IntRC); |
937 | 0 | BuildMI(B, MI, DL, TII->get(A2_or), TmpR2) |
938 | 0 | .addReg(Op1.getReg(), RS1, HiSR) |
939 | 0 | .addReg(TmpR1); |
940 | 0 | BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR) |
941 | 0 | .addReg(TmpR2) |
942 | 0 | .addReg(Op2.getReg(), RS2, HiSR) |
943 | 0 | .addImm(S); |
944 | 0 | } else if (0 S == 320 ) { |
945 | 0 | // DR = or (R1, asl(R2, #32)) |
946 | 0 | // -> or R1, R2.lo |
947 | 0 | // LoR = R1.lo |
948 | 0 | // HiR = or R1.hi, R2.lo |
949 | 0 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR) |
950 | 0 | .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR); |
951 | 0 | BuildMI(B, MI, DL, TII->get(A2_or), HiR) |
952 | 0 | .addReg(Op1.getReg(), RS1, HiSR) |
953 | 0 | .addReg(Op2.getReg(), RS2, LoSR); |
954 | 0 | } else if (0 S < 640 ) { |
955 | 0 | // DR = or (R1, asl(R2, #s)) |
956 | 0 | // |
957 | 0 | // LoR = R1:lo |
958 | 0 | // HiR = or (R1:hi, asl(R2:lo, #s-32)) |
959 | 0 | S -= 32; |
960 | 0 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR) |
961 | 0 | .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR); |
962 | 0 | BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR) |
963 | 0 | .addReg(Op1.getReg(), RS1, HiSR) |
964 | 0 | .addReg(Op2.getReg(), RS2, LoSR) |
965 | 0 | .addImm(S); |
966 | 0 | } |
967 | 0 | } |
968 | | |
969 | | bool HexagonSplitDoubleRegs::splitInstr(MachineInstr *MI, |
970 | 289 | const UUPairMap &PairMap) { |
971 | 289 | using namespace Hexagon; |
972 | 289 | |
973 | 289 | DEBUG(dbgs() << "Splitting: " << *MI); |
974 | 289 | bool Split = false; |
975 | 289 | unsigned Opc = MI->getOpcode(); |
976 | 289 | |
977 | 289 | switch (Opc) { |
978 | 94 | case TargetOpcode::PHI: |
979 | 94 | case TargetOpcode::COPY: { |
980 | 94 | unsigned DstR = MI->getOperand(0).getReg(); |
981 | 94 | if (MRI->getRegClass(DstR) == DoubleRC94 ) { |
982 | 11 | createHalfInstr(Opc, MI, PairMap, isub_lo); |
983 | 11 | createHalfInstr(Opc, MI, PairMap, isub_hi); |
984 | 11 | Split = true; |
985 | 11 | } |
986 | 94 | break; |
987 | 94 | } |
988 | 5 | case A2_andp: |
989 | 5 | createHalfInstr(A2_and, MI, PairMap, isub_lo); |
990 | 5 | createHalfInstr(A2_and, MI, PairMap, isub_hi); |
991 | 5 | Split = true; |
992 | 5 | break; |
993 | 1 | case A2_orp: |
994 | 1 | createHalfInstr(A2_or, MI, PairMap, isub_lo); |
995 | 1 | createHalfInstr(A2_or, MI, PairMap, isub_hi); |
996 | 1 | Split = true; |
997 | 1 | break; |
998 | 10 | case A2_xorp: |
999 | 10 | createHalfInstr(A2_xor, MI, PairMap, isub_lo); |
1000 | 10 | createHalfInstr(A2_xor, MI, PairMap, isub_hi); |
1001 | 10 | Split = true; |
1002 | 10 | break; |
1003 | 94 | |
1004 | 0 | case L2_loadrd_io: |
1005 | 0 | case L2_loadrd_pi: |
1006 | 0 | case S2_storerd_io: |
1007 | 0 | case S2_storerd_pi: |
1008 | 0 | splitMemRef(MI, PairMap); |
1009 | 0 | Split = true; |
1010 | 0 | break; |
1011 | 0 |
|
1012 | 50 | case A2_tfrpi: |
1013 | 50 | case CONST64: |
1014 | 50 | splitImmediate(MI, PairMap); |
1015 | 50 | Split = true; |
1016 | 50 | break; |
1017 | 50 | |
1018 | 95 | case A2_combineii: |
1019 | 95 | case A4_combineir: |
1020 | 95 | case A4_combineii: |
1021 | 95 | case A4_combineri: |
1022 | 95 | case A2_combinew: |
1023 | 95 | splitCombine(MI, PairMap); |
1024 | 95 | Split = true; |
1025 | 95 | break; |
1026 | 95 | |
1027 | 5 | case A2_sxtw: |
1028 | 5 | splitExt(MI, PairMap); |
1029 | 5 | Split = true; |
1030 | 5 | break; |
1031 | 95 | |
1032 | 29 | case S2_asl_i_p: |
1033 | 29 | case S2_asr_i_p: |
1034 | 29 | case S2_lsr_i_p: |
1035 | 29 | splitShift(MI, PairMap); |
1036 | 29 | Split = true; |
1037 | 29 | break; |
1038 | 29 | |
1039 | 0 | case S2_asl_i_p_or: |
1040 | 0 | splitAslOr(MI, PairMap); |
1041 | 0 | Split = true; |
1042 | 0 | break; |
1043 | 29 | |
1044 | 0 | default: |
1045 | 0 | llvm_unreachable("Instruction not splitable"); |
1046 | 0 | return false; |
1047 | 289 | } |
1048 | 289 | |
1049 | 289 | return Split; |
1050 | 289 | } |
1051 | | |
1052 | | void HexagonSplitDoubleRegs::replaceSubregUses(MachineInstr *MI, |
1053 | 126 | const UUPairMap &PairMap) { |
1054 | 354 | for (auto &Op : MI->operands()) { |
1055 | 354 | if (!Op.isReg() || 354 !Op.isUse()293 || !Op.getSubReg()167 ) |
1056 | 257 | continue; |
1057 | 97 | unsigned R = Op.getReg(); |
1058 | 97 | UUPairMap::const_iterator F = PairMap.find(R); |
1059 | 97 | if (F == PairMap.end()) |
1060 | 0 | continue; |
1061 | 97 | const UUPair &P = F->second; |
1062 | 97 | switch (Op.getSubReg()) { |
1063 | 61 | case Hexagon::isub_lo: |
1064 | 61 | Op.setReg(P.first); |
1065 | 61 | break; |
1066 | 36 | case Hexagon::isub_hi: |
1067 | 36 | Op.setReg(P.second); |
1068 | 36 | break; |
1069 | 97 | } |
1070 | 97 | Op.setSubReg(0); |
1071 | 97 | } |
1072 | 126 | } |
1073 | | |
1074 | | void HexagonSplitDoubleRegs::collapseRegPairs(MachineInstr *MI, |
1075 | 159 | const UUPairMap &PairMap) { |
1076 | 159 | MachineBasicBlock &B = *MI->getParent(); |
1077 | 159 | DebugLoc DL = MI->getDebugLoc(); |
1078 | 159 | |
1079 | 534 | for (auto &Op : MI->operands()) { |
1080 | 534 | if (!Op.isReg() || 534 !Op.isUse()467 ) |
1081 | 216 | continue; |
1082 | 318 | unsigned R = Op.getReg(); |
1083 | 318 | if (!TargetRegisterInfo::isVirtualRegister(R)) |
1084 | 1 | continue; |
1085 | 317 | if (317 MRI->getRegClass(R) != DoubleRC || 317 Op.getSubReg()277 ) |
1086 | 40 | continue; |
1087 | 277 | UUPairMap::const_iterator F = PairMap.find(R); |
1088 | 277 | if (F == PairMap.end()) |
1089 | 118 | continue; |
1090 | 159 | const UUPair &Pr = F->second; |
1091 | 159 | unsigned NewDR = MRI->createVirtualRegister(DoubleRC); |
1092 | 159 | BuildMI(B, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), NewDR) |
1093 | 159 | .addReg(Pr.first) |
1094 | 159 | .addImm(Hexagon::isub_lo) |
1095 | 159 | .addReg(Pr.second) |
1096 | 159 | .addImm(Hexagon::isub_hi); |
1097 | 159 | Op.setReg(NewDR); |
1098 | 159 | } |
1099 | 159 | } |
1100 | | |
1101 | 177 | bool HexagonSplitDoubleRegs::splitPartition(const USet &Part) { |
1102 | 177 | using MISet = std::set<MachineInstr *>; |
1103 | 177 | |
1104 | 177 | const TargetRegisterClass *IntRC = &Hexagon::IntRegsRegClass; |
1105 | 177 | bool Changed = false; |
1106 | 177 | |
1107 | 177 | DEBUG(dbgs() << "Splitting partition: "; dump_partition(dbgs(), Part, *TRI); |
1108 | 177 | dbgs() << '\n'); |
1109 | 177 | |
1110 | 177 | UUPairMap PairMap; |
1111 | 177 | |
1112 | 177 | MISet SplitIns; |
1113 | 206 | for (unsigned DR : Part) { |
1114 | 206 | MachineInstr *DefI = MRI->getVRegDef(DR); |
1115 | 206 | SplitIns.insert(DefI); |
1116 | 206 | |
1117 | 206 | // Collect all instructions, including fixed ones. We won't split them, |
1118 | 206 | // but we need to visit them again to insert the REG_SEQUENCE instructions. |
1119 | 206 | for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end(); |
1120 | 477 | U != W477 ; ++U271 ) |
1121 | 271 | SplitIns.insert(U->getParent()); |
1122 | 206 | |
1123 | 206 | unsigned LoR = MRI->createVirtualRegister(IntRC); |
1124 | 206 | unsigned HiR = MRI->createVirtualRegister(IntRC); |
1125 | 206 | DEBUG(dbgs() << "Created mapping: " << PrintReg(DR, TRI) << " -> " |
1126 | 206 | << PrintReg(HiR, TRI) << ':' << PrintReg(LoR, TRI) << '\n'); |
1127 | 206 | PairMap.insert(std::make_pair(DR, UUPair(LoR, HiR))); |
1128 | 206 | } |
1129 | 177 | |
1130 | 177 | MISet Erase; |
1131 | 448 | for (auto MI : SplitIns) { |
1132 | 448 | if (isFixedInstr(MI)448 ) { |
1133 | 159 | collapseRegPairs(MI, PairMap); |
1134 | 448 | } else { |
1135 | 289 | bool Done = splitInstr(MI, PairMap); |
1136 | 289 | if (Done) |
1137 | 206 | Erase.insert(MI); |
1138 | 289 | Changed |= Done; |
1139 | 289 | } |
1140 | 448 | } |
1141 | 177 | |
1142 | 206 | for (unsigned DR : Part) { |
1143 | 206 | // Before erasing "double" instructions, revisit all uses of the double |
1144 | 206 | // registers in this partition, and replace all uses of them with subre- |
1145 | 206 | // gisters, with the corresponding single registers. |
1146 | 206 | MISet Uses; |
1147 | 206 | for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end(); |
1148 | 332 | U != W332 ; ++U126 ) |
1149 | 126 | Uses.insert(U->getParent()); |
1150 | 206 | for (auto M : Uses) |
1151 | 126 | replaceSubregUses(M, PairMap); |
1152 | 206 | } |
1153 | 177 | |
1154 | 206 | for (auto MI : Erase) { |
1155 | 206 | MachineBasicBlock *B = MI->getParent(); |
1156 | 206 | B->erase(MI); |
1157 | 206 | } |
1158 | 177 | |
1159 | 177 | return Changed; |
1160 | 177 | } |
1161 | | |
1162 | 839 | bool HexagonSplitDoubleRegs::runOnMachineFunction(MachineFunction &MF) { |
1163 | 839 | DEBUG(dbgs() << "Splitting double registers in function: " |
1164 | 839 | << MF.getName() << '\n'); |
1165 | 839 | |
1166 | 839 | if (skipFunction(*MF.getFunction())) |
1167 | 0 | return false; |
1168 | 839 | |
1169 | 839 | auto &ST = MF.getSubtarget<HexagonSubtarget>(); |
1170 | 839 | TRI = ST.getRegisterInfo(); |
1171 | 839 | TII = ST.getInstrInfo(); |
1172 | 839 | MRI = &MF.getRegInfo(); |
1173 | 839 | MLI = &getAnalysis<MachineLoopInfo>(); |
1174 | 839 | |
1175 | 839 | UUSetMap P2Rs; |
1176 | 839 | LoopRegMap IRM; |
1177 | 839 | |
1178 | 839 | collectIndRegs(IRM); |
1179 | 839 | partitionRegisters(P2Rs); |
1180 | 839 | |
1181 | 839 | DEBUG({ |
1182 | 839 | dbgs() << "Register partitioning: (partition #0 is fixed)\n"; |
1183 | 839 | for (UUSetMap::iterator I = P2Rs.begin(), E = P2Rs.end(); I != E; ++I) { |
1184 | 839 | dbgs() << '#' << I->first << " -> "; |
1185 | 839 | dump_partition(dbgs(), I->second, *TRI); |
1186 | 839 | dbgs() << '\n'; |
1187 | 839 | } |
1188 | 839 | }); |
1189 | 839 | |
1190 | 839 | bool Changed = false; |
1191 | 839 | int Limit = MaxHSDR; |
1192 | 839 | |
1193 | 1.21k | for (UUSetMap::iterator I = P2Rs.begin(), E = P2Rs.end(); I != E1.21k ; ++I372 ) { |
1194 | 372 | if (I->first == 0) |
1195 | 142 | continue; |
1196 | 230 | if (230 Limit >= 0 && 230 Counter >= Limit0 ) |
1197 | 0 | break; |
1198 | 230 | USet &Part = I->second; |
1199 | 230 | DEBUG(dbgs() << "Calculating profit for partition #" << I->first << '\n'); |
1200 | 230 | if (!isProfitable(Part, IRM)) |
1201 | 53 | continue; |
1202 | 177 | Counter++; |
1203 | 177 | Changed |= splitPartition(Part); |
1204 | 177 | } |
1205 | 839 | |
1206 | 839 | return Changed; |
1207 | 839 | } |
1208 | | |
1209 | 392 | FunctionPass *llvm::createHexagonSplitDoubleRegs() { |
1210 | 392 | return new HexagonSplitDoubleRegs(); |
1211 | 392 | } |