/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- HexagonSplitDouble.cpp ---------------------------------------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | |
9 | | #define DEBUG_TYPE "hsdr" |
10 | | |
11 | | #include "HexagonInstrInfo.h" |
12 | | #include "HexagonRegisterInfo.h" |
13 | | #include "HexagonSubtarget.h" |
14 | | #include "llvm/ADT/BitVector.h" |
15 | | #include "llvm/ADT/STLExtras.h" |
16 | | #include "llvm/ADT/SmallVector.h" |
17 | | #include "llvm/ADT/StringRef.h" |
18 | | #include "llvm/CodeGen/MachineBasicBlock.h" |
19 | | #include "llvm/CodeGen/MachineFunction.h" |
20 | | #include "llvm/CodeGen/MachineFunctionPass.h" |
21 | | #include "llvm/CodeGen/MachineInstr.h" |
22 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
23 | | #include "llvm/CodeGen/MachineLoopInfo.h" |
24 | | #include "llvm/CodeGen/MachineMemOperand.h" |
25 | | #include "llvm/CodeGen/MachineOperand.h" |
26 | | #include "llvm/CodeGen/MachineRegisterInfo.h" |
27 | | #include "llvm/CodeGen/TargetRegisterInfo.h" |
28 | | #include "llvm/Config/llvm-config.h" |
29 | | #include "llvm/IR/DebugLoc.h" |
30 | | #include "llvm/Pass.h" |
31 | | #include "llvm/Support/CommandLine.h" |
32 | | #include "llvm/Support/Compiler.h" |
33 | | #include "llvm/Support/Debug.h" |
34 | | #include "llvm/Support/ErrorHandling.h" |
35 | | #include "llvm/Support/raw_ostream.h" |
36 | | #include <algorithm> |
37 | | #include <cassert> |
38 | | #include <cstdint> |
39 | | #include <limits> |
40 | | #include <map> |
41 | | #include <set> |
42 | | #include <utility> |
43 | | #include <vector> |
44 | | |
45 | | using namespace llvm; |
46 | | |
47 | | namespace llvm { |
48 | | |
49 | | FunctionPass *createHexagonSplitDoubleRegs(); |
50 | | void initializeHexagonSplitDoubleRegsPass(PassRegistry&); |
51 | | |
52 | | } // end namespace llvm |
53 | | |
54 | | static cl::opt<int> MaxHSDR("max-hsdr", cl::Hidden, cl::init(-1), |
55 | | cl::desc("Maximum number of split partitions")); |
56 | | static cl::opt<bool> MemRefsFixed("hsdr-no-mem", cl::Hidden, cl::init(true), |
57 | | cl::desc("Do not split loads or stores")); |
58 | | static cl::opt<bool> SplitAll("hsdr-split-all", cl::Hidden, cl::init(false), |
59 | | cl::desc("Split all partitions")); |
60 | | |
61 | | namespace { |
62 | | |
63 | | class HexagonSplitDoubleRegs : public MachineFunctionPass { |
64 | | public: |
65 | | static char ID; |
66 | | |
67 | 849 | HexagonSplitDoubleRegs() : MachineFunctionPass(ID) {} |
68 | | |
69 | 3.33k | StringRef getPassName() const override { |
70 | 3.33k | return "Hexagon Split Double Registers"; |
71 | 3.33k | } |
72 | | |
73 | 842 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
74 | 842 | AU.addRequired<MachineLoopInfo>(); |
75 | 842 | AU.addPreserved<MachineLoopInfo>(); |
76 | 842 | MachineFunctionPass::getAnalysisUsage(AU); |
77 | 842 | } |
78 | | |
79 | | bool runOnMachineFunction(MachineFunction &MF) override; |
80 | | |
81 | | private: |
82 | | static const TargetRegisterClass *const DoubleRC; |
83 | | |
84 | | const HexagonRegisterInfo *TRI = nullptr; |
85 | | const HexagonInstrInfo *TII = nullptr; |
86 | | const MachineLoopInfo *MLI; |
87 | | MachineRegisterInfo *MRI; |
88 | | |
89 | | using USet = std::set<unsigned>; |
90 | | using UUSetMap = std::map<unsigned, USet>; |
91 | | using UUPair = std::pair<unsigned, unsigned>; |
92 | | using UUPairMap = std::map<unsigned, UUPair>; |
93 | | using LoopRegMap = std::map<const MachineLoop *, USet>; |
94 | | |
95 | | bool isInduction(unsigned Reg, LoopRegMap &IRM) const; |
96 | | bool isVolatileInstr(const MachineInstr *MI) const; |
97 | | bool isFixedInstr(const MachineInstr *MI) const; |
98 | | void partitionRegisters(UUSetMap &P2Rs); |
99 | | int32_t profit(const MachineInstr *MI) const; |
100 | | int32_t profit(unsigned Reg) const; |
101 | | bool isProfitable(const USet &Part, LoopRegMap &IRM) const; |
102 | | |
103 | | void collectIndRegsForLoop(const MachineLoop *L, USet &Rs); |
104 | | void collectIndRegs(LoopRegMap &IRM); |
105 | | |
106 | | void createHalfInstr(unsigned Opc, MachineInstr *MI, |
107 | | const UUPairMap &PairMap, unsigned SubR); |
108 | | void splitMemRef(MachineInstr *MI, const UUPairMap &PairMap); |
109 | | void splitImmediate(MachineInstr *MI, const UUPairMap &PairMap); |
110 | | void splitCombine(MachineInstr *MI, const UUPairMap &PairMap); |
111 | | void splitExt(MachineInstr *MI, const UUPairMap &PairMap); |
112 | | void splitShift(MachineInstr *MI, const UUPairMap &PairMap); |
113 | | void splitAslOr(MachineInstr *MI, const UUPairMap &PairMap); |
114 | | bool splitInstr(MachineInstr *MI, const UUPairMap &PairMap); |
115 | | void replaceSubregUses(MachineInstr *MI, const UUPairMap &PairMap); |
116 | | void collapseRegPairs(MachineInstr *MI, const UUPairMap &PairMap); |
117 | | bool splitPartition(const USet &Part); |
118 | | |
119 | | static int Counter; |
120 | | |
121 | | static void dump_partition(raw_ostream&, const USet&, |
122 | | const TargetRegisterInfo&); |
123 | | }; |
124 | | |
125 | | } // end anonymous namespace |
126 | | |
127 | | char HexagonSplitDoubleRegs::ID; |
128 | | int HexagonSplitDoubleRegs::Counter = 0; |
129 | | const TargetRegisterClass *const HexagonSplitDoubleRegs::DoubleRC = |
130 | | &Hexagon::DoubleRegsRegClass; |
131 | | |
132 | | INITIALIZE_PASS(HexagonSplitDoubleRegs, "hexagon-split-double", |
133 | | "Hexagon Split Double Registers", false, false) |
134 | | |
135 | | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) |
136 | | LLVM_DUMP_METHOD void HexagonSplitDoubleRegs::dump_partition(raw_ostream &os, |
137 | | const USet &Part, const TargetRegisterInfo &TRI) { |
138 | | dbgs() << '{'; |
139 | | for (auto I : Part) |
140 | | dbgs() << ' ' << printReg(I, &TRI); |
141 | | dbgs() << " }"; |
142 | | } |
143 | | #endif |
144 | | |
145 | 536 | bool HexagonSplitDoubleRegs::isInduction(unsigned Reg, LoopRegMap &IRM) const { |
146 | 536 | for (auto I : IRM) { |
147 | 22 | const USet &Rs = I.second; |
148 | 22 | if (Rs.find(Reg) != Rs.end()) |
149 | 8 | return true; |
150 | 22 | } |
151 | 536 | return false528 ; |
152 | 536 | } |
153 | | |
154 | 0 | bool HexagonSplitDoubleRegs::isVolatileInstr(const MachineInstr *MI) const { |
155 | 0 | for (auto &MO : MI->memoperands()) |
156 | 0 | if (MO->isVolatile() || MO->isAtomic()) |
157 | 0 | return true; |
158 | 0 | return false; |
159 | 0 | } |
160 | | |
161 | 4.53k | bool HexagonSplitDoubleRegs::isFixedInstr(const MachineInstr *MI) const { |
162 | 4.53k | if (MI->mayLoad() || MI->mayStore()4.07k ) |
163 | 597 | if (MemRefsFixed || isVolatileInstr(MI)0 ) |
164 | 597 | return true; |
165 | 3.93k | if (MI->isDebugInstr()) |
166 | 0 | return false; |
167 | 3.93k | |
168 | 3.93k | unsigned Opc = MI->getOpcode(); |
169 | 3.93k | switch (Opc) { |
170 | 3.93k | default: |
171 | 1.35k | return true; |
172 | 3.93k | |
173 | 3.93k | case TargetOpcode::PHI: |
174 | 1.49k | case TargetOpcode::COPY: |
175 | 1.49k | break; |
176 | 1.49k | |
177 | 1.49k | case Hexagon::L2_loadrd_io: |
178 | 0 | // Not handling stack stores (only reg-based addresses). |
179 | 0 | if (MI->getOperand(1).isReg()) |
180 | 0 | break; |
181 | 0 | return true; |
182 | 0 | case Hexagon::S2_storerd_io: |
183 | 0 | // Not handling stack stores (only reg-based addresses). |
184 | 0 | if (MI->getOperand(0).isReg()) |
185 | 0 | break; |
186 | 0 | return true; |
187 | 1.08k | case Hexagon::L2_loadrd_pi: |
188 | 1.08k | case Hexagon::S2_storerd_pi: |
189 | 1.08k | |
190 | 1.08k | case Hexagon::A2_tfrpi: |
191 | 1.08k | case Hexagon::A2_combineii: |
192 | 1.08k | case Hexagon::A4_combineir: |
193 | 1.08k | case Hexagon::A4_combineii: |
194 | 1.08k | case Hexagon::A4_combineri: |
195 | 1.08k | case Hexagon::A2_combinew: |
196 | 1.08k | case Hexagon::CONST64: |
197 | 1.08k | |
198 | 1.08k | case Hexagon::A2_sxtw: |
199 | 1.08k | |
200 | 1.08k | case Hexagon::A2_andp: |
201 | 1.08k | case Hexagon::A2_orp: |
202 | 1.08k | case Hexagon::A2_xorp: |
203 | 1.08k | case Hexagon::S2_asl_i_p_or: |
204 | 1.08k | case Hexagon::S2_asl_i_p: |
205 | 1.08k | case Hexagon::S2_asr_i_p: |
206 | 1.08k | case Hexagon::S2_lsr_i_p: |
207 | 1.08k | break; |
208 | 2.58k | } |
209 | 2.58k | |
210 | 6.49k | for (auto &Op : MI->operands())2.58k { |
211 | 6.49k | if (!Op.isReg()) |
212 | 1.19k | continue; |
213 | 5.29k | unsigned R = Op.getReg(); |
214 | 5.29k | if (!TargetRegisterInfo::isVirtualRegister(R)) |
215 | 656 | return true; |
216 | 5.29k | } |
217 | 2.58k | return false1.92k ; |
218 | 2.58k | } |
219 | | |
220 | 3.32k | void HexagonSplitDoubleRegs::partitionRegisters(UUSetMap &P2Rs) { |
221 | 3.32k | using UUMap = std::map<unsigned, unsigned>; |
222 | 3.32k | using UVect = std::vector<unsigned>; |
223 | 3.32k | |
224 | 3.32k | unsigned NumRegs = MRI->getNumVirtRegs(); |
225 | 3.32k | BitVector DoubleRegs(NumRegs); |
226 | 33.4k | for (unsigned i = 0; i < NumRegs; ++i30.1k ) { |
227 | 30.1k | unsigned R = TargetRegisterInfo::index2VirtReg(i); |
228 | 30.1k | if (MRI->getRegClass(R) == DoubleRC) |
229 | 2.35k | DoubleRegs.set(i); |
230 | 30.1k | } |
231 | 3.32k | |
232 | 3.32k | BitVector FixedRegs(NumRegs); |
233 | 5.68k | for (int x = DoubleRegs.find_first(); x >= 0; x = DoubleRegs.find_next(x)2.35k ) { |
234 | 2.35k | unsigned R = TargetRegisterInfo::index2VirtReg(x); |
235 | 2.35k | MachineInstr *DefI = MRI->getVRegDef(R); |
236 | 2.35k | // In some cases a register may exist, but never be defined or used. |
237 | 2.35k | // It should never appear anywhere, but mark it as "fixed", just to be |
238 | 2.35k | // safe. |
239 | 2.35k | if (!DefI || isFixedInstr(DefI)2.35k ) |
240 | 1.82k | FixedRegs.set(x); |
241 | 2.35k | } |
242 | 3.32k | |
243 | 3.32k | UUSetMap AssocMap; |
244 | 5.68k | for (int x = DoubleRegs.find_first(); x >= 0; x = DoubleRegs.find_next(x)2.35k ) { |
245 | 2.35k | if (FixedRegs[x]) |
246 | 1.82k | continue; |
247 | 536 | unsigned R = TargetRegisterInfo::index2VirtReg(x); |
248 | 536 | LLVM_DEBUG(dbgs() << printReg(R, TRI) << " ~~"); |
249 | 536 | USet &Asc = AssocMap[R]; |
250 | 536 | for (auto U = MRI->use_nodbg_begin(R), Z = MRI->use_nodbg_end(); |
251 | 1.23k | U != Z; ++U695 ) { |
252 | 695 | MachineOperand &Op = *U; |
253 | 695 | MachineInstr *UseI = Op.getParent(); |
254 | 695 | if (isFixedInstr(UseI)) |
255 | 300 | continue; |
256 | 1.51k | for (unsigned i = 0, n = UseI->getNumOperands(); 395 i < n; ++i1.12k ) { |
257 | 1.12k | MachineOperand &MO = UseI->getOperand(i); |
258 | 1.12k | // Skip non-registers or registers with subregisters. |
259 | 1.12k | if (&MO == &Op || !MO.isReg()728 || MO.getSubReg()551 ) |
260 | 572 | continue; |
261 | 551 | unsigned T = MO.getReg(); |
262 | 551 | if (!TargetRegisterInfo::isVirtualRegister(T)) { |
263 | 0 | FixedRegs.set(x); |
264 | 0 | continue; |
265 | 0 | } |
266 | 551 | if (MRI->getRegClass(T) != DoubleRC) |
267 | 223 | continue; |
268 | 328 | unsigned u = TargetRegisterInfo::virtReg2Index(T); |
269 | 328 | if (FixedRegs[u]) |
270 | 46 | continue; |
271 | 282 | LLVM_DEBUG(dbgs() << ' ' << printReg(T, TRI)); |
272 | 282 | Asc.insert(T); |
273 | 282 | // Make it symmetric. |
274 | 282 | AssocMap[T].insert(R); |
275 | 282 | } |
276 | 395 | } |
277 | 536 | LLVM_DEBUG(dbgs() << '\n'); |
278 | 536 | } |
279 | 3.32k | |
280 | 3.32k | UUMap R2P; |
281 | 3.32k | unsigned NextP = 1; |
282 | 3.32k | USet Visited; |
283 | 5.68k | for (int x = DoubleRegs.find_first(); x >= 0; x = DoubleRegs.find_next(x)2.35k ) { |
284 | 2.35k | unsigned R = TargetRegisterInfo::index2VirtReg(x); |
285 | 2.35k | if (Visited.count(R)) |
286 | 151 | continue; |
287 | 2.20k | // Create a new partition for R. |
288 | 2.20k | unsigned ThisP = FixedRegs[x] ? 01.82k : NextP++385 ; |
289 | 2.20k | UVect WorkQ; |
290 | 2.20k | WorkQ.push_back(R); |
291 | 4.82k | for (unsigned i = 0; i < WorkQ.size(); ++i2.62k ) { |
292 | 2.62k | unsigned T = WorkQ[i]; |
293 | 2.62k | if (Visited.count(T)) |
294 | 263 | continue; |
295 | 2.35k | R2P[T] = ThisP; |
296 | 2.35k | Visited.insert(T); |
297 | 2.35k | // Add all registers associated with T. |
298 | 2.35k | USet &Asc = AssocMap[T]; |
299 | 2.77k | for (USet::iterator J = Asc.begin(), F = Asc.end(); J != F; ++J414 ) |
300 | 414 | WorkQ.push_back(*J); |
301 | 2.35k | } |
302 | 2.20k | } |
303 | 3.32k | |
304 | 3.32k | for (auto I : R2P) |
305 | 2.35k | P2Rs[I.second].insert(I.first); |
306 | 3.32k | } |
307 | | |
308 | 462 | static inline int32_t profitImm(unsigned Imm) { |
309 | 462 | int32_t P = 0; |
310 | 462 | if (Imm == 0 || Imm == 0xFFFFFFFF246 ) |
311 | 273 | P += 10; |
312 | 462 | return P; |
313 | 462 | } |
314 | | |
315 | 1.00k | int32_t HexagonSplitDoubleRegs::profit(const MachineInstr *MI) const { |
316 | 1.00k | unsigned ImmX = 0; |
317 | 1.00k | unsigned Opc = MI->getOpcode(); |
318 | 1.00k | switch (Opc) { |
319 | 1.00k | case TargetOpcode::PHI: |
320 | 99 | for (const auto &Op : MI->operands()) |
321 | 99 | if (!Op.getSubReg()) |
322 | 99 | return 0; |
323 | 99 | return 100 ; |
324 | 234 | case TargetOpcode::COPY: |
325 | 234 | if (MI->getOperand(1).getSubReg() != 0) |
326 | 223 | return 10; |
327 | 11 | return 0; |
328 | 11 | |
329 | 11 | case Hexagon::L2_loadrd_io: |
330 | 0 | case Hexagon::S2_storerd_io: |
331 | 0 | return -1; |
332 | 0 | case Hexagon::L2_loadrd_pi: |
333 | 0 | case Hexagon::S2_storerd_pi: |
334 | 0 | return 2; |
335 | 0 |
|
336 | 229 | case Hexagon::A2_tfrpi: |
337 | 229 | case Hexagon::CONST64: { |
338 | 229 | uint64_t D = MI->getOperand(1).getImm(); |
339 | 229 | unsigned Lo = D & 0xFFFFFFFFULL; |
340 | 229 | unsigned Hi = D >> 32; |
341 | 229 | return profitImm(Lo) + profitImm(Hi); |
342 | 229 | } |
343 | 229 | case Hexagon::A2_combineii: |
344 | 2 | case Hexagon::A4_combineii: { |
345 | 2 | const MachineOperand &Op1 = MI->getOperand(1); |
346 | 2 | const MachineOperand &Op2 = MI->getOperand(2); |
347 | 2 | int32_t Prof1 = Op1.isImm() ? profitImm(Op1.getImm()) : 00 ; |
348 | 2 | int32_t Prof2 = Op2.isImm() ? profitImm(Op2.getImm()) : 00 ; |
349 | 2 | return Prof1 + Prof2; |
350 | 2 | } |
351 | 2 | case Hexagon::A4_combineri: |
352 | 0 | ImmX++; |
353 | 0 | // Fall through into A4_combineir. |
354 | 0 | LLVM_FALLTHROUGH; |
355 | 102 | case Hexagon::A4_combineir: { |
356 | 102 | ImmX++; |
357 | 102 | const MachineOperand &OpX = MI->getOperand(ImmX); |
358 | 102 | if (OpX.isImm()) { |
359 | 102 | int64_t V = OpX.getImm(); |
360 | 102 | if (V == 0 || V == -11 ) |
361 | 101 | return 10; |
362 | 1 | } |
363 | 1 | // Fall through into A2_combinew. |
364 | 1 | LLVM_FALLTHROUGH; |
365 | 1 | } |
366 | 9 | case Hexagon::A2_combinew: |
367 | 9 | return 2; |
368 | 1 | |
369 | 20 | case Hexagon::A2_sxtw: |
370 | 20 | return 3; |
371 | 1 | |
372 | 116 | case Hexagon::A2_andp: |
373 | 116 | case Hexagon::A2_orp: |
374 | 116 | case Hexagon::A2_xorp: { |
375 | 116 | unsigned Rs = MI->getOperand(1).getReg(); |
376 | 116 | unsigned Rt = MI->getOperand(2).getReg(); |
377 | 116 | return profit(Rs) + profit(Rt); |
378 | 116 | } |
379 | 116 | |
380 | 116 | case Hexagon::S2_asl_i_p_or: { |
381 | 85 | unsigned S = MI->getOperand(3).getImm(); |
382 | 85 | if (S == 0 || S == 32) |
383 | 21 | return 10; |
384 | 64 | return -1; |
385 | 64 | } |
386 | 113 | case Hexagon::S2_asl_i_p: |
387 | 113 | case Hexagon::S2_asr_i_p: |
388 | 113 | case Hexagon::S2_lsr_i_p: |
389 | 113 | unsigned S = MI->getOperand(2).getImm(); |
390 | 113 | if (S == 0 || S == 32) |
391 | 64 | return 10; |
392 | 49 | if (S == 16) |
393 | 17 | return 5; |
394 | 32 | if (S == 48) |
395 | 7 | return 7; |
396 | 25 | return -10; |
397 | 0 | } |
398 | 0 | |
399 | 0 | return 0; |
400 | 0 | } |
401 | | |
402 | 232 | int32_t HexagonSplitDoubleRegs::profit(unsigned Reg) const { |
403 | 232 | assert(TargetRegisterInfo::isVirtualRegister(Reg)); |
404 | 232 | |
405 | 232 | const MachineInstr *DefI = MRI->getVRegDef(Reg); |
406 | 232 | switch (DefI->getOpcode()) { |
407 | 232 | case Hexagon::A2_tfrpi: |
408 | 77 | case Hexagon::CONST64: |
409 | 77 | case Hexagon::A2_combineii: |
410 | 77 | case Hexagon::A4_combineii: |
411 | 77 | case Hexagon::A4_combineri: |
412 | 77 | case Hexagon::A4_combineir: |
413 | 77 | case Hexagon::A2_combinew: |
414 | 77 | return profit(DefI); |
415 | 155 | default: |
416 | 155 | break; |
417 | 155 | } |
418 | 155 | return 0; |
419 | 155 | } |
420 | | |
421 | | bool HexagonSplitDoubleRegs::isProfitable(const USet &Part, LoopRegMap &IRM) |
422 | 385 | const { |
423 | 385 | unsigned FixedNum = 0, LoopPhiNum = 0; |
424 | 385 | int32_t TotalP = 0; |
425 | 385 | |
426 | 536 | for (unsigned DR : Part) { |
427 | 536 | MachineInstr *DefI = MRI->getVRegDef(DR); |
428 | 536 | int32_t P = profit(DefI); |
429 | 536 | if (P == std::numeric_limits<int>::min()) |
430 | 0 | return false; |
431 | 536 | TotalP += P; |
432 | 536 | // Reduce the profitability of splitting induction registers. |
433 | 536 | if (isInduction(DR, IRM)) |
434 | 8 | TotalP -= 30; |
435 | 536 | |
436 | 536 | for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end(); |
437 | 1.23k | U != W; ++U695 ) { |
438 | 695 | MachineInstr *UseI = U->getParent(); |
439 | 695 | if (isFixedInstr(UseI)) { |
440 | 300 | FixedNum++; |
441 | 300 | // Calculate the cost of generating REG_SEQUENCE instructions. |
442 | 952 | for (auto &Op : UseI->operands()) { |
443 | 952 | if (Op.isReg() && Part.count(Op.getReg())761 ) |
444 | 300 | if (Op.getSubReg()) |
445 | 1 | TotalP -= 2; |
446 | 952 | } |
447 | 300 | continue; |
448 | 300 | } |
449 | 395 | // If a register from this partition is used in a fixed instruction, |
450 | 395 | // and there is also a register in this partition that is used in |
451 | 395 | // a loop phi node, then decrease the splitting profit as this can |
452 | 395 | // confuse the modulo scheduler. |
453 | 395 | if (UseI->isPHI()) { |
454 | 50 | const MachineBasicBlock *PB = UseI->getParent(); |
455 | 50 | const MachineLoop *L = MLI->getLoopFor(PB); |
456 | 50 | if (L && L->getHeader() == PB34 ) |
457 | 30 | LoopPhiNum++; |
458 | 50 | } |
459 | 395 | // Splittable instruction. |
460 | 395 | int32_t P = profit(UseI); |
461 | 395 | if (P == std::numeric_limits<int>::min()) |
462 | 0 | return false; |
463 | 395 | TotalP += P; |
464 | 395 | } |
465 | 536 | } |
466 | 385 | |
467 | 385 | if (FixedNum > 0 && LoopPhiNum > 0236 ) |
468 | 20 | TotalP -= 20*LoopPhiNum; |
469 | 385 | |
470 | 385 | LLVM_DEBUG(dbgs() << "Partition profit: " << TotalP << '\n'); |
471 | 385 | if (SplitAll) |
472 | 0 | return true; |
473 | 385 | return TotalP > 0; |
474 | 385 | } |
475 | | |
476 | | void HexagonSplitDoubleRegs::collectIndRegsForLoop(const MachineLoop *L, |
477 | 417 | USet &Rs) { |
478 | 417 | const MachineBasicBlock *HB = L->getHeader(); |
479 | 417 | const MachineBasicBlock *LB = L->getLoopLatch(); |
480 | 417 | if (!HB || !LB) |
481 | 3 | return; |
482 | 414 | |
483 | 414 | // Examine the latch branch. Expect it to be a conditional branch to |
484 | 414 | // the header (either "br-cond header" or "br-cond exit; br header"). |
485 | 414 | MachineBasicBlock *TB = nullptr, *FB = nullptr; |
486 | 414 | MachineBasicBlock *TmpLB = const_cast<MachineBasicBlock*>(LB); |
487 | 414 | SmallVector<MachineOperand,2> Cond; |
488 | 414 | bool BadLB = TII->analyzeBranch(*TmpLB, TB, FB, Cond, false); |
489 | 414 | // Only analyzable conditional branches. HII::analyzeBranch will put |
490 | 414 | // the branch opcode as the first element of Cond, and the predicate |
491 | 414 | // operand as the second. |
492 | 414 | if (BadLB || Cond.size() != 2) |
493 | 31 | return; |
494 | 383 | // Only simple jump-conditional (with or without negation). |
495 | 383 | if (!TII->PredOpcodeHasJMP_c(Cond[0].getImm())) |
496 | 0 | return; |
497 | 383 | // Must go to the header. |
498 | 383 | if (TB != HB && FB != HB30 ) |
499 | 0 | return; |
500 | 383 | assert(Cond[1].isReg() && "Unexpected Cond vector from analyzeBranch"); |
501 | 383 | // Expect a predicate register. |
502 | 383 | unsigned PR = Cond[1].getReg(); |
503 | 383 | assert(MRI->getRegClass(PR) == &Hexagon::PredRegsRegClass); |
504 | 383 | |
505 | 383 | // Get the registers on which the loop controlling compare instruction |
506 | 383 | // depends. |
507 | 383 | unsigned CmpR1 = 0, CmpR2 = 0; |
508 | 383 | const MachineInstr *CmpI = MRI->getVRegDef(PR); |
509 | 669 | while (CmpI->getOpcode() == Hexagon::C2_not) |
510 | 286 | CmpI = MRI->getVRegDef(CmpI->getOperand(1).getReg()); |
511 | 383 | |
512 | 383 | int Mask = 0, Val = 0; |
513 | 383 | bool OkCI = TII->analyzeCompare(*CmpI, CmpR1, CmpR2, Mask, Val); |
514 | 383 | if (!OkCI) |
515 | 102 | return; |
516 | 281 | // Eliminate non-double input registers. |
517 | 281 | if (CmpR1 && MRI->getRegClass(CmpR1) != DoubleRC) |
518 | 277 | CmpR1 = 0; |
519 | 281 | if (CmpR2 && MRI->getRegClass(CmpR2) != DoubleRC81 ) |
520 | 77 | CmpR2 = 0; |
521 | 281 | if (!CmpR1 && !CmpR2277 ) |
522 | 277 | return; |
523 | 4 | |
524 | 4 | // Now examine the top of the loop: the phi nodes that could poten- |
525 | 4 | // tially define loop induction registers. The registers defined by |
526 | 4 | // such a phi node would be used in a 64-bit add, which then would |
527 | 4 | // be used in the loop compare instruction. |
528 | 4 | |
529 | 4 | // Get the set of all double registers defined by phi nodes in the |
530 | 4 | // loop header. |
531 | 4 | using UVect = std::vector<unsigned>; |
532 | 4 | |
533 | 4 | UVect DP; |
534 | 13 | for (auto &MI : *HB) { |
535 | 13 | if (!MI.isPHI()) |
536 | 4 | break; |
537 | 9 | const MachineOperand &MD = MI.getOperand(0); |
538 | 9 | unsigned R = MD.getReg(); |
539 | 9 | if (MRI->getRegClass(R) == DoubleRC) |
540 | 6 | DP.push_back(R); |
541 | 9 | } |
542 | 4 | if (DP.empty()) |
543 | 0 | return; |
544 | 4 | |
545 | 6 | auto NoIndOp = [this, CmpR1, CmpR2] (unsigned R) -> bool 4 { |
546 | 6 | for (auto I = MRI->use_nodbg_begin(R), E = MRI->use_nodbg_end(); |
547 | 15 | I != E; ++I9 ) { |
548 | 13 | const MachineInstr *UseI = I->getParent(); |
549 | 13 | if (UseI->getOpcode() != Hexagon::A2_addp) |
550 | 9 | continue; |
551 | 4 | // Get the output from the add. If it is one of the inputs to the |
552 | 4 | // loop-controlling compare instruction, then R is likely an induc- |
553 | 4 | // tion register. |
554 | 4 | unsigned T = UseI->getOperand(0).getReg(); |
555 | 4 | if (T == CmpR1 || T == CmpR23 ) |
556 | 4 | return false; |
557 | 4 | } |
558 | 6 | return true2 ; |
559 | 6 | }; |
560 | 4 | UVect::iterator End = llvm::remove_if(DP, NoIndOp); |
561 | 4 | Rs.insert(DP.begin(), End); |
562 | 4 | Rs.insert(CmpR1); |
563 | 4 | Rs.insert(CmpR2); |
564 | 4 | |
565 | 4 | LLVM_DEBUG({ |
566 | 4 | dbgs() << "For loop at " << printMBBReference(*HB) << " ind regs: "; |
567 | 4 | dump_partition(dbgs(), Rs, *TRI); |
568 | 4 | dbgs() << '\n'; |
569 | 4 | }); |
570 | 4 | } |
571 | | |
572 | 3.32k | void HexagonSplitDoubleRegs::collectIndRegs(LoopRegMap &IRM) { |
573 | 3.32k | using LoopVector = std::vector<MachineLoop *>; |
574 | 3.32k | |
575 | 3.32k | LoopVector WorkQ; |
576 | 3.32k | |
577 | 3.32k | for (auto I : *MLI) |
578 | 372 | WorkQ.push_back(I); |
579 | 3.74k | for (unsigned i = 0; i < WorkQ.size(); ++i417 ) { |
580 | 417 | for (auto I : *WorkQ[i]) |
581 | 45 | WorkQ.push_back(I); |
582 | 417 | } |
583 | 3.32k | |
584 | 3.32k | USet Rs; |
585 | 3.74k | for (unsigned i = 0, n = WorkQ.size(); i < n; ++i417 ) { |
586 | 417 | MachineLoop *L = WorkQ[i]; |
587 | 417 | Rs.clear(); |
588 | 417 | collectIndRegsForLoop(L, Rs); |
589 | 417 | if (!Rs.empty()) |
590 | 4 | IRM.insert(std::make_pair(L, Rs)); |
591 | 417 | } |
592 | 3.32k | } |
593 | | |
594 | | void HexagonSplitDoubleRegs::createHalfInstr(unsigned Opc, MachineInstr *MI, |
595 | 114 | const UUPairMap &PairMap, unsigned SubR) { |
596 | 114 | MachineBasicBlock &B = *MI->getParent(); |
597 | 114 | DebugLoc DL = MI->getDebugLoc(); |
598 | 114 | MachineInstr *NewI = BuildMI(B, MI, DL, TII->get(Opc)); |
599 | 114 | |
600 | 416 | for (auto &Op : MI->operands()) { |
601 | 416 | if (!Op.isReg()) { |
602 | 80 | NewI->addOperand(Op); |
603 | 80 | continue; |
604 | 80 | } |
605 | 336 | // For register operands, set the subregister. |
606 | 336 | unsigned R = Op.getReg(); |
607 | 336 | unsigned SR = Op.getSubReg(); |
608 | 336 | bool isVirtReg = TargetRegisterInfo::isVirtualRegister(R); |
609 | 336 | bool isKill = Op.isKill(); |
610 | 336 | if (isVirtReg && MRI->getRegClass(R) == DoubleRC) { |
611 | 336 | isKill = false; |
612 | 336 | UUPairMap::const_iterator F = PairMap.find(R); |
613 | 336 | if (F == PairMap.end()) { |
614 | 82 | SR = SubR; |
615 | 254 | } else { |
616 | 254 | const UUPair &P = F->second; |
617 | 254 | R = (SubR == Hexagon::isub_lo) ? P.first127 : P.second127 ; |
618 | 254 | SR = 0; |
619 | 254 | } |
620 | 336 | } |
621 | 336 | auto CO = MachineOperand::CreateReg(R, Op.isDef(), Op.isImplicit(), isKill, |
622 | 336 | Op.isDead(), Op.isUndef(), Op.isEarlyClobber(), SR, Op.isDebug(), |
623 | 336 | Op.isInternalRead()); |
624 | 336 | NewI->addOperand(CO); |
625 | 336 | } |
626 | 114 | } |
627 | | |
628 | | void HexagonSplitDoubleRegs::splitMemRef(MachineInstr *MI, |
629 | 0 | const UUPairMap &PairMap) { |
630 | 0 | bool Load = MI->mayLoad(); |
631 | 0 | unsigned OrigOpc = MI->getOpcode(); |
632 | 0 | bool PostInc = (OrigOpc == Hexagon::L2_loadrd_pi || |
633 | 0 | OrigOpc == Hexagon::S2_storerd_pi); |
634 | 0 | MachineInstr *LowI, *HighI; |
635 | 0 | MachineBasicBlock &B = *MI->getParent(); |
636 | 0 | DebugLoc DL = MI->getDebugLoc(); |
637 | 0 |
|
638 | 0 | // Index of the base-address-register operand. |
639 | 0 | unsigned AdrX = PostInc ? (Load ? 2 : 1) |
640 | 0 | : (Load ? 1 : 0); |
641 | 0 | MachineOperand &AdrOp = MI->getOperand(AdrX); |
642 | 0 | unsigned RSA = getRegState(AdrOp); |
643 | 0 | MachineOperand &ValOp = Load ? MI->getOperand(0) |
644 | 0 | : (PostInc ? MI->getOperand(3) |
645 | 0 | : MI->getOperand(2)); |
646 | 0 | UUPairMap::const_iterator F = PairMap.find(ValOp.getReg()); |
647 | 0 | assert(F != PairMap.end()); |
648 | 0 |
|
649 | 0 | if (Load) { |
650 | 0 | const UUPair &P = F->second; |
651 | 0 | int64_t Off = PostInc ? 0 : MI->getOperand(2).getImm(); |
652 | 0 | LowI = BuildMI(B, MI, DL, TII->get(Hexagon::L2_loadri_io), P.first) |
653 | 0 | .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg()) |
654 | 0 | .addImm(Off); |
655 | 0 | HighI = BuildMI(B, MI, DL, TII->get(Hexagon::L2_loadri_io), P.second) |
656 | 0 | .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg()) |
657 | 0 | .addImm(Off+4); |
658 | 0 | } else { |
659 | 0 | const UUPair &P = F->second; |
660 | 0 | int64_t Off = PostInc ? 0 : MI->getOperand(1).getImm(); |
661 | 0 | LowI = BuildMI(B, MI, DL, TII->get(Hexagon::S2_storeri_io)) |
662 | 0 | .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg()) |
663 | 0 | .addImm(Off) |
664 | 0 | .addReg(P.first); |
665 | 0 | HighI = BuildMI(B, MI, DL, TII->get(Hexagon::S2_storeri_io)) |
666 | 0 | .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg()) |
667 | 0 | .addImm(Off+4) |
668 | 0 | .addReg(P.second); |
669 | 0 | } |
670 | 0 |
|
671 | 0 | if (PostInc) { |
672 | 0 | // Create the increment of the address register. |
673 | 0 | int64_t Inc = Load ? MI->getOperand(3).getImm() |
674 | 0 | : MI->getOperand(2).getImm(); |
675 | 0 | MachineOperand &UpdOp = Load ? MI->getOperand(1) : MI->getOperand(0); |
676 | 0 | const TargetRegisterClass *RC = MRI->getRegClass(UpdOp.getReg()); |
677 | 0 | unsigned NewR = MRI->createVirtualRegister(RC); |
678 | 0 | assert(!UpdOp.getSubReg() && "Def operand with subreg"); |
679 | 0 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_addi), NewR) |
680 | 0 | .addReg(AdrOp.getReg(), RSA) |
681 | 0 | .addImm(Inc); |
682 | 0 | MRI->replaceRegWith(UpdOp.getReg(), NewR); |
683 | 0 | // The original instruction will be deleted later. |
684 | 0 | } |
685 | 0 |
|
686 | 0 | // Generate a new pair of memory-operands. |
687 | 0 | MachineFunction &MF = *B.getParent(); |
688 | 0 | for (auto &MO : MI->memoperands()) { |
689 | 0 | const MachinePointerInfo &Ptr = MO->getPointerInfo(); |
690 | 0 | MachineMemOperand::Flags F = MO->getFlags(); |
691 | 0 | int A = MO->getAlignment(); |
692 | 0 |
|
693 | 0 | auto *Tmp1 = MF.getMachineMemOperand(Ptr, F, 4/*size*/, A); |
694 | 0 | LowI->addMemOperand(MF, Tmp1); |
695 | 0 | auto *Tmp2 = MF.getMachineMemOperand(Ptr, F, 4/*size*/, std::min(A, 4)); |
696 | 0 | HighI->addMemOperand(MF, Tmp2); |
697 | 0 | } |
698 | 0 | } |
699 | | |
700 | | void HexagonSplitDoubleRegs::splitImmediate(MachineInstr *MI, |
701 | 114 | const UUPairMap &PairMap) { |
702 | 114 | MachineOperand &Op0 = MI->getOperand(0); |
703 | 114 | MachineOperand &Op1 = MI->getOperand(1); |
704 | 114 | assert(Op0.isReg() && Op1.isImm()); |
705 | 114 | uint64_t V = Op1.getImm(); |
706 | 114 | |
707 | 114 | MachineBasicBlock &B = *MI->getParent(); |
708 | 114 | DebugLoc DL = MI->getDebugLoc(); |
709 | 114 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
710 | 114 | assert(F != PairMap.end()); |
711 | 114 | const UUPair &P = F->second; |
712 | 114 | |
713 | 114 | // The operand to A2_tfrsi can only have 32 significant bits. Immediate |
714 | 114 | // values in MachineOperand are stored as 64-bit integers, and so the |
715 | 114 | // value -1 may be represented either as 64-bit -1, or 4294967295. Both |
716 | 114 | // will have the 32 higher bits truncated in the end, but -1 will remain |
717 | 114 | // as -1, while the latter may appear to be a large unsigned value |
718 | 114 | // requiring a constant extender. The casting to int32_t will select the |
719 | 114 | // former representation. (The same reasoning applies to all 32-bit |
720 | 114 | // values.) |
721 | 114 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.first) |
722 | 114 | .addImm(int32_t(V & 0xFFFFFFFFULL)); |
723 | 114 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.second) |
724 | 114 | .addImm(int32_t(V >> 32)); |
725 | 114 | } |
726 | | |
727 | | void HexagonSplitDoubleRegs::splitCombine(MachineInstr *MI, |
728 | 103 | const UUPairMap &PairMap) { |
729 | 103 | MachineOperand &Op0 = MI->getOperand(0); |
730 | 103 | MachineOperand &Op1 = MI->getOperand(1); |
731 | 103 | MachineOperand &Op2 = MI->getOperand(2); |
732 | 103 | assert(Op0.isReg()); |
733 | 103 | |
734 | 103 | MachineBasicBlock &B = *MI->getParent(); |
735 | 103 | DebugLoc DL = MI->getDebugLoc(); |
736 | 103 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
737 | 103 | assert(F != PairMap.end()); |
738 | 103 | const UUPair &P = F->second; |
739 | 103 | |
740 | 103 | if (!Op1.isReg()) { |
741 | 95 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.second) |
742 | 95 | .add(Op1); |
743 | 95 | } else { |
744 | 8 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.second) |
745 | 8 | .addReg(Op1.getReg(), getRegState(Op1), Op1.getSubReg()); |
746 | 8 | } |
747 | 103 | |
748 | 103 | if (!Op2.isReg()) { |
749 | 3 | BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.first) |
750 | 3 | .add(Op2); |
751 | 100 | } else { |
752 | 100 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.first) |
753 | 100 | .addReg(Op2.getReg(), getRegState(Op2), Op2.getSubReg()); |
754 | 100 | } |
755 | 103 | } |
756 | | |
757 | | void HexagonSplitDoubleRegs::splitExt(MachineInstr *MI, |
758 | 20 | const UUPairMap &PairMap) { |
759 | 20 | MachineOperand &Op0 = MI->getOperand(0); |
760 | 20 | MachineOperand &Op1 = MI->getOperand(1); |
761 | 20 | assert(Op0.isReg() && Op1.isReg()); |
762 | 20 | |
763 | 20 | MachineBasicBlock &B = *MI->getParent(); |
764 | 20 | DebugLoc DL = MI->getDebugLoc(); |
765 | 20 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
766 | 20 | assert(F != PairMap.end()); |
767 | 20 | const UUPair &P = F->second; |
768 | 20 | unsigned RS = getRegState(Op1); |
769 | 20 | |
770 | 20 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.first) |
771 | 20 | .addReg(Op1.getReg(), RS & ~RegState::Kill, Op1.getSubReg()); |
772 | 20 | BuildMI(B, MI, DL, TII->get(Hexagon::S2_asr_i_r), P.second) |
773 | 20 | .addReg(Op1.getReg(), RS, Op1.getSubReg()) |
774 | 20 | .addImm(31); |
775 | 20 | } |
776 | | |
777 | | void HexagonSplitDoubleRegs::splitShift(MachineInstr *MI, |
778 | 74 | const UUPairMap &PairMap) { |
779 | 74 | using namespace Hexagon; |
780 | 74 | |
781 | 74 | MachineOperand &Op0 = MI->getOperand(0); |
782 | 74 | MachineOperand &Op1 = MI->getOperand(1); |
783 | 74 | MachineOperand &Op2 = MI->getOperand(2); |
784 | 74 | assert(Op0.isReg() && Op1.isReg() && Op2.isImm()); |
785 | 74 | int64_t Sh64 = Op2.getImm(); |
786 | 74 | assert(Sh64 >= 0 && Sh64 < 64); |
787 | 74 | unsigned S = Sh64; |
788 | 74 | |
789 | 74 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
790 | 74 | assert(F != PairMap.end()); |
791 | 74 | const UUPair &P = F->second; |
792 | 74 | unsigned LoR = P.first; |
793 | 74 | unsigned HiR = P.second; |
794 | 74 | |
795 | 74 | unsigned Opc = MI->getOpcode(); |
796 | 74 | bool Right = (Opc == S2_lsr_i_p || Opc == S2_asr_i_p8 ); |
797 | 74 | bool Left = !Right; |
798 | 74 | bool Signed = (Opc == S2_asr_i_p); |
799 | 74 | |
800 | 74 | MachineBasicBlock &B = *MI->getParent(); |
801 | 74 | DebugLoc DL = MI->getDebugLoc(); |
802 | 74 | unsigned RS = getRegState(Op1); |
803 | 74 | unsigned ShiftOpc = Left ? S2_asl_i_r3 |
804 | 74 | : (Signed 71 ? S2_asr_i_r5 : S2_lsr_i_r66 ); |
805 | 74 | unsigned LoSR = isub_lo; |
806 | 74 | unsigned HiSR = isub_hi; |
807 | 74 | |
808 | 74 | if (S == 0) { |
809 | 0 | // No shift, subregister copy. |
810 | 0 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR) |
811 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR); |
812 | 0 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), HiR) |
813 | 0 | .addReg(Op1.getReg(), RS, HiSR); |
814 | 74 | } else if (S < 32) { |
815 | 13 | const TargetRegisterClass *IntRC = &IntRegsRegClass; |
816 | 13 | unsigned TmpR = MRI->createVirtualRegister(IntRC); |
817 | 13 | // Expansion: |
818 | 13 | // Shift left: DR = shl R, #s |
819 | 13 | // LoR = shl R.lo, #s |
820 | 13 | // TmpR = extractu R.lo, #s, #32-s |
821 | 13 | // HiR = or (TmpR, asl(R.hi, #s)) |
822 | 13 | // Shift right: DR = shr R, #s |
823 | 13 | // HiR = shr R.hi, #s |
824 | 13 | // TmpR = shr R.lo, #s |
825 | 13 | // LoR = insert TmpR, R.hi, #s, #32-s |
826 | 13 | |
827 | 13 | // Shift left: |
828 | 13 | // LoR = shl R.lo, #s |
829 | 13 | // Shift right: |
830 | 13 | // TmpR = shr R.lo, #s |
831 | 13 | |
832 | 13 | // Make a special case for A2_aslh and A2_asrh (they are predicable as |
833 | 13 | // opposed to S2_asl_i_r/S2_asr_i_r). |
834 | 13 | if (S == 16 && Left11 ) |
835 | 1 | BuildMI(B, MI, DL, TII->get(A2_aslh), LoR) |
836 | 1 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR); |
837 | 12 | else if (S == 16 && Signed10 ) |
838 | 0 | BuildMI(B, MI, DL, TII->get(A2_asrh), TmpR) |
839 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR); |
840 | 12 | else |
841 | 12 | BuildMI(B, MI, DL, TII->get(ShiftOpc), (Left ? LoR0 : TmpR)) |
842 | 12 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR) |
843 | 12 | .addImm(S); |
844 | 13 | |
845 | 13 | if (Left) { |
846 | 1 | // TmpR = extractu R.lo, #s, #32-s |
847 | 1 | BuildMI(B, MI, DL, TII->get(S2_extractu), TmpR) |
848 | 1 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR) |
849 | 1 | .addImm(S) |
850 | 1 | .addImm(32-S); |
851 | 1 | // HiR = or (TmpR, asl(R.hi, #s)) |
852 | 1 | BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR) |
853 | 1 | .addReg(TmpR) |
854 | 1 | .addReg(Op1.getReg(), RS, HiSR) |
855 | 1 | .addImm(S); |
856 | 12 | } else { |
857 | 12 | // HiR = shr R.hi, #s |
858 | 12 | BuildMI(B, MI, DL, TII->get(ShiftOpc), HiR) |
859 | 12 | .addReg(Op1.getReg(), RS & ~RegState::Kill, HiSR) |
860 | 12 | .addImm(S); |
861 | 12 | // LoR = insert TmpR, R.hi, #s, #32-s |
862 | 12 | BuildMI(B, MI, DL, TII->get(S2_insert), LoR) |
863 | 12 | .addReg(TmpR) |
864 | 12 | .addReg(Op1.getReg(), RS, HiSR) |
865 | 12 | .addImm(S) |
866 | 12 | .addImm(32-S); |
867 | 12 | } |
868 | 61 | } else if (S == 32) { |
869 | 53 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), (Left ? HiR2 : LoR51 )) |
870 | 53 | .addReg(Op1.getReg(), RS & ~RegState::Kill, (Left ? LoSR2 : HiSR51 )); |
871 | 53 | if (!Signed) |
872 | 49 | BuildMI(B, MI, DL, TII->get(A2_tfrsi), (Left ? LoR2 : HiR47 )) |
873 | 49 | .addImm(0); |
874 | 4 | else // Must be right shift. |
875 | 4 | BuildMI(B, MI, DL, TII->get(S2_asr_i_r), HiR) |
876 | 4 | .addReg(Op1.getReg(), RS, HiSR) |
877 | 4 | .addImm(31); |
878 | 53 | } else if (8 S < 648 ) { |
879 | 8 | S -= 32; |
880 | 8 | if (S == 16 && Left6 ) |
881 | 0 | BuildMI(B, MI, DL, TII->get(A2_aslh), HiR) |
882 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR); |
883 | 8 | else if (S == 16 && Signed6 ) |
884 | 0 | BuildMI(B, MI, DL, TII->get(A2_asrh), LoR) |
885 | 0 | .addReg(Op1.getReg(), RS & ~RegState::Kill, HiSR); |
886 | 8 | else |
887 | 8 | BuildMI(B, MI, DL, TII->get(ShiftOpc), (Left ? HiR0 : LoR)) |
888 | 8 | .addReg(Op1.getReg(), RS & ~RegState::Kill, (Left ? LoSR0 : HiSR)) |
889 | 8 | .addImm(S); |
890 | 8 | |
891 | 8 | if (Signed) |
892 | 0 | BuildMI(B, MI, DL, TII->get(S2_asr_i_r), HiR) |
893 | 0 | .addReg(Op1.getReg(), RS, HiSR) |
894 | 0 | .addImm(31); |
895 | 8 | else |
896 | 8 | BuildMI(B, MI, DL, TII->get(A2_tfrsi), (Left ? LoR0 : HiR)) |
897 | 8 | .addImm(0); |
898 | 8 | } |
899 | 74 | } |
900 | | |
901 | | void HexagonSplitDoubleRegs::splitAslOr(MachineInstr *MI, |
902 | 26 | const UUPairMap &PairMap) { |
903 | 26 | using namespace Hexagon; |
904 | 26 | |
905 | 26 | MachineOperand &Op0 = MI->getOperand(0); |
906 | 26 | MachineOperand &Op1 = MI->getOperand(1); |
907 | 26 | MachineOperand &Op2 = MI->getOperand(2); |
908 | 26 | MachineOperand &Op3 = MI->getOperand(3); |
909 | 26 | assert(Op0.isReg() && Op1.isReg() && Op2.isReg() && Op3.isImm()); |
910 | 26 | int64_t Sh64 = Op3.getImm(); |
911 | 26 | assert(Sh64 >= 0 && Sh64 < 64); |
912 | 26 | unsigned S = Sh64; |
913 | 26 | |
914 | 26 | UUPairMap::const_iterator F = PairMap.find(Op0.getReg()); |
915 | 26 | assert(F != PairMap.end()); |
916 | 26 | const UUPair &P = F->second; |
917 | 26 | unsigned LoR = P.first; |
918 | 26 | unsigned HiR = P.second; |
919 | 26 | |
920 | 26 | MachineBasicBlock &B = *MI->getParent(); |
921 | 26 | DebugLoc DL = MI->getDebugLoc(); |
922 | 26 | unsigned RS1 = getRegState(Op1); |
923 | 26 | unsigned RS2 = getRegState(Op2); |
924 | 26 | const TargetRegisterClass *IntRC = &IntRegsRegClass; |
925 | 26 | |
926 | 26 | unsigned LoSR = isub_lo; |
927 | 26 | unsigned HiSR = isub_hi; |
928 | 26 | |
929 | 26 | // Op0 = S2_asl_i_p_or Op1, Op2, Op3 |
930 | 26 | // means: Op0 = or (Op1, asl(Op2, Op3)) |
931 | 26 | |
932 | 26 | // Expansion of |
933 | 26 | // DR = or (R1, asl(R2, #s)) |
934 | 26 | // |
935 | 26 | // LoR = or (R1.lo, asl(R2.lo, #s)) |
936 | 26 | // Tmp1 = extractu R2.lo, #s, #32-s |
937 | 26 | // Tmp2 = or R1.hi, Tmp1 |
938 | 26 | // HiR = or (Tmp2, asl(R2.hi, #s)) |
939 | 26 | |
940 | 26 | if (S == 0) { |
941 | 0 | // DR = or (R1, asl(R2, #0)) |
942 | 0 | // -> or (R1, R2) |
943 | 0 | // i.e. LoR = or R1.lo, R2.lo |
944 | 0 | // HiR = or R1.hi, R2.hi |
945 | 0 | BuildMI(B, MI, DL, TII->get(A2_or), LoR) |
946 | 0 | .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR) |
947 | 0 | .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR); |
948 | 0 | BuildMI(B, MI, DL, TII->get(A2_or), HiR) |
949 | 0 | .addReg(Op1.getReg(), RS1, HiSR) |
950 | 0 | .addReg(Op2.getReg(), RS2, HiSR); |
951 | 26 | } else if (S < 32) { |
952 | 16 | BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), LoR) |
953 | 16 | .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR) |
954 | 16 | .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR) |
955 | 16 | .addImm(S); |
956 | 16 | unsigned TmpR1 = MRI->createVirtualRegister(IntRC); |
957 | 16 | BuildMI(B, MI, DL, TII->get(S2_extractu), TmpR1) |
958 | 16 | .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR) |
959 | 16 | .addImm(S) |
960 | 16 | .addImm(32-S); |
961 | 16 | unsigned TmpR2 = MRI->createVirtualRegister(IntRC); |
962 | 16 | BuildMI(B, MI, DL, TII->get(A2_or), TmpR2) |
963 | 16 | .addReg(Op1.getReg(), RS1, HiSR) |
964 | 16 | .addReg(TmpR1); |
965 | 16 | BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR) |
966 | 16 | .addReg(TmpR2) |
967 | 16 | .addReg(Op2.getReg(), RS2, HiSR) |
968 | 16 | .addImm(S); |
969 | 16 | } else if (10 S == 3210 ) { |
970 | 7 | // DR = or (R1, asl(R2, #32)) |
971 | 7 | // -> or R1, R2.lo |
972 | 7 | // LoR = R1.lo |
973 | 7 | // HiR = or R1.hi, R2.lo |
974 | 7 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR) |
975 | 7 | .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR); |
976 | 7 | BuildMI(B, MI, DL, TII->get(A2_or), HiR) |
977 | 7 | .addReg(Op1.getReg(), RS1, HiSR) |
978 | 7 | .addReg(Op2.getReg(), RS2, LoSR); |
979 | 7 | } else if (3 S < 643 ) { |
980 | 3 | // DR = or (R1, asl(R2, #s)) |
981 | 3 | // |
982 | 3 | // LoR = R1:lo |
983 | 3 | // HiR = or (R1:hi, asl(R2:lo, #s-32)) |
984 | 3 | S -= 32; |
985 | 3 | BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR) |
986 | 3 | .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR); |
987 | 3 | BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR) |
988 | 3 | .addReg(Op1.getReg(), RS1, HiSR) |
989 | 3 | .addReg(Op2.getReg(), RS2, LoSR) |
990 | 3 | .addImm(S); |
991 | 3 | } |
992 | 26 | } |
993 | | |
994 | | bool HexagonSplitDoubleRegs::splitInstr(MachineInstr *MI, |
995 | 600 | const UUPairMap &PairMap) { |
996 | 600 | using namespace Hexagon; |
997 | 600 | |
998 | 600 | LLVM_DEBUG(dbgs() << "Splitting: " << *MI); |
999 | 600 | bool Split = false; |
1000 | 600 | unsigned Opc = MI->getOpcode(); |
1001 | 600 | |
1002 | 600 | switch (Opc) { |
1003 | 600 | case TargetOpcode::PHI: |
1004 | 230 | case TargetOpcode::COPY: { |
1005 | 230 | unsigned DstR = MI->getOperand(0).getReg(); |
1006 | 230 | if (MRI->getRegClass(DstR) == DoubleRC) { |
1007 | 24 | createHalfInstr(Opc, MI, PairMap, isub_lo); |
1008 | 24 | createHalfInstr(Opc, MI, PairMap, isub_hi); |
1009 | 24 | Split = true; |
1010 | 24 | } |
1011 | 230 | break; |
1012 | 230 | } |
1013 | 230 | case A2_andp: |
1014 | 25 | createHalfInstr(A2_and, MI, PairMap, isub_lo); |
1015 | 25 | createHalfInstr(A2_and, MI, PairMap, isub_hi); |
1016 | 25 | Split = true; |
1017 | 25 | break; |
1018 | 230 | case A2_orp: |
1019 | 6 | createHalfInstr(A2_or, MI, PairMap, isub_lo); |
1020 | 6 | createHalfInstr(A2_or, MI, PairMap, isub_hi); |
1021 | 6 | Split = true; |
1022 | 6 | break; |
1023 | 230 | case A2_xorp: |
1024 | 2 | createHalfInstr(A2_xor, MI, PairMap, isub_lo); |
1025 | 2 | createHalfInstr(A2_xor, MI, PairMap, isub_hi); |
1026 | 2 | Split = true; |
1027 | 2 | break; |
1028 | 230 | |
1029 | 230 | case L2_loadrd_io: |
1030 | 0 | case L2_loadrd_pi: |
1031 | 0 | case S2_storerd_io: |
1032 | 0 | case S2_storerd_pi: |
1033 | 0 | splitMemRef(MI, PairMap); |
1034 | 0 | Split = true; |
1035 | 0 | break; |
1036 | 0 |
|
1037 | 114 | case A2_tfrpi: |
1038 | 114 | case CONST64: |
1039 | 114 | splitImmediate(MI, PairMap); |
1040 | 114 | Split = true; |
1041 | 114 | break; |
1042 | 114 | |
1043 | 114 | case A2_combineii: |
1044 | 103 | case A4_combineir: |
1045 | 103 | case A4_combineii: |
1046 | 103 | case A4_combineri: |
1047 | 103 | case A2_combinew: |
1048 | 103 | splitCombine(MI, PairMap); |
1049 | 103 | Split = true; |
1050 | 103 | break; |
1051 | 103 | |
1052 | 103 | case A2_sxtw: |
1053 | 20 | splitExt(MI, PairMap); |
1054 | 20 | Split = true; |
1055 | 20 | break; |
1056 | 103 | |
1057 | 103 | case S2_asl_i_p: |
1058 | 74 | case S2_asr_i_p: |
1059 | 74 | case S2_lsr_i_p: |
1060 | 74 | splitShift(MI, PairMap); |
1061 | 74 | Split = true; |
1062 | 74 | break; |
1063 | 74 | |
1064 | 74 | case S2_asl_i_p_or: |
1065 | 26 | splitAslOr(MI, PairMap); |
1066 | 26 | Split = true; |
1067 | 26 | break; |
1068 | 74 | |
1069 | 74 | default: |
1070 | 0 | llvm_unreachable("Instruction not splitable"); |
1071 | 74 | return false0 ; |
1072 | 600 | } |
1073 | 600 | |
1074 | 600 | return Split; |
1075 | 600 | } |
1076 | | |
1077 | | void HexagonSplitDoubleRegs::replaceSubregUses(MachineInstr *MI, |
1078 | 457 | const UUPairMap &PairMap) { |
1079 | 1.31k | for (auto &Op : MI->operands()) { |
1080 | 1.31k | if (!Op.isReg() || !Op.isUse()1.09k || !Op.getSubReg()640 ) |
1081 | 961 | continue; |
1082 | 349 | unsigned R = Op.getReg(); |
1083 | 349 | UUPairMap::const_iterator F = PairMap.find(R); |
1084 | 349 | if (F == PairMap.end()) |
1085 | 1 | continue; |
1086 | 348 | const UUPair &P = F->second; |
1087 | 348 | switch (Op.getSubReg()) { |
1088 | 348 | case Hexagon::isub_lo: |
1089 | 231 | Op.setReg(P.first); |
1090 | 231 | break; |
1091 | 348 | case Hexagon::isub_hi: |
1092 | 117 | Op.setReg(P.second); |
1093 | 117 | break; |
1094 | 348 | } |
1095 | 348 | Op.setSubReg(0); |
1096 | 348 | } |
1097 | 457 | } |
1098 | | |
1099 | | void HexagonSplitDoubleRegs::collapseRegPairs(MachineInstr *MI, |
1100 | 193 | const UUPairMap &PairMap) { |
1101 | 193 | MachineBasicBlock &B = *MI->getParent(); |
1102 | 193 | DebugLoc DL = MI->getDebugLoc(); |
1103 | 193 | |
1104 | 622 | for (auto &Op : MI->operands()) { |
1105 | 622 | if (!Op.isReg() || !Op.isUse()463 ) |
1106 | 325 | continue; |
1107 | 297 | unsigned R = Op.getReg(); |
1108 | 297 | if (!TargetRegisterInfo::isVirtualRegister(R)) |
1109 | 4 | continue; |
1110 | 293 | if (MRI->getRegClass(R) != DoubleRC || Op.getSubReg()254 ) |
1111 | 40 | continue; |
1112 | 253 | UUPairMap::const_iterator F = PairMap.find(R); |
1113 | 253 | if (F == PairMap.end()) |
1114 | 61 | continue; |
1115 | 192 | const UUPair &Pr = F->second; |
1116 | 192 | unsigned NewDR = MRI->createVirtualRegister(DoubleRC); |
1117 | 192 | BuildMI(B, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), NewDR) |
1118 | 192 | .addReg(Pr.first) |
1119 | 192 | .addImm(Hexagon::isub_lo) |
1120 | 192 | .addReg(Pr.second) |
1121 | 192 | .addImm(Hexagon::isub_hi); |
1122 | 192 | Op.setReg(NewDR); |
1123 | 192 | } |
1124 | 193 | } |
1125 | | |
1126 | 272 | bool HexagonSplitDoubleRegs::splitPartition(const USet &Part) { |
1127 | 272 | using MISet = std::set<MachineInstr *>; |
1128 | 272 | |
1129 | 272 | const TargetRegisterClass *IntRC = &Hexagon::IntRegsRegClass; |
1130 | 272 | bool Changed = false; |
1131 | 272 | |
1132 | 272 | LLVM_DEBUG(dbgs() << "Splitting partition: "; |
1133 | 272 | dump_partition(dbgs(), Part, *TRI); dbgs() << '\n'); |
1134 | 272 | |
1135 | 272 | UUPairMap PairMap; |
1136 | 272 | |
1137 | 272 | MISet SplitIns; |
1138 | 394 | for (unsigned DR : Part) { |
1139 | 394 | MachineInstr *DefI = MRI->getVRegDef(DR); |
1140 | 394 | SplitIns.insert(DefI); |
1141 | 394 | |
1142 | 394 | // Collect all instructions, including fixed ones. We won't split them, |
1143 | 394 | // but we need to visit them again to insert the REG_SEQUENCE instructions. |
1144 | 394 | for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end(); |
1145 | 933 | U != W; ++U539 ) |
1146 | 539 | SplitIns.insert(U->getParent()); |
1147 | 394 | |
1148 | 394 | unsigned LoR = MRI->createVirtualRegister(IntRC); |
1149 | 394 | unsigned HiR = MRI->createVirtualRegister(IntRC); |
1150 | 394 | LLVM_DEBUG(dbgs() << "Created mapping: " << printReg(DR, TRI) << " -> " |
1151 | 394 | << printReg(HiR, TRI) << ':' << printReg(LoR, TRI) |
1152 | 394 | << '\n'); |
1153 | 394 | PairMap.insert(std::make_pair(DR, UUPair(LoR, HiR))); |
1154 | 394 | } |
1155 | 272 | |
1156 | 272 | MISet Erase; |
1157 | 793 | for (auto MI : SplitIns) { |
1158 | 793 | if (isFixedInstr(MI)) { |
1159 | 193 | collapseRegPairs(MI, PairMap); |
1160 | 600 | } else { |
1161 | 600 | bool Done = splitInstr(MI, PairMap); |
1162 | 600 | if (Done) |
1163 | 394 | Erase.insert(MI); |
1164 | 600 | Changed |= Done; |
1165 | 600 | } |
1166 | 793 | } |
1167 | 272 | |
1168 | 394 | for (unsigned DR : Part) { |
1169 | 394 | // Before erasing "double" instructions, revisit all uses of the double |
1170 | 394 | // registers in this partition, and replace all uses of them with subre- |
1171 | 394 | // gisters, with the corresponding single registers. |
1172 | 394 | MISet Uses; |
1173 | 394 | for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end(); |
1174 | 863 | U != W; ++U469 ) |
1175 | 469 | Uses.insert(U->getParent()); |
1176 | 394 | for (auto M : Uses) |
1177 | 457 | replaceSubregUses(M, PairMap); |
1178 | 394 | } |
1179 | 272 | |
1180 | 394 | for (auto MI : Erase) { |
1181 | 394 | MachineBasicBlock *B = MI->getParent(); |
1182 | 394 | B->erase(MI); |
1183 | 394 | } |
1184 | 272 | |
1185 | 272 | return Changed; |
1186 | 272 | } |
1187 | | |
1188 | 3.33k | bool HexagonSplitDoubleRegs::runOnMachineFunction(MachineFunction &MF) { |
1189 | 3.33k | if (skipFunction(MF.getFunction())) |
1190 | 10 | return false; |
1191 | 3.32k | |
1192 | 3.32k | LLVM_DEBUG(dbgs() << "Splitting double registers in function: " |
1193 | 3.32k | << MF.getName() << '\n'); |
1194 | 3.32k | |
1195 | 3.32k | auto &ST = MF.getSubtarget<HexagonSubtarget>(); |
1196 | 3.32k | TRI = ST.getRegisterInfo(); |
1197 | 3.32k | TII = ST.getInstrInfo(); |
1198 | 3.32k | MRI = &MF.getRegInfo(); |
1199 | 3.32k | MLI = &getAnalysis<MachineLoopInfo>(); |
1200 | 3.32k | |
1201 | 3.32k | UUSetMap P2Rs; |
1202 | 3.32k | LoopRegMap IRM; |
1203 | 3.32k | |
1204 | 3.32k | collectIndRegs(IRM); |
1205 | 3.32k | partitionRegisters(P2Rs); |
1206 | 3.32k | |
1207 | 3.32k | LLVM_DEBUG({ |
1208 | 3.32k | dbgs() << "Register partitioning: (partition #0 is fixed)\n"; |
1209 | 3.32k | for (UUSetMap::iterator I = P2Rs.begin(), E = P2Rs.end(); I != E; ++I) { |
1210 | 3.32k | dbgs() << '#' << I->first << " -> "; |
1211 | 3.32k | dump_partition(dbgs(), I->second, *TRI); |
1212 | 3.32k | dbgs() << '\n'; |
1213 | 3.32k | } |
1214 | 3.32k | }); |
1215 | 3.32k | |
1216 | 3.32k | bool Changed = false; |
1217 | 3.32k | int Limit = MaxHSDR; |
1218 | 3.32k | |
1219 | 4.14k | for (UUSetMap::iterator I = P2Rs.begin(), E = P2Rs.end(); I != E; ++I820 ) { |
1220 | 820 | if (I->first == 0) |
1221 | 435 | continue; |
1222 | 385 | if (Limit >= 0 && Counter >= Limit0 ) |
1223 | 0 | break; |
1224 | 385 | USet &Part = I->second; |
1225 | 385 | LLVM_DEBUG(dbgs() << "Calculating profit for partition #" << I->first |
1226 | 385 | << '\n'); |
1227 | 385 | if (!isProfitable(Part, IRM)) |
1228 | 113 | continue; |
1229 | 272 | Counter++; |
1230 | 272 | Changed |= splitPartition(Part); |
1231 | 272 | } |
1232 | 3.32k | |
1233 | 3.32k | return Changed; |
1234 | 3.32k | } |
1235 | | |
1236 | 848 | FunctionPass *llvm::createHexagonSplitDoubleRegs() { |
1237 | 848 | return new HexagonSplitDoubleRegs(); |
1238 | 848 | } |