/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | // |
10 | | // This file implements integer type expansion and promotion for LegalizeTypes. |
11 | | // Promotion is the act of changing a computation in an illegal type into a |
12 | | // computation in a larger type. For example, implementing i8 arithmetic in an |
13 | | // i32 register (often needed on powerpc). |
14 | | // Expansion is the act of changing a computation in an illegal type into a |
15 | | // computation in two identical registers of a smaller type. For example, |
16 | | // implementing i64 arithmetic in two i32 registers (often needed on 32-bit |
17 | | // targets). |
18 | | // |
19 | | //===----------------------------------------------------------------------===// |
20 | | |
21 | | #include "LegalizeTypes.h" |
22 | | #include "llvm/IR/DerivedTypes.h" |
23 | | #include "llvm/Support/ErrorHandling.h" |
24 | | #include "llvm/Support/KnownBits.h" |
25 | | #include "llvm/Support/raw_ostream.h" |
26 | | using namespace llvm; |
27 | | |
28 | | #define DEBUG_TYPE "legalize-types" |
29 | | |
30 | | //===----------------------------------------------------------------------===// |
31 | | // Integer Result Promotion |
32 | | //===----------------------------------------------------------------------===// |
33 | | |
34 | | /// PromoteIntegerResult - This method is called when a result of a node is |
35 | | /// found to be in need of promotion to a larger type. At this point, the node |
36 | | /// may also have invalid operands or may have other results that need |
37 | | /// expansion, we just know that (at least) one result needs promotion. |
38 | 1.38M | void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { |
39 | 1.38M | DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n"); |
40 | 1.38M | SDValue Res = SDValue(); |
41 | 1.38M | |
42 | 1.38M | // See if the target wants to custom expand this node. |
43 | 1.38M | if (CustomLowerNode(N, N->getValueType(ResNo), true)) |
44 | 181 | return; |
45 | 1.38M | |
46 | 1.38M | switch (N->getOpcode()) { |
47 | 0 | default: |
48 | | #ifndef NDEBUG |
49 | | dbgs() << "PromoteIntegerResult #" << ResNo << ": "; |
50 | | N->dump(&DAG); dbgs() << "\n"; |
51 | | #endif |
52 | 0 | llvm_unreachable("Do not know how to promote this operator!"); |
53 | 0 | case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break; |
54 | 63 | case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break; |
55 | 63 | case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break; |
56 | 4.88k | case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break; |
57 | 15 | case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break; |
58 | 88 | case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break; |
59 | 0 | case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break; |
60 | 458k | case ISD::Constant: Res = PromoteIntRes_Constant(N); break; |
61 | 8 | case ISD::CTLZ_ZERO_UNDEF: |
62 | 8 | case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; |
63 | 30 | case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; |
64 | 20 | case ISD::CTTZ_ZERO_UNDEF: |
65 | 20 | case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; |
66 | 80.4k | case ISD::EXTRACT_VECTOR_ELT: |
67 | 80.4k | Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; |
68 | 224k | case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break; |
69 | 4 | case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N)); |
70 | 4 | break; |
71 | 10 | case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N)); |
72 | 10 | break; |
73 | 13.6k | case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break; |
74 | 78 | case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; |
75 | 0 | case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; |
76 | 339k | case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; |
77 | 92 | case ISD::SMIN: |
78 | 92 | case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; |
79 | 50 | case ISD::UMIN: |
80 | 50 | case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break; |
81 | 50 | |
82 | 1.62k | case ISD::SHL: Res = PromoteIntRes_SHL(N); break; |
83 | 60 | case ISD::SIGN_EXTEND_INREG: |
84 | 60 | Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break; |
85 | 298 | case ISD::SRA: Res = PromoteIntRes_SRA(N); break; |
86 | 6.11k | case ISD::SRL: Res = PromoteIntRes_SRL(N); break; |
87 | 148k | case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break; |
88 | 2.63k | case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break; |
89 | 36 | case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break; |
90 | 50 | |
91 | 23.6k | case ISD::EXTRACT_SUBVECTOR: |
92 | 23.6k | Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break; |
93 | 104 | case ISD::VECTOR_SHUFFLE: |
94 | 104 | Res = PromoteIntRes_VECTOR_SHUFFLE(N); break; |
95 | 184 | case ISD::INSERT_VECTOR_ELT: |
96 | 184 | Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break; |
97 | 1.80k | case ISD::BUILD_VECTOR: |
98 | 1.80k | Res = PromoteIntRes_BUILD_VECTOR(N); break; |
99 | 26 | case ISD::SCALAR_TO_VECTOR: |
100 | 26 | Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break; |
101 | 1.83k | case ISD::CONCAT_VECTORS: |
102 | 1.83k | Res = PromoteIntRes_CONCAT_VECTORS(N); break; |
103 | 50 | |
104 | 3 | case ISD::ANY_EXTEND_VECTOR_INREG: |
105 | 3 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
106 | 3 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
107 | 3 | Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; |
108 | 3 | |
109 | 28.8k | case ISD::SIGN_EXTEND: |
110 | 28.8k | case ISD::ZERO_EXTEND: |
111 | 28.8k | case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; |
112 | 28.8k | |
113 | 216 | case ISD::FP_TO_SINT: |
114 | 216 | case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break; |
115 | 216 | |
116 | 803 | case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break; |
117 | 216 | |
118 | 38.5k | case ISD::AND: |
119 | 38.5k | case ISD::OR: |
120 | 38.5k | case ISD::XOR: |
121 | 38.5k | case ISD::ADD: |
122 | 38.5k | case ISD::SUB: |
123 | 38.5k | case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break; |
124 | 38.5k | |
125 | 303 | case ISD::SDIV: |
126 | 303 | case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break; |
127 | 303 | |
128 | 332 | case ISD::UDIV: |
129 | 332 | case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break; |
130 | 332 | |
131 | 105 | case ISD::SADDO: |
132 | 105 | case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break; |
133 | 266 | case ISD::UADDO: |
134 | 266 | case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break; |
135 | 1.19k | case ISD::SMULO: |
136 | 1.19k | case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break; |
137 | 1.19k | |
138 | 5 | case ISD::ADDCARRY: |
139 | 5 | case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break; |
140 | 5 | |
141 | 335 | case ISD::ATOMIC_LOAD: |
142 | 335 | Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break; |
143 | 5 | |
144 | 973 | case ISD::ATOMIC_LOAD_ADD: |
145 | 973 | case ISD::ATOMIC_LOAD_SUB: |
146 | 973 | case ISD::ATOMIC_LOAD_AND: |
147 | 973 | case ISD::ATOMIC_LOAD_OR: |
148 | 973 | case ISD::ATOMIC_LOAD_XOR: |
149 | 973 | case ISD::ATOMIC_LOAD_NAND: |
150 | 973 | case ISD::ATOMIC_LOAD_MIN: |
151 | 973 | case ISD::ATOMIC_LOAD_MAX: |
152 | 973 | case ISD::ATOMIC_LOAD_UMIN: |
153 | 973 | case ISD::ATOMIC_LOAD_UMAX: |
154 | 973 | case ISD::ATOMIC_SWAP: |
155 | 973 | Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break; |
156 | 973 | |
157 | 1.80k | case ISD::ATOMIC_CMP_SWAP: |
158 | 1.80k | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
159 | 1.80k | Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo); |
160 | 1.80k | break; |
161 | 1.38M | } |
162 | 1.38M | |
163 | 1.38M | // If the result is null then the sub-method took care of registering it. |
164 | 1.38M | if (1.38M Res.getNode()1.38M ) |
165 | 1.38M | SetPromotedInteger(SDValue(N, ResNo), Res); |
166 | 1.38M | } |
167 | | |
168 | | SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N, |
169 | 0 | unsigned ResNo) { |
170 | 0 | SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); |
171 | 0 | return GetPromotedInteger(Op); |
172 | 0 | } |
173 | | |
174 | 63 | SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) { |
175 | 63 | // Sign-extend the new bits, and continue the assertion. |
176 | 63 | SDValue Op = SExtPromotedInteger(N->getOperand(0)); |
177 | 63 | return DAG.getNode(ISD::AssertSext, SDLoc(N), |
178 | 63 | Op.getValueType(), Op, N->getOperand(1)); |
179 | 63 | } |
180 | | |
181 | 63 | SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) { |
182 | 63 | // Zero the new bits, and continue the assertion. |
183 | 63 | SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
184 | 63 | return DAG.getNode(ISD::AssertZext, SDLoc(N), |
185 | 63 | Op.getValueType(), Op, N->getOperand(1)); |
186 | 63 | } |
187 | | |
188 | 335 | SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) { |
189 | 335 | EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
190 | 335 | SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), |
191 | 335 | N->getMemoryVT(), ResVT, |
192 | 335 | N->getChain(), N->getBasePtr(), |
193 | 335 | N->getMemOperand()); |
194 | 335 | // Legalize the chain result - switch anything that used the old chain to |
195 | 335 | // use the new one. |
196 | 335 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
197 | 335 | return Res; |
198 | 335 | } |
199 | | |
200 | 973 | SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) { |
201 | 973 | SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
202 | 973 | SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N), |
203 | 973 | N->getMemoryVT(), |
204 | 973 | N->getChain(), N->getBasePtr(), |
205 | 973 | Op2, N->getMemOperand()); |
206 | 973 | // Legalize the chain result - switch anything that used the old chain to |
207 | 973 | // use the new one. |
208 | 973 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
209 | 973 | return Res; |
210 | 973 | } |
211 | | |
212 | | SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, |
213 | 1.80k | unsigned ResNo) { |
214 | 1.80k | if (ResNo == 11.80k ) { |
215 | 1.67k | assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
216 | 1.67k | EVT SVT = getSetCCResultType(N->getOperand(2).getValueType()); |
217 | 1.67k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
218 | 1.67k | |
219 | 1.67k | // Only use the result of getSetCCResultType if it is legal, |
220 | 1.67k | // otherwise just use the promoted result type (NVT). |
221 | 1.67k | if (!TLI.isTypeLegal(SVT)) |
222 | 0 | SVT = NVT; |
223 | 1.67k | |
224 | 1.67k | SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other); |
225 | 1.67k | SDValue Res = DAG.getAtomicCmpSwap( |
226 | 1.67k | ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs, |
227 | 1.67k | N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3), |
228 | 1.67k | N->getMemOperand()); |
229 | 1.67k | ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); |
230 | 1.67k | ReplaceValueWith(SDValue(N, 2), Res.getValue(2)); |
231 | 1.67k | return Res.getValue(1); |
232 | 1.67k | } |
233 | 137 | |
234 | 137 | SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
235 | 137 | SDValue Op3 = GetPromotedInteger(N->getOperand(3)); |
236 | 137 | SDVTList VTs = |
237 | 137 | DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other); |
238 | 137 | SDValue Res = DAG.getAtomicCmpSwap( |
239 | 137 | N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(), |
240 | 137 | N->getBasePtr(), Op2, Op3, N->getMemOperand()); |
241 | 137 | // Update the use to N with the newly created Res. |
242 | 411 | for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults411 ; ++i274 ) |
243 | 274 | ReplaceValueWith(SDValue(N, i), Res.getValue(i)); |
244 | 1.80k | return Res; |
245 | 1.80k | } |
246 | | |
247 | 4.88k | SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { |
248 | 4.88k | SDValue InOp = N->getOperand(0); |
249 | 4.88k | EVT InVT = InOp.getValueType(); |
250 | 4.88k | EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); |
251 | 4.88k | EVT OutVT = N->getValueType(0); |
252 | 4.88k | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
253 | 4.88k | SDLoc dl(N); |
254 | 4.88k | |
255 | 4.88k | switch (getTypeAction(InVT)) { |
256 | 774 | case TargetLowering::TypeLegal: |
257 | 774 | break; |
258 | 542 | case TargetLowering::TypePromoteInteger: |
259 | 542 | if (NOutVT.bitsEq(NInVT) && 542 !NOutVT.isVector()58 && !NInVT.isVector()0 ) |
260 | 542 | // The input promotes to the same size. Convert the promoted value. |
261 | 0 | return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); |
262 | 542 | break; |
263 | 0 | case TargetLowering::TypeSoftenFloat: |
264 | 0 | // Promote the integer operand by hand. |
265 | 0 | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp)); |
266 | 434 | case TargetLowering::TypePromoteFloat: { |
267 | 434 | // Convert the promoted float by hand. |
268 | 434 | SDValue PromotedOp = GetPromotedFloat(InOp); |
269 | 434 | return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp); |
270 | 0 | break; |
271 | 542 | } |
272 | 357 | case TargetLowering::TypeExpandInteger: |
273 | 357 | case TargetLowering::TypeExpandFloat: |
274 | 357 | break; |
275 | 2.31k | case TargetLowering::TypeScalarizeVector: |
276 | 2.31k | // Convert the element to an integer and promote it by hand. |
277 | 2.31k | if (!NOutVT.isVector()) |
278 | 2.27k | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, |
279 | 2.27k | BitConvertToInteger(GetScalarizedVector(InOp))); |
280 | 37 | break; |
281 | 457 | case TargetLowering::TypeSplitVector: { |
282 | 457 | // For example, i32 = BITCAST v2i16 on alpha. Convert the split |
283 | 457 | // pieces of the input into integers and reassemble in the final type. |
284 | 457 | SDValue Lo, Hi; |
285 | 457 | GetSplitVector(N->getOperand(0), Lo, Hi); |
286 | 457 | Lo = BitConvertToInteger(Lo); |
287 | 457 | Hi = BitConvertToInteger(Hi); |
288 | 457 | |
289 | 457 | if (DAG.getDataLayout().isBigEndian()) |
290 | 45 | std::swap(Lo, Hi); |
291 | 457 | |
292 | 457 | InOp = DAG.getNode(ISD::ANY_EXTEND, dl, |
293 | 457 | EVT::getIntegerVT(*DAG.getContext(), |
294 | 457 | NOutVT.getSizeInBits()), |
295 | 457 | JoinIntegers(Lo, Hi)); |
296 | 457 | return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); |
297 | 37 | } |
298 | 7 | case TargetLowering::TypeWidenVector: |
299 | 7 | // The input is widened to the same size. Convert to the widened value. |
300 | 7 | // Make sure that the outgoing value is not a vector, because this would |
301 | 7 | // make us bitcast between two vectors which are legalized in different ways. |
302 | 7 | if (NOutVT.bitsEq(NInVT) && 7 !NOutVT.isVector()7 ) |
303 | 5 | return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp)); |
304 | 1.71k | } |
305 | 1.71k | |
306 | 1.71k | return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, |
307 | 1.71k | CreateStackStoreLoad(InOp, OutVT)); |
308 | 1.71k | } |
309 | | |
310 | 88 | SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) { |
311 | 88 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
312 | 88 | EVT OVT = N->getValueType(0); |
313 | 88 | EVT NVT = Op.getValueType(); |
314 | 88 | SDLoc dl(N); |
315 | 88 | |
316 | 88 | unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
317 | 88 | return DAG.getNode( |
318 | 88 | ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op), |
319 | 88 | DAG.getConstant(DiffBits, dl, |
320 | 88 | TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); |
321 | 88 | } |
322 | | |
323 | 15 | SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) { |
324 | 15 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
325 | 15 | EVT OVT = N->getValueType(0); |
326 | 15 | EVT NVT = Op.getValueType(); |
327 | 15 | SDLoc dl(N); |
328 | 15 | |
329 | 15 | unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(); |
330 | 15 | return DAG.getNode( |
331 | 15 | ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), |
332 | 15 | DAG.getConstant(DiffBits, dl, |
333 | 15 | TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); |
334 | 15 | } |
335 | | |
336 | 0 | SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) { |
337 | 0 | // The pair element type may be legal, or may not promote to the same type as |
338 | 0 | // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases. |
339 | 0 | return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), |
340 | 0 | TLI.getTypeToTransformTo(*DAG.getContext(), |
341 | 0 | N->getValueType(0)), JoinIntegers(N->getOperand(0), |
342 | 0 | N->getOperand(1))); |
343 | 0 | } |
344 | | |
345 | 458k | SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) { |
346 | 458k | EVT VT = N->getValueType(0); |
347 | 458k | // FIXME there is no actual debug info here |
348 | 458k | SDLoc dl(N); |
349 | 458k | // Zero extend things like i1, sign extend everything else. It shouldn't |
350 | 458k | // matter in theory which one we pick, but this tends to give better code? |
351 | 458k | unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND386k : ISD::ZERO_EXTEND71.6k ; |
352 | 458k | SDValue Result = DAG.getNode(Opc, dl, |
353 | 458k | TLI.getTypeToTransformTo(*DAG.getContext(), VT), |
354 | 458k | SDValue(N, 0)); |
355 | 458k | assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?"); |
356 | 458k | return Result; |
357 | 458k | } |
358 | | |
359 | 8 | SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { |
360 | 8 | // Zero extend to the promoted type and do the count there. |
361 | 8 | SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
362 | 8 | SDLoc dl(N); |
363 | 8 | EVT OVT = N->getValueType(0); |
364 | 8 | EVT NVT = Op.getValueType(); |
365 | 8 | Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); |
366 | 8 | // Subtract off the extra leading bits in the bigger type. |
367 | 8 | return DAG.getNode( |
368 | 8 | ISD::SUB, dl, NVT, Op, |
369 | 8 | DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, |
370 | 8 | NVT)); |
371 | 8 | } |
372 | | |
373 | 30 | SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) { |
374 | 30 | // Zero extend to the promoted type and do the count there. |
375 | 30 | SDValue Op = ZExtPromotedInteger(N->getOperand(0)); |
376 | 30 | return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op); |
377 | 30 | } |
378 | | |
379 | 20 | SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { |
380 | 20 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
381 | 20 | EVT OVT = N->getValueType(0); |
382 | 20 | EVT NVT = Op.getValueType(); |
383 | 20 | SDLoc dl(N); |
384 | 20 | if (N->getOpcode() == ISD::CTTZ20 ) { |
385 | 10 | // The count is the same in the promoted type except if the original |
386 | 10 | // value was zero. This can be handled by setting the bit just off |
387 | 10 | // the top of the original type. |
388 | 10 | auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(), |
389 | 10 | OVT.getScalarSizeInBits()); |
390 | 10 | Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT)); |
391 | 10 | } |
392 | 20 | return DAG.getNode(N->getOpcode(), dl, NVT, Op); |
393 | 20 | } |
394 | | |
395 | 80.4k | SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { |
396 | 80.4k | SDLoc dl(N); |
397 | 80.4k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
398 | 80.4k | return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0), |
399 | 80.4k | N->getOperand(1)); |
400 | 80.4k | } |
401 | | |
402 | 216 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) { |
403 | 216 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
404 | 216 | unsigned NewOpc = N->getOpcode(); |
405 | 216 | SDLoc dl(N); |
406 | 216 | |
407 | 216 | // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is |
408 | 216 | // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT |
409 | 216 | // and SINT conversions are Custom, there is no way to tell which is |
410 | 216 | // preferable. We choose SINT because that's the right thing on PPC.) |
411 | 216 | if (N->getOpcode() == ISD::FP_TO_UINT && |
412 | 136 | !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) && |
413 | 125 | TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)) |
414 | 94 | NewOpc = ISD::FP_TO_SINT; |
415 | 216 | |
416 | 216 | SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0)); |
417 | 216 | |
418 | 216 | // Assert that the converted value fits in the original type. If it doesn't |
419 | 216 | // (eg: because the value being converted is too big), then the result of the |
420 | 216 | // original operation was undefined anyway, so the assert is still correct. |
421 | 216 | // |
422 | 216 | // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example: |
423 | 216 | // before legalization: fp-to-uint16, 65534. -> 0xfffe |
424 | 216 | // after legalization: fp-to-sint32, 65534. -> 0x0000fffe |
425 | 216 | return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? |
426 | 216 | ISD::AssertZext136 : ISD::AssertSext80 , dl, NVT, Res, |
427 | 216 | DAG.getValueType(N->getValueType(0).getScalarType())); |
428 | 216 | } |
429 | | |
430 | 803 | SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) { |
431 | 803 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
432 | 803 | SDLoc dl(N); |
433 | 803 | |
434 | 803 | return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
435 | 803 | } |
436 | | |
437 | 28.8k | SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) { |
438 | 28.8k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
439 | 28.8k | SDLoc dl(N); |
440 | 28.8k | |
441 | 28.8k | if (getTypeAction(N->getOperand(0).getValueType()) |
442 | 28.8k | == TargetLowering::TypePromoteInteger) { |
443 | 28.0k | SDValue Res = GetPromotedInteger(N->getOperand(0)); |
444 | 28.0k | assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!"); |
445 | 28.0k | |
446 | 28.0k | // If the result and operand types are the same after promotion, simplify |
447 | 28.0k | // to an in-register extension. |
448 | 28.0k | if (NVT == Res.getValueType()28.0k ) { |
449 | 28.0k | // The high bits are not guaranteed to be anything. Insert an extend. |
450 | 28.0k | if (N->getOpcode() == ISD::SIGN_EXTEND) |
451 | 469 | return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, |
452 | 469 | DAG.getValueType(N->getOperand(0).getValueType())); |
453 | 27.5k | if (27.5k N->getOpcode() == ISD::ZERO_EXTEND27.5k ) |
454 | 1.31k | return DAG.getZeroExtendInReg(Res, dl, |
455 | 1.31k | N->getOperand(0).getValueType().getScalarType()); |
456 | 0 | assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); |
457 | 26.2k | return Res; |
458 | 26.2k | } |
459 | 28.0k | } |
460 | 801 | |
461 | 801 | // Otherwise, just extend the original operand all the way to the larger type. |
462 | 801 | return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
463 | 801 | } |
464 | | |
465 | 224k | SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { |
466 | 224k | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); |
467 | 224k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
468 | 224k | ISD::LoadExtType ExtType = |
469 | 224k | ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD223k : N->getExtensionType()1.17k ; |
470 | 224k | SDLoc dl(N); |
471 | 224k | SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(), |
472 | 224k | N->getMemoryVT(), N->getMemOperand()); |
473 | 224k | |
474 | 224k | // Legalize the chain result - switch anything that used the old chain to |
475 | 224k | // use the new one. |
476 | 224k | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
477 | 224k | return Res; |
478 | 224k | } |
479 | | |
480 | 4 | SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { |
481 | 4 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
482 | 4 | SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0()); |
483 | 4 | |
484 | 4 | SDLoc dl(N); |
485 | 4 | SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), |
486 | 4 | N->getMask(), ExtSrc0, N->getMemoryVT(), |
487 | 4 | N->getMemOperand(), ISD::SEXTLOAD); |
488 | 4 | // Legalize the chain result - switch anything that used the old chain to |
489 | 4 | // use the new one. |
490 | 4 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
491 | 4 | return Res; |
492 | 4 | } |
493 | | |
494 | 10 | SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { |
495 | 10 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
496 | 10 | SDValue ExtSrc0 = GetPromotedInteger(N->getValue()); |
497 | 10 | assert(NVT == ExtSrc0.getValueType() && |
498 | 10 | "Gather result type and the passThru agrument type should be the same"); |
499 | 10 | |
500 | 10 | SDLoc dl(N); |
501 | 10 | SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(), |
502 | 10 | N->getIndex()}; |
503 | 10 | SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other), |
504 | 10 | N->getMemoryVT(), dl, Ops, |
505 | 10 | N->getMemOperand()); |
506 | 10 | // Legalize the chain result - switch anything that used the old chain to |
507 | 10 | // use the new one. |
508 | 10 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
509 | 10 | return Res; |
510 | 10 | } |
511 | | |
512 | | /// Promote the overflow flag of an overflowing arithmetic node. |
513 | 1.54k | SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { |
514 | 1.54k | // Simply change the return type of the boolean result. |
515 | 1.54k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); |
516 | 1.54k | EVT ValueVTs[] = { N->getValueType(0), NVT }; |
517 | 1.54k | SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; |
518 | 1.54k | unsigned NumOps = N->getNumOperands(); |
519 | 1.54k | assert(NumOps <= 3 && "Too many operands"); |
520 | 1.54k | if (NumOps == 3) |
521 | 5 | Ops[2] = N->getOperand(2); |
522 | 1.54k | |
523 | 1.54k | SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), |
524 | 1.54k | DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps)); |
525 | 1.54k | |
526 | 1.54k | // Modified the sum result - switch anything that used the old sum to use |
527 | 1.54k | // the new one. |
528 | 1.54k | ReplaceValueWith(SDValue(N, 0), Res); |
529 | 1.54k | |
530 | 1.54k | return SDValue(Res.getNode(), 1); |
531 | 1.54k | } |
532 | | |
533 | 105 | SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) { |
534 | 105 | if (ResNo == 1) |
535 | 103 | return PromoteIntRes_Overflow(N); |
536 | 2 | |
537 | 2 | // The operation overflowed iff the result in the larger type is not the |
538 | 2 | // sign extension of its truncation to the original type. |
539 | 2 | SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
540 | 2 | SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
541 | 2 | EVT OVT = N->getOperand(0).getValueType(); |
542 | 2 | EVT NVT = LHS.getValueType(); |
543 | 2 | SDLoc dl(N); |
544 | 2 | |
545 | 2 | // Do the arithmetic in the larger type. |
546 | 2 | unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD2 : ISD::SUB0 ; |
547 | 105 | SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); |
548 | 105 | |
549 | 105 | // Calculate the overflow flag: sign extend the arithmetic result from |
550 | 105 | // the original type. |
551 | 105 | SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, |
552 | 105 | DAG.getValueType(OVT)); |
553 | 105 | // Overflowed if and only if this is not equal to Res. |
554 | 105 | Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); |
555 | 105 | |
556 | 105 | // Use the calculated overflow everywhere. |
557 | 105 | ReplaceValueWith(SDValue(N, 1), Ofl); |
558 | 105 | |
559 | 105 | return Res; |
560 | 105 | } |
561 | | |
562 | 13.6k | SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) { |
563 | 13.6k | SDValue LHS = GetPromotedInteger(N->getOperand(1)); |
564 | 13.6k | SDValue RHS = GetPromotedInteger(N->getOperand(2)); |
565 | 13.6k | return DAG.getSelect(SDLoc(N), |
566 | 13.6k | LHS.getValueType(), N->getOperand(0), LHS, RHS); |
567 | 13.6k | } |
568 | | |
569 | 78 | SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) { |
570 | 78 | SDValue Mask = N->getOperand(0); |
571 | 78 | EVT OpTy = N->getOperand(1).getValueType(); |
572 | 78 | |
573 | 78 | // Promote all the way up to the canonical SetCC type. |
574 | 78 | Mask = PromoteTargetBoolean(Mask, OpTy); |
575 | 78 | SDValue LHS = GetPromotedInteger(N->getOperand(1)); |
576 | 78 | SDValue RHS = GetPromotedInteger(N->getOperand(2)); |
577 | 78 | return DAG.getNode(ISD::VSELECT, SDLoc(N), |
578 | 78 | LHS.getValueType(), Mask, LHS, RHS); |
579 | 78 | } |
580 | | |
581 | 0 | SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) { |
582 | 0 | SDValue LHS = GetPromotedInteger(N->getOperand(2)); |
583 | 0 | SDValue RHS = GetPromotedInteger(N->getOperand(3)); |
584 | 0 | return DAG.getNode(ISD::SELECT_CC, SDLoc(N), |
585 | 0 | LHS.getValueType(), N->getOperand(0), |
586 | 0 | N->getOperand(1), LHS, RHS, N->getOperand(4)); |
587 | 0 | } |
588 | | |
589 | 339k | SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { |
590 | 339k | EVT SVT = getSetCCResultType(N->getOperand(0).getValueType()); |
591 | 339k | |
592 | 339k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
593 | 339k | |
594 | 339k | // Only use the result of getSetCCResultType if it is legal, |
595 | 339k | // otherwise just use the promoted result type (NVT). |
596 | 339k | if (!TLI.isTypeLegal(SVT)) |
597 | 1.63k | SVT = NVT; |
598 | 339k | |
599 | 339k | SDLoc dl(N); |
600 | 339k | assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() && |
601 | 339k | "Vector compare must return a vector result!"); |
602 | 339k | |
603 | 339k | SDValue LHS = N->getOperand(0); |
604 | 339k | SDValue RHS = N->getOperand(1); |
605 | 339k | if (LHS.getValueType() != RHS.getValueType()339k ) { |
606 | 1 | if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger && |
607 | 0 | !LHS.getValueType().isVector()) |
608 | 0 | LHS = GetPromotedInteger(LHS); |
609 | 1 | if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger && |
610 | 1 | !RHS.getValueType().isVector()) |
611 | 1 | RHS = GetPromotedInteger(RHS); |
612 | 1 | } |
613 | 339k | |
614 | 339k | // Get the SETCC result using the canonical SETCC type. |
615 | 339k | SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS, |
616 | 339k | N->getOperand(2)); |
617 | 339k | |
618 | 339k | // Convert to the expected type. |
619 | 339k | return DAG.getSExtOrTrunc(SetCC, dl, NVT); |
620 | 339k | } |
621 | | |
622 | 1.62k | SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) { |
623 | 1.62k | SDValue LHS = N->getOperand(0); |
624 | 1.62k | SDValue RHS = N->getOperand(1); |
625 | 1.62k | if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) |
626 | 1.62k | LHS = GetPromotedInteger(LHS); |
627 | 1.62k | if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
628 | 51 | RHS = ZExtPromotedInteger(RHS); |
629 | 1.62k | return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS); |
630 | 1.62k | } |
631 | | |
632 | 60 | SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) { |
633 | 60 | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
634 | 60 | return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), |
635 | 60 | Op.getValueType(), Op, N->getOperand(1)); |
636 | 60 | } |
637 | | |
638 | 38.5k | SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) { |
639 | 38.5k | // The input may have strange things in the top bits of the registers, but |
640 | 38.5k | // these operations don't care. They may have weird bits going out, but |
641 | 38.5k | // that too is okay if they are integer operations. |
642 | 38.5k | SDValue LHS = GetPromotedInteger(N->getOperand(0)); |
643 | 38.5k | SDValue RHS = GetPromotedInteger(N->getOperand(1)); |
644 | 38.5k | return DAG.getNode(N->getOpcode(), SDLoc(N), |
645 | 38.5k | LHS.getValueType(), LHS, RHS); |
646 | 38.5k | } |
647 | | |
648 | 395 | SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) { |
649 | 395 | // Sign extend the input. |
650 | 395 | SDValue LHS = SExtPromotedInteger(N->getOperand(0)); |
651 | 395 | SDValue RHS = SExtPromotedInteger(N->getOperand(1)); |
652 | 395 | return DAG.getNode(N->getOpcode(), SDLoc(N), |
653 | 395 | LHS.getValueType(), LHS, RHS); |
654 | 395 | } |
655 | | |
656 | 382 | SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) { |
657 | 382 | // Zero extend the input. |
658 | 382 | SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
659 | 382 | SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); |
660 | 382 | return DAG.getNode(N->getOpcode(), SDLoc(N), |
661 | 382 | LHS.getValueType(), LHS, RHS); |
662 | 382 | } |
663 | | |
664 | 298 | SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) { |
665 | 298 | SDValue LHS = N->getOperand(0); |
666 | 298 | SDValue RHS = N->getOperand(1); |
667 | 298 | // The input value must be properly sign extended. |
668 | 298 | if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) |
669 | 298 | LHS = SExtPromotedInteger(LHS); |
670 | 298 | if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
671 | 42 | RHS = ZExtPromotedInteger(RHS); |
672 | 298 | return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS); |
673 | 298 | } |
674 | | |
675 | 6.11k | SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) { |
676 | 6.11k | SDValue LHS = N->getOperand(0); |
677 | 6.11k | SDValue RHS = N->getOperand(1); |
678 | 6.11k | // The input value must be properly zero extended. |
679 | 6.11k | if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger) |
680 | 6.11k | LHS = ZExtPromotedInteger(LHS); |
681 | 6.11k | if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger) |
682 | 56 | RHS = ZExtPromotedInteger(RHS); |
683 | 6.11k | return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS); |
684 | 6.11k | } |
685 | | |
686 | 148k | SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) { |
687 | 148k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
688 | 148k | SDValue Res; |
689 | 148k | SDValue InOp = N->getOperand(0); |
690 | 148k | SDLoc dl(N); |
691 | 148k | |
692 | 148k | switch (getTypeAction(InOp.getValueType())) { |
693 | 0 | default: 0 llvm_unreachable0 ("Unknown type action!"); |
694 | 142k | case TargetLowering::TypeLegal: |
695 | 142k | case TargetLowering::TypeExpandInteger: |
696 | 142k | Res = InOp; |
697 | 142k | break; |
698 | 5.43k | case TargetLowering::TypePromoteInteger: |
699 | 5.43k | Res = GetPromotedInteger(InOp); |
700 | 5.43k | break; |
701 | 13 | case TargetLowering::TypeSplitVector: { |
702 | 13 | EVT InVT = InOp.getValueType(); |
703 | 13 | assert(InVT.isVector() && "Cannot split scalar types"); |
704 | 13 | unsigned NumElts = InVT.getVectorNumElements(); |
705 | 13 | assert(NumElts == NVT.getVectorNumElements() && |
706 | 13 | "Dst and Src must have the same number of elements"); |
707 | 13 | assert(isPowerOf2_32(NumElts) && |
708 | 13 | "Promoted vector type must be a power of two"); |
709 | 13 | |
710 | 13 | SDValue EOp1, EOp2; |
711 | 13 | GetSplitVector(InOp, EOp1, EOp2); |
712 | 13 | |
713 | 13 | EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(), |
714 | 13 | NumElts/2); |
715 | 13 | EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1); |
716 | 13 | EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2); |
717 | 13 | |
718 | 13 | return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2); |
719 | 142k | } |
720 | 1 | case TargetLowering::TypeWidenVector: { |
721 | 1 | SDValue WideInOp = GetWidenedVector(InOp); |
722 | 1 | |
723 | 1 | // Truncate widened InOp. |
724 | 1 | unsigned NumElem = WideInOp.getValueType().getVectorNumElements(); |
725 | 1 | EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), |
726 | 1 | N->getValueType(0).getScalarType(), NumElem); |
727 | 1 | SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp); |
728 | 1 | |
729 | 1 | // Zero extend so that the elements are of same type as those of NVT |
730 | 1 | EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(), |
731 | 1 | NumElem); |
732 | 1 | SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc); |
733 | 1 | |
734 | 1 | // Extract the low NVT subvector. |
735 | 1 | MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); |
736 | 1 | SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy); |
737 | 1 | return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx); |
738 | 148k | } |
739 | 148k | } |
740 | 148k | |
741 | 148k | // Truncate to NVT instead of VT |
742 | 148k | return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res); |
743 | 148k | } |
744 | | |
745 | 266 | SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { |
746 | 266 | if (ResNo == 1) |
747 | 239 | return PromoteIntRes_Overflow(N); |
748 | 27 | |
749 | 27 | // The operation overflowed iff the result in the larger type is not the |
750 | 27 | // zero extension of its truncation to the original type. |
751 | 27 | SDValue LHS = ZExtPromotedInteger(N->getOperand(0)); |
752 | 27 | SDValue RHS = ZExtPromotedInteger(N->getOperand(1)); |
753 | 27 | EVT OVT = N->getOperand(0).getValueType(); |
754 | 27 | EVT NVT = LHS.getValueType(); |
755 | 27 | SDLoc dl(N); |
756 | 27 | |
757 | 27 | // Do the arithmetic in the larger type. |
758 | 27 | unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD25 : ISD::SUB2 ; |
759 | 266 | SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS); |
760 | 266 | |
761 | 266 | // Calculate the overflow flag: zero extend the arithmetic result from |
762 | 266 | // the original type. |
763 | 266 | SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); |
764 | 266 | // Overflowed if and only if this is not equal to Res. |
765 | 266 | Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); |
766 | 266 | |
767 | 266 | // Use the calculated overflow everywhere. |
768 | 266 | ReplaceValueWith(SDValue(N, 1), Ofl); |
769 | 266 | |
770 | 266 | return Res; |
771 | 266 | } |
772 | | |
773 | 5 | SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { |
774 | 5 | if (ResNo == 1) |
775 | 5 | return PromoteIntRes_Overflow(N); |
776 | 0 | llvm_unreachable0 ("Not implemented"); |
777 | 0 | } |
778 | | |
779 | 1.19k | SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { |
780 | 1.19k | // Promote the overflow bit trivially. |
781 | 1.19k | if (ResNo == 1) |
782 | 1.19k | return PromoteIntRes_Overflow(N); |
783 | 1 | |
784 | 1 | SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); |
785 | 1 | SDLoc DL(N); |
786 | 1 | EVT SmallVT = LHS.getValueType(); |
787 | 1 | |
788 | 1 | // To determine if the result overflowed in a larger type, we extend the |
789 | 1 | // input to the larger type, do the multiply (checking if it overflows), |
790 | 1 | // then also check the high bits of the result to see if overflow happened |
791 | 1 | // there. |
792 | 1 | if (N->getOpcode() == ISD::SMULO1 ) { |
793 | 1 | LHS = SExtPromotedInteger(LHS); |
794 | 1 | RHS = SExtPromotedInteger(RHS); |
795 | 1 | } else { |
796 | 0 | LHS = ZExtPromotedInteger(LHS); |
797 | 0 | RHS = ZExtPromotedInteger(RHS); |
798 | 0 | } |
799 | 1 | SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1)); |
800 | 1 | SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS); |
801 | 1 | |
802 | 1 | // Overflow occurred if it occurred in the larger type, or if the high part |
803 | 1 | // of the result does not zero/sign-extend the low part. Check this second |
804 | 1 | // possibility first. |
805 | 1 | SDValue Overflow; |
806 | 1 | if (N->getOpcode() == ISD::UMULO1 ) { |
807 | 0 | // Unsigned overflow occurred if the high part is non-zero. |
808 | 0 | SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, |
809 | 0 | DAG.getIntPtrConstant(SmallVT.getSizeInBits(), |
810 | 0 | DL)); |
811 | 0 | Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, |
812 | 0 | DAG.getConstant(0, DL, Hi.getValueType()), |
813 | 0 | ISD::SETNE); |
814 | 1 | } else { |
815 | 1 | // Signed overflow occurred if the high part does not sign extend the low. |
816 | 1 | SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(), |
817 | 1 | Mul, DAG.getValueType(SmallVT)); |
818 | 1 | Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE); |
819 | 1 | } |
820 | 1.19k | |
821 | 1.19k | // The only other way for overflow to occur is if the multiplication in the |
822 | 1.19k | // larger type itself overflowed. |
823 | 1.19k | Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow, |
824 | 1.19k | SDValue(Mul.getNode(), 1)); |
825 | 1.19k | |
826 | 1.19k | // Use the calculated overflow everywhere. |
827 | 1.19k | ReplaceValueWith(SDValue(N, 1), Overflow); |
828 | 1.19k | return Mul; |
829 | 1.19k | } |
830 | | |
831 | 2.63k | SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) { |
832 | 2.63k | return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), |
833 | 2.63k | N->getValueType(0))); |
834 | 2.63k | } |
835 | | |
836 | 36 | SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) { |
837 | 36 | SDValue Chain = N->getOperand(0); // Get the chain. |
838 | 36 | SDValue Ptr = N->getOperand(1); // Get the pointer. |
839 | 36 | EVT VT = N->getValueType(0); |
840 | 36 | SDLoc dl(N); |
841 | 36 | |
842 | 36 | MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT); |
843 | 36 | unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT); |
844 | 36 | // The argument is passed as NumRegs registers of type RegVT. |
845 | 36 | |
846 | 36 | SmallVector<SDValue, 8> Parts(NumRegs); |
847 | 72 | for (unsigned i = 0; i < NumRegs72 ; ++i36 ) { |
848 | 36 | Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2), |
849 | 36 | N->getConstantOperandVal(3)); |
850 | 36 | Chain = Parts[i].getValue(1); |
851 | 36 | } |
852 | 36 | |
853 | 36 | // Handle endianness of the load. |
854 | 36 | if (DAG.getDataLayout().isBigEndian()) |
855 | 18 | std::reverse(Parts.begin(), Parts.end()); |
856 | 36 | |
857 | 36 | // Assemble the parts in the promoted type. |
858 | 36 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
859 | 36 | SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]); |
860 | 36 | for (unsigned i = 1; i < NumRegs36 ; ++i0 ) { |
861 | 0 | SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]); |
862 | 0 | // Shift it to the right position and "or" it in. |
863 | 0 | Part = DAG.getNode(ISD::SHL, dl, NVT, Part, |
864 | 0 | DAG.getConstant(i * RegVT.getSizeInBits(), dl, |
865 | 0 | TLI.getPointerTy(DAG.getDataLayout()))); |
866 | 0 | Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part); |
867 | 0 | } |
868 | 36 | |
869 | 36 | // Modified the chain result - switch anything that used the old chain to |
870 | 36 | // use the new one. |
871 | 36 | ReplaceValueWith(SDValue(N, 1), Chain); |
872 | 36 | |
873 | 36 | return Res; |
874 | 36 | } |
875 | | |
876 | | //===----------------------------------------------------------------------===// |
877 | | // Integer Operand Promotion |
878 | | //===----------------------------------------------------------------------===// |
879 | | |
880 | | /// PromoteIntegerOperand - This method is called when the specified operand of |
881 | | /// the specified node is found to need promotion. At this point, all of the |
882 | | /// result types of the node are known to be legal, but other operands of the |
883 | | /// node may need promotion or expansion as well as the specified one. |
884 | 1.41M | bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { |
885 | 1.41M | DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n"); |
886 | 1.41M | SDValue Res = SDValue(); |
887 | 1.41M | |
888 | 1.41M | if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) |
889 | 385 | return false; |
890 | 1.41M | |
891 | 1.41M | switch (N->getOpcode()) { |
892 | 0 | default: |
893 | | #ifndef NDEBUG |
894 | | dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": "; |
895 | | N->dump(&DAG); dbgs() << "\n"; |
896 | | #endif |
897 | 0 | llvm_unreachable("Do not know how to promote this operator's operand!"); |
898 | 1.41M | |
899 | 108k | case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break; |
900 | 257 | case ISD::ATOMIC_STORE: |
901 | 257 | Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N)); |
902 | 257 | break; |
903 | 1.26k | case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break; |
904 | 10.3k | case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break; |
905 | 287k | case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break; |
906 | 0 | case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break; |
907 | 18.8k | case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break; |
908 | 1.84k | case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break; |
909 | 38.0k | case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break; |
910 | 173 | case ISD::INSERT_VECTOR_ELT: |
911 | 173 | Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break; |
912 | 21 | case ISD::SCALAR_TO_VECTOR: |
913 | 21 | Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break; |
914 | 30.9k | case ISD::VSELECT: |
915 | 30.9k | case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; |
916 | 14.2k | case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; |
917 | 236k | case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; |
918 | 11.8k | case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; |
919 | 8.91k | case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; |
920 | 286k | case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N), |
921 | 286k | OpNo); break; |
922 | 22 | case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N), |
923 | 22 | OpNo); break; |
924 | 45 | case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N), |
925 | 45 | OpNo); break; |
926 | 42 | case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N), |
927 | 42 | OpNo); break; |
928 | 22 | case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N), |
929 | 22 | OpNo); break; |
930 | 2.88k | case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break; |
931 | 12.1k | case ISD::FP16_TO_FP: |
932 | 12.1k | case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; |
933 | 347k | case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; |
934 | 42 | case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; |
935 | 12.1k | |
936 | 0 | case ISD::SHL: |
937 | 0 | case ISD::SRA: |
938 | 0 | case ISD::SRL: |
939 | 0 | case ISD::ROTL: |
940 | 0 | case ISD::ROTR: Res = PromoteIntOp_Shift(N); break; |
941 | 0 |
|
942 | 5 | case ISD::ADDCARRY: |
943 | 5 | case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break; |
944 | 1.41M | } |
945 | 1.41M | |
946 | 1.41M | // If the result is null, the sub-method took care of registering results etc. |
947 | 1.41M | if (1.41M !Res.getNode()1.41M ) return false2 ; |
948 | 1.41M | |
949 | 1.41M | // If the result is N, the sub-method updated N in place. Tell the legalizer |
950 | 1.41M | // core about this. |
951 | 1.41M | if (1.41M Res.getNode() == N1.41M ) |
952 | 618k | return true; |
953 | 800k | |
954 | 1.41M | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
955 | 800k | "Invalid operand expansion"); |
956 | 800k | |
957 | 800k | ReplaceValueWith(SDValue(N, 0), Res); |
958 | 800k | return false; |
959 | 800k | } |
960 | | |
961 | | /// PromoteSetCCOperands - Promote the operands of a comparison. This code is |
962 | | /// shared among BR_CC, SELECT_CC, and SETCC handlers. |
963 | | void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS, |
964 | 261k | ISD::CondCode CCCode) { |
965 | 261k | // We have to insert explicit sign or zero extends. Note that we could |
966 | 261k | // insert sign extends for ALL conditions, but zero extend is cheaper on |
967 | 261k | // many machines (an AND instead of two shifts), so prefer it. |
968 | 261k | switch (CCCode) { |
969 | 0 | default: 0 llvm_unreachable0 ("Unknown integer comparison!"); |
970 | 226k | case ISD::SETEQ: |
971 | 226k | case ISD::SETNE: { |
972 | 226k | SDValue OpL = GetPromotedInteger(NewLHS); |
973 | 226k | SDValue OpR = GetPromotedInteger(NewRHS); |
974 | 226k | |
975 | 226k | // We would prefer to promote the comparison operand with sign extension. |
976 | 226k | // If the width of OpL/OpR excluding the duplicated sign bits is no greater |
977 | 226k | // than the width of NewLHS/NewRH, we can avoid inserting real truncate |
978 | 226k | // instruction, which is redudant eventually. |
979 | 226k | unsigned OpLEffectiveBits = |
980 | 226k | OpL.getValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1; |
981 | 226k | unsigned OpREffectiveBits = |
982 | 226k | OpR.getValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1; |
983 | 226k | if (OpLEffectiveBits <= NewLHS.getValueSizeInBits() && |
984 | 226k | OpREffectiveBits <= NewRHS.getValueSizeInBits()7.99k ) { |
985 | 7.96k | NewLHS = OpL; |
986 | 7.96k | NewRHS = OpR; |
987 | 226k | } else { |
988 | 219k | NewLHS = ZExtPromotedInteger(NewLHS); |
989 | 219k | NewRHS = ZExtPromotedInteger(NewRHS); |
990 | 219k | } |
991 | 226k | break; |
992 | 226k | } |
993 | 23.5k | case ISD::SETUGE: |
994 | 23.5k | case ISD::SETUGT: |
995 | 23.5k | case ISD::SETULE: |
996 | 23.5k | case ISD::SETULT: |
997 | 23.5k | // ALL of these operations will work if we either sign or zero extend |
998 | 23.5k | // the operands (including the unsigned comparisons!). Zero extend is |
999 | 23.5k | // usually a simpler/cheaper operation, so prefer it. |
1000 | 23.5k | NewLHS = ZExtPromotedInteger(NewLHS); |
1001 | 23.5k | NewRHS = ZExtPromotedInteger(NewRHS); |
1002 | 23.5k | break; |
1003 | 10.8k | case ISD::SETGE: |
1004 | 10.8k | case ISD::SETGT: |
1005 | 10.8k | case ISD::SETLT: |
1006 | 10.8k | case ISD::SETLE: |
1007 | 10.8k | NewLHS = SExtPromotedInteger(NewLHS); |
1008 | 10.8k | NewRHS = SExtPromotedInteger(NewRHS); |
1009 | 10.8k | break; |
1010 | 261k | } |
1011 | 261k | } |
1012 | | |
1013 | 108k | SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) { |
1014 | 108k | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1015 | 108k | return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op); |
1016 | 108k | } |
1017 | | |
1018 | 257 | SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) { |
1019 | 257 | SDValue Op2 = GetPromotedInteger(N->getOperand(2)); |
1020 | 257 | return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(), |
1021 | 257 | N->getChain(), N->getBasePtr(), Op2, N->getMemOperand()); |
1022 | 257 | } |
1023 | | |
1024 | 1.26k | SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) { |
1025 | 1.26k | // This should only occur in unusual situations like bitcasting to an |
1026 | 1.26k | // x86_fp80, so just turn it into a store+load |
1027 | 1.26k | return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0)); |
1028 | 1.26k | } |
1029 | | |
1030 | 10.3k | SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) { |
1031 | 10.3k | assert(OpNo == 2 && "Don't know how to promote this operand!"); |
1032 | 10.3k | |
1033 | 10.3k | SDValue LHS = N->getOperand(2); |
1034 | 10.3k | SDValue RHS = N->getOperand(3); |
1035 | 10.3k | PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get()); |
1036 | 10.3k | |
1037 | 10.3k | // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always |
1038 | 10.3k | // legal types. |
1039 | 10.3k | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1040 | 10.3k | N->getOperand(1), LHS, RHS, N->getOperand(4)), |
1041 | 10.3k | 0); |
1042 | 10.3k | } |
1043 | | |
1044 | 287k | SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) { |
1045 | 287k | assert(OpNo == 1 && "only know how to promote condition"); |
1046 | 287k | |
1047 | 287k | // Promote all the way up to the canonical SetCC type. |
1048 | 287k | SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other); |
1049 | 287k | |
1050 | 287k | // The chain (Op#0) and basic block destination (Op#2) are always legal types. |
1051 | 287k | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond, |
1052 | 287k | N->getOperand(2)), 0); |
1053 | 287k | } |
1054 | | |
1055 | 0 | SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) { |
1056 | 0 | // Since the result type is legal, the operands must promote to it. |
1057 | 0 | EVT OVT = N->getOperand(0).getValueType(); |
1058 | 0 | SDValue Lo = ZExtPromotedInteger(N->getOperand(0)); |
1059 | 0 | SDValue Hi = GetPromotedInteger(N->getOperand(1)); |
1060 | 0 | assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?"); |
1061 | 0 | SDLoc dl(N); |
1062 | 0 |
|
1063 | 0 | Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi, |
1064 | 0 | DAG.getConstant(OVT.getSizeInBits(), dl, |
1065 | 0 | TLI.getPointerTy(DAG.getDataLayout()))); |
1066 | 0 | return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi); |
1067 | 0 | } |
1068 | | |
1069 | 18.8k | SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) { |
1070 | 18.8k | // The vector type is legal but the element type is not. This implies |
1071 | 18.8k | // that the vector is a power-of-two in length and that the element |
1072 | 18.8k | // type does not have a strange size (eg: it is not i1). |
1073 | 18.8k | EVT VecVT = N->getValueType(0); |
1074 | 18.8k | unsigned NumElts = VecVT.getVectorNumElements(); |
1075 | 18.8k | assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) && |
1076 | 18.8k | "Legal vector of one illegal element?"); |
1077 | 18.8k | |
1078 | 18.8k | // Promote the inserted value. The type does not need to match the |
1079 | 18.8k | // vector element type. Check that any extra bits introduced will be |
1080 | 18.8k | // truncated away. |
1081 | 18.8k | assert(N->getOperand(0).getValueSizeInBits() >= |
1082 | 18.8k | N->getValueType(0).getScalarSizeInBits() && |
1083 | 18.8k | "Type of inserted value narrower than vector element type!"); |
1084 | 18.8k | |
1085 | 18.8k | SmallVector<SDValue, 16> NewOps; |
1086 | 169k | for (unsigned i = 0; i < NumElts169k ; ++i150k ) |
1087 | 150k | NewOps.push_back(GetPromotedInteger(N->getOperand(i))); |
1088 | 18.8k | |
1089 | 18.8k | return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
1090 | 18.8k | } |
1091 | | |
1092 | | SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, |
1093 | 173 | unsigned OpNo) { |
1094 | 173 | if (OpNo == 1173 ) { |
1095 | 173 | // Promote the inserted value. This is valid because the type does not |
1096 | 173 | // have to match the vector element type. |
1097 | 173 | |
1098 | 173 | // Check that any extra bits introduced will be truncated away. |
1099 | 173 | assert(N->getOperand(1).getValueSizeInBits() >= |
1100 | 173 | N->getValueType(0).getScalarSizeInBits() && |
1101 | 173 | "Type of inserted value narrower than vector element type!"); |
1102 | 173 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1103 | 173 | GetPromotedInteger(N->getOperand(1)), |
1104 | 173 | N->getOperand(2)), |
1105 | 173 | 0); |
1106 | 173 | } |
1107 | 0 |
|
1108 | 173 | assert(OpNo == 2 && "Different operand and result vector types?"); |
1109 | 0 |
|
1110 | 0 | // Promote the index. |
1111 | 0 | SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N), |
1112 | 0 | TLI.getVectorIdxTy(DAG.getDataLayout())); |
1113 | 0 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1114 | 0 | N->getOperand(1), Idx), 0); |
1115 | 0 | } |
1116 | | |
1117 | 21 | SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) { |
1118 | 21 | // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote |
1119 | 21 | // the operand in place. |
1120 | 21 | return SDValue(DAG.UpdateNodeOperands(N, |
1121 | 21 | GetPromotedInteger(N->getOperand(0))), 0); |
1122 | 21 | } |
1123 | | |
1124 | 30.9k | SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { |
1125 | 30.9k | assert(OpNo == 0 && "Only know how to promote the condition!"); |
1126 | 30.9k | SDValue Cond = N->getOperand(0); |
1127 | 30.9k | EVT OpTy = N->getOperand(1).getValueType(); |
1128 | 30.9k | |
1129 | 30.9k | if (N->getOpcode() == ISD::VSELECT) |
1130 | 2.12k | if (SDValue 2.12k Res2.12k = WidenVSELECTAndMask(N)) |
1131 | 1.64k | return Res; |
1132 | 29.2k | |
1133 | 29.2k | // Promote all the way up to the canonical SetCC type. |
1134 | 29.2k | EVT OpVT = N->getOpcode() == ISD::SELECT ? 29.2k OpTy.getScalarType()28.8k : OpTy484 ; |
1135 | 30.9k | Cond = PromoteTargetBoolean(Cond, OpVT); |
1136 | 30.9k | |
1137 | 30.9k | return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), |
1138 | 30.9k | N->getOperand(2)), 0); |
1139 | 30.9k | } |
1140 | | |
1141 | 14.2k | SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) { |
1142 | 14.2k | assert(OpNo == 0 && "Don't know how to promote this operand!"); |
1143 | 14.2k | |
1144 | 14.2k | SDValue LHS = N->getOperand(0); |
1145 | 14.2k | SDValue RHS = N->getOperand(1); |
1146 | 14.2k | PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get()); |
1147 | 14.2k | |
1148 | 14.2k | // The CC (#4) and the possible return values (#2 and #3) have legal types. |
1149 | 14.2k | return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2), |
1150 | 14.2k | N->getOperand(3), N->getOperand(4)), 0); |
1151 | 14.2k | } |
1152 | | |
1153 | 236k | SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) { |
1154 | 236k | assert(OpNo == 0 && "Don't know how to promote this operand!"); |
1155 | 236k | |
1156 | 236k | SDValue LHS = N->getOperand(0); |
1157 | 236k | SDValue RHS = N->getOperand(1); |
1158 | 236k | PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get()); |
1159 | 236k | |
1160 | 236k | // The CC (#2) is always legal. |
1161 | 236k | return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0); |
1162 | 236k | } |
1163 | | |
1164 | 0 | SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) { |
1165 | 0 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
1166 | 0 | ZExtPromotedInteger(N->getOperand(1))), 0); |
1167 | 0 | } |
1168 | | |
1169 | 11.8k | SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { |
1170 | 11.8k | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1171 | 11.8k | SDLoc dl(N); |
1172 | 11.8k | Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); |
1173 | 11.8k | return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), |
1174 | 11.8k | Op, DAG.getValueType(N->getOperand(0).getValueType())); |
1175 | 11.8k | } |
1176 | | |
1177 | 8.91k | SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { |
1178 | 8.91k | return SDValue(DAG.UpdateNodeOperands(N, |
1179 | 8.91k | SExtPromotedInteger(N->getOperand(0))), 0); |
1180 | 8.91k | } |
1181 | | |
1182 | 286k | SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){ |
1183 | 286k | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); |
1184 | 286k | SDValue Ch = N->getChain(), Ptr = N->getBasePtr(); |
1185 | 286k | SDLoc dl(N); |
1186 | 286k | |
1187 | 286k | SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value. |
1188 | 286k | |
1189 | 286k | // Truncate the value and store the result. |
1190 | 286k | return DAG.getTruncStore(Ch, dl, Val, Ptr, |
1191 | 286k | N->getMemoryVT(), N->getMemOperand()); |
1192 | 286k | } |
1193 | | |
1194 | | SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, |
1195 | 25 | unsigned OpNo) { |
1196 | 25 | |
1197 | 25 | SDValue DataOp = N->getValue(); |
1198 | 25 | EVT DataVT = DataOp.getValueType(); |
1199 | 25 | SDValue Mask = N->getMask(); |
1200 | 25 | SDLoc dl(N); |
1201 | 25 | |
1202 | 25 | bool TruncateStore = false; |
1203 | 25 | if (OpNo == 225 ) { |
1204 | 21 | // Mask comes before the data operand. If the data operand is legal, we just |
1205 | 21 | // promote the mask. |
1206 | 21 | // When the data operand has illegal type, we should legalize the data |
1207 | 21 | // operand first. The mask will be promoted/splitted/widened according to |
1208 | 21 | // the data operand type. |
1209 | 21 | if (TLI.isTypeLegal(DataVT)) |
1210 | 14 | Mask = PromoteTargetBoolean(Mask, DataVT); |
1211 | 7 | else { |
1212 | 7 | if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger) |
1213 | 3 | return PromoteIntOp_MSTORE(N, 3); |
1214 | 7 | |
1215 | 4 | else if (4 getTypeAction(DataVT) == TargetLowering::TypeWidenVector4 ) |
1216 | 4 | return WidenVecOp_MSTORE(N, 3); |
1217 | 4 | |
1218 | 0 | else { |
1219 | 0 | assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector); |
1220 | 0 | return SplitVecOp_MSTORE(N, 3); |
1221 | 0 | } |
1222 | 25 | } |
1223 | 0 | } else { // Data operand |
1224 | 4 | assert(OpNo == 3 && "Unexpected operand for promotion"); |
1225 | 4 | DataOp = GetPromotedInteger(DataOp); |
1226 | 4 | Mask = PromoteTargetBoolean(Mask, DataOp.getValueType()); |
1227 | 4 | TruncateStore = true; |
1228 | 4 | } |
1229 | 25 | |
1230 | 18 | return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask, |
1231 | 18 | N->getMemoryVT(), N->getMemOperand(), |
1232 | 18 | TruncateStore, N->isCompressingStore()); |
1233 | 25 | } |
1234 | | |
1235 | | SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, |
1236 | 45 | unsigned OpNo) { |
1237 | 45 | assert(OpNo == 2 && "Only know how to promote the mask!"); |
1238 | 45 | EVT DataVT = N->getValueType(0); |
1239 | 45 | SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
1240 | 45 | SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end()); |
1241 | 45 | NewOps[OpNo] = Mask; |
1242 | 45 | return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
1243 | 45 | } |
1244 | | |
1245 | | SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N, |
1246 | 42 | unsigned OpNo) { |
1247 | 42 | |
1248 | 42 | SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); |
1249 | 42 | if (OpNo == 242 ) { |
1250 | 20 | // The Mask |
1251 | 20 | EVT DataVT = N->getValueType(0); |
1252 | 20 | NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
1253 | 20 | } else |
1254 | 22 | NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); |
1255 | 42 | |
1256 | 42 | SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
1257 | 42 | // updated in place. |
1258 | 42 | if (Res.getNode() == N) |
1259 | 40 | return Res; |
1260 | 2 | |
1261 | 2 | ReplaceValueWith(SDValue(N, 0), Res.getValue(0)); |
1262 | 2 | ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); |
1263 | 2 | return SDValue(); |
1264 | 2 | } |
1265 | | |
1266 | | SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, |
1267 | 22 | unsigned OpNo) { |
1268 | 22 | SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end()); |
1269 | 22 | if (OpNo == 222 ) { |
1270 | 8 | // The Mask |
1271 | 8 | EVT DataVT = N->getValue().getValueType(); |
1272 | 8 | NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT); |
1273 | 8 | } else |
1274 | 14 | NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo)); |
1275 | 22 | return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); |
1276 | 22 | } |
1277 | | |
1278 | 2.88k | SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) { |
1279 | 2.88k | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1280 | 2.88k | return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op); |
1281 | 2.88k | } |
1282 | | |
1283 | 12.1k | SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) { |
1284 | 12.1k | return SDValue(DAG.UpdateNodeOperands(N, |
1285 | 12.1k | ZExtPromotedInteger(N->getOperand(0))), 0); |
1286 | 12.1k | } |
1287 | | |
1288 | 347k | SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { |
1289 | 347k | SDLoc dl(N); |
1290 | 347k | SDValue Op = GetPromotedInteger(N->getOperand(0)); |
1291 | 347k | Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); |
1292 | 347k | return DAG.getZeroExtendInReg(Op, dl, |
1293 | 347k | N->getOperand(0).getValueType().getScalarType()); |
1294 | 347k | } |
1295 | | |
1296 | 5 | SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { |
1297 | 5 | assert(OpNo == 2 && "Don't know how to promote this operand!"); |
1298 | 5 | |
1299 | 5 | SDValue LHS = N->getOperand(0); |
1300 | 5 | SDValue RHS = N->getOperand(1); |
1301 | 5 | SDValue Carry = N->getOperand(2); |
1302 | 5 | SDLoc DL(N); |
1303 | 5 | |
1304 | 5 | auto VT = getSetCCResultType(LHS.getValueType()); |
1305 | 5 | TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT); |
1306 | 5 | switch (BoolType) { |
1307 | 0 | case TargetLoweringBase::UndefinedBooleanContent: |
1308 | 0 | Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT); |
1309 | 0 | break; |
1310 | 5 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
1311 | 5 | Carry = DAG.getZExtOrTrunc(Carry, DL, VT); |
1312 | 5 | break; |
1313 | 0 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
1314 | 0 | Carry = DAG.getSExtOrTrunc(Carry, DL, VT); |
1315 | 0 | break; |
1316 | 5 | } |
1317 | 5 | |
1318 | 5 | return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); |
1319 | 5 | } |
1320 | | |
1321 | | //===----------------------------------------------------------------------===// |
1322 | | // Integer Result Expansion |
1323 | | //===----------------------------------------------------------------------===// |
1324 | | |
1325 | | /// ExpandIntegerResult - This method is called when the specified result of the |
1326 | | /// specified node is found to need expansion. At this point, the node may also |
1327 | | /// have invalid operands or may have other results that need promotion, we just |
1328 | | /// know that (at least) one result needs expansion. |
1329 | 182k | void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { |
1330 | 182k | DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n"); |
1331 | 182k | SDValue Lo, Hi; |
1332 | 182k | Lo = Hi = SDValue(); |
1333 | 182k | |
1334 | 182k | // See if the target wants to custom expand this node. |
1335 | 182k | if (CustomLowerNode(N, N->getValueType(ResNo), true)) |
1336 | 1.21k | return; |
1337 | 181k | |
1338 | 181k | switch (N->getOpcode()) { |
1339 | 0 | default: |
1340 | | #ifndef NDEBUG |
1341 | | dbgs() << "ExpandIntegerResult #" << ResNo << ": "; |
1342 | | N->dump(&DAG); dbgs() << "\n"; |
1343 | | #endif |
1344 | 0 | llvm_unreachable("Do not know how to expand the result of this operator!"); |
1345 | 181k | |
1346 | 40 | case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; |
1347 | 749 | case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; |
1348 | 620 | case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; |
1349 | 1.63k | case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; |
1350 | 181k | |
1351 | 9.88k | case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; |
1352 | 21.8k | case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; |
1353 | 2.76k | case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; |
1354 | 1.81k | case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; |
1355 | 14 | case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; |
1356 | 181k | |
1357 | 4.47k | case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break; |
1358 | 317 | case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break; |
1359 | 315 | case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; |
1360 | 1 | case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; |
1361 | 47 | case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; |
1362 | 11.7k | case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; |
1363 | 100 | case ISD::CTLZ_ZERO_UNDEF: |
1364 | 100 | case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; |
1365 | 11 | case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; |
1366 | 8 | case ISD::CTTZ_ZERO_UNDEF: |
1367 | 8 | case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; |
1368 | 1 | case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; |
1369 | 89 | case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; |
1370 | 81 | case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; |
1371 | 7.26k | case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break; |
1372 | 1.67k | case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; |
1373 | 2 | case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; |
1374 | 107 | case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break; |
1375 | 1.82k | case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break; |
1376 | 156 | case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break; |
1377 | 73 | case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break; |
1378 | 12.3k | case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break; |
1379 | 111 | case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break; |
1380 | 73 | case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break; |
1381 | 9.29k | case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break; |
1382 | 11 | case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break; |
1383 | 8 | |
1384 | 41 | case ISD::ATOMIC_LOAD_ADD: |
1385 | 41 | case ISD::ATOMIC_LOAD_SUB: |
1386 | 41 | case ISD::ATOMIC_LOAD_AND: |
1387 | 41 | case ISD::ATOMIC_LOAD_OR: |
1388 | 41 | case ISD::ATOMIC_LOAD_XOR: |
1389 | 41 | case ISD::ATOMIC_LOAD_NAND: |
1390 | 41 | case ISD::ATOMIC_LOAD_MIN: |
1391 | 41 | case ISD::ATOMIC_LOAD_MAX: |
1392 | 41 | case ISD::ATOMIC_LOAD_UMIN: |
1393 | 41 | case ISD::ATOMIC_LOAD_UMAX: |
1394 | 41 | case ISD::ATOMIC_SWAP: |
1395 | 41 | case ISD::ATOMIC_CMP_SWAP: { |
1396 | 41 | std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N); |
1397 | 41 | SplitInteger(Tmp.first, Lo, Hi); |
1398 | 41 | ReplaceValueWith(SDValue(N, 1), Tmp.second); |
1399 | 41 | break; |
1400 | 41 | } |
1401 | 25 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
1402 | 25 | AtomicSDNode *AN = cast<AtomicSDNode>(N); |
1403 | 25 | SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other); |
1404 | 25 | SDValue Tmp = DAG.getAtomicCmpSwap( |
1405 | 25 | ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs, |
1406 | 25 | N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3), |
1407 | 25 | AN->getMemOperand()); |
1408 | 25 | |
1409 | 25 | // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine |
1410 | 25 | // success simply by comparing the loaded value against the ingoing |
1411 | 25 | // comparison. |
1412 | 25 | SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp, |
1413 | 25 | N->getOperand(2), ISD::SETEQ); |
1414 | 25 | |
1415 | 25 | SplitInteger(Tmp, Lo, Hi); |
1416 | 25 | ReplaceValueWith(SDValue(N, 1), Success); |
1417 | 25 | ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1)); |
1418 | 25 | break; |
1419 | 41 | } |
1420 | 41 | |
1421 | 20.2k | case ISD::AND: |
1422 | 20.2k | case ISD::OR: |
1423 | 20.2k | case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break; |
1424 | 20.2k | |
1425 | 9 | case ISD::UMAX: |
1426 | 9 | case ISD::SMAX: |
1427 | 9 | case ISD::UMIN: |
1428 | 9 | case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break; |
1429 | 9 | |
1430 | 5.38k | case ISD::ADD: |
1431 | 5.38k | case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break; |
1432 | 5.38k | |
1433 | 7 | case ISD::ADDC: |
1434 | 7 | case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break; |
1435 | 7 | |
1436 | 7 | case ISD::ADDE: |
1437 | 7 | case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break; |
1438 | 7 | |
1439 | 760 | case ISD::ADDCARRY: |
1440 | 760 | case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break; |
1441 | 760 | |
1442 | 64.5k | case ISD::SHL: |
1443 | 64.5k | case ISD::SRA: |
1444 | 64.5k | case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break; |
1445 | 64.5k | |
1446 | 20 | case ISD::SADDO: |
1447 | 20 | case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break; |
1448 | 547 | case ISD::UADDO: |
1449 | 547 | case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break; |
1450 | 11 | case ISD::UMULO: |
1451 | 11 | case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break; |
1452 | 181k | } |
1453 | 181k | |
1454 | 181k | // If Lo/Hi is null, the sub-method took care of registering results etc. |
1455 | 181k | if (181k Lo.getNode()181k ) |
1456 | 181k | SetExpandedInteger(SDValue(N, ResNo), Lo, Hi); |
1457 | 182k | } |
1458 | | |
1459 | | /// Lower an atomic node to the appropriate builtin call. |
1460 | 41 | std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) { |
1461 | 41 | unsigned Opc = Node->getOpcode(); |
1462 | 41 | MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); |
1463 | 41 | RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); |
1464 | 41 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); |
1465 | 41 | |
1466 | 41 | return ExpandChainLibCall(LC, Node, false); |
1467 | 41 | } |
1468 | | |
1469 | | /// N is a shift by a value that needs to be expanded, |
1470 | | /// and the shift amount is a constant 'Amt'. Expand the operation. |
1471 | | void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt, |
1472 | 63.1k | SDValue &Lo, SDValue &Hi) { |
1473 | 63.1k | SDLoc DL(N); |
1474 | 63.1k | // Expand the incoming operand to be shifted, so that we have its parts |
1475 | 63.1k | SDValue InL, InH; |
1476 | 63.1k | GetExpandedInteger(N->getOperand(0), InL, InH); |
1477 | 63.1k | |
1478 | 63.1k | // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization |
1479 | 63.1k | // splitted a vector shift, like this: <op1, op2> SHL <0, 2>. |
1480 | 63.1k | if (!Amt63.1k ) { |
1481 | 0 | Lo = InL; |
1482 | 0 | Hi = InH; |
1483 | 0 | return; |
1484 | 0 | } |
1485 | 63.1k | |
1486 | 63.1k | EVT NVT = InL.getValueType(); |
1487 | 63.1k | unsigned VTBits = N->getValueType(0).getSizeInBits(); |
1488 | 63.1k | unsigned NVTBits = NVT.getSizeInBits(); |
1489 | 63.1k | EVT ShTy = N->getOperand(1).getValueType(); |
1490 | 63.1k | |
1491 | 63.1k | if (N->getOpcode() == ISD::SHL63.1k ) { |
1492 | 14.0k | if (Amt.ugt(VTBits)14.0k ) { |
1493 | 2 | Lo = Hi = DAG.getConstant(0, DL, NVT); |
1494 | 14.0k | } else if (14.0k Amt.ugt(NVTBits)14.0k ) { |
1495 | 2.77k | Lo = DAG.getConstant(0, DL, NVT); |
1496 | 2.77k | Hi = DAG.getNode(ISD::SHL, DL, |
1497 | 2.77k | NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
1498 | 14.0k | } else if (11.2k Amt == NVTBits11.2k ) { |
1499 | 10.2k | Lo = DAG.getConstant(0, DL, NVT); |
1500 | 10.2k | Hi = InL; |
1501 | 11.2k | } else { |
1502 | 964 | Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy)); |
1503 | 964 | Hi = DAG.getNode(ISD::OR, DL, NVT, |
1504 | 964 | DAG.getNode(ISD::SHL, DL, NVT, InH, |
1505 | 964 | DAG.getConstant(Amt, DL, ShTy)), |
1506 | 964 | DAG.getNode(ISD::SRL, DL, NVT, InL, |
1507 | 964 | DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
1508 | 964 | } |
1509 | 14.0k | return; |
1510 | 14.0k | } |
1511 | 49.1k | |
1512 | 49.1k | if (49.1k N->getOpcode() == ISD::SRL49.1k ) { |
1513 | 48.7k | if (Amt.ugt(VTBits)48.7k ) { |
1514 | 2 | Lo = Hi = DAG.getConstant(0, DL, NVT); |
1515 | 48.7k | } else if (48.7k Amt.ugt(NVTBits)48.7k ) { |
1516 | 447 | Lo = DAG.getNode(ISD::SRL, DL, |
1517 | 447 | NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
1518 | 447 | Hi = DAG.getConstant(0, DL, NVT); |
1519 | 48.7k | } else if (48.2k Amt == NVTBits48.2k ) { |
1520 | 30.2k | Lo = InH; |
1521 | 30.2k | Hi = DAG.getConstant(0, DL, NVT); |
1522 | 48.2k | } else { |
1523 | 18.0k | Lo = DAG.getNode(ISD::OR, DL, NVT, |
1524 | 18.0k | DAG.getNode(ISD::SRL, DL, NVT, InL, |
1525 | 18.0k | DAG.getConstant(Amt, DL, ShTy)), |
1526 | 18.0k | DAG.getNode(ISD::SHL, DL, NVT, InH, |
1527 | 18.0k | DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
1528 | 18.0k | Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); |
1529 | 18.0k | } |
1530 | 48.7k | return; |
1531 | 48.7k | } |
1532 | 411 | |
1533 | 49.1k | assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
1534 | 411 | if (Amt.ugt(VTBits)411 ) { |
1535 | 2 | Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, |
1536 | 2 | DAG.getConstant(NVTBits - 1, DL, ShTy)); |
1537 | 411 | } else if (409 Amt.ugt(NVTBits)409 ) { |
1538 | 354 | Lo = DAG.getNode(ISD::SRA, DL, NVT, InH, |
1539 | 354 | DAG.getConstant(Amt - NVTBits, DL, ShTy)); |
1540 | 354 | Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, |
1541 | 354 | DAG.getConstant(NVTBits - 1, DL, ShTy)); |
1542 | 409 | } else if (55 Amt == NVTBits55 ) { |
1543 | 3 | Lo = InH; |
1544 | 3 | Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, |
1545 | 3 | DAG.getConstant(NVTBits - 1, DL, ShTy)); |
1546 | 55 | } else { |
1547 | 52 | Lo = DAG.getNode(ISD::OR, DL, NVT, |
1548 | 52 | DAG.getNode(ISD::SRL, DL, NVT, InL, |
1549 | 52 | DAG.getConstant(Amt, DL, ShTy)), |
1550 | 52 | DAG.getNode(ISD::SHL, DL, NVT, InH, |
1551 | 52 | DAG.getConstant(-Amt + NVTBits, DL, ShTy))); |
1552 | 52 | Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy)); |
1553 | 52 | } |
1554 | 63.1k | } |
1555 | | |
1556 | | /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify |
1557 | | /// this shift based on knowledge of the high bit of the shift amount. If we |
1558 | | /// can tell this, we know that it is >= 32 or < 32, without knowing the actual |
1559 | | /// shift amount. |
1560 | | bool DAGTypeLegalizer:: |
1561 | 1.37k | ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
1562 | 1.37k | SDValue Amt = N->getOperand(1); |
1563 | 1.37k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
1564 | 1.37k | EVT ShTy = Amt.getValueType(); |
1565 | 1.37k | unsigned ShBits = ShTy.getScalarSizeInBits(); |
1566 | 1.37k | unsigned NVTBits = NVT.getScalarSizeInBits(); |
1567 | 1.37k | assert(isPowerOf2_32(NVTBits) && |
1568 | 1.37k | "Expanded integer type size not a power of two!"); |
1569 | 1.37k | SDLoc dl(N); |
1570 | 1.37k | |
1571 | 1.37k | APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits)); |
1572 | 1.37k | KnownBits Known; |
1573 | 1.37k | DAG.computeKnownBits(N->getOperand(1), Known); |
1574 | 1.37k | |
1575 | 1.37k | // If we don't know anything about the high bits, exit. |
1576 | 1.37k | if (((Known.Zero|Known.One) & HighBitMask) == 0) |
1577 | 1.26k | return false; |
1578 | 113 | |
1579 | 113 | // Get the incoming operand to be shifted. |
1580 | 113 | SDValue InL, InH; |
1581 | 113 | GetExpandedInteger(N->getOperand(0), InL, InH); |
1582 | 113 | |
1583 | 113 | // If we know that any of the high bits of the shift amount are one, then we |
1584 | 113 | // can do this as a couple of simple shifts. |
1585 | 113 | if (Known.One.intersects(HighBitMask)113 ) { |
1586 | 2 | // Mask out the high bit, which we know is set. |
1587 | 2 | Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt, |
1588 | 2 | DAG.getConstant(~HighBitMask, dl, ShTy)); |
1589 | 2 | |
1590 | 2 | switch (N->getOpcode()) { |
1591 | 0 | default: 0 llvm_unreachable0 ("Unknown shift"); |
1592 | 0 | case ISD::SHL: |
1593 | 0 | Lo = DAG.getConstant(0, dl, NVT); // Low part is zero. |
1594 | 0 | Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part. |
1595 | 0 | return true; |
1596 | 2 | case ISD::SRL: |
1597 | 2 | Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero. |
1598 | 2 | Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part. |
1599 | 2 | return true; |
1600 | 0 | case ISD::SRA: |
1601 | 0 | Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part. |
1602 | 0 | DAG.getConstant(NVTBits - 1, dl, ShTy)); |
1603 | 0 | Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part. |
1604 | 0 | return true; |
1605 | 111 | } |
1606 | 111 | } |
1607 | 111 | |
1608 | 111 | // If we know that all of the high bits of the shift amount are zero, then we |
1609 | 111 | // can do this as a couple of simple shifts. |
1610 | 111 | if (111 HighBitMask.isSubsetOf(Known.Zero)111 ) { |
1611 | 11 | // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined |
1612 | 11 | // shift if x is zero. We can use XOR here because x is known to be smaller |
1613 | 11 | // than 32. |
1614 | 11 | SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt, |
1615 | 11 | DAG.getConstant(NVTBits - 1, dl, ShTy)); |
1616 | 11 | |
1617 | 11 | unsigned Op1, Op2; |
1618 | 11 | switch (N->getOpcode()) { |
1619 | 0 | default: 0 llvm_unreachable0 ("Unknown shift"); |
1620 | 5 | case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break; |
1621 | 6 | case ISD::SRL: |
1622 | 6 | case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break; |
1623 | 11 | } |
1624 | 11 | |
1625 | 11 | // When shifting right the arithmetic for Lo and Hi is swapped. |
1626 | 11 | if (11 N->getOpcode() != ISD::SHL11 ) |
1627 | 6 | std::swap(InL, InH); |
1628 | 11 | |
1629 | 11 | // Use a little trick to get the bits that move from Lo to Hi. First |
1630 | 11 | // shift by one bit. |
1631 | 11 | SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy)); |
1632 | 11 | // Then compute the remaining shift with amount-1. |
1633 | 11 | SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2); |
1634 | 11 | |
1635 | 11 | Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt); |
1636 | 11 | Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2); |
1637 | 11 | |
1638 | 11 | if (N->getOpcode() != ISD::SHL) |
1639 | 6 | std::swap(Hi, Lo); |
1640 | 11 | return true; |
1641 | 11 | } |
1642 | 100 | |
1643 | 100 | return false; |
1644 | 100 | } |
1645 | | |
1646 | | /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift |
1647 | | /// of any size. |
1648 | | bool DAGTypeLegalizer:: |
1649 | 64 | ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) { |
1650 | 64 | SDValue Amt = N->getOperand(1); |
1651 | 64 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
1652 | 64 | EVT ShTy = Amt.getValueType(); |
1653 | 64 | unsigned NVTBits = NVT.getSizeInBits(); |
1654 | 64 | assert(isPowerOf2_32(NVTBits) && |
1655 | 64 | "Expanded integer type size not a power of two!"); |
1656 | 64 | SDLoc dl(N); |
1657 | 64 | |
1658 | 64 | // Get the incoming operand to be shifted. |
1659 | 64 | SDValue InL, InH; |
1660 | 64 | GetExpandedInteger(N->getOperand(0), InL, InH); |
1661 | 64 | |
1662 | 64 | SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy); |
1663 | 64 | SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode); |
1664 | 64 | SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt); |
1665 | 64 | SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy), |
1666 | 64 | Amt, NVBitsNode, ISD::SETULT); |
1667 | 64 | SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy), |
1668 | 64 | Amt, DAG.getConstant(0, dl, ShTy), |
1669 | 64 | ISD::SETEQ); |
1670 | 64 | |
1671 | 64 | SDValue LoS, HiS, LoL, HiL; |
1672 | 64 | switch (N->getOpcode()) { |
1673 | 0 | default: 0 llvm_unreachable0 ("Unknown shift"); |
1674 | 26 | case ISD::SHL: |
1675 | 26 | // Short: ShAmt < NVTBits |
1676 | 26 | LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); |
1677 | 26 | HiS = DAG.getNode(ISD::OR, dl, NVT, |
1678 | 26 | DAG.getNode(ISD::SHL, dl, NVT, InH, Amt), |
1679 | 26 | DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack)); |
1680 | 26 | |
1681 | 26 | // Long: ShAmt >= NVTBits |
1682 | 26 | LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero. |
1683 | 26 | HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part. |
1684 | 26 | |
1685 | 26 | Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL); |
1686 | 26 | Hi = DAG.getSelect(dl, NVT, isZero, InH, |
1687 | 26 | DAG.getSelect(dl, NVT, isShort, HiS, HiL)); |
1688 | 26 | return true; |
1689 | 21 | case ISD::SRL: |
1690 | 21 | // Short: ShAmt < NVTBits |
1691 | 21 | HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); |
1692 | 21 | LoS = DAG.getNode(ISD::OR, dl, NVT, |
1693 | 21 | DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), |
1694 | 21 | // FIXME: If Amt is zero, the following shift generates an undefined result |
1695 | 21 | // on some architectures. |
1696 | 21 | DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); |
1697 | 21 | |
1698 | 21 | // Long: ShAmt >= NVTBits |
1699 | 21 | HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero. |
1700 | 21 | LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part. |
1701 | 21 | |
1702 | 21 | Lo = DAG.getSelect(dl, NVT, isZero, InL, |
1703 | 21 | DAG.getSelect(dl, NVT, isShort, LoS, LoL)); |
1704 | 21 | Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); |
1705 | 21 | return true; |
1706 | 17 | case ISD::SRA: |
1707 | 17 | // Short: ShAmt < NVTBits |
1708 | 17 | HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); |
1709 | 17 | LoS = DAG.getNode(ISD::OR, dl, NVT, |
1710 | 17 | DAG.getNode(ISD::SRL, dl, NVT, InL, Amt), |
1711 | 17 | DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack)); |
1712 | 17 | |
1713 | 17 | // Long: ShAmt >= NVTBits |
1714 | 17 | HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part. |
1715 | 17 | DAG.getConstant(NVTBits - 1, dl, ShTy)); |
1716 | 17 | LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part. |
1717 | 17 | |
1718 | 17 | Lo = DAG.getSelect(dl, NVT, isZero, InL, |
1719 | 17 | DAG.getSelect(dl, NVT, isShort, LoS, LoL)); |
1720 | 17 | Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL); |
1721 | 17 | return true; |
1722 | 0 | } |
1723 | 0 | } |
1724 | | |
1725 | 9 | static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) { |
1726 | 9 | |
1727 | 9 | switch (Op) { |
1728 | 0 | default: 0 llvm_unreachable0 ("invalid min/max opcode"); |
1729 | 2 | case ISD::SMAX: |
1730 | 2 | return std::make_pair(ISD::SETGT, ISD::UMAX); |
1731 | 3 | case ISD::UMAX: |
1732 | 3 | return std::make_pair(ISD::SETUGT, ISD::UMAX); |
1733 | 2 | case ISD::SMIN: |
1734 | 2 | return std::make_pair(ISD::SETLT, ISD::UMIN); |
1735 | 2 | case ISD::UMIN: |
1736 | 2 | return std::make_pair(ISD::SETULT, ISD::UMIN); |
1737 | 0 | } |
1738 | 0 | } |
1739 | | |
1740 | | void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N, |
1741 | 9 | SDValue &Lo, SDValue &Hi) { |
1742 | 9 | SDLoc DL(N); |
1743 | 9 | ISD::NodeType LoOpc; |
1744 | 9 | ISD::CondCode CondC; |
1745 | 9 | std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode()); |
1746 | 9 | |
1747 | 9 | // Expand the subcomponents. |
1748 | 9 | SDValue LHSL, LHSH, RHSL, RHSH; |
1749 | 9 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
1750 | 9 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
1751 | 9 | |
1752 | 9 | // Value types |
1753 | 9 | EVT NVT = LHSL.getValueType(); |
1754 | 9 | EVT CCT = getSetCCResultType(NVT); |
1755 | 9 | |
1756 | 9 | // Hi part is always the same op |
1757 | 9 | Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH}); |
1758 | 9 | |
1759 | 9 | // We need to know whether to select Lo part that corresponds to 'winning' |
1760 | 9 | // Hi part or if Hi parts are equal. |
1761 | 9 | SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC); |
1762 | 9 | SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ); |
1763 | 9 | |
1764 | 9 | // Lo part corresponding to the 'winning' Hi part |
1765 | 9 | SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL); |
1766 | 9 | |
1767 | 9 | // Recursed Lo part if Hi parts are equal, this uses unsigned version |
1768 | 9 | SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL}); |
1769 | 9 | |
1770 | 9 | Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp); |
1771 | 9 | } |
1772 | | |
1773 | | void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N, |
1774 | 5.38k | SDValue &Lo, SDValue &Hi) { |
1775 | 5.38k | SDLoc dl(N); |
1776 | 5.38k | // Expand the subcomponents. |
1777 | 5.38k | SDValue LHSL, LHSH, RHSL, RHSH; |
1778 | 5.38k | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
1779 | 5.38k | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
1780 | 5.38k | |
1781 | 5.38k | EVT NVT = LHSL.getValueType(); |
1782 | 5.38k | SDValue LoOps[2] = { LHSL, RHSL }; |
1783 | 5.38k | SDValue HiOps[3] = { LHSH, RHSH }; |
1784 | 5.38k | |
1785 | 5.38k | bool HasOpCarry = TLI.isOperationLegalOrCustom( |
1786 | 5.38k | N->getOpcode() == ISD::ADD ? ISD::ADDCARRY3.78k : ISD::SUBCARRY1.60k , |
1787 | 5.38k | TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
1788 | 5.38k | if (HasOpCarry5.38k ) { |
1789 | 4.18k | SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT)); |
1790 | 4.18k | if (N->getOpcode() == ISD::ADD4.18k ) { |
1791 | 3.36k | Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); |
1792 | 3.36k | HiOps[2] = Lo.getValue(1); |
1793 | 3.36k | Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps); |
1794 | 4.18k | } else { |
1795 | 821 | Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); |
1796 | 821 | HiOps[2] = Lo.getValue(1); |
1797 | 821 | Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps); |
1798 | 821 | } |
1799 | 4.18k | return; |
1800 | 4.18k | } |
1801 | 1.19k | |
1802 | 1.19k | // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support |
1803 | 1.19k | // them. TODO: Teach operation legalization how to expand unsupported |
1804 | 1.19k | // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate |
1805 | 1.19k | // a carry of type MVT::Glue, but there doesn't seem to be any way to |
1806 | 1.19k | // generate a value of this type in the expanded code sequence. |
1807 | 1.19k | bool hasCarry = |
1808 | 1.19k | TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? |
1809 | 1.19k | ISD::ADDC418 : ISD::SUBC779 , |
1810 | 1.19k | TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
1811 | 1.19k | |
1812 | 1.19k | if (hasCarry1.19k ) { |
1813 | 253 | SDVTList VTList = DAG.getVTList(NVT, MVT::Glue); |
1814 | 253 | if (N->getOpcode() == ISD::ADD253 ) { |
1815 | 166 | Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); |
1816 | 166 | HiOps[2] = Lo.getValue(1); |
1817 | 166 | Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); |
1818 | 253 | } else { |
1819 | 87 | Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); |
1820 | 87 | HiOps[2] = Lo.getValue(1); |
1821 | 87 | Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); |
1822 | 87 | } |
1823 | 253 | return; |
1824 | 253 | } |
1825 | 944 | |
1826 | 944 | bool hasOVF = |
1827 | 944 | TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ? |
1828 | 944 | ISD::UADDO252 : ISD::USUBO692 , |
1829 | 944 | TLI.getTypeToExpandTo(*DAG.getContext(), NVT)); |
1830 | 944 | TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT); |
1831 | 944 | |
1832 | 944 | if (hasOVF944 ) { |
1833 | 686 | EVT OvfVT = getSetCCResultType(NVT); |
1834 | 686 | SDVTList VTList = DAG.getVTList(NVT, OvfVT); |
1835 | 686 | int RevOpc; |
1836 | 686 | if (N->getOpcode() == ISD::ADD686 ) { |
1837 | 59 | RevOpc = ISD::SUB; |
1838 | 59 | Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps); |
1839 | 59 | Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); |
1840 | 686 | } else { |
1841 | 627 | RevOpc = ISD::ADD; |
1842 | 627 | Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps); |
1843 | 627 | Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); |
1844 | 627 | } |
1845 | 686 | SDValue OVF = Lo.getValue(1); |
1846 | 686 | |
1847 | 686 | switch (BoolType) { |
1848 | 0 | case TargetLoweringBase::UndefinedBooleanContent: |
1849 | 0 | OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF); |
1850 | 0 | LLVM_FALLTHROUGH; |
1851 | 0 | case TargetLoweringBase::ZeroOrOneBooleanContent: |
1852 | 0 | OVF = DAG.getZExtOrTrunc(OVF, dl, NVT); |
1853 | 0 | Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF); |
1854 | 0 | break; |
1855 | 686 | case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: |
1856 | 686 | OVF = DAG.getSExtOrTrunc(OVF, dl, NVT); |
1857 | 686 | Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF); |
1858 | 686 | } |
1859 | 686 | return; |
1860 | 258 | } |
1861 | 258 | |
1862 | 258 | if (258 N->getOpcode() == ISD::ADD258 ) { |
1863 | 193 | Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps); |
1864 | 193 | Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2)); |
1865 | 193 | SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0], |
1866 | 193 | ISD::SETULT); |
1867 | 193 | |
1868 | 193 | if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent193 ) { |
1869 | 191 | SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT); |
1870 | 191 | Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry); |
1871 | 191 | return; |
1872 | 191 | } |
1873 | 2 | |
1874 | 2 | SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1, |
1875 | 2 | DAG.getConstant(1, dl, NVT), |
1876 | 2 | DAG.getConstant(0, dl, NVT)); |
1877 | 2 | SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1], |
1878 | 2 | ISD::SETULT); |
1879 | 2 | SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2, |
1880 | 2 | DAG.getConstant(1, dl, NVT), Carry1); |
1881 | 2 | Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2); |
1882 | 258 | } else { |
1883 | 65 | Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps); |
1884 | 65 | Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2)); |
1885 | 65 | SDValue Cmp = |
1886 | 65 | DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()), |
1887 | 65 | LoOps[0], LoOps[1], ISD::SETULT); |
1888 | 65 | |
1889 | 65 | SDValue Borrow; |
1890 | 65 | if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) |
1891 | 64 | Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT); |
1892 | 65 | else |
1893 | 1 | Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT), |
1894 | 1 | DAG.getConstant(0, dl, NVT)); |
1895 | 65 | |
1896 | 65 | Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow); |
1897 | 65 | } |
1898 | 5.38k | } |
1899 | | |
1900 | | void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N, |
1901 | 7 | SDValue &Lo, SDValue &Hi) { |
1902 | 7 | // Expand the subcomponents. |
1903 | 7 | SDValue LHSL, LHSH, RHSL, RHSH; |
1904 | 7 | SDLoc dl(N); |
1905 | 7 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
1906 | 7 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
1907 | 7 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
1908 | 7 | SDValue LoOps[2] = { LHSL, RHSL }; |
1909 | 7 | SDValue HiOps[3] = { LHSH, RHSH }; |
1910 | 7 | |
1911 | 7 | if (N->getOpcode() == ISD::ADDC7 ) { |
1912 | 5 | Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps); |
1913 | 5 | HiOps[2] = Lo.getValue(1); |
1914 | 5 | Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps); |
1915 | 7 | } else { |
1916 | 2 | Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps); |
1917 | 2 | HiOps[2] = Lo.getValue(1); |
1918 | 2 | Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps); |
1919 | 2 | } |
1920 | 7 | |
1921 | 7 | // Legalized the flag result - switch anything that used the old flag to |
1922 | 7 | // use the new one. |
1923 | 7 | ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
1924 | 7 | } |
1925 | | |
1926 | | void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N, |
1927 | 7 | SDValue &Lo, SDValue &Hi) { |
1928 | 7 | // Expand the subcomponents. |
1929 | 7 | SDValue LHSL, LHSH, RHSL, RHSH; |
1930 | 7 | SDLoc dl(N); |
1931 | 7 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
1932 | 7 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
1933 | 7 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue); |
1934 | 7 | SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; |
1935 | 7 | SDValue HiOps[3] = { LHSH, RHSH }; |
1936 | 7 | |
1937 | 7 | Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
1938 | 7 | HiOps[2] = Lo.getValue(1); |
1939 | 7 | Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); |
1940 | 7 | |
1941 | 7 | // Legalized the flag result - switch anything that used the old flag to |
1942 | 7 | // use the new one. |
1943 | 7 | ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
1944 | 7 | } |
1945 | | |
1946 | | void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N, |
1947 | 547 | SDValue &Lo, SDValue &Hi) { |
1948 | 547 | SDValue LHS = N->getOperand(0); |
1949 | 547 | SDValue RHS = N->getOperand(1); |
1950 | 547 | SDLoc dl(N); |
1951 | 547 | |
1952 | 547 | SDValue Ovf; |
1953 | 547 | |
1954 | 547 | bool HasOpCarry = TLI.isOperationLegalOrCustom( |
1955 | 547 | N->getOpcode() == ISD::ADD ? ISD::ADDCARRY0 : ISD::SUBCARRY547 , |
1956 | 547 | TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType())); |
1957 | 547 | |
1958 | 547 | if (HasOpCarry547 ) { |
1959 | 541 | // Expand the subcomponents. |
1960 | 541 | SDValue LHSL, LHSH, RHSL, RHSH; |
1961 | 541 | GetExpandedInteger(LHS, LHSL, LHSH); |
1962 | 541 | GetExpandedInteger(RHS, RHSL, RHSH); |
1963 | 541 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
1964 | 541 | SDValue LoOps[2] = { LHSL, RHSL }; |
1965 | 541 | SDValue HiOps[3] = { LHSH, RHSH }; |
1966 | 541 | |
1967 | 541 | unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY497 : ISD::SUBCARRY44 ; |
1968 | 541 | Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
1969 | 541 | HiOps[2] = Lo.getValue(1); |
1970 | 541 | Hi = DAG.getNode(Opc, dl, VTList, HiOps); |
1971 | 541 | |
1972 | 541 | Ovf = Hi.getValue(1); |
1973 | 547 | } else { |
1974 | 6 | // Expand the result by simply replacing it with the equivalent |
1975 | 6 | // non-overflow-checking operation. |
1976 | 6 | auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD3 : ISD::SUB3 ; |
1977 | 6 | SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS); |
1978 | 6 | SplitInteger(Sum, Lo, Hi); |
1979 | 6 | |
1980 | 6 | // Calculate the overflow: addition overflows iff a + b < a, and subtraction |
1981 | 6 | // overflows iff a - b > a. |
1982 | 6 | auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT3 : ISD::SETUGT3 ; |
1983 | 6 | Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond); |
1984 | 6 | } |
1985 | 547 | |
1986 | 547 | // Legalized the flag result - switch anything that used the old flag to |
1987 | 547 | // use the new one. |
1988 | 547 | ReplaceValueWith(SDValue(N, 1), Ovf); |
1989 | 547 | } |
1990 | | |
1991 | | void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N, |
1992 | 760 | SDValue &Lo, SDValue &Hi) { |
1993 | 760 | // Expand the subcomponents. |
1994 | 760 | SDValue LHSL, LHSH, RHSL, RHSH; |
1995 | 760 | SDLoc dl(N); |
1996 | 760 | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
1997 | 760 | GetExpandedInteger(N->getOperand(1), RHSL, RHSH); |
1998 | 760 | SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1)); |
1999 | 760 | SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) }; |
2000 | 760 | SDValue HiOps[3] = { LHSH, RHSH, SDValue() }; |
2001 | 760 | |
2002 | 760 | Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps); |
2003 | 760 | HiOps[2] = Lo.getValue(1); |
2004 | 760 | Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps); |
2005 | 760 | |
2006 | 760 | // Legalized the flag result - switch anything that used the old flag to |
2007 | 760 | // use the new one. |
2008 | 760 | ReplaceValueWith(SDValue(N, 1), Hi.getValue(1)); |
2009 | 760 | } |
2010 | | |
2011 | | void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N, |
2012 | 4.47k | SDValue &Lo, SDValue &Hi) { |
2013 | 4.47k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2014 | 4.47k | SDLoc dl(N); |
2015 | 4.47k | SDValue Op = N->getOperand(0); |
2016 | 4.47k | if (Op.getValueType().bitsLE(NVT)4.47k ) { |
2017 | 4.41k | // The low part is any extension of the input (which degenerates to a copy). |
2018 | 4.41k | Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op); |
2019 | 4.41k | Hi = DAG.getUNDEF(NVT); // The high part is undefined. |
2020 | 4.47k | } else { |
2021 | 63 | // For example, extension of an i48 to an i64. The operand type necessarily |
2022 | 63 | // promotes to the result type, so will end up being expanded too. |
2023 | 63 | assert(getTypeAction(Op.getValueType()) == |
2024 | 63 | TargetLowering::TypePromoteInteger && |
2025 | 63 | "Only know how to promote this result!"); |
2026 | 63 | SDValue Res = GetPromotedInteger(Op); |
2027 | 63 | assert(Res.getValueType() == N->getValueType(0) && |
2028 | 63 | "Operand over promoted?"); |
2029 | 63 | // Split the promoted operand. This will simplify when it is expanded. |
2030 | 63 | SplitInteger(Res, Lo, Hi); |
2031 | 63 | } |
2032 | 4.47k | } |
2033 | | |
2034 | | void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N, |
2035 | 317 | SDValue &Lo, SDValue &Hi) { |
2036 | 317 | SDLoc dl(N); |
2037 | 317 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2038 | 317 | EVT NVT = Lo.getValueType(); |
2039 | 317 | EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); |
2040 | 317 | unsigned NVTBits = NVT.getSizeInBits(); |
2041 | 317 | unsigned EVTBits = EVT.getSizeInBits(); |
2042 | 317 | |
2043 | 317 | if (NVTBits < EVTBits317 ) { |
2044 | 254 | Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi, |
2045 | 254 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
2046 | 254 | EVTBits - NVTBits))); |
2047 | 317 | } else { |
2048 | 63 | Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT)); |
2049 | 63 | // The high part replicates the sign bit of Lo, make it explicit. |
2050 | 63 | Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
2051 | 63 | DAG.getConstant(NVTBits - 1, dl, |
2052 | 63 | TLI.getPointerTy(DAG.getDataLayout()))); |
2053 | 63 | } |
2054 | 317 | } |
2055 | | |
2056 | | void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N, |
2057 | 315 | SDValue &Lo, SDValue &Hi) { |
2058 | 315 | SDLoc dl(N); |
2059 | 315 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2060 | 315 | EVT NVT = Lo.getValueType(); |
2061 | 315 | EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); |
2062 | 315 | unsigned NVTBits = NVT.getSizeInBits(); |
2063 | 315 | unsigned EVTBits = EVT.getSizeInBits(); |
2064 | 315 | |
2065 | 315 | if (NVTBits < EVTBits315 ) { |
2066 | 252 | Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi, |
2067 | 252 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
2068 | 252 | EVTBits - NVTBits))); |
2069 | 315 | } else { |
2070 | 63 | Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT)); |
2071 | 63 | // The high part must be zero, make it explicit. |
2072 | 63 | Hi = DAG.getConstant(0, dl, NVT); |
2073 | 63 | } |
2074 | 315 | } |
2075 | | |
2076 | | void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N, |
2077 | 1 | SDValue &Lo, SDValue &Hi) { |
2078 | 1 | SDLoc dl(N); |
2079 | 1 | GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. |
2080 | 1 | Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo); |
2081 | 1 | Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi); |
2082 | 1 | } |
2083 | | |
2084 | | void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N, |
2085 | 47 | SDValue &Lo, SDValue &Hi) { |
2086 | 47 | SDLoc dl(N); |
2087 | 47 | GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands. |
2088 | 47 | Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo); |
2089 | 47 | Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi); |
2090 | 47 | } |
2091 | | |
2092 | | void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, |
2093 | 11.7k | SDValue &Lo, SDValue &Hi) { |
2094 | 11.7k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2095 | 11.7k | unsigned NBitWidth = NVT.getSizeInBits(); |
2096 | 11.7k | auto Constant = cast<ConstantSDNode>(N); |
2097 | 11.7k | const APInt &Cst = Constant->getAPIntValue(); |
2098 | 11.7k | bool IsTarget = Constant->isTargetOpcode(); |
2099 | 11.7k | bool IsOpaque = Constant->isOpaque(); |
2100 | 11.7k | SDLoc dl(N); |
2101 | 11.7k | Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque); |
2102 | 11.7k | Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget, |
2103 | 11.7k | IsOpaque); |
2104 | 11.7k | } |
2105 | | |
2106 | | void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, |
2107 | 100 | SDValue &Lo, SDValue &Hi) { |
2108 | 100 | SDLoc dl(N); |
2109 | 100 | // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32) |
2110 | 100 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2111 | 100 | EVT NVT = Lo.getValueType(); |
2112 | 100 | |
2113 | 100 | SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi, |
2114 | 100 | DAG.getConstant(0, dl, NVT), ISD::SETNE); |
2115 | 100 | |
2116 | 100 | SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); |
2117 | 100 | SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); |
2118 | 100 | |
2119 | 100 | Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ, |
2120 | 100 | DAG.getNode(ISD::ADD, dl, NVT, LoLZ, |
2121 | 100 | DAG.getConstant(NVT.getSizeInBits(), dl, |
2122 | 100 | NVT))); |
2123 | 100 | Hi = DAG.getConstant(0, dl, NVT); |
2124 | 100 | } |
2125 | | |
2126 | | void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N, |
2127 | 11 | SDValue &Lo, SDValue &Hi) { |
2128 | 11 | SDLoc dl(N); |
2129 | 11 | // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo) |
2130 | 11 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2131 | 11 | EVT NVT = Lo.getValueType(); |
2132 | 11 | Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo), |
2133 | 11 | DAG.getNode(ISD::CTPOP, dl, NVT, Hi)); |
2134 | 11 | Hi = DAG.getConstant(0, dl, NVT); |
2135 | 11 | } |
2136 | | |
2137 | | void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, |
2138 | 8 | SDValue &Lo, SDValue &Hi) { |
2139 | 8 | SDLoc dl(N); |
2140 | 8 | // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32) |
2141 | 8 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2142 | 8 | EVT NVT = Lo.getValueType(); |
2143 | 8 | |
2144 | 8 | SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, |
2145 | 8 | DAG.getConstant(0, dl, NVT), ISD::SETNE); |
2146 | 8 | |
2147 | 8 | SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); |
2148 | 8 | SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); |
2149 | 8 | |
2150 | 8 | Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ, |
2151 | 8 | DAG.getNode(ISD::ADD, dl, NVT, HiLZ, |
2152 | 8 | DAG.getConstant(NVT.getSizeInBits(), dl, |
2153 | 8 | NVT))); |
2154 | 8 | Hi = DAG.getConstant(0, dl, NVT); |
2155 | 8 | } |
2156 | | |
2157 | | void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo, |
2158 | 1 | SDValue &Hi) { |
2159 | 1 | SDLoc dl(N); |
2160 | 1 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2161 | 1 | unsigned NBitWidth = NVT.getSizeInBits(); |
2162 | 1 | |
2163 | 1 | EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); |
2164 | 1 | Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT); |
2165 | 1 | // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS |
2166 | 1 | Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
2167 | 1 | DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy)); |
2168 | 1 | } |
2169 | | |
2170 | | void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo, |
2171 | 89 | SDValue &Hi) { |
2172 | 89 | SDLoc dl(N); |
2173 | 89 | EVT VT = N->getValueType(0); |
2174 | 89 | |
2175 | 89 | SDValue Op = N->getOperand(0); |
2176 | 89 | if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) |
2177 | 6 | Op = GetPromotedFloat(Op); |
2178 | 89 | |
2179 | 89 | RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT); |
2180 | 89 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!"); |
2181 | 89 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first, |
2182 | 89 | Lo, Hi); |
2183 | 89 | } |
2184 | | |
2185 | | void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, |
2186 | 81 | SDValue &Hi) { |
2187 | 81 | SDLoc dl(N); |
2188 | 81 | EVT VT = N->getValueType(0); |
2189 | 81 | |
2190 | 81 | SDValue Op = N->getOperand(0); |
2191 | 81 | if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) |
2192 | 4 | Op = GetPromotedFloat(Op); |
2193 | 81 | |
2194 | 81 | RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT); |
2195 | 81 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!"); |
2196 | 81 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first, |
2197 | 81 | Lo, Hi); |
2198 | 81 | } |
2199 | | |
2200 | | void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, |
2201 | 7.26k | SDValue &Lo, SDValue &Hi) { |
2202 | 7.26k | if (ISD::isNormalLoad(N)7.26k ) { |
2203 | 6.49k | ExpandRes_NormalLoad(N, Lo, Hi); |
2204 | 6.49k | return; |
2205 | 6.49k | } |
2206 | 778 | |
2207 | 7.26k | assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); |
2208 | 778 | |
2209 | 778 | EVT VT = N->getValueType(0); |
2210 | 778 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
2211 | 778 | SDValue Ch = N->getChain(); |
2212 | 778 | SDValue Ptr = N->getBasePtr(); |
2213 | 778 | ISD::LoadExtType ExtType = N->getExtensionType(); |
2214 | 778 | unsigned Alignment = N->getAlignment(); |
2215 | 778 | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
2216 | 778 | AAMDNodes AAInfo = N->getAAInfo(); |
2217 | 778 | SDLoc dl(N); |
2218 | 778 | |
2219 | 778 | assert(NVT.isByteSized() && "Expanded type not byte sized!"); |
2220 | 778 | |
2221 | 778 | if (N->getMemoryVT().bitsLE(NVT)778 ) { |
2222 | 454 | EVT MemVT = N->getMemoryVT(); |
2223 | 454 | |
2224 | 454 | Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT, |
2225 | 454 | Alignment, MMOFlags, AAInfo); |
2226 | 454 | |
2227 | 454 | // Remember the chain. |
2228 | 454 | Ch = Lo.getValue(1); |
2229 | 454 | |
2230 | 454 | if (ExtType == ISD::SEXTLOAD454 ) { |
2231 | 86 | // The high part is obtained by SRA'ing all but one of the bits of the |
2232 | 86 | // lo part. |
2233 | 86 | unsigned LoSize = Lo.getValueSizeInBits(); |
2234 | 86 | Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo, |
2235 | 86 | DAG.getConstant(LoSize - 1, dl, |
2236 | 86 | TLI.getPointerTy(DAG.getDataLayout()))); |
2237 | 454 | } else if (368 ExtType == ISD::ZEXTLOAD368 ) { |
2238 | 236 | // The high part is just a zero. |
2239 | 236 | Hi = DAG.getConstant(0, dl, NVT); |
2240 | 368 | } else { |
2241 | 132 | assert(ExtType == ISD::EXTLOAD && "Unknown extload!"); |
2242 | 132 | // The high part is undefined. |
2243 | 132 | Hi = DAG.getUNDEF(NVT); |
2244 | 132 | } |
2245 | 778 | } else if (324 DAG.getDataLayout().isLittleEndian()324 ) { |
2246 | 316 | // Little-endian - low bits are at low addresses. |
2247 | 316 | Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags, |
2248 | 316 | AAInfo); |
2249 | 316 | |
2250 | 316 | unsigned ExcessBits = |
2251 | 316 | N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
2252 | 316 | EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); |
2253 | 316 | |
2254 | 316 | // Increment the pointer to the other half. |
2255 | 316 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
2256 | 316 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
2257 | 316 | DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); |
2258 | 316 | Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, |
2259 | 316 | N->getPointerInfo().getWithOffset(IncrementSize), NEVT, |
2260 | 316 | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); |
2261 | 316 | |
2262 | 316 | // Build a factor node to remember that this load is independent of the |
2263 | 316 | // other one. |
2264 | 316 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
2265 | 316 | Hi.getValue(1)); |
2266 | 324 | } else { |
2267 | 8 | // Big-endian - high bits are at low addresses. Favor aligned loads at |
2268 | 8 | // the cost of some bit-fiddling. |
2269 | 8 | EVT MemVT = N->getMemoryVT(); |
2270 | 8 | unsigned EBytes = MemVT.getStoreSize(); |
2271 | 8 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
2272 | 8 | unsigned ExcessBits = (EBytes - IncrementSize)*8; |
2273 | 8 | |
2274 | 8 | // Load both the high bits and maybe some of the low bits. |
2275 | 8 | Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), |
2276 | 8 | EVT::getIntegerVT(*DAG.getContext(), |
2277 | 8 | MemVT.getSizeInBits() - ExcessBits), |
2278 | 8 | Alignment, MMOFlags, AAInfo); |
2279 | 8 | |
2280 | 8 | // Increment the pointer to the other half. |
2281 | 8 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
2282 | 8 | DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); |
2283 | 8 | // Load the rest of the low bits. |
2284 | 8 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr, |
2285 | 8 | N->getPointerInfo().getWithOffset(IncrementSize), |
2286 | 8 | EVT::getIntegerVT(*DAG.getContext(), ExcessBits), |
2287 | 8 | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); |
2288 | 8 | |
2289 | 8 | // Build a factor node to remember that this load is independent of the |
2290 | 8 | // other one. |
2291 | 8 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
2292 | 8 | Hi.getValue(1)); |
2293 | 8 | |
2294 | 8 | if (ExcessBits < NVT.getSizeInBits()8 ) { |
2295 | 7 | // Transfer low bits from the bottom of Hi to the top of Lo. |
2296 | 7 | Lo = DAG.getNode( |
2297 | 7 | ISD::OR, dl, NVT, Lo, |
2298 | 7 | DAG.getNode(ISD::SHL, dl, NVT, Hi, |
2299 | 7 | DAG.getConstant(ExcessBits, dl, |
2300 | 7 | TLI.getPointerTy(DAG.getDataLayout())))); |
2301 | 7 | // Move high bits to the right position in Hi. |
2302 | 7 | Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA0 : ISD::SRL7 , dl, NVT, |
2303 | 7 | Hi, |
2304 | 7 | DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, |
2305 | 7 | TLI.getPointerTy(DAG.getDataLayout()))); |
2306 | 7 | } |
2307 | 324 | } |
2308 | 7.26k | |
2309 | 7.26k | // Legalize the chain result - switch anything that used the old chain to |
2310 | 7.26k | // use the new one. |
2311 | 7.26k | ReplaceValueWith(SDValue(N, 1), Ch); |
2312 | 7.26k | } |
2313 | | |
2314 | | void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N, |
2315 | 20.2k | SDValue &Lo, SDValue &Hi) { |
2316 | 20.2k | SDLoc dl(N); |
2317 | 20.2k | SDValue LL, LH, RL, RH; |
2318 | 20.2k | GetExpandedInteger(N->getOperand(0), LL, LH); |
2319 | 20.2k | GetExpandedInteger(N->getOperand(1), RL, RH); |
2320 | 20.2k | Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL); |
2321 | 20.2k | Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH); |
2322 | 20.2k | } |
2323 | | |
2324 | | void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N, |
2325 | 1.67k | SDValue &Lo, SDValue &Hi) { |
2326 | 1.67k | EVT VT = N->getValueType(0); |
2327 | 1.67k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
2328 | 1.67k | SDLoc dl(N); |
2329 | 1.67k | |
2330 | 1.67k | SDValue LL, LH, RL, RH; |
2331 | 1.67k | GetExpandedInteger(N->getOperand(0), LL, LH); |
2332 | 1.67k | GetExpandedInteger(N->getOperand(1), RL, RH); |
2333 | 1.67k | |
2334 | 1.67k | if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, |
2335 | 1.67k | TargetLowering::MulExpansionKind::OnlyLegalOrCustom, |
2336 | 1.67k | LL, LH, RL, RH)) |
2337 | 1.27k | return; |
2338 | 400 | |
2339 | 400 | // If nothing else, we can make a libcall. |
2340 | 400 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
2341 | 400 | if (VT == MVT::i16) |
2342 | 0 | LC = RTLIB::MUL_I16; |
2343 | 400 | else if (400 VT == MVT::i32400 ) |
2344 | 7 | LC = RTLIB::MUL_I32; |
2345 | 393 | else if (393 VT == MVT::i64393 ) |
2346 | 105 | LC = RTLIB::MUL_I64; |
2347 | 288 | else if (288 VT == MVT::i128288 ) |
2348 | 194 | LC = RTLIB::MUL_I128; |
2349 | 400 | |
2350 | 400 | if (LC == RTLIB::UNKNOWN_LIBCALL || 400 !TLI.getLibcallName(LC)306 ) { |
2351 | 100 | // We'll expand the multiplication by brute force because we have no other |
2352 | 100 | // options. This is a trivially-generalized version of the code from |
2353 | 100 | // Hacker's Delight (itself derived from Knuth's Algorithm M from section |
2354 | 100 | // 4.3.1). |
2355 | 100 | unsigned Bits = NVT.getSizeInBits(); |
2356 | 100 | unsigned HalfBits = Bits >> 1; |
2357 | 100 | SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl, |
2358 | 100 | NVT); |
2359 | 100 | SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask); |
2360 | 100 | SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask); |
2361 | 100 | |
2362 | 100 | SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL); |
2363 | 100 | SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask); |
2364 | 100 | |
2365 | 100 | EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); |
2366 | 100 | if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)100 ) { |
2367 | 2 | // The type from TLI is too small to fit the shift amount we want. |
2368 | 2 | // Override it with i32. The shift will have to be legalized. |
2369 | 2 | ShiftAmtTy = MVT::i32; |
2370 | 2 | } |
2371 | 100 | SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy); |
2372 | 100 | SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift); |
2373 | 100 | SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift); |
2374 | 100 | SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift); |
2375 | 100 | |
2376 | 100 | SDValue U = DAG.getNode(ISD::ADD, dl, NVT, |
2377 | 100 | DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH); |
2378 | 100 | SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask); |
2379 | 100 | SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift); |
2380 | 100 | |
2381 | 100 | SDValue V = DAG.getNode(ISD::ADD, dl, NVT, |
2382 | 100 | DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL); |
2383 | 100 | SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift); |
2384 | 100 | |
2385 | 100 | SDValue W = DAG.getNode(ISD::ADD, dl, NVT, |
2386 | 100 | DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH), |
2387 | 100 | DAG.getNode(ISD::ADD, dl, NVT, UH, VH)); |
2388 | 100 | Lo = DAG.getNode(ISD::ADD, dl, NVT, TL, |
2389 | 100 | DAG.getNode(ISD::SHL, dl, NVT, V, Shift)); |
2390 | 100 | |
2391 | 100 | Hi = DAG.getNode(ISD::ADD, dl, NVT, W, |
2392 | 100 | DAG.getNode(ISD::ADD, dl, NVT, |
2393 | 100 | DAG.getNode(ISD::MUL, dl, NVT, RH, LL), |
2394 | 100 | DAG.getNode(ISD::MUL, dl, NVT, RL, LH))); |
2395 | 100 | return; |
2396 | 100 | } |
2397 | 300 | |
2398 | 300 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
2399 | 300 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first, |
2400 | 300 | Lo, Hi); |
2401 | 300 | } |
2402 | | |
2403 | | void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo, |
2404 | 2 | SDValue &Hi) { |
2405 | 2 | SDLoc DL(N); |
2406 | 2 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2407 | 2 | SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other); |
2408 | 2 | SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0)); |
2409 | 2 | Lo = R.getValue(0); |
2410 | 2 | Hi = R.getValue(1); |
2411 | 2 | ReplaceValueWith(SDValue(N, 1), R.getValue(2)); |
2412 | 2 | } |
2413 | | |
2414 | | void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node, |
2415 | 20 | SDValue &Lo, SDValue &Hi) { |
2416 | 20 | SDValue LHS = Node->getOperand(0); |
2417 | 20 | SDValue RHS = Node->getOperand(1); |
2418 | 20 | SDLoc dl(Node); |
2419 | 20 | |
2420 | 20 | // Expand the result by simply replacing it with the equivalent |
2421 | 20 | // non-overflow-checking operation. |
2422 | 20 | SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? |
2423 | 20 | ISD::ADD15 : ISD::SUB5 , dl, LHS.getValueType(), |
2424 | 20 | LHS, RHS); |
2425 | 20 | SplitInteger(Sum, Lo, Hi); |
2426 | 20 | |
2427 | 20 | // Compute the overflow. |
2428 | 20 | // |
2429 | 20 | // LHSSign -> LHS >= 0 |
2430 | 20 | // RHSSign -> RHS >= 0 |
2431 | 20 | // SumSign -> Sum >= 0 |
2432 | 20 | // |
2433 | 20 | // Add: |
2434 | 20 | // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) |
2435 | 20 | // Sub: |
2436 | 20 | // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) |
2437 | 20 | // |
2438 | 20 | EVT OType = Node->getValueType(1); |
2439 | 20 | SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); |
2440 | 20 | |
2441 | 20 | SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); |
2442 | 20 | SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); |
2443 | 20 | SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, |
2444 | 20 | Node->getOpcode() == ISD::SADDO ? |
2445 | 20 | ISD::SETEQ15 : ISD::SETNE5 ); |
2446 | 20 | |
2447 | 20 | SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); |
2448 | 20 | SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); |
2449 | 20 | |
2450 | 20 | SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); |
2451 | 20 | |
2452 | 20 | // Use the calculated overflow everywhere. |
2453 | 20 | ReplaceValueWith(SDValue(Node, 1), Cmp); |
2454 | 20 | } |
2455 | | |
2456 | | void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N, |
2457 | 107 | SDValue &Lo, SDValue &Hi) { |
2458 | 107 | EVT VT = N->getValueType(0); |
2459 | 107 | SDLoc dl(N); |
2460 | 107 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
2461 | 107 | |
2462 | 107 | if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom107 ) { |
2463 | 3 | SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
2464 | 3 | SplitInteger(Res.getValue(0), Lo, Hi); |
2465 | 3 | return; |
2466 | 3 | } |
2467 | 104 | |
2468 | 104 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
2469 | 104 | if (VT == MVT::i16) |
2470 | 0 | LC = RTLIB::SDIV_I16; |
2471 | 104 | else if (104 VT == MVT::i32104 ) |
2472 | 1 | LC = RTLIB::SDIV_I32; |
2473 | 103 | else if (103 VT == MVT::i64103 ) |
2474 | 72 | LC = RTLIB::SDIV_I64; |
2475 | 31 | else if (31 VT == MVT::i12831 ) |
2476 | 31 | LC = RTLIB::SDIV_I128; |
2477 | 107 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!"); |
2478 | 107 | |
2479 | 107 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi); |
2480 | 107 | } |
2481 | | |
2482 | | void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, |
2483 | 64.5k | SDValue &Lo, SDValue &Hi) { |
2484 | 64.5k | EVT VT = N->getValueType(0); |
2485 | 64.5k | SDLoc dl(N); |
2486 | 64.5k | |
2487 | 64.5k | // If we can emit an efficient shift operation, do so now. Check to see if |
2488 | 64.5k | // the RHS is a constant. |
2489 | 64.5k | if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1))) |
2490 | 63.1k | return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi); |
2491 | 1.37k | |
2492 | 1.37k | // If we can determine that the high bit of the shift is zero or one, even if |
2493 | 1.37k | // the low bits are variable, emit this shift in an optimized form. |
2494 | 1.37k | if (1.37k ExpandShiftWithKnownAmountBit(N, Lo, Hi)1.37k ) |
2495 | 13 | return; |
2496 | 1.36k | |
2497 | 1.36k | // If this target supports shift_PARTS, use it. First, map to the _PARTS opc. |
2498 | 1.36k | unsigned PartsOpc; |
2499 | 1.36k | if (N->getOpcode() == ISD::SHL1.36k ) { |
2500 | 761 | PartsOpc = ISD::SHL_PARTS; |
2501 | 1.36k | } else if (604 N->getOpcode() == ISD::SRL604 ) { |
2502 | 515 | PartsOpc = ISD::SRL_PARTS; |
2503 | 604 | } else { |
2504 | 89 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
2505 | 89 | PartsOpc = ISD::SRA_PARTS; |
2506 | 89 | } |
2507 | 1.36k | |
2508 | 1.36k | // Next check to see if the target supports this SHL_PARTS operation or if it |
2509 | 1.36k | // will custom expand it. |
2510 | 1.36k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
2511 | 1.36k | TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); |
2512 | 1.36k | if ((Action == TargetLowering::Legal && 1.36k TLI.isTypeLegal(NVT)64 ) || |
2513 | 1.36k | Action == TargetLowering::Custom1.36k ) { |
2514 | 1.30k | // Expand the subcomponents. |
2515 | 1.30k | SDValue LHSL, LHSH; |
2516 | 1.30k | GetExpandedInteger(N->getOperand(0), LHSL, LHSH); |
2517 | 1.30k | EVT VT = LHSL.getValueType(); |
2518 | 1.30k | |
2519 | 1.30k | // If the shift amount operand is coming from a vector legalization it may |
2520 | 1.30k | // have an illegal type. Fix that first by casting the operand, otherwise |
2521 | 1.30k | // the new SHL_PARTS operation would need further legalization. |
2522 | 1.30k | SDValue ShiftOp = N->getOperand(1); |
2523 | 1.30k | EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2524 | 1.30k | assert(ShiftTy.getScalarSizeInBits() >= |
2525 | 1.30k | Log2_32_Ceil(VT.getScalarSizeInBits()) && |
2526 | 1.30k | "ShiftAmountTy is too small to cover the range of this type!"); |
2527 | 1.30k | if (ShiftOp.getValueType() != ShiftTy) |
2528 | 55 | ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy); |
2529 | 1.30k | |
2530 | 1.30k | SDValue Ops[] = { LHSL, LHSH, ShiftOp }; |
2531 | 1.30k | Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops); |
2532 | 1.30k | Hi = Lo.getValue(1); |
2533 | 1.30k | return; |
2534 | 1.30k | } |
2535 | 65 | |
2536 | 65 | // Otherwise, emit a libcall. |
2537 | 65 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
2538 | 65 | bool isSigned; |
2539 | 65 | if (N->getOpcode() == ISD::SHL65 ) { |
2540 | 27 | isSigned = false; /*sign irrelevant*/ |
2541 | 27 | if (VT == MVT::i16) |
2542 | 0 | LC = RTLIB::SHL_I16; |
2543 | 27 | else if (27 VT == MVT::i3227 ) |
2544 | 0 | LC = RTLIB::SHL_I32; |
2545 | 27 | else if (27 VT == MVT::i6427 ) |
2546 | 0 | LC = RTLIB::SHL_I64; |
2547 | 27 | else if (27 VT == MVT::i12827 ) |
2548 | 18 | LC = RTLIB::SHL_I128; |
2549 | 65 | } else if (38 N->getOpcode() == ISD::SRL38 ) { |
2550 | 21 | isSigned = false; |
2551 | 21 | if (VT == MVT::i16) |
2552 | 0 | LC = RTLIB::SRL_I16; |
2553 | 21 | else if (21 VT == MVT::i3221 ) |
2554 | 0 | LC = RTLIB::SRL_I32; |
2555 | 21 | else if (21 VT == MVT::i6421 ) |
2556 | 0 | LC = RTLIB::SRL_I64; |
2557 | 21 | else if (21 VT == MVT::i12821 ) |
2558 | 15 | LC = RTLIB::SRL_I128; |
2559 | 38 | } else { |
2560 | 17 | assert(N->getOpcode() == ISD::SRA && "Unknown shift!"); |
2561 | 17 | isSigned = true; |
2562 | 17 | if (VT == MVT::i16) |
2563 | 0 | LC = RTLIB::SRA_I16; |
2564 | 17 | else if (17 VT == MVT::i3217 ) |
2565 | 0 | LC = RTLIB::SRA_I32; |
2566 | 17 | else if (17 VT == MVT::i6417 ) |
2567 | 0 | LC = RTLIB::SRA_I64; |
2568 | 17 | else if (17 VT == MVT::i12817 ) |
2569 | 14 | LC = RTLIB::SRA_I128; |
2570 | 38 | } |
2571 | 65 | |
2572 | 65 | if (LC != RTLIB::UNKNOWN_LIBCALL && 65 TLI.getLibcallName(LC)47 ) { |
2573 | 1 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
2574 | 1 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi); |
2575 | 1 | return; |
2576 | 1 | } |
2577 | 64 | |
2578 | 64 | if (64 !ExpandShiftWithUnknownAmountBit(N, Lo, Hi)64 ) |
2579 | 0 | llvm_unreachable("Unsupported shift!"); |
2580 | 64.5k | } |
2581 | | |
2582 | | void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N, |
2583 | 1.82k | SDValue &Lo, SDValue &Hi) { |
2584 | 1.82k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2585 | 1.82k | SDLoc dl(N); |
2586 | 1.82k | SDValue Op = N->getOperand(0); |
2587 | 1.82k | if (Op.getValueType().bitsLE(NVT)1.82k ) { |
2588 | 1.82k | // The low part is sign extension of the input (degenerates to a copy). |
2589 | 1.82k | Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0)); |
2590 | 1.82k | // The high part is obtained by SRA'ing all but one of the bits of low part. |
2591 | 1.82k | unsigned LoSize = NVT.getSizeInBits(); |
2592 | 1.82k | Hi = DAG.getNode( |
2593 | 1.82k | ISD::SRA, dl, NVT, Lo, |
2594 | 1.82k | DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
2595 | 1.82k | } else { |
2596 | 1 | // For example, extension of an i48 to an i64. The operand type necessarily |
2597 | 1 | // promotes to the result type, so will end up being expanded too. |
2598 | 1 | assert(getTypeAction(Op.getValueType()) == |
2599 | 1 | TargetLowering::TypePromoteInteger && |
2600 | 1 | "Only know how to promote this result!"); |
2601 | 1 | SDValue Res = GetPromotedInteger(Op); |
2602 | 1 | assert(Res.getValueType() == N->getValueType(0) && |
2603 | 1 | "Operand over promoted?"); |
2604 | 1 | // Split the promoted operand. This will simplify when it is expanded. |
2605 | 1 | SplitInteger(Res, Lo, Hi); |
2606 | 1 | unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
2607 | 1 | Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, |
2608 | 1 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
2609 | 1 | ExcessBits))); |
2610 | 1 | } |
2611 | 1.82k | } |
2612 | | |
2613 | | void DAGTypeLegalizer:: |
2614 | 156 | ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) { |
2615 | 156 | SDLoc dl(N); |
2616 | 156 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
2617 | 156 | EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT(); |
2618 | 156 | |
2619 | 156 | if (EVT.bitsLE(Lo.getValueType())156 ) { |
2620 | 87 | // sext_inreg the low part if needed. |
2621 | 87 | Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo, |
2622 | 87 | N->getOperand(1)); |
2623 | 87 | |
2624 | 87 | // The high part gets the sign extension from the lo-part. This handles |
2625 | 87 | // things like sextinreg V:i64 from i8. |
2626 | 87 | Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo, |
2627 | 87 | DAG.getConstant(Hi.getValueSizeInBits() - 1, dl, |
2628 | 87 | TLI.getPointerTy(DAG.getDataLayout()))); |
2629 | 156 | } else { |
2630 | 69 | // For example, extension of an i48 to an i64. Leave the low part alone, |
2631 | 69 | // sext_inreg the high part. |
2632 | 69 | unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits(); |
2633 | 69 | Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi, |
2634 | 69 | DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), |
2635 | 69 | ExcessBits))); |
2636 | 69 | } |
2637 | 156 | } |
2638 | | |
2639 | | void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N, |
2640 | 73 | SDValue &Lo, SDValue &Hi) { |
2641 | 73 | EVT VT = N->getValueType(0); |
2642 | 73 | SDLoc dl(N); |
2643 | 73 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
2644 | 73 | |
2645 | 73 | if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom73 ) { |
2646 | 9 | SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
2647 | 9 | SplitInteger(Res.getValue(1), Lo, Hi); |
2648 | 9 | return; |
2649 | 9 | } |
2650 | 64 | |
2651 | 64 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
2652 | 64 | if (VT == MVT::i16) |
2653 | 0 | LC = RTLIB::SREM_I16; |
2654 | 64 | else if (64 VT == MVT::i3264 ) |
2655 | 1 | LC = RTLIB::SREM_I32; |
2656 | 63 | else if (63 VT == MVT::i6463 ) |
2657 | 46 | LC = RTLIB::SREM_I64; |
2658 | 17 | else if (17 VT == MVT::i12817 ) |
2659 | 17 | LC = RTLIB::SREM_I128; |
2660 | 73 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!"); |
2661 | 73 | |
2662 | 73 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi); |
2663 | 73 | } |
2664 | | |
2665 | | void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N, |
2666 | 12.3k | SDValue &Lo, SDValue &Hi) { |
2667 | 12.3k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2668 | 12.3k | SDLoc dl(N); |
2669 | 12.3k | Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0)); |
2670 | 12.3k | Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(), |
2671 | 12.3k | N->getOperand(0), |
2672 | 12.3k | DAG.getConstant(NVT.getSizeInBits(), dl, |
2673 | 12.3k | TLI.getPointerTy(DAG.getDataLayout()))); |
2674 | 12.3k | Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi); |
2675 | 12.3k | } |
2676 | | |
2677 | | void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N, |
2678 | 11 | SDValue &Lo, SDValue &Hi) { |
2679 | 11 | EVT VT = N->getValueType(0); |
2680 | 11 | SDLoc dl(N); |
2681 | 11 | |
2682 | 11 | // A divide for UMULO should be faster than a function call. |
2683 | 11 | if (N->getOpcode() == ISD::UMULO11 ) { |
2684 | 2 | SDValue LHS = N->getOperand(0), RHS = N->getOperand(1); |
2685 | 2 | |
2686 | 2 | SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS); |
2687 | 2 | SplitInteger(MUL, Lo, Hi); |
2688 | 2 | |
2689 | 2 | // A divide for UMULO will be faster than a function call. Select to |
2690 | 2 | // make sure we aren't using 0. |
2691 | 2 | SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT), |
2692 | 2 | RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ); |
2693 | 2 | SDValue NotZero = DAG.getSelect(dl, VT, isZero, |
2694 | 2 | DAG.getConstant(1, dl, VT), RHS); |
2695 | 2 | SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero); |
2696 | 2 | SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS, |
2697 | 2 | ISD::SETNE); |
2698 | 2 | Overflow = DAG.getSelect(dl, N->getValueType(1), isZero, |
2699 | 2 | DAG.getConstant(0, dl, N->getValueType(1)), |
2700 | 2 | Overflow); |
2701 | 2 | ReplaceValueWith(SDValue(N, 1), Overflow); |
2702 | 2 | return; |
2703 | 2 | } |
2704 | 9 | |
2705 | 9 | Type *RetTy = VT.getTypeForEVT(*DAG.getContext()); |
2706 | 9 | EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); |
2707 | 9 | Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext()); |
2708 | 9 | |
2709 | 9 | // Replace this with a libcall that will check overflow. |
2710 | 9 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
2711 | 9 | if (VT == MVT::i32) |
2712 | 0 | LC = RTLIB::MULO_I32; |
2713 | 9 | else if (9 VT == MVT::i649 ) |
2714 | 7 | LC = RTLIB::MULO_I64; |
2715 | 2 | else if (2 VT == MVT::i1282 ) |
2716 | 2 | LC = RTLIB::MULO_I128; |
2717 | 9 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!"); |
2718 | 9 | |
2719 | 9 | SDValue Temp = DAG.CreateStackTemporary(PtrVT); |
2720 | 9 | // Temporary for the overflow value, default it to zero. |
2721 | 9 | SDValue Chain = |
2722 | 9 | DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp, |
2723 | 9 | MachinePointerInfo()); |
2724 | 9 | |
2725 | 9 | TargetLowering::ArgListTy Args; |
2726 | 9 | TargetLowering::ArgListEntry Entry; |
2727 | 18 | for (const SDValue &Op : N->op_values()) { |
2728 | 18 | EVT ArgVT = Op.getValueType(); |
2729 | 18 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
2730 | 18 | Entry.Node = Op; |
2731 | 18 | Entry.Ty = ArgTy; |
2732 | 18 | Entry.IsSExt = true; |
2733 | 18 | Entry.IsZExt = false; |
2734 | 18 | Args.push_back(Entry); |
2735 | 18 | } |
2736 | 11 | |
2737 | 11 | // Also pass the address of the overflow check. |
2738 | 11 | Entry.Node = Temp; |
2739 | 11 | Entry.Ty = PtrTy->getPointerTo(); |
2740 | 11 | Entry.IsSExt = true; |
2741 | 11 | Entry.IsZExt = false; |
2742 | 11 | Args.push_back(Entry); |
2743 | 11 | |
2744 | 11 | SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT); |
2745 | 11 | |
2746 | 11 | TargetLowering::CallLoweringInfo CLI(DAG); |
2747 | 11 | CLI.setDebugLoc(dl) |
2748 | 11 | .setChain(Chain) |
2749 | 11 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args)) |
2750 | 11 | .setSExtResult(); |
2751 | 11 | |
2752 | 11 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2753 | 11 | |
2754 | 11 | SplitInteger(CallInfo.first, Lo, Hi); |
2755 | 11 | SDValue Temp2 = |
2756 | 11 | DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo()); |
2757 | 11 | SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2, |
2758 | 11 | DAG.getConstant(0, dl, PtrVT), |
2759 | 11 | ISD::SETNE); |
2760 | 11 | // Use the overflow from the libcall everywhere. |
2761 | 11 | ReplaceValueWith(SDValue(N, 1), Ofl); |
2762 | 11 | } |
2763 | | |
2764 | | void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N, |
2765 | 111 | SDValue &Lo, SDValue &Hi) { |
2766 | 111 | EVT VT = N->getValueType(0); |
2767 | 111 | SDLoc dl(N); |
2768 | 111 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
2769 | 111 | |
2770 | 111 | if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom111 ) { |
2771 | 5 | SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
2772 | 5 | SplitInteger(Res.getValue(0), Lo, Hi); |
2773 | 5 | return; |
2774 | 5 | } |
2775 | 106 | |
2776 | 106 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
2777 | 106 | if (VT == MVT::i16) |
2778 | 0 | LC = RTLIB::UDIV_I16; |
2779 | 106 | else if (106 VT == MVT::i32106 ) |
2780 | 1 | LC = RTLIB::UDIV_I32; |
2781 | 105 | else if (105 VT == MVT::i64105 ) |
2782 | 73 | LC = RTLIB::UDIV_I64; |
2783 | 32 | else if (32 VT == MVT::i12832 ) |
2784 | 32 | LC = RTLIB::UDIV_I128; |
2785 | 111 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!"); |
2786 | 111 | |
2787 | 111 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi); |
2788 | 111 | } |
2789 | | |
2790 | | void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N, |
2791 | 73 | SDValue &Lo, SDValue &Hi) { |
2792 | 73 | EVT VT = N->getValueType(0); |
2793 | 73 | SDLoc dl(N); |
2794 | 73 | SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) }; |
2795 | 73 | |
2796 | 73 | if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom73 ) { |
2797 | 11 | SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops); |
2798 | 11 | SplitInteger(Res.getValue(1), Lo, Hi); |
2799 | 11 | return; |
2800 | 11 | } |
2801 | 62 | |
2802 | 62 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
2803 | 62 | if (VT == MVT::i16) |
2804 | 0 | LC = RTLIB::UREM_I16; |
2805 | 62 | else if (62 VT == MVT::i3262 ) |
2806 | 1 | LC = RTLIB::UREM_I32; |
2807 | 61 | else if (61 VT == MVT::i6461 ) |
2808 | 45 | LC = RTLIB::UREM_I64; |
2809 | 16 | else if (16 VT == MVT::i12816 ) |
2810 | 16 | LC = RTLIB::UREM_I128; |
2811 | 73 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!"); |
2812 | 73 | |
2813 | 73 | SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi); |
2814 | 73 | } |
2815 | | |
2816 | | void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N, |
2817 | 9.29k | SDValue &Lo, SDValue &Hi) { |
2818 | 9.29k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); |
2819 | 9.29k | SDLoc dl(N); |
2820 | 9.29k | SDValue Op = N->getOperand(0); |
2821 | 9.29k | if (Op.getValueType().bitsLE(NVT)9.29k ) { |
2822 | 9.29k | // The low part is zero extension of the input (degenerates to a copy). |
2823 | 9.29k | Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0)); |
2824 | 9.29k | Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero. |
2825 | 9.29k | } else { |
2826 | 0 | // For example, extension of an i48 to an i64. The operand type necessarily |
2827 | 0 | // promotes to the result type, so will end up being expanded too. |
2828 | 0 | assert(getTypeAction(Op.getValueType()) == |
2829 | 0 | TargetLowering::TypePromoteInteger && |
2830 | 0 | "Only know how to promote this result!"); |
2831 | 0 | SDValue Res = GetPromotedInteger(Op); |
2832 | 0 | assert(Res.getValueType() == N->getValueType(0) && |
2833 | 0 | "Operand over promoted?"); |
2834 | 0 | // Split the promoted operand. This will simplify when it is expanded. |
2835 | 0 | SplitInteger(Res, Lo, Hi); |
2836 | 0 | unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits(); |
2837 | 0 | Hi = DAG.getZeroExtendInReg(Hi, dl, |
2838 | 0 | EVT::getIntegerVT(*DAG.getContext(), |
2839 | 0 | ExcessBits)); |
2840 | 0 | } |
2841 | 9.29k | } |
2842 | | |
2843 | | void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, |
2844 | 11 | SDValue &Lo, SDValue &Hi) { |
2845 | 11 | SDLoc dl(N); |
2846 | 11 | EVT VT = cast<AtomicSDNode>(N)->getMemoryVT(); |
2847 | 11 | SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other); |
2848 | 11 | SDValue Zero = DAG.getConstant(0, dl, VT); |
2849 | 11 | SDValue Swap = DAG.getAtomicCmpSwap( |
2850 | 11 | ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, |
2851 | 11 | cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0), |
2852 | 11 | N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand()); |
2853 | 11 | |
2854 | 11 | ReplaceValueWith(SDValue(N, 0), Swap.getValue(0)); |
2855 | 11 | ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); |
2856 | 11 | } |
2857 | | |
2858 | | //===----------------------------------------------------------------------===// |
2859 | | // Integer Operand Expansion |
2860 | | //===----------------------------------------------------------------------===// |
2861 | | |
2862 | | /// ExpandIntegerOperand - This method is called when the specified operand of |
2863 | | /// the specified node is found to need expansion. At this point, all of the |
2864 | | /// result types of the node are known to be legal, but other operands of the |
2865 | | /// node may need promotion or expansion as well as the specified one. |
2866 | 148k | bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { |
2867 | 148k | DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n"); |
2868 | 148k | SDValue Res = SDValue(); |
2869 | 148k | |
2870 | 148k | if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) |
2871 | 782 | return false; |
2872 | 148k | |
2873 | 148k | switch (N->getOpcode()) { |
2874 | 0 | default: |
2875 | | #ifndef NDEBUG |
2876 | | dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; |
2877 | | N->dump(&DAG); dbgs() << "\n"; |
2878 | | #endif |
2879 | 0 | llvm_unreachable("Do not know how to expand this operator's operand!"); |
2880 | 148k | |
2881 | 3.77k | case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; |
2882 | 0 | case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; |
2883 | 1.03k | case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; |
2884 | 32.4k | case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; |
2885 | 35 | case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break; |
2886 | 39 | case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break; |
2887 | 1.84k | case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break; |
2888 | 4.23k | case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break; |
2889 | 0 | case ISD::SETCCE: Res = ExpandIntOp_SETCCE(N); break; |
2890 | 24 | case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break; |
2891 | 42 | case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break; |
2892 | 7.89k | case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break; |
2893 | 94.7k | case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break; |
2894 | 39 | case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break; |
2895 | 148k | |
2896 | 1.93k | case ISD::SHL: |
2897 | 1.93k | case ISD::SRA: |
2898 | 1.93k | case ISD::SRL: |
2899 | 1.93k | case ISD::ROTL: |
2900 | 1.93k | case ISD::ROTR: Res = ExpandIntOp_Shift(N); break; |
2901 | 1 | case ISD::RETURNADDR: |
2902 | 1 | case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break; |
2903 | 1 | |
2904 | 11 | case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break; |
2905 | 148k | } |
2906 | 148k | |
2907 | 148k | // If the result is null, the sub-method took care of registering results etc. |
2908 | 148k | if (148k !Res.getNode()148k ) return false0 ; |
2909 | 148k | |
2910 | 148k | // If the result is N, the sub-method updated N in place. Tell the legalizer |
2911 | 148k | // core about this. |
2912 | 148k | if (148k Res.getNode() == N148k ) |
2913 | 6.01k | return true; |
2914 | 142k | |
2915 | 148k | assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && |
2916 | 142k | "Invalid operand expansion"); |
2917 | 142k | |
2918 | 142k | ReplaceValueWith(SDValue(N, 0), Res); |
2919 | 142k | return false; |
2920 | 142k | } |
2921 | | |
2922 | | /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code |
2923 | | /// is shared among BR_CC, SELECT_CC, and SETCC handlers. |
2924 | | void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS, |
2925 | | SDValue &NewRHS, |
2926 | | ISD::CondCode &CCCode, |
2927 | 6.07k | const SDLoc &dl) { |
2928 | 6.07k | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
2929 | 6.07k | GetExpandedInteger(NewLHS, LHSLo, LHSHi); |
2930 | 6.07k | GetExpandedInteger(NewRHS, RHSLo, RHSHi); |
2931 | 6.07k | |
2932 | 6.07k | if (CCCode == ISD::SETEQ || 6.07k CCCode == ISD::SETNE5.31k ) { |
2933 | 1.93k | if (RHSLo == RHSHi1.93k ) { |
2934 | 946 | if (ConstantSDNode *RHSCST946 = dyn_cast<ConstantSDNode>(RHSLo)) { |
2935 | 943 | if (RHSCST->isAllOnesValue()943 ) { |
2936 | 0 | // Equality comparison to -1. |
2937 | 0 | NewLHS = DAG.getNode(ISD::AND, dl, |
2938 | 0 | LHSLo.getValueType(), LHSLo, LHSHi); |
2939 | 0 | NewRHS = RHSLo; |
2940 | 0 | return; |
2941 | 0 | } |
2942 | 1.93k | } |
2943 | 946 | } |
2944 | 1.93k | |
2945 | 1.93k | NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo); |
2946 | 1.93k | NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi); |
2947 | 1.93k | NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS); |
2948 | 1.93k | NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType()); |
2949 | 1.93k | return; |
2950 | 1.93k | } |
2951 | 4.14k | |
2952 | 4.14k | // If this is a comparison of the sign bit, just look at the top part. |
2953 | 4.14k | // X > -1, x < 0 |
2954 | 4.14k | if (ConstantSDNode *4.14k CST4.14k = dyn_cast<ConstantSDNode>(NewRHS)) |
2955 | 1.30k | if (1.30k (CCCode == ISD::SETLT && 1.30k CST->isNullValue()403 ) || // X < 0 |
2956 | 1.30k | (CCCode == ISD::SETGT && 996 CST->isAllOnesValue()65 )) { // X > -1 |
2957 | 337 | NewLHS = LHSHi; |
2958 | 337 | NewRHS = RHSHi; |
2959 | 337 | return; |
2960 | 337 | } |
2961 | 3.80k | |
2962 | 3.80k | // FIXME: This generated code sucks. |
2963 | 3.80k | ISD::CondCode LowCC; |
2964 | 3.80k | switch (CCCode) { |
2965 | 0 | default: 0 llvm_unreachable0 ("Unknown integer setcc!"); |
2966 | 731 | case ISD::SETLT: |
2967 | 731 | case ISD::SETULT: LowCC = ISD::SETULT; break; |
2968 | 780 | case ISD::SETGT: |
2969 | 780 | case ISD::SETUGT: LowCC = ISD::SETUGT; break; |
2970 | 365 | case ISD::SETLE: |
2971 | 365 | case ISD::SETULE: LowCC = ISD::SETULE; break; |
2972 | 1.93k | case ISD::SETGE: |
2973 | 1.93k | case ISD::SETUGE: LowCC = ISD::SETUGE; break; |
2974 | 3.80k | } |
2975 | 3.80k | |
2976 | 3.80k | // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison |
2977 | 3.80k | // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands |
2978 | 3.80k | // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp; |
2979 | 3.80k | |
2980 | 3.80k | // NOTE: on targets without efficient SELECT of bools, we can always use |
2981 | 3.80k | // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3) |
2982 | 3.80k | TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true, |
2983 | 3.80k | nullptr); |
2984 | 3.80k | SDValue LoCmp, HiCmp; |
2985 | 3.80k | if (TLI.isTypeLegal(LHSLo.getValueType()) && |
2986 | 3.78k | TLI.isTypeLegal(RHSLo.getValueType())) |
2987 | 3.78k | LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo, |
2988 | 3.78k | RHSLo, LowCC, false, DagCombineInfo, dl); |
2989 | 3.80k | if (!LoCmp.getNode()) |
2990 | 2.99k | LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, |
2991 | 2.99k | RHSLo, LowCC); |
2992 | 3.80k | if (TLI.isTypeLegal(LHSHi.getValueType()) && |
2993 | 3.78k | TLI.isTypeLegal(RHSHi.getValueType())) |
2994 | 3.78k | HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi, |
2995 | 3.78k | RHSHi, CCCode, false, DagCombineInfo, dl); |
2996 | 3.80k | if (!HiCmp.getNode()) |
2997 | 3.25k | HiCmp = |
2998 | 3.25k | DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()), |
2999 | 3.25k | LHSHi, RHSHi, DAG.getCondCode(CCCode)); |
3000 | 3.80k | |
3001 | 3.80k | ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode()); |
3002 | 3.80k | ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode()); |
3003 | 3.80k | |
3004 | 3.67k | bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE || |
3005 | 3.80k | CCCode == ISD::SETUGE3.59k || CCCode == ISD::SETULE1.74k ); |
3006 | 3.80k | |
3007 | 3.80k | if ((EqAllowed && 3.80k (HiCmpC && 2.29k HiCmpC->isNullValue()27 )) || |
3008 | 3.80k | (!EqAllowed && 3.80k ((HiCmpC && 1.51k (HiCmpC->getAPIntValue() == 1)229 ) || |
3009 | 3.80k | (LoCmpC && 1.51k LoCmpC->isNullValue()188 )))) { |
3010 | 190 | // For LE / GE, if high part is known false, ignore the low part. |
3011 | 190 | // For LT / GT: if low part is known false, return the high part. |
3012 | 190 | // if high part is known true, ignore the low part. |
3013 | 190 | NewLHS = HiCmp; |
3014 | 190 | NewRHS = SDValue(); |
3015 | 190 | return; |
3016 | 190 | } |
3017 | 3.61k | |
3018 | 3.61k | if (3.61k LHSHi == RHSHi3.61k ) { |
3019 | 69 | // Comparing the low bits is enough. |
3020 | 69 | NewLHS = LoCmp; |
3021 | 69 | NewRHS = SDValue(); |
3022 | 69 | return; |
3023 | 69 | } |
3024 | 3.54k | |
3025 | 3.54k | // Lower with SETCCE or SETCCCARRY if the target supports it. |
3026 | 3.54k | EVT HiVT = LHSHi.getValueType(); |
3027 | 3.54k | EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT); |
3028 | 3.54k | bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT); |
3029 | 3.54k | |
3030 | 3.54k | // FIXME: Make all targets support this, then remove the other lowering. |
3031 | 3.54k | if (HasSETCCCARRY || |
3032 | 3.54k | TLI.getOperationAction(ISD::SETCCE, ExpandVT) == TargetLowering::Custom2.96k ) { |
3033 | 1.57k | // SETCCE/SETCCCARRY can detect < and >= directly. For > and <=, flip |
3034 | 1.57k | // operands and condition code. |
3035 | 1.57k | bool FlipOperands = false; |
3036 | 1.57k | switch (CCCode) { |
3037 | 135 | case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break; |
3038 | 381 | case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break; |
3039 | 108 | case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break; |
3040 | 210 | case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break; |
3041 | 744 | default: break; |
3042 | 1.57k | } |
3043 | 1.57k | if (1.57k FlipOperands1.57k ) { |
3044 | 834 | std::swap(LHSLo, RHSLo); |
3045 | 834 | std::swap(LHSHi, RHSHi); |
3046 | 834 | } |
3047 | 1.57k | // Perform a wide subtraction, feeding the carry from the low part into |
3048 | 1.57k | // SETCCE/SETCCCARRY. The SETCCE/SETCCCARRY operation is essentially |
3049 | 1.57k | // looking at the high part of the result of LHS - RHS. It is negative |
3050 | 1.57k | // iff LHS < RHS. It is zero or positive iff LHS >= RHS. |
3051 | 1.57k | EVT LoVT = LHSLo.getValueType(); |
3052 | 1.57k | SDVTList VTList = DAG.getVTList( |
3053 | 1.57k | LoVT, HasSETCCCARRY ? getSetCCResultType(LoVT)579 : MVT::Glue999 ); |
3054 | 1.57k | SDValue LowCmp = DAG.getNode(HasSETCCCARRY ? ISD::USUBO579 : ISD::SUBC999 , dl, |
3055 | 1.57k | VTList, LHSLo, RHSLo); |
3056 | 1.57k | SDValue Res = DAG.getNode(HasSETCCCARRY ? ISD::SETCCCARRY579 : ISD::SETCCE999 , dl, |
3057 | 1.57k | getSetCCResultType(HiVT), LHSHi, RHSHi, |
3058 | 1.57k | LowCmp.getValue(1), DAG.getCondCode(CCCode)); |
3059 | 1.57k | NewLHS = Res; |
3060 | 1.57k | NewRHS = SDValue(); |
3061 | 1.57k | return; |
3062 | 1.57k | } |
3063 | 1.97k | |
3064 | 1.97k | NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ, |
3065 | 1.97k | false, DagCombineInfo, dl); |
3066 | 1.97k | if (!NewLHS.getNode()) |
3067 | 1.94k | NewLHS = |
3068 | 1.94k | DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ); |
3069 | 6.07k | NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp); |
3070 | 6.07k | NewRHS = SDValue(); |
3071 | 6.07k | } |
3072 | | |
3073 | 0 | SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) { |
3074 | 0 | SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); |
3075 | 0 | ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); |
3076 | 0 | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
3077 | 0 |
|
3078 | 0 | // If ExpandSetCCOperands returned a scalar, we need to compare the result |
3079 | 0 | // against zero to select between true and false values. |
3080 | 0 | if (!NewRHS.getNode()0 ) { |
3081 | 0 | NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); |
3082 | 0 | CCCode = ISD::SETNE; |
3083 | 0 | } |
3084 | 0 |
|
3085 | 0 | // Update N to have the operands specified. |
3086 | 0 | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), |
3087 | 0 | DAG.getCondCode(CCCode), NewLHS, NewRHS, |
3088 | 0 | N->getOperand(4)), 0); |
3089 | 0 | } |
3090 | | |
3091 | 1.84k | SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) { |
3092 | 1.84k | SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); |
3093 | 1.84k | ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); |
3094 | 1.84k | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
3095 | 1.84k | |
3096 | 1.84k | // If ExpandSetCCOperands returned a scalar, we need to compare the result |
3097 | 1.84k | // against zero to select between true and false values. |
3098 | 1.84k | if (!NewRHS.getNode()1.84k ) { |
3099 | 1.81k | NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); |
3100 | 1.81k | CCCode = ISD::SETNE; |
3101 | 1.81k | } |
3102 | 1.84k | |
3103 | 1.84k | // Update N to have the operands specified. |
3104 | 1.84k | return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, |
3105 | 1.84k | N->getOperand(2), N->getOperand(3), |
3106 | 1.84k | DAG.getCondCode(CCCode)), 0); |
3107 | 1.84k | } |
3108 | | |
3109 | 4.23k | SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) { |
3110 | 4.23k | SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); |
3111 | 4.23k | ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); |
3112 | 4.23k | IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N)); |
3113 | 4.23k | |
3114 | 4.23k | // If ExpandSetCCOperands returned a scalar, use it. |
3115 | 4.23k | if (!NewRHS.getNode()4.23k ) { |
3116 | 1.99k | assert(NewLHS.getValueType() == N->getValueType(0) && |
3117 | 1.99k | "Unexpected setcc expansion!"); |
3118 | 1.99k | return NewLHS; |
3119 | 1.99k | } |
3120 | 2.23k | |
3121 | 2.23k | // Otherwise, update N to have the operands specified. |
3122 | 2.23k | return SDValue( |
3123 | 2.23k | DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0); |
3124 | 2.23k | } |
3125 | | |
3126 | 0 | SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) { |
3127 | 0 | SDValue LHS = N->getOperand(0); |
3128 | 0 | SDValue RHS = N->getOperand(1); |
3129 | 0 | SDValue Carry = N->getOperand(2); |
3130 | 0 | SDValue Cond = N->getOperand(3); |
3131 | 0 | SDLoc dl = SDLoc(N); |
3132 | 0 |
|
3133 | 0 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
3134 | 0 | GetExpandedInteger(LHS, LHSLo, LHSHi); |
3135 | 0 | GetExpandedInteger(RHS, RHSLo, RHSHi); |
3136 | 0 |
|
3137 | 0 | // Expand to a SUBE for the low part and a smaller SETCCE for the high. |
3138 | 0 | SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue); |
3139 | 0 | SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry); |
3140 | 0 | return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi, |
3141 | 0 | LowCmp.getValue(1), Cond); |
3142 | 0 | } |
3143 | | |
3144 | 24 | SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) { |
3145 | 24 | SDValue LHS = N->getOperand(0); |
3146 | 24 | SDValue RHS = N->getOperand(1); |
3147 | 24 | SDValue Carry = N->getOperand(2); |
3148 | 24 | SDValue Cond = N->getOperand(3); |
3149 | 24 | SDLoc dl = SDLoc(N); |
3150 | 24 | |
3151 | 24 | SDValue LHSLo, LHSHi, RHSLo, RHSHi; |
3152 | 24 | GetExpandedInteger(LHS, LHSLo, LHSHi); |
3153 | 24 | GetExpandedInteger(RHS, RHSLo, RHSHi); |
3154 | 24 | |
3155 | 24 | // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high. |
3156 | 24 | SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType()); |
3157 | 24 | SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry); |
3158 | 24 | return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi, |
3159 | 24 | LowCmp.getValue(1), Cond); |
3160 | 24 | } |
3161 | | |
3162 | 1.93k | SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) { |
3163 | 1.93k | // The value being shifted is legal, but the shift amount is too big. |
3164 | 1.93k | // It follows that either the result of the shift is undefined, or the |
3165 | 1.93k | // upper half of the shift amount is zero. Just use the lower half. |
3166 | 1.93k | SDValue Lo, Hi; |
3167 | 1.93k | GetExpandedInteger(N->getOperand(1), Lo, Hi); |
3168 | 1.93k | return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0); |
3169 | 1.93k | } |
3170 | | |
3171 | 1 | SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) { |
3172 | 1 | // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This |
3173 | 1 | // surely makes pretty nice problems on 8/16 bit targets. Just truncate this |
3174 | 1 | // constant to valid type. |
3175 | 1 | SDValue Lo, Hi; |
3176 | 1 | GetExpandedInteger(N->getOperand(0), Lo, Hi); |
3177 | 1 | return SDValue(DAG.UpdateNodeOperands(N, Lo), 0); |
3178 | 1 | } |
3179 | | |
3180 | 42 | SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) { |
3181 | 42 | SDValue Op = N->getOperand(0); |
3182 | 42 | EVT DstVT = N->getValueType(0); |
3183 | 42 | RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT); |
3184 | 42 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
3185 | 42 | "Don't know how to expand this SINT_TO_FP!"); |
3186 | 42 | return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first; |
3187 | 42 | } |
3188 | | |
3189 | 7.89k | SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) { |
3190 | 7.89k | if (ISD::isNormalStore(N)) |
3191 | 6.50k | return ExpandOp_NormalStore(N, OpNo); |
3192 | 1.38k | |
3193 | 7.89k | assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); |
3194 | 1.38k | assert(OpNo == 1 && "Can only expand the stored value so far"); |
3195 | 1.38k | |
3196 | 1.38k | EVT VT = N->getOperand(1).getValueType(); |
3197 | 1.38k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
3198 | 1.38k | SDValue Ch = N->getChain(); |
3199 | 1.38k | SDValue Ptr = N->getBasePtr(); |
3200 | 1.38k | unsigned Alignment = N->getAlignment(); |
3201 | 1.38k | MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags(); |
3202 | 1.38k | AAMDNodes AAInfo = N->getAAInfo(); |
3203 | 1.38k | SDLoc dl(N); |
3204 | 1.38k | SDValue Lo, Hi; |
3205 | 1.38k | |
3206 | 1.38k | assert(NVT.isByteSized() && "Expanded type not byte sized!"); |
3207 | 1.38k | |
3208 | 1.38k | if (N->getMemoryVT().bitsLE(NVT)1.38k ) { |
3209 | 336 | GetExpandedInteger(N->getValue(), Lo, Hi); |
3210 | 336 | return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), |
3211 | 336 | N->getMemoryVT(), Alignment, MMOFlags, AAInfo); |
3212 | 336 | } |
3213 | 1.04k | |
3214 | 1.04k | if (1.04k DAG.getDataLayout().isLittleEndian()1.04k ) { |
3215 | 1.04k | // Little-endian - low bits are at low addresses. |
3216 | 1.04k | GetExpandedInteger(N->getValue(), Lo, Hi); |
3217 | 1.04k | |
3218 | 1.04k | Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags, |
3219 | 1.04k | AAInfo); |
3220 | 1.04k | |
3221 | 1.04k | unsigned ExcessBits = |
3222 | 1.04k | N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits(); |
3223 | 1.04k | EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits); |
3224 | 1.04k | |
3225 | 1.04k | // Increment the pointer to the other half. |
3226 | 1.04k | unsigned IncrementSize = NVT.getSizeInBits()/8; |
3227 | 1.04k | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
3228 | 1.04k | DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); |
3229 | 1.04k | Hi = DAG.getTruncStore( |
3230 | 1.04k | Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT, |
3231 | 1.04k | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); |
3232 | 1.04k | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
3233 | 1.04k | } |
3234 | 6 | |
3235 | 6 | // Big-endian - high bits are at low addresses. Favor aligned stores at |
3236 | 6 | // the cost of some bit-fiddling. |
3237 | 6 | GetExpandedInteger(N->getValue(), Lo, Hi); |
3238 | 6 | |
3239 | 6 | EVT ExtVT = N->getMemoryVT(); |
3240 | 6 | unsigned EBytes = ExtVT.getStoreSize(); |
3241 | 6 | unsigned IncrementSize = NVT.getSizeInBits()/8; |
3242 | 6 | unsigned ExcessBits = (EBytes - IncrementSize)*8; |
3243 | 6 | EVT HiVT = EVT::getIntegerVT(*DAG.getContext(), |
3244 | 6 | ExtVT.getSizeInBits() - ExcessBits); |
3245 | 6 | |
3246 | 6 | if (ExcessBits < NVT.getSizeInBits()6 ) { |
3247 | 5 | // Transfer high bits from the top of Lo to the bottom of Hi. |
3248 | 5 | Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi, |
3249 | 5 | DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl, |
3250 | 5 | TLI.getPointerTy(DAG.getDataLayout()))); |
3251 | 5 | Hi = DAG.getNode( |
3252 | 5 | ISD::OR, dl, NVT, Hi, |
3253 | 5 | DAG.getNode(ISD::SRL, dl, NVT, Lo, |
3254 | 5 | DAG.getConstant(ExcessBits, dl, |
3255 | 5 | TLI.getPointerTy(DAG.getDataLayout())))); |
3256 | 5 | } |
3257 | 7.89k | |
3258 | 7.89k | // Store both the high bits and maybe some of the low bits. |
3259 | 7.89k | Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment, |
3260 | 7.89k | MMOFlags, AAInfo); |
3261 | 7.89k | |
3262 | 7.89k | // Increment the pointer to the other half. |
3263 | 7.89k | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
3264 | 7.89k | DAG.getConstant(IncrementSize, dl, Ptr.getValueType())); |
3265 | 7.89k | // Store the lowest ExcessBits bits in the second half. |
3266 | 7.89k | Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, |
3267 | 7.89k | N->getPointerInfo().getWithOffset(IncrementSize), |
3268 | 7.89k | EVT::getIntegerVT(*DAG.getContext(), ExcessBits), |
3269 | 7.89k | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); |
3270 | 7.89k | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
3271 | 7.89k | } |
3272 | | |
3273 | 94.7k | SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) { |
3274 | 94.7k | SDValue InL, InH; |
3275 | 94.7k | GetExpandedInteger(N->getOperand(0), InL, InH); |
3276 | 94.7k | // Just truncate the low part of the source. |
3277 | 94.7k | return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL); |
3278 | 94.7k | } |
3279 | | |
3280 | 39 | SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) { |
3281 | 39 | SDValue Op = N->getOperand(0); |
3282 | 39 | EVT SrcVT = Op.getValueType(); |
3283 | 39 | EVT DstVT = N->getValueType(0); |
3284 | 39 | SDLoc dl(N); |
3285 | 39 | |
3286 | 39 | // The following optimization is valid only if every value in SrcVT (when |
3287 | 39 | // treated as signed) is representable in DstVT. Check that the mantissa |
3288 | 39 | // size of DstVT is >= than the number of bits in SrcVT -1. |
3289 | 39 | const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT); |
3290 | 39 | if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 && |
3291 | 39 | TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom0 ){ |
3292 | 0 | // Do a signed conversion then adjust the result. |
3293 | 0 | SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op); |
3294 | 0 | SignedConv = TLI.LowerOperation(SignedConv, DAG); |
3295 | 0 |
|
3296 | 0 | // The result of the signed conversion needs adjusting if the 'sign bit' of |
3297 | 0 | // the incoming integer was set. To handle this, we dynamically test to see |
3298 | 0 | // if it is set, and, if so, add a fudge factor. |
3299 | 0 |
|
3300 | 0 | const uint64_t F32TwoE32 = 0x4F800000ULL; |
3301 | 0 | const uint64_t F32TwoE64 = 0x5F800000ULL; |
3302 | 0 | const uint64_t F32TwoE128 = 0x7F800000ULL; |
3303 | 0 |
|
3304 | 0 | APInt FF(32, 0); |
3305 | 0 | if (SrcVT == MVT::i32) |
3306 | 0 | FF = APInt(32, F32TwoE32); |
3307 | 0 | else if (0 SrcVT == MVT::i640 ) |
3308 | 0 | FF = APInt(32, F32TwoE64); |
3309 | 0 | else if (0 SrcVT == MVT::i1280 ) |
3310 | 0 | FF = APInt(32, F32TwoE128); |
3311 | 0 | else |
3312 | 0 | llvm_unreachable("Unsupported UINT_TO_FP!"); |
3313 | 0 |
|
3314 | 0 | // Check whether the sign bit is set. |
3315 | 0 | SDValue Lo, Hi; |
3316 | 0 | GetExpandedInteger(Op, Lo, Hi); |
3317 | 0 | SDValue SignSet = DAG.getSetCC(dl, |
3318 | 0 | getSetCCResultType(Hi.getValueType()), |
3319 | 0 | Hi, |
3320 | 0 | DAG.getConstant(0, dl, Hi.getValueType()), |
3321 | 0 | ISD::SETLT); |
3322 | 0 |
|
3323 | 0 | // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits. |
3324 | 0 | SDValue FudgePtr = |
3325 | 0 | DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)), |
3326 | 0 | TLI.getPointerTy(DAG.getDataLayout())); |
3327 | 0 |
|
3328 | 0 | // Get a pointer to FF if the sign bit was set, or to 0 otherwise. |
3329 | 0 | SDValue Zero = DAG.getIntPtrConstant(0, dl); |
3330 | 0 | SDValue Four = DAG.getIntPtrConstant(4, dl); |
3331 | 0 | if (DAG.getDataLayout().isBigEndian()) |
3332 | 0 | std::swap(Zero, Four); |
3333 | 0 | SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, |
3334 | 0 | Zero, Four); |
3335 | 0 | unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment(); |
3336 | 0 | FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(), |
3337 | 0 | FudgePtr, Offset); |
3338 | 0 | Alignment = std::min(Alignment, 4u); |
3339 | 0 |
|
3340 | 0 | // Load the value out, extending it from f32 to the destination float type. |
3341 | 0 | // FIXME: Avoid the extend by constructing the right constant pool? |
3342 | 0 | SDValue Fudge = DAG.getExtLoad( |
3343 | 0 | ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr, |
3344 | 0 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, |
3345 | 0 | Alignment); |
3346 | 0 | return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge); |
3347 | 39 | } |
3348 | 39 | |
3349 | 39 | // Otherwise, use a libcall. |
3350 | 39 | RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT); |
3351 | 39 | assert(LC != RTLIB::UNKNOWN_LIBCALL && |
3352 | 39 | "Don't know how to expand this UINT_TO_FP!"); |
3353 | 39 | return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first; |
3354 | 39 | } |
3355 | | |
3356 | 11 | SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { |
3357 | 11 | SDLoc dl(N); |
3358 | 11 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
3359 | 11 | cast<AtomicSDNode>(N)->getMemoryVT(), |
3360 | 11 | N->getOperand(0), |
3361 | 11 | N->getOperand(1), N->getOperand(2), |
3362 | 11 | cast<AtomicSDNode>(N)->getMemOperand()); |
3363 | 11 | return Swap.getValue(1); |
3364 | 11 | } |
3365 | | |
3366 | | |
3367 | 23.6k | SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { |
3368 | 23.6k | SDValue InOp0 = N->getOperand(0); |
3369 | 23.6k | EVT InVT = InOp0.getValueType(); |
3370 | 23.6k | |
3371 | 23.6k | EVT OutVT = N->getValueType(0); |
3372 | 23.6k | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
3373 | 23.6k | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
3374 | 23.6k | unsigned OutNumElems = OutVT.getVectorNumElements(); |
3375 | 23.6k | EVT NOutVTElem = NOutVT.getVectorElementType(); |
3376 | 23.6k | |
3377 | 23.6k | SDLoc dl(N); |
3378 | 23.6k | SDValue BaseIdx = N->getOperand(1); |
3379 | 23.6k | |
3380 | 23.6k | SmallVector<SDValue, 8> Ops; |
3381 | 23.6k | Ops.reserve(OutNumElems); |
3382 | 84.0k | for (unsigned i = 0; i != OutNumElems84.0k ; ++i60.4k ) { |
3383 | 60.4k | |
3384 | 60.4k | // Extract the element from the original vector. |
3385 | 60.4k | SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(), |
3386 | 60.4k | BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType())); |
3387 | 60.4k | SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
3388 | 60.4k | InVT.getVectorElementType(), N->getOperand(0), Index); |
3389 | 60.4k | |
3390 | 60.4k | SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); |
3391 | 60.4k | // Insert the converted element to the new vector. |
3392 | 60.4k | Ops.push_back(Op); |
3393 | 60.4k | } |
3394 | 23.6k | |
3395 | 23.6k | return DAG.getBuildVector(NOutVT, dl, Ops); |
3396 | 23.6k | } |
3397 | | |
3398 | | |
3399 | 104 | SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) { |
3400 | 104 | ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N); |
3401 | 104 | EVT VT = N->getValueType(0); |
3402 | 104 | SDLoc dl(N); |
3403 | 104 | |
3404 | 104 | ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements()); |
3405 | 104 | |
3406 | 104 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
3407 | 104 | SDValue V1 = GetPromotedInteger(N->getOperand(1)); |
3408 | 104 | EVT OutVT = V0.getValueType(); |
3409 | 104 | |
3410 | 104 | return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask); |
3411 | 104 | } |
3412 | | |
3413 | | |
3414 | 1.80k | SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) { |
3415 | 1.80k | EVT OutVT = N->getValueType(0); |
3416 | 1.80k | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
3417 | 1.80k | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
3418 | 1.80k | unsigned NumElems = N->getNumOperands(); |
3419 | 1.80k | EVT NOutVTElem = NOutVT.getVectorElementType(); |
3420 | 1.80k | |
3421 | 1.80k | SDLoc dl(N); |
3422 | 1.80k | |
3423 | 1.80k | SmallVector<SDValue, 8> Ops; |
3424 | 1.80k | Ops.reserve(NumElems); |
3425 | 16.3k | for (unsigned i = 0; i != NumElems16.3k ; ++i14.5k ) { |
3426 | 14.5k | SDValue Op; |
3427 | 14.5k | // BUILD_VECTOR integer operand types are allowed to be larger than the |
3428 | 14.5k | // result's element type. This may still be true after the promotion. For |
3429 | 14.5k | // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to |
3430 | 14.5k | // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>. |
3431 | 14.5k | if (N->getOperand(i).getValueType().bitsLT(NOutVTElem)) |
3432 | 10.1k | Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i)); |
3433 | 14.5k | else |
3434 | 4.42k | Op = N->getOperand(i); |
3435 | 14.5k | Ops.push_back(Op); |
3436 | 14.5k | } |
3437 | 1.80k | |
3438 | 1.80k | return DAG.getBuildVector(NOutVT, dl, Ops); |
3439 | 1.80k | } |
3440 | | |
3441 | 26 | SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) { |
3442 | 26 | |
3443 | 26 | SDLoc dl(N); |
3444 | 26 | |
3445 | 26 | assert(!N->getOperand(0).getValueType().isVector() && |
3446 | 26 | "Input must be a scalar"); |
3447 | 26 | |
3448 | 26 | EVT OutVT = N->getValueType(0); |
3449 | 26 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
3450 | 26 | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
3451 | 26 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
3452 | 26 | |
3453 | 26 | SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0)); |
3454 | 26 | |
3455 | 26 | return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op); |
3456 | 26 | } |
3457 | | |
3458 | 1.83k | SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) { |
3459 | 1.83k | SDLoc dl(N); |
3460 | 1.83k | |
3461 | 1.83k | EVT OutVT = N->getValueType(0); |
3462 | 1.83k | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
3463 | 1.83k | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
3464 | 1.83k | |
3465 | 1.83k | EVT InElemTy = OutVT.getVectorElementType(); |
3466 | 1.83k | EVT OutElemTy = NOutVT.getVectorElementType(); |
3467 | 1.83k | |
3468 | 1.83k | unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements(); |
3469 | 1.83k | unsigned NumOutElem = NOutVT.getVectorNumElements(); |
3470 | 1.83k | unsigned NumOperands = N->getNumOperands(); |
3471 | 1.83k | assert(NumElem * NumOperands == NumOutElem && |
3472 | 1.83k | "Unexpected number of elements"); |
3473 | 1.83k | |
3474 | 1.83k | // Take the elements from the first vector. |
3475 | 1.83k | SmallVector<SDValue, 8> Ops(NumOutElem); |
3476 | 7.43k | for (unsigned i = 0; i < NumOperands7.43k ; ++i5.59k ) { |
3477 | 5.59k | SDValue Op = N->getOperand(i); |
3478 | 32.2k | for (unsigned j = 0; j < NumElem32.2k ; ++j26.6k ) { |
3479 | 26.6k | SDValue Ext = DAG.getNode( |
3480 | 26.6k | ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op, |
3481 | 26.6k | DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
3482 | 26.6k | Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext); |
3483 | 26.6k | } |
3484 | 5.59k | } |
3485 | 1.83k | |
3486 | 1.83k | return DAG.getBuildVector(NOutVT, dl, Ops); |
3487 | 1.83k | } |
3488 | | |
3489 | 3 | SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) { |
3490 | 3 | EVT VT = N->getValueType(0); |
3491 | 3 | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
3492 | 3 | assert(NVT.isVector() && "This type must be promoted to a vector type"); |
3493 | 3 | |
3494 | 3 | SDLoc dl(N); |
3495 | 3 | |
3496 | 3 | // For operands whose TypeAction is to promote, extend the promoted node |
3497 | 3 | // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion |
3498 | 3 | // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to |
3499 | 3 | // type.. |
3500 | 3 | if (getTypeAction(N->getOperand(0).getValueType()) |
3501 | 3 | == TargetLowering::TypePromoteInteger) { |
3502 | 3 | SDValue Promoted; |
3503 | 3 | |
3504 | 3 | switch(N->getOpcode()) { |
3505 | 1 | case ISD::SIGN_EXTEND_VECTOR_INREG: |
3506 | 1 | Promoted = SExtPromotedInteger(N->getOperand(0)); |
3507 | 1 | break; |
3508 | 1 | case ISD::ZERO_EXTEND_VECTOR_INREG: |
3509 | 1 | Promoted = ZExtPromotedInteger(N->getOperand(0)); |
3510 | 1 | break; |
3511 | 1 | case ISD::ANY_EXTEND_VECTOR_INREG: |
3512 | 1 | Promoted = GetPromotedInteger(N->getOperand(0)); |
3513 | 1 | break; |
3514 | 0 | default: |
3515 | 0 | llvm_unreachable("Node has unexpected Opcode"); |
3516 | 3 | } |
3517 | 3 | return DAG.getNode(N->getOpcode(), dl, NVT, Promoted); |
3518 | 3 | } |
3519 | 0 |
|
3520 | 0 | // Directly extend to the appropriate transform-to type. |
3521 | 0 | return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); |
3522 | 0 | } |
3523 | | |
3524 | 184 | SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { |
3525 | 184 | EVT OutVT = N->getValueType(0); |
3526 | 184 | EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); |
3527 | 184 | assert(NOutVT.isVector() && "This type must be promoted to a vector type"); |
3528 | 184 | |
3529 | 184 | EVT NOutVTElem = NOutVT.getVectorElementType(); |
3530 | 184 | |
3531 | 184 | SDLoc dl(N); |
3532 | 184 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
3533 | 184 | |
3534 | 184 | SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl, |
3535 | 184 | NOutVTElem, N->getOperand(1)); |
3536 | 184 | return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT, |
3537 | 184 | V0, ConvElem, N->getOperand(2)); |
3538 | 184 | } |
3539 | | |
3540 | 38.0k | SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { |
3541 | 38.0k | SDLoc dl(N); |
3542 | 38.0k | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
3543 | 38.0k | SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, |
3544 | 38.0k | TLI.getVectorIdxTy(DAG.getDataLayout())); |
3545 | 38.0k | SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, |
3546 | 38.0k | V0->getValueType(0).getScalarType(), V0, V1); |
3547 | 38.0k | |
3548 | 38.0k | // EXTRACT_VECTOR_ELT can return types which are wider than the incoming |
3549 | 38.0k | // element types. If this is the case then we need to expand the outgoing |
3550 | 38.0k | // value and not truncate it. |
3551 | 38.0k | return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0)); |
3552 | 38.0k | } |
3553 | | |
3554 | 42 | SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) { |
3555 | 42 | SDLoc dl(N); |
3556 | 42 | SDValue V0 = GetPromotedInteger(N->getOperand(0)); |
3557 | 42 | MVT InVT = V0.getValueType().getSimpleVT(); |
3558 | 42 | MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(), |
3559 | 42 | N->getValueType(0).getVectorNumElements()); |
3560 | 42 | SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1)); |
3561 | 42 | return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext); |
3562 | 42 | } |
3563 | | |
3564 | 1.84k | SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) { |
3565 | 1.84k | SDLoc dl(N); |
3566 | 1.84k | unsigned NumElems = N->getNumOperands(); |
3567 | 1.84k | |
3568 | 1.84k | EVT RetSclrTy = N->getValueType(0).getVectorElementType(); |
3569 | 1.84k | |
3570 | 1.84k | SmallVector<SDValue, 8> NewOps; |
3571 | 1.84k | NewOps.reserve(NumElems); |
3572 | 1.84k | |
3573 | 1.84k | // For each incoming vector |
3574 | 7.63k | for (unsigned VecIdx = 0; VecIdx != NumElems7.63k ; ++VecIdx5.78k ) { |
3575 | 5.78k | SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx)); |
3576 | 5.78k | EVT SclrTy = Incoming->getValueType(0).getVectorElementType(); |
3577 | 5.78k | unsigned NumElem = Incoming->getValueType(0).getVectorNumElements(); |
3578 | 5.78k | |
3579 | 27.5k | for (unsigned i=0; i<NumElem27.5k ; ++i21.7k ) { |
3580 | 21.7k | // Extract element from incoming vector |
3581 | 21.7k | SDValue Ex = DAG.getNode( |
3582 | 21.7k | ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming, |
3583 | 21.7k | DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
3584 | 21.7k | SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex); |
3585 | 21.7k | NewOps.push_back(Tr); |
3586 | 21.7k | } |
3587 | 5.78k | } |
3588 | 1.84k | |
3589 | 1.84k | return DAG.getBuildVector(N->getValueType(0), dl, NewOps); |
3590 | 1.84k | } |