/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This file implements the SelectionDAG::Legalize method. |
10 | | // |
11 | | //===----------------------------------------------------------------------===// |
12 | | |
13 | | #include "llvm/ADT/APFloat.h" |
14 | | #include "llvm/ADT/APInt.h" |
15 | | #include "llvm/ADT/ArrayRef.h" |
16 | | #include "llvm/ADT/SetVector.h" |
17 | | #include "llvm/ADT/SmallPtrSet.h" |
18 | | #include "llvm/ADT/SmallSet.h" |
19 | | #include "llvm/ADT/SmallVector.h" |
20 | | #include "llvm/CodeGen/ISDOpcodes.h" |
21 | | #include "llvm/CodeGen/MachineFunction.h" |
22 | | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
23 | | #include "llvm/CodeGen/MachineMemOperand.h" |
24 | | #include "llvm/CodeGen/RuntimeLibcalls.h" |
25 | | #include "llvm/CodeGen/SelectionDAG.h" |
26 | | #include "llvm/CodeGen/SelectionDAGNodes.h" |
27 | | #include "llvm/CodeGen/TargetFrameLowering.h" |
28 | | #include "llvm/CodeGen/TargetLowering.h" |
29 | | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
30 | | #include "llvm/CodeGen/ValueTypes.h" |
31 | | #include "llvm/IR/CallingConv.h" |
32 | | #include "llvm/IR/Constants.h" |
33 | | #include "llvm/IR/DataLayout.h" |
34 | | #include "llvm/IR/DerivedTypes.h" |
35 | | #include "llvm/IR/Function.h" |
36 | | #include "llvm/IR/Metadata.h" |
37 | | #include "llvm/IR/Type.h" |
38 | | #include "llvm/Support/Casting.h" |
39 | | #include "llvm/Support/Compiler.h" |
40 | | #include "llvm/Support/Debug.h" |
41 | | #include "llvm/Support/ErrorHandling.h" |
42 | | #include "llvm/Support/MachineValueType.h" |
43 | | #include "llvm/Support/MathExtras.h" |
44 | | #include "llvm/Support/raw_ostream.h" |
45 | | #include "llvm/Target/TargetMachine.h" |
46 | | #include "llvm/Target/TargetOptions.h" |
47 | | #include <algorithm> |
48 | | #include <cassert> |
49 | | #include <cstdint> |
50 | | #include <tuple> |
51 | | #include <utility> |
52 | | |
53 | | using namespace llvm; |
54 | | |
55 | | #define DEBUG_TYPE "legalizedag" |
56 | | |
57 | | namespace { |
58 | | |
59 | | /// Keeps track of state when getting the sign of a floating-point value as an |
60 | | /// integer. |
61 | | struct FloatSignAsInt { |
62 | | EVT FloatVT; |
63 | | SDValue Chain; |
64 | | SDValue FloatPtr; |
65 | | SDValue IntPtr; |
66 | | MachinePointerInfo IntPointerInfo; |
67 | | MachinePointerInfo FloatPointerInfo; |
68 | | SDValue IntValue; |
69 | | APInt SignMask; |
70 | | uint8_t SignBit; |
71 | | }; |
72 | | |
73 | | //===----------------------------------------------------------------------===// |
74 | | /// This takes an arbitrary SelectionDAG as input and |
75 | | /// hacks on it until the target machine can handle it. This involves |
76 | | /// eliminating value sizes the machine cannot handle (promoting small sizes to |
77 | | /// large sizes or splitting up large values into small values) as well as |
78 | | /// eliminating operations the machine cannot handle. |
79 | | /// |
80 | | /// This code also does a small amount of optimization and recognition of idioms |
81 | | /// as part of its processing. For example, if a target does not support a |
82 | | /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this |
83 | | /// will attempt merge setcc and brc instructions into brcc's. |
84 | | class SelectionDAGLegalize { |
85 | | const TargetMachine &TM; |
86 | | const TargetLowering &TLI; |
87 | | SelectionDAG &DAG; |
88 | | |
89 | | /// The set of nodes which have already been legalized. We hold a |
90 | | /// reference to it in order to update as necessary on node deletion. |
91 | | SmallPtrSetImpl<SDNode *> &LegalizedNodes; |
92 | | |
93 | | /// A set of all the nodes updated during legalization. |
94 | | SmallSetVector<SDNode *, 16> *UpdatedNodes; |
95 | | |
96 | 2.15k | EVT getSetCCResultType(EVT VT) const { |
97 | 2.15k | return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); |
98 | 2.15k | } |
99 | | |
100 | | // Libcall insertion helpers. |
101 | | |
102 | | public: |
103 | | SelectionDAGLegalize(SelectionDAG &DAG, |
104 | | SmallPtrSetImpl<SDNode *> &LegalizedNodes, |
105 | | SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) |
106 | | : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), |
107 | 28.7M | LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} |
108 | | |
109 | | /// Legalizes the given operation. |
110 | | void LegalizeOp(SDNode *Node); |
111 | | |
112 | | private: |
113 | | SDValue OptimizeFloatStore(StoreSDNode *ST); |
114 | | |
115 | | void LegalizeLoadOps(SDNode *Node); |
116 | | void LegalizeStoreOps(SDNode *Node); |
117 | | |
118 | | /// Some targets cannot handle a variable |
119 | | /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it |
120 | | /// is necessary to spill the vector being inserted into to memory, perform |
121 | | /// the insert there, and then read the result back. |
122 | | SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, |
123 | | const SDLoc &dl); |
124 | | SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, |
125 | | const SDLoc &dl); |
126 | | |
127 | | /// Return a vector shuffle operation which |
128 | | /// performs the same shuffe in terms of order or result bytes, but on a type |
129 | | /// whose vector element type is narrower than the original shuffle type. |
130 | | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> |
131 | | SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl, |
132 | | SDValue N1, SDValue N2, |
133 | | ArrayRef<int> Mask) const; |
134 | | |
135 | | bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, |
136 | | bool &NeedInvert, const SDLoc &dl); |
137 | | |
138 | | SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); |
139 | | |
140 | | std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, |
141 | | SDNode *Node, bool isSigned); |
142 | | SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, |
143 | | RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, |
144 | | RTLIB::Libcall Call_F128, |
145 | | RTLIB::Libcall Call_PPCF128); |
146 | | SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, |
147 | | RTLIB::Libcall Call_I8, |
148 | | RTLIB::Libcall Call_I16, |
149 | | RTLIB::Libcall Call_I32, |
150 | | RTLIB::Libcall Call_I64, |
151 | | RTLIB::Libcall Call_I128); |
152 | | SDValue ExpandArgFPLibCall(SDNode *Node, |
153 | | RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, |
154 | | RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, |
155 | | RTLIB::Libcall Call_PPCF128); |
156 | | void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
157 | | void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
158 | | |
159 | | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
160 | | const SDLoc &dl); |
161 | | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
162 | | const SDLoc &dl, SDValue ChainIn); |
163 | | SDValue ExpandBUILD_VECTOR(SDNode *Node); |
164 | | SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); |
165 | | void ExpandDYNAMIC_STACKALLOC(SDNode *Node, |
166 | | SmallVectorImpl<SDValue> &Results); |
167 | | void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, |
168 | | SDValue Value) const; |
169 | | SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL, |
170 | | SDValue NewIntValue) const; |
171 | | SDValue ExpandFCOPYSIGN(SDNode *Node) const; |
172 | | SDValue ExpandFABS(SDNode *Node) const; |
173 | | SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT, |
174 | | const SDLoc &dl); |
175 | | SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, |
176 | | const SDLoc &dl); |
177 | | SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, |
178 | | const SDLoc &dl); |
179 | | |
180 | | SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl); |
181 | | SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl); |
182 | | |
183 | | SDValue ExpandExtractFromVectorThroughStack(SDValue Op); |
184 | | SDValue ExpandInsertToVectorThroughStack(SDValue Op); |
185 | | SDValue ExpandVectorBuildThroughStack(SDNode* Node); |
186 | | |
187 | | SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); |
188 | | SDValue ExpandConstant(ConstantSDNode *CP); |
189 | | |
190 | | // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall |
191 | | bool ExpandNode(SDNode *Node); |
192 | | void ConvertNodeToLibcall(SDNode *Node); |
193 | | void PromoteNode(SDNode *Node); |
194 | | |
195 | | public: |
196 | | // Node replacement helpers |
197 | | |
198 | 1.87M | void ReplacedNode(SDNode *N) { |
199 | 1.87M | LegalizedNodes.erase(N); |
200 | 1.87M | if (UpdatedNodes) |
201 | 11.9k | UpdatedNodes->insert(N); |
202 | 1.87M | } |
203 | | |
204 | 336 | void ReplaceNode(SDNode *Old, SDNode *New) { |
205 | 336 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
206 | 336 | dbgs() << " with: "; New->dump(&DAG)); |
207 | 336 | |
208 | 336 | assert(Old->getNumValues() == New->getNumValues() && |
209 | 336 | "Replacing one node with another that produces a different number " |
210 | 336 | "of values!"); |
211 | 336 | DAG.ReplaceAllUsesWith(Old, New); |
212 | 336 | if (UpdatedNodes) |
213 | 5 | UpdatedNodes->insert(New); |
214 | 336 | ReplacedNode(Old); |
215 | 336 | } |
216 | | |
217 | 1.62M | void ReplaceNode(SDValue Old, SDValue New) { |
218 | 1.62M | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
219 | 1.62M | dbgs() << " with: "; New->dump(&DAG)); |
220 | 1.62M | |
221 | 1.62M | DAG.ReplaceAllUsesWith(Old, New); |
222 | 1.62M | if (UpdatedNodes) |
223 | 6.28k | UpdatedNodes->insert(New.getNode()); |
224 | 1.62M | ReplacedNode(Old.getNode()); |
225 | 1.62M | } |
226 | | |
227 | 196k | void ReplaceNode(SDNode *Old, const SDValue *New) { |
228 | 196k | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); |
229 | 196k | |
230 | 196k | DAG.ReplaceAllUsesWith(Old, New); |
231 | 461k | for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i265k ) { |
232 | 265k | LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: "); |
233 | 265k | New[i]->dump(&DAG)); |
234 | 265k | if (UpdatedNodes) |
235 | 7.12k | UpdatedNodes->insert(New[i].getNode()); |
236 | 265k | } |
237 | 196k | ReplacedNode(Old); |
238 | 196k | } |
239 | | }; |
240 | | |
241 | | } // end anonymous namespace |
242 | | |
243 | | /// Return a vector shuffle operation which |
244 | | /// performs the same shuffle in terms of order or result bytes, but on a type |
245 | | /// whose vector element type is narrower than the original shuffle type. |
246 | | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> |
247 | | SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType( |
248 | | EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, |
249 | 1.19k | ArrayRef<int> Mask) const { |
250 | 1.19k | unsigned NumMaskElts = VT.getVectorNumElements(); |
251 | 1.19k | unsigned NumDestElts = NVT.getVectorNumElements(); |
252 | 1.19k | unsigned NumEltsGrowth = NumDestElts / NumMaskElts; |
253 | 1.19k | |
254 | 1.19k | assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); |
255 | 1.19k | |
256 | 1.19k | if (NumEltsGrowth == 1) |
257 | 0 | return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); |
258 | 1.19k | |
259 | 1.19k | SmallVector<int, 8> NewMask; |
260 | 9.49k | for (unsigned i = 0; i != NumMaskElts; ++i8.29k ) { |
261 | 8.29k | int Idx = Mask[i]; |
262 | 29.6k | for (unsigned j = 0; j != NumEltsGrowth; ++j21.3k ) { |
263 | 21.3k | if (Idx < 0) |
264 | 6.25k | NewMask.push_back(-1); |
265 | 15.1k | else |
266 | 15.1k | NewMask.push_back(Idx * NumEltsGrowth + j); |
267 | 21.3k | } |
268 | 8.29k | } |
269 | 1.19k | assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); |
270 | 1.19k | assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); |
271 | 1.19k | return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask); |
272 | 1.19k | } |
273 | | |
274 | | /// Expands the ConstantFP node to an integer constant or |
275 | | /// a load from the constant pool. |
276 | | SDValue |
277 | 12.4k | SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { |
278 | 12.4k | bool Extend = false; |
279 | 12.4k | SDLoc dl(CFP); |
280 | 12.4k | |
281 | 12.4k | // If a FP immediate is precise when represented as a float and if the |
282 | 12.4k | // target can do an extending load from float to double, we put it into |
283 | 12.4k | // the constant pool as a float, even if it's is statically typed as a |
284 | 12.4k | // double. This shrinks FP constants and canonicalizes them for targets where |
285 | 12.4k | // an FP extending load is the same cost as a normal load (such as on the x87 |
286 | 12.4k | // fp stack or PPC FP unit). |
287 | 12.4k | EVT VT = CFP->getValueType(0); |
288 | 12.4k | ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); |
289 | 12.4k | if (!UseCP) { |
290 | 0 | assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); |
291 | 0 | return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, |
292 | 0 | (VT == MVT::f64) ? MVT::i64 : MVT::i32); |
293 | 0 | } |
294 | 12.4k | |
295 | 12.4k | APFloat APF = CFP->getValueAPF(); |
296 | 12.4k | EVT OrigVT = VT; |
297 | 12.4k | EVT SVT = VT; |
298 | 12.4k | |
299 | 12.4k | // We don't want to shrink SNaNs. Converting the SNaN back to its real type |
300 | 12.4k | // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ). |
301 | 12.4k | if (!APF.isSignaling()) { |
302 | 21.8k | while (SVT != MVT::f32 && SVT != MVT::f169.69k ) { |
303 | 9.38k | SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); |
304 | 9.38k | if (ConstantFPSDNode::isValueValidForType(SVT, APF) && |
305 | 9.38k | // Only do this if the target has a native EXTLOAD instruction from |
306 | 9.38k | // smaller type. |
307 | 9.38k | TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT)3.19k && |
308 | 9.38k | TLI.ShouldShrinkFPConstant(OrigVT)1.39k ) { |
309 | 1.32k | Type *SType = SVT.getTypeForEVT(*DAG.getContext()); |
310 | 1.32k | LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); |
311 | 1.32k | VT = SVT; |
312 | 1.32k | Extend = true; |
313 | 1.32k | } |
314 | 9.38k | } |
315 | 12.4k | } |
316 | 12.4k | |
317 | 12.4k | SDValue CPIdx = |
318 | 12.4k | DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout())); |
319 | 12.4k | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); |
320 | 12.4k | if (Extend) { |
321 | 992 | SDValue Result = DAG.getExtLoad( |
322 | 992 | ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, |
323 | 992 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT, |
324 | 992 | Alignment); |
325 | 992 | return Result; |
326 | 992 | } |
327 | 11.4k | SDValue Result = DAG.getLoad( |
328 | 11.4k | OrigVT, dl, DAG.getEntryNode(), CPIdx, |
329 | 11.4k | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); |
330 | 11.4k | return Result; |
331 | 11.4k | } |
332 | | |
333 | | /// Expands the Constant node to a load from the constant pool. |
334 | 0 | SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) { |
335 | 0 | SDLoc dl(CP); |
336 | 0 | EVT VT = CP->getValueType(0); |
337 | 0 | SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), |
338 | 0 | TLI.getPointerTy(DAG.getDataLayout())); |
339 | 0 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); |
340 | 0 | SDValue Result = DAG.getLoad( |
341 | 0 | VT, dl, DAG.getEntryNode(), CPIdx, |
342 | 0 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); |
343 | 0 | return Result; |
344 | 0 | } |
345 | | |
346 | | /// Some target cannot handle a variable insertion index for the |
347 | | /// INSERT_VECTOR_ELT instruction. In this case, it |
348 | | /// is necessary to spill the vector being inserted into to memory, perform |
349 | | /// the insert there, and then read the result back. |
350 | | SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, |
351 | | SDValue Val, |
352 | | SDValue Idx, |
353 | 32 | const SDLoc &dl) { |
354 | 32 | SDValue Tmp1 = Vec; |
355 | 32 | SDValue Tmp2 = Val; |
356 | 32 | SDValue Tmp3 = Idx; |
357 | 32 | |
358 | 32 | // If the target doesn't support this, we have to spill the input vector |
359 | 32 | // to a temporary stack slot, update the element, then reload it. This is |
360 | 32 | // badness. We could also load the value into a vector register (either |
361 | 32 | // with a "move to register" or "extload into register" instruction, then |
362 | 32 | // permute it into place, if the idx is a constant and if the idx is |
363 | 32 | // supported by the target. |
364 | 32 | EVT VT = Tmp1.getValueType(); |
365 | 32 | EVT EltVT = VT.getVectorElementType(); |
366 | 32 | SDValue StackPtr = DAG.CreateStackTemporary(VT); |
367 | 32 | |
368 | 32 | int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
369 | 32 | |
370 | 32 | // Store the vector. |
371 | 32 | SDValue Ch = DAG.getStore( |
372 | 32 | DAG.getEntryNode(), dl, Tmp1, StackPtr, |
373 | 32 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); |
374 | 32 | |
375 | 32 | SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3); |
376 | 32 | |
377 | 32 | // Store the scalar value. |
378 | 32 | Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT); |
379 | 32 | // Load the updated vector. |
380 | 32 | return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack( |
381 | 32 | DAG.getMachineFunction(), SPFI)); |
382 | 32 | } |
383 | | |
384 | | SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, |
385 | | SDValue Idx, |
386 | 1.88k | const SDLoc &dl) { |
387 | 1.88k | if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { |
388 | 1.85k | // SCALAR_TO_VECTOR requires that the type of the value being inserted |
389 | 1.85k | // match the element type of the vector being created, except for |
390 | 1.85k | // integers in which case the inserted value can be over width. |
391 | 1.85k | EVT EltVT = Vec.getValueType().getVectorElementType(); |
392 | 1.85k | if (Val.getValueType() == EltVT || |
393 | 1.85k | (1 EltVT.isInteger()1 && Val.getValueType().bitsGE(EltVT)1 )) { |
394 | 1.85k | SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, |
395 | 1.85k | Vec.getValueType(), Val); |
396 | 1.85k | |
397 | 1.85k | unsigned NumElts = Vec.getValueType().getVectorNumElements(); |
398 | 1.85k | // We generate a shuffle of InVec and ScVec, so the shuffle mask |
399 | 1.85k | // should be 0,1,2,3,4,5... with the appropriate element replaced with |
400 | 1.85k | // elt 0 of the RHS. |
401 | 1.85k | SmallVector<int, 8> ShufOps; |
402 | 7.96k | for (unsigned i = 0; i != NumElts; ++i6.10k ) |
403 | 6.10k | ShufOps.push_back(i != InsertPos->getZExtValue() ? i4.24k : NumElts1.85k ); |
404 | 1.85k | |
405 | 1.85k | return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps); |
406 | 1.85k | } |
407 | 32 | } |
408 | 32 | return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); |
409 | 32 | } |
410 | | |
411 | 1.77M | SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { |
412 | 1.77M | LLVM_DEBUG(dbgs() << "Optimizing float store operations\n"); |
413 | 1.77M | // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' |
414 | 1.77M | // FIXME: We shouldn't do this for TargetConstantFP's. |
415 | 1.77M | // FIXME: move this to the DAG Combiner! Note that we can't regress due |
416 | 1.77M | // to phase ordering between legalized code and the dag combiner. This |
417 | 1.77M | // probably means that we need to integrate dag combiner and legalizer |
418 | 1.77M | // together. |
419 | 1.77M | // We generally can't do this one for long doubles. |
420 | 1.77M | SDValue Chain = ST->getChain(); |
421 | 1.77M | SDValue Ptr = ST->getBasePtr(); |
422 | 1.77M | unsigned Alignment = ST->getAlignment(); |
423 | 1.77M | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
424 | 1.77M | AAMDNodes AAInfo = ST->getAAInfo(); |
425 | 1.77M | SDLoc dl(ST); |
426 | 1.77M | if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { |
427 | 226 | if (CFP->getValueType(0) == MVT::f32 && |
428 | 226 | TLI.isTypeLegal(MVT::i32)0 ) { |
429 | 0 | SDValue Con = DAG.getConstant(CFP->getValueAPF(). |
430 | 0 | bitcastToAPInt().zextOrTrunc(32), |
431 | 0 | SDLoc(CFP), MVT::i32); |
432 | 0 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment, |
433 | 0 | MMOFlags, AAInfo); |
434 | 0 | } |
435 | 226 | |
436 | 226 | if (CFP->getValueType(0) == MVT::f64) { |
437 | 13 | // If this target supports 64-bit registers, do a single 64-bit store. |
438 | 13 | if (TLI.isTypeLegal(MVT::i64)) { |
439 | 5 | SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). |
440 | 5 | zextOrTrunc(64), SDLoc(CFP), MVT::i64); |
441 | 5 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), |
442 | 5 | Alignment, MMOFlags, AAInfo); |
443 | 5 | } |
444 | 8 | |
445 | 8 | if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { |
446 | 0 | // Otherwise, if the target supports 32-bit registers, use 2 32-bit |
447 | 0 | // stores. If the target supports neither 32- nor 64-bits, this |
448 | 0 | // xform is certainly not worth it. |
449 | 0 | const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); |
450 | 0 | SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32); |
451 | 0 | SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32); |
452 | 0 | if (DAG.getDataLayout().isBigEndian()) |
453 | 0 | std::swap(Lo, Hi); |
454 | 0 |
|
455 | 0 | Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment, |
456 | 0 | MMOFlags, AAInfo); |
457 | 0 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
458 | 0 | DAG.getConstant(4, dl, Ptr.getValueType())); |
459 | 0 | Hi = DAG.getStore(Chain, dl, Hi, Ptr, |
460 | 0 | ST->getPointerInfo().getWithOffset(4), |
461 | 0 | MinAlign(Alignment, 4U), MMOFlags, AAInfo); |
462 | 0 |
|
463 | 0 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
464 | 0 | } |
465 | 1.77M | } |
466 | 226 | } |
467 | 1.77M | return SDValue(nullptr, 0); |
468 | 1.77M | } |
469 | | |
470 | 2.00M | void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { |
471 | 2.00M | StoreSDNode *ST = cast<StoreSDNode>(Node); |
472 | 2.00M | SDValue Chain = ST->getChain(); |
473 | 2.00M | SDValue Ptr = ST->getBasePtr(); |
474 | 2.00M | SDLoc dl(Node); |
475 | 2.00M | |
476 | 2.00M | unsigned Alignment = ST->getAlignment(); |
477 | 2.00M | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
478 | 2.00M | AAMDNodes AAInfo = ST->getAAInfo(); |
479 | 2.00M | |
480 | 2.00M | if (!ST->isTruncatingStore()) { |
481 | 1.77M | LLVM_DEBUG(dbgs() << "Legalizing store operation\n"); |
482 | 1.77M | if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { |
483 | 5 | ReplaceNode(ST, OptStore); |
484 | 5 | return; |
485 | 5 | } |
486 | 1.77M | |
487 | 1.77M | SDValue Value = ST->getValue(); |
488 | 1.77M | MVT VT = Value.getSimpleValueType(); |
489 | 1.77M | switch (TLI.getOperationAction(ISD::STORE, VT)) { |
490 | 1.77M | default: 0 llvm_unreachable0 ("This action is not supported yet!"); |
491 | 1.77M | case TargetLowering::Legal: { |
492 | 1.48M | // If this is an unaligned store and the target doesn't support it, |
493 | 1.48M | // expand it. |
494 | 1.48M | EVT MemVT = ST->getMemoryVT(); |
495 | 1.48M | const DataLayout &DL = DAG.getDataLayout(); |
496 | 1.48M | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, |
497 | 1.48M | *ST->getMemOperand())) { |
498 | 226 | LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n"); |
499 | 226 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
500 | 226 | ReplaceNode(SDValue(ST, 0), Result); |
501 | 226 | } else |
502 | 1.48M | LLVM_DEBUG(dbgs() << "Legal store\n"); |
503 | 1.48M | break; |
504 | 1.77M | } |
505 | 1.77M | case TargetLowering::Custom: { |
506 | 148k | LLVM_DEBUG(dbgs() << "Trying custom lowering\n"); |
507 | 148k | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); |
508 | 148k | if (Res && Res != SDValue(Node, 0)20.1k ) |
509 | 17.6k | ReplaceNode(SDValue(Node, 0), Res); |
510 | 148k | return; |
511 | 1.77M | } |
512 | 1.77M | case TargetLowering::Promote: { |
513 | 148k | MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); |
514 | 148k | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
515 | 148k | "Can only promote stores to same size type"); |
516 | 148k | Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); |
517 | 148k | SDValue Result = |
518 | 148k | DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
519 | 148k | Alignment, MMOFlags, AAInfo); |
520 | 148k | ReplaceNode(SDValue(Node, 0), Result); |
521 | 148k | break; |
522 | 1.63M | } |
523 | 1.63M | } |
524 | 1.63M | return; |
525 | 1.63M | } |
526 | 223k | |
527 | 223k | LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n"); |
528 | 223k | SDValue Value = ST->getValue(); |
529 | 223k | EVT StVT = ST->getMemoryVT(); |
530 | 223k | unsigned StWidth = StVT.getSizeInBits(); |
531 | 223k | auto &DL = DAG.getDataLayout(); |
532 | 223k | |
533 | 223k | if (StWidth != StVT.getStoreSizeInBits()) { |
534 | 10.6k | // Promote to a byte-sized store with upper bits zero if not |
535 | 10.6k | // storing an integral number of bytes. For example, promote |
536 | 10.6k | // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) |
537 | 10.6k | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), |
538 | 10.6k | StVT.getStoreSizeInBits()); |
539 | 10.6k | Value = DAG.getZeroExtendInReg(Value, dl, StVT); |
540 | 10.6k | SDValue Result = |
541 | 10.6k | DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, |
542 | 10.6k | Alignment, MMOFlags, AAInfo); |
543 | 10.6k | ReplaceNode(SDValue(Node, 0), Result); |
544 | 212k | } else if (StWidth & (StWidth - 1)) { |
545 | 1.01k | // If not storing a power-of-2 number of bits, expand as two stores. |
546 | 1.01k | assert(!StVT.isVector() && "Unsupported truncstore!"); |
547 | 1.01k | unsigned LogStWidth = Log2_32(StWidth); |
548 | 1.01k | assert(LogStWidth < 32); |
549 | 1.01k | unsigned RoundWidth = 1 << LogStWidth; |
550 | 1.01k | assert(RoundWidth < StWidth); |
551 | 1.01k | unsigned ExtraWidth = StWidth - RoundWidth; |
552 | 1.01k | assert(ExtraWidth < RoundWidth); |
553 | 1.01k | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
554 | 1.01k | "Store size not an integral number of bytes!"); |
555 | 1.01k | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); |
556 | 1.01k | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); |
557 | 1.01k | SDValue Lo, Hi; |
558 | 1.01k | unsigned IncrementSize; |
559 | 1.01k | |
560 | 1.01k | if (DL.isLittleEndian()) { |
561 | 1.00k | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) |
562 | 1.00k | // Store the bottom RoundWidth bits. |
563 | 1.00k | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
564 | 1.00k | RoundVT, Alignment, MMOFlags, AAInfo); |
565 | 1.00k | |
566 | 1.00k | // Store the remaining ExtraWidth bits. |
567 | 1.00k | IncrementSize = RoundWidth / 8; |
568 | 1.00k | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
569 | 1.00k | DAG.getConstant(IncrementSize, dl, |
570 | 1.00k | Ptr.getValueType())); |
571 | 1.00k | Hi = DAG.getNode( |
572 | 1.00k | ISD::SRL, dl, Value.getValueType(), Value, |
573 | 1.00k | DAG.getConstant(RoundWidth, dl, |
574 | 1.00k | TLI.getShiftAmountTy(Value.getValueType(), DL))); |
575 | 1.00k | Hi = DAG.getTruncStore( |
576 | 1.00k | Chain, dl, Hi, Ptr, |
577 | 1.00k | ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT, |
578 | 1.00k | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); |
579 | 1.00k | } else { |
580 | 12 | // Big endian - avoid unaligned stores. |
581 | 12 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X |
582 | 12 | // Store the top RoundWidth bits. |
583 | 12 | Hi = DAG.getNode( |
584 | 12 | ISD::SRL, dl, Value.getValueType(), Value, |
585 | 12 | DAG.getConstant(ExtraWidth, dl, |
586 | 12 | TLI.getShiftAmountTy(Value.getValueType(), DL))); |
587 | 12 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), |
588 | 12 | RoundVT, Alignment, MMOFlags, AAInfo); |
589 | 12 | |
590 | 12 | // Store the remaining ExtraWidth bits. |
591 | 12 | IncrementSize = RoundWidth / 8; |
592 | 12 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
593 | 12 | DAG.getConstant(IncrementSize, dl, |
594 | 12 | Ptr.getValueType())); |
595 | 12 | Lo = DAG.getTruncStore( |
596 | 12 | Chain, dl, Value, Ptr, |
597 | 12 | ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT, |
598 | 12 | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); |
599 | 12 | } |
600 | 1.01k | |
601 | 1.01k | // The order of the stores doesn't matter. |
602 | 1.01k | SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
603 | 1.01k | ReplaceNode(SDValue(Node, 0), Result); |
604 | 211k | } else { |
605 | 211k | switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { |
606 | 211k | default: 0 llvm_unreachable0 ("This action is not supported yet!"); |
607 | 211k | case TargetLowering::Legal: { |
608 | 207k | EVT MemVT = ST->getMemoryVT(); |
609 | 207k | // If this is an unaligned store and the target doesn't support it, |
610 | 207k | // expand it. |
611 | 207k | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, |
612 | 207k | *ST->getMemOperand())) { |
613 | 440 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
614 | 440 | ReplaceNode(SDValue(ST, 0), Result); |
615 | 440 | } |
616 | 207k | break; |
617 | 211k | } |
618 | 211k | case TargetLowering::Custom: { |
619 | 1.72k | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); |
620 | 1.72k | if (Res && Res != SDValue(Node, 0)1.55k ) |
621 | 1.55k | ReplaceNode(SDValue(Node, 0), Res); |
622 | 1.72k | return; |
623 | 211k | } |
624 | 211k | case TargetLowering::Expand: |
625 | 1.90k | assert(!StVT.isVector() && |
626 | 1.90k | "Vector Stores are handled in LegalizeVectorOps"); |
627 | 1.90k | |
628 | 1.90k | SDValue Result; |
629 | 1.90k | |
630 | 1.90k | // TRUNCSTORE:i16 i32 -> STORE i16 |
631 | 1.90k | if (TLI.isTypeLegal(StVT)) { |
632 | 1.87k | Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); |
633 | 1.87k | Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
634 | 1.87k | Alignment, MMOFlags, AAInfo); |
635 | 1.87k | } else { |
636 | 28 | // The in-memory type isn't legal. Truncate to the type it would promote |
637 | 28 | // to, and then do a truncstore. |
638 | 28 | Value = DAG.getNode(ISD::TRUNCATE, dl, |
639 | 28 | TLI.getTypeToTransformTo(*DAG.getContext(), StVT), |
640 | 28 | Value); |
641 | 28 | Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
642 | 28 | StVT, Alignment, MMOFlags, AAInfo); |
643 | 28 | } |
644 | 1.90k | |
645 | 1.90k | ReplaceNode(SDValue(Node, 0), Result); |
646 | 1.90k | break; |
647 | 211k | } |
648 | 211k | } |
649 | 223k | } |
650 | | |
651 | 1.83M | void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { |
652 | 1.83M | LoadSDNode *LD = cast<LoadSDNode>(Node); |
653 | 1.83M | SDValue Chain = LD->getChain(); // The chain. |
654 | 1.83M | SDValue Ptr = LD->getBasePtr(); // The base pointer. |
655 | 1.83M | SDValue Value; // The value returned by the load op. |
656 | 1.83M | SDLoc dl(Node); |
657 | 1.83M | |
658 | 1.83M | ISD::LoadExtType ExtType = LD->getExtensionType(); |
659 | 1.83M | if (ExtType == ISD::NON_EXTLOAD) { |
660 | 1.46M | LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n"); |
661 | 1.46M | MVT VT = Node->getSimpleValueType(0); |
662 | 1.46M | SDValue RVal = SDValue(Node, 0); |
663 | 1.46M | SDValue RChain = SDValue(Node, 1); |
664 | 1.46M | |
665 | 1.46M | switch (TLI.getOperationAction(Node->getOpcode(), VT)) { |
666 | 1.46M | default: 0 llvm_unreachable0 ("This action is not supported yet!"); |
667 | 1.46M | case TargetLowering::Legal: { |
668 | 1.27M | EVT MemVT = LD->getMemoryVT(); |
669 | 1.27M | const DataLayout &DL = DAG.getDataLayout(); |
670 | 1.27M | // If this is an unaligned load and the target doesn't support it, |
671 | 1.27M | // expand it. |
672 | 1.27M | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, |
673 | 1.27M | *LD->getMemOperand())) { |
674 | 291 | std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); |
675 | 291 | } |
676 | 1.27M | break; |
677 | 1.46M | } |
678 | 1.46M | case TargetLowering::Custom: |
679 | 148k | if (SDValue Res = TLI.LowerOperation(RVal, DAG)) { |
680 | 19.1k | RVal = Res; |
681 | 19.1k | RChain = Res.getValue(1); |
682 | 19.1k | } |
683 | 148k | break; |
684 | 1.46M | |
685 | 1.46M | case TargetLowering::Promote: { |
686 | 38.2k | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); |
687 | 38.2k | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
688 | 38.2k | "Can only promote loads to same size type"); |
689 | 38.2k | |
690 | 38.2k | SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); |
691 | 38.2k | RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); |
692 | 38.2k | RChain = Res.getValue(1); |
693 | 38.2k | break; |
694 | 1.46M | } |
695 | 1.46M | } |
696 | 1.46M | if (RChain.getNode() != Node) { |
697 | 50.7k | assert(RVal.getNode() != Node && "Load must be completely replaced"); |
698 | 50.7k | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); |
699 | 50.7k | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); |
700 | 50.7k | if (UpdatedNodes) { |
701 | 40 | UpdatedNodes->insert(RVal.getNode()); |
702 | 40 | UpdatedNodes->insert(RChain.getNode()); |
703 | 40 | } |
704 | 50.7k | ReplacedNode(Node); |
705 | 50.7k | } |
706 | 1.46M | return; |
707 | 1.46M | } |
708 | 369k | |
709 | 369k | LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n"); |
710 | 369k | EVT SrcVT = LD->getMemoryVT(); |
711 | 369k | unsigned SrcWidth = SrcVT.getSizeInBits(); |
712 | 369k | unsigned Alignment = LD->getAlignment(); |
713 | 369k | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); |
714 | 369k | AAMDNodes AAInfo = LD->getAAInfo(); |
715 | 369k | |
716 | 369k | if (SrcWidth != SrcVT.getStoreSizeInBits() && |
717 | 369k | // Some targets pretend to have an i1 loading operation, and actually |
718 | 369k | // load an i8. This trick is correct for ZEXTLOAD because the top 7 |
719 | 369k | // bits are guaranteed to be zero; it helps the optimizers understand |
720 | 369k | // that these bits are zero. It is also useful for EXTLOAD, since it |
721 | 369k | // tells the optimizers that those bits are undefined. It would be |
722 | 369k | // nice to have an effective generic way of getting these benefits... |
723 | 369k | // Until such a way is found, don't insist on promoting i1 here. |
724 | 369k | (2.15k SrcVT != MVT::i12.15k || |
725 | 2.15k | TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == |
726 | 1.77k | TargetLowering::Promote)) { |
727 | 550 | // Promote to a byte-sized load if not loading an integral number of |
728 | 550 | // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. |
729 | 550 | unsigned NewWidth = SrcVT.getStoreSizeInBits(); |
730 | 550 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); |
731 | 550 | SDValue Ch; |
732 | 550 | |
733 | 550 | // The extra bits are guaranteed to be zero, since we stored them that |
734 | 550 | // way. A zext load from NVT thus automatically gives zext from SrcVT. |
735 | 550 | |
736 | 550 | ISD::LoadExtType NewExtType = |
737 | 550 | ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD149 : ISD::EXTLOAD401 ; |
738 | 550 | |
739 | 550 | SDValue Result = |
740 | 550 | DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr, |
741 | 550 | LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo); |
742 | 550 | |
743 | 550 | Ch = Result.getValue(1); // The chain. |
744 | 550 | |
745 | 550 | if (ExtType == ISD::SEXTLOAD) |
746 | 103 | // Having the top bits zero doesn't help when sign extending. |
747 | 103 | Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, |
748 | 103 | Result.getValueType(), |
749 | 103 | Result, DAG.getValueType(SrcVT)); |
750 | 447 | else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()298 ) |
751 | 206 | // All the top bits are guaranteed to be zero - inform the optimizers. |
752 | 206 | Result = DAG.getNode(ISD::AssertZext, dl, |
753 | 206 | Result.getValueType(), Result, |
754 | 206 | DAG.getValueType(SrcVT)); |
755 | 550 | |
756 | 550 | Value = Result; |
757 | 550 | Chain = Ch; |
758 | 368k | } else if (SrcWidth & (SrcWidth - 1)) { |
759 | 531 | // If not loading a power-of-2 number of bits, expand as two loads. |
760 | 531 | assert(!SrcVT.isVector() && "Unsupported extload!"); |
761 | 531 | unsigned LogSrcWidth = Log2_32(SrcWidth); |
762 | 531 | assert(LogSrcWidth < 32); |
763 | 531 | unsigned RoundWidth = 1 << LogSrcWidth; |
764 | 531 | assert(RoundWidth < SrcWidth); |
765 | 531 | unsigned ExtraWidth = SrcWidth - RoundWidth; |
766 | 531 | assert(ExtraWidth < RoundWidth); |
767 | 531 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
768 | 531 | "Load size not an integral number of bytes!"); |
769 | 531 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); |
770 | 531 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); |
771 | 531 | SDValue Lo, Hi, Ch; |
772 | 531 | unsigned IncrementSize; |
773 | 531 | auto &DL = DAG.getDataLayout(); |
774 | 531 | |
775 | 531 | if (DL.isLittleEndian()) { |
776 | 505 | // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) |
777 | 505 | // Load the bottom RoundWidth bits. |
778 | 505 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, |
779 | 505 | LD->getPointerInfo(), RoundVT, Alignment, MMOFlags, |
780 | 505 | AAInfo); |
781 | 505 | |
782 | 505 | // Load the remaining ExtraWidth bits. |
783 | 505 | IncrementSize = RoundWidth / 8; |
784 | 505 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
785 | 505 | DAG.getConstant(IncrementSize, dl, |
786 | 505 | Ptr.getValueType())); |
787 | 505 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
788 | 505 | LD->getPointerInfo().getWithOffset(IncrementSize), |
789 | 505 | ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags, |
790 | 505 | AAInfo); |
791 | 505 | |
792 | 505 | // Build a factor node to remember that this load is independent of |
793 | 505 | // the other one. |
794 | 505 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
795 | 505 | Hi.getValue(1)); |
796 | 505 | |
797 | 505 | // Move the top bits to the right place. |
798 | 505 | Hi = DAG.getNode( |
799 | 505 | ISD::SHL, dl, Hi.getValueType(), Hi, |
800 | 505 | DAG.getConstant(RoundWidth, dl, |
801 | 505 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); |
802 | 505 | |
803 | 505 | // Join the hi and lo parts. |
804 | 505 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); |
805 | 505 | } else { |
806 | 26 | // Big endian - avoid unaligned loads. |
807 | 26 | // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 |
808 | 26 | // Load the top RoundWidth bits. |
809 | 26 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
810 | 26 | LD->getPointerInfo(), RoundVT, Alignment, MMOFlags, |
811 | 26 | AAInfo); |
812 | 26 | |
813 | 26 | // Load the remaining ExtraWidth bits. |
814 | 26 | IncrementSize = RoundWidth / 8; |
815 | 26 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
816 | 26 | DAG.getConstant(IncrementSize, dl, |
817 | 26 | Ptr.getValueType())); |
818 | 26 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, |
819 | 26 | LD->getPointerInfo().getWithOffset(IncrementSize), |
820 | 26 | ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags, |
821 | 26 | AAInfo); |
822 | 26 | |
823 | 26 | // Build a factor node to remember that this load is independent of |
824 | 26 | // the other one. |
825 | 26 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
826 | 26 | Hi.getValue(1)); |
827 | 26 | |
828 | 26 | // Move the top bits to the right place. |
829 | 26 | Hi = DAG.getNode( |
830 | 26 | ISD::SHL, dl, Hi.getValueType(), Hi, |
831 | 26 | DAG.getConstant(ExtraWidth, dl, |
832 | 26 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); |
833 | 26 | |
834 | 26 | // Join the hi and lo parts. |
835 | 26 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); |
836 | 26 | } |
837 | 531 | |
838 | 531 | Chain = Ch; |
839 | 368k | } else { |
840 | 368k | bool isCustom = false; |
841 | 368k | switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), |
842 | 368k | SrcVT.getSimpleVT())) { |
843 | 368k | default: 0 llvm_unreachable0 ("This action is not supported yet!"); |
844 | 368k | case TargetLowering::Custom: |
845 | 6.51k | isCustom = true; |
846 | 6.51k | LLVM_FALLTHROUGH; |
847 | 367k | case TargetLowering::Legal: |
848 | 367k | Value = SDValue(Node, 0); |
849 | 367k | Chain = SDValue(Node, 1); |
850 | 367k | |
851 | 367k | if (isCustom) { |
852 | 6.51k | if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { |
853 | 4.47k | Value = Res; |
854 | 4.47k | Chain = Res.getValue(1); |
855 | 4.47k | } |
856 | 361k | } else { |
857 | 361k | // If this is an unaligned load and the target doesn't support it, |
858 | 361k | // expand it. |
859 | 361k | EVT MemVT = LD->getMemoryVT(); |
860 | 361k | const DataLayout &DL = DAG.getDataLayout(); |
861 | 361k | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, |
862 | 361k | *LD->getMemOperand())) { |
863 | 523 | std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); |
864 | 523 | } |
865 | 361k | } |
866 | 367k | break; |
867 | 6.51k | |
868 | 6.51k | case TargetLowering::Expand: { |
869 | 481 | EVT DestVT = Node->getValueType(0); |
870 | 481 | if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { |
871 | 455 | // If the source type is not legal, see if there is a legal extload to |
872 | 455 | // an intermediate type that we can then extend further. |
873 | 455 | EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); |
874 | 455 | if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT? |
875 | 455 | TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)93 ) { |
876 | 455 | // If we are loading a legal type, this is a non-extload followed by a |
877 | 455 | // full extend. |
878 | 455 | ISD::LoadExtType MidExtType = |
879 | 455 | (LoadVT == SrcVT) ? ISD::NON_EXTLOAD362 : ExtType93 ; |
880 | 455 | |
881 | 455 | SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, |
882 | 455 | SrcVT, LD->getMemOperand()); |
883 | 455 | unsigned ExtendOp = |
884 | 455 | ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); |
885 | 455 | Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); |
886 | 455 | Chain = Load.getValue(1); |
887 | 455 | break; |
888 | 455 | } |
889 | 0 | |
890 | 0 | // Handle the special case of fp16 extloads. EXTLOAD doesn't have the |
891 | 0 | // normal undefined upper bits behavior to allow using an in-reg extend |
892 | 0 | // with the illegal FP type, so load as an integer and do the |
893 | 0 | // from-integer conversion. |
894 | 0 | if (SrcVT.getScalarType() == MVT::f16) { |
895 | 0 | EVT ISrcVT = SrcVT.changeTypeToInteger(); |
896 | 0 | EVT IDestVT = DestVT.changeTypeToInteger(); |
897 | 0 | EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT()); |
898 | 0 |
|
899 | 0 | SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT, |
900 | 0 | Chain, Ptr, ISrcVT, |
901 | 0 | LD->getMemOperand()); |
902 | 0 | Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result); |
903 | 0 | Chain = Result.getValue(1); |
904 | 0 | break; |
905 | 0 | } |
906 | 26 | } |
907 | 26 | |
908 | 26 | assert(!SrcVT.isVector() && |
909 | 26 | "Vector Loads are handled in LegalizeVectorOps"); |
910 | 26 | |
911 | 26 | // FIXME: This does not work for vectors on most targets. Sign- |
912 | 26 | // and zero-extend operations are currently folded into extending |
913 | 26 | // loads, whether they are legal or not, and then we end up here |
914 | 26 | // without any support for legalizing them. |
915 | 26 | assert(ExtType != ISD::EXTLOAD && |
916 | 26 | "EXTLOAD should always be supported!"); |
917 | 26 | // Turn the unsupported load into an EXTLOAD followed by an |
918 | 26 | // explicit zero/sign extend inreg. |
919 | 26 | SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, |
920 | 26 | Node->getValueType(0), |
921 | 26 | Chain, Ptr, SrcVT, |
922 | 26 | LD->getMemOperand()); |
923 | 26 | SDValue ValRes; |
924 | 26 | if (ExtType == ISD::SEXTLOAD) |
925 | 25 | ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, |
926 | 25 | Result.getValueType(), |
927 | 25 | Result, DAG.getValueType(SrcVT)); |
928 | 1 | else |
929 | 1 | ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); |
930 | 26 | Value = ValRes; |
931 | 26 | Chain = Result.getValue(1); |
932 | 26 | break; |
933 | 26 | } |
934 | 368k | } |
935 | 368k | } |
936 | 369k | |
937 | 369k | // Since loads produce two values, make sure to remember that we legalized |
938 | 369k | // both of them. |
939 | 369k | if (Chain.getNode() != Node) { |
940 | 6.54k | assert(Value.getNode() != Node && "Load must be completely replaced"); |
941 | 6.54k | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); |
942 | 6.54k | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); |
943 | 6.54k | if (UpdatedNodes) { |
944 | 1 | UpdatedNodes->insert(Value.getNode()); |
945 | 1 | UpdatedNodes->insert(Chain.getNode()); |
946 | 1 | } |
947 | 6.54k | ReplacedNode(Node); |
948 | 6.54k | } |
949 | 369k | } |
950 | | |
951 | | /// Return a legal replacement for the given operation, with all legal operands. |
952 | 55.6M | void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { |
953 | 55.6M | LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); |
954 | 55.6M | |
955 | 55.6M | // Allow illegal target nodes and illegal registers. |
956 | 55.6M | if (Node->getOpcode() == ISD::TargetConstant || |
957 | 55.6M | Node->getOpcode() == ISD::Register54.4M ) |
958 | 9.04M | return; |
959 | 46.6M | |
960 | | #ifndef NDEBUG |
961 | | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) |
962 | | assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == |
963 | | TargetLowering::TypeLegal || |
964 | | TLI.isTypeLegal(Node->getValueType(i))) && |
965 | | "Unexpected illegal type!"); |
966 | | |
967 | | for (const SDValue &Op : Node->op_values()) |
968 | | assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == |
969 | | TargetLowering::TypeLegal || |
970 | | TLI.isTypeLegal(Op.getValueType()) || |
971 | | Op.getOpcode() == ISD::TargetConstant || |
972 | | Op.getOpcode() == ISD::Register) && |
973 | | "Unexpected illegal type!"); |
974 | | #endif |
975 | | |
976 | 46.6M | // Figure out the correct action; the way to query this varies by opcode |
977 | 46.6M | TargetLowering::LegalizeAction Action = TargetLowering::Legal; |
978 | 46.6M | bool SimpleFinishLegalizing = true; |
979 | 46.6M | switch (Node->getOpcode()) { |
980 | 46.6M | case ISD::INTRINSIC_W_CHAIN: |
981 | 226k | case ISD::INTRINSIC_WO_CHAIN: |
982 | 226k | case ISD::INTRINSIC_VOID: |
983 | 226k | case ISD::STACKSAVE: |
984 | 226k | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); |
985 | 226k | break; |
986 | 226k | case ISD::GET_DYNAMIC_AREA_OFFSET: |
987 | 4 | Action = TLI.getOperationAction(Node->getOpcode(), |
988 | 4 | Node->getValueType(0)); |
989 | 4 | break; |
990 | 226k | case ISD::VAARG: |
991 | 361 | Action = TLI.getOperationAction(Node->getOpcode(), |
992 | 361 | Node->getValueType(0)); |
993 | 361 | if (Action != TargetLowering::Promote) |
994 | 360 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); |
995 | 361 | break; |
996 | 628k | case ISD::FP_TO_FP16: |
997 | 628k | case ISD::SINT_TO_FP: |
998 | 628k | case ISD::UINT_TO_FP: |
999 | 628k | case ISD::EXTRACT_VECTOR_ELT: |
1000 | 628k | case ISD::LROUND: |
1001 | 628k | case ISD::LLROUND: |
1002 | 628k | case ISD::LRINT: |
1003 | 628k | case ISD::LLRINT: |
1004 | 628k | Action = TLI.getOperationAction(Node->getOpcode(), |
1005 | 628k | Node->getOperand(0).getValueType()); |
1006 | 628k | break; |
1007 | 628k | case ISD::FP_ROUND_INREG: |
1008 | 36.8k | case ISD::SIGN_EXTEND_INREG: { |
1009 | 36.8k | EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
1010 | 36.8k | Action = TLI.getOperationAction(Node->getOpcode(), InnerType); |
1011 | 36.8k | break; |
1012 | 36.8k | } |
1013 | 36.8k | case ISD::ATOMIC_STORE: |
1014 | 13.4k | Action = TLI.getOperationAction(Node->getOpcode(), |
1015 | 13.4k | Node->getOperand(2).getValueType()); |
1016 | 13.4k | break; |
1017 | 529k | case ISD::SELECT_CC: |
1018 | 529k | case ISD::SETCC: |
1019 | 529k | case ISD::BR_CC: { |
1020 | 529k | unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 442.4k : |
1021 | 529k | Node->getOpcode() == ISD::SETCC 487k ? 269.5k : 1417k ; |
1022 | 529k | unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2417k : 0111k ; |
1023 | 529k | MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); |
1024 | 529k | ISD::CondCode CCCode = |
1025 | 529k | cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); |
1026 | 529k | Action = TLI.getCondCodeAction(CCCode, OpVT); |
1027 | 529k | if (Action == TargetLowering::Legal) { |
1028 | 528k | if (Node->getOpcode() == ISD::SELECT_CC) |
1029 | 42.1k | Action = TLI.getOperationAction(Node->getOpcode(), |
1030 | 42.1k | Node->getValueType(0)); |
1031 | 486k | else |
1032 | 486k | Action = TLI.getOperationAction(Node->getOpcode(), OpVT); |
1033 | 528k | } |
1034 | 529k | break; |
1035 | 529k | } |
1036 | 3.83M | case ISD::LOAD: |
1037 | 3.83M | case ISD::STORE: |
1038 | 3.83M | // FIXME: Model these properly. LOAD and STORE are complicated, and |
1039 | 3.83M | // STORE expects the unlegalized operand in some cases. |
1040 | 3.83M | SimpleFinishLegalizing = false; |
1041 | 3.83M | break; |
1042 | 3.83M | case ISD::CALLSEQ_START: |
1043 | 1.58M | case ISD::CALLSEQ_END: |
1044 | 1.58M | // FIXME: This shouldn't be necessary. These nodes have special properties |
1045 | 1.58M | // dealing with the recursive nature of legalization. Removing this |
1046 | 1.58M | // special case should be done as part of making LegalizeDAG non-recursive. |
1047 | 1.58M | SimpleFinishLegalizing = false; |
1048 | 1.58M | break; |
1049 | 1.58M | case ISD::EXTRACT_ELEMENT: |
1050 | 36.5k | case ISD::FLT_ROUNDS_: |
1051 | 36.5k | case ISD::MERGE_VALUES: |
1052 | 36.5k | case ISD::EH_RETURN: |
1053 | 36.5k | case ISD::FRAME_TO_ARGS_OFFSET: |
1054 | 36.5k | case ISD::EH_DWARF_CFA: |
1055 | 36.5k | case ISD::EH_SJLJ_SETJMP: |
1056 | 36.5k | case ISD::EH_SJLJ_LONGJMP: |
1057 | 36.5k | case ISD::EH_SJLJ_SETUP_DISPATCH: |
1058 | 36.5k | // These operations lie about being legal: when they claim to be legal, |
1059 | 36.5k | // they should actually be expanded. |
1060 | 36.5k | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1061 | 36.5k | if (Action == TargetLowering::Legal) |
1062 | 33.8k | Action = TargetLowering::Expand; |
1063 | 36.5k | break; |
1064 | 36.5k | case ISD::INIT_TRAMPOLINE: |
1065 | 16.9k | case ISD::ADJUST_TRAMPOLINE: |
1066 | 16.9k | case ISD::FRAMEADDR: |
1067 | 16.9k | case ISD::RETURNADDR: |
1068 | 16.9k | case ISD::ADDROFRETURNADDR: |
1069 | 16.9k | case ISD::SPONENTRY: |
1070 | 16.9k | // These operations lie about being legal: when they claim to be legal, |
1071 | 16.9k | // they should actually be custom-lowered. |
1072 | 16.9k | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1073 | 16.9k | if (Action == TargetLowering::Legal) |
1074 | 16.9k | Action = TargetLowering::Custom; |
1075 | 16.9k | break; |
1076 | 16.9k | case ISD::READCYCLECOUNTER: |
1077 | 33 | // READCYCLECOUNTER returns an i64, even if type legalization might have |
1078 | 33 | // expanded that to several smaller types. |
1079 | 33 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); |
1080 | 33 | break; |
1081 | 16.9k | case ISD::READ_REGISTER: |
1082 | 780 | case ISD::WRITE_REGISTER: |
1083 | 780 | // Named register is legal in the DAG, but blocked by register name |
1084 | 780 | // selection if not implemented by target (to chose the correct register) |
1085 | 780 | // They'll be converted to Copy(To/From)Reg. |
1086 | 780 | Action = TargetLowering::Legal; |
1087 | 780 | break; |
1088 | 780 | case ISD::DEBUGTRAP: |
1089 | 58 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1090 | 58 | if (Action == TargetLowering::Expand) { |
1091 | 2 | // replace ISD::DEBUGTRAP with ISD::TRAP |
1092 | 2 | SDValue NewVal; |
1093 | 2 | NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), |
1094 | 2 | Node->getOperand(0)); |
1095 | 2 | ReplaceNode(Node, NewVal.getNode()); |
1096 | 2 | LegalizeOp(NewVal.getNode()); |
1097 | 2 | return; |
1098 | 2 | } |
1099 | 56 | break; |
1100 | 3.50k | case ISD::STRICT_FADD: |
1101 | 3.50k | case ISD::STRICT_FSUB: |
1102 | 3.50k | case ISD::STRICT_FMUL: |
1103 | 3.50k | case ISD::STRICT_FDIV: |
1104 | 3.50k | case ISD::STRICT_FREM: |
1105 | 3.50k | case ISD::STRICT_FSQRT: |
1106 | 3.50k | case ISD::STRICT_FMA: |
1107 | 3.50k | case ISD::STRICT_FPOW: |
1108 | 3.50k | case ISD::STRICT_FPOWI: |
1109 | 3.50k | case ISD::STRICT_FSIN: |
1110 | 3.50k | case ISD::STRICT_FCOS: |
1111 | 3.50k | case ISD::STRICT_FEXP: |
1112 | 3.50k | case ISD::STRICT_FEXP2: |
1113 | 3.50k | case ISD::STRICT_FLOG: |
1114 | 3.50k | case ISD::STRICT_FLOG10: |
1115 | 3.50k | case ISD::STRICT_FLOG2: |
1116 | 3.50k | case ISD::STRICT_FRINT: |
1117 | 3.50k | case ISD::STRICT_FNEARBYINT: |
1118 | 3.50k | case ISD::STRICT_FMAXNUM: |
1119 | 3.50k | case ISD::STRICT_FMINNUM: |
1120 | 3.50k | case ISD::STRICT_FCEIL: |
1121 | 3.50k | case ISD::STRICT_FFLOOR: |
1122 | 3.50k | case ISD::STRICT_FROUND: |
1123 | 3.50k | case ISD::STRICT_FTRUNC: |
1124 | 3.50k | case ISD::STRICT_FP_ROUND: |
1125 | 3.50k | case ISD::STRICT_FP_EXTEND: |
1126 | 3.50k | // These pseudo-ops get legalized as if they were their non-strict |
1127 | 3.50k | // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT |
1128 | 3.50k | // is also legal, but if ISD::FSQRT requires expansion then so does |
1129 | 3.50k | // ISD::STRICT_FSQRT. |
1130 | 3.50k | Action = TLI.getStrictFPOperationAction(Node->getOpcode(), |
1131 | 3.50k | Node->getValueType(0)); |
1132 | 3.50k | break; |
1133 | 4.61k | case ISD::SADDSAT: |
1134 | 4.61k | case ISD::UADDSAT: |
1135 | 4.61k | case ISD::SSUBSAT: |
1136 | 4.61k | case ISD::USUBSAT: { |
1137 | 4.61k | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1138 | 4.61k | break; |
1139 | 4.61k | } |
1140 | 4.61k | case ISD::SMULFIX: |
1141 | 74 | case ISD::SMULFIXSAT: |
1142 | 74 | case ISD::UMULFIX: { |
1143 | 74 | unsigned Scale = Node->getConstantOperandVal(2); |
1144 | 74 | Action = TLI.getFixedPointOperationAction(Node->getOpcode(), |
1145 | 74 | Node->getValueType(0), Scale); |
1146 | 74 | break; |
1147 | 74 | } |
1148 | 113 | case ISD::MSCATTER: |
1149 | 113 | Action = TLI.getOperationAction(Node->getOpcode(), |
1150 | 113 | cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); |
1151 | 113 | break; |
1152 | 1.25k | case ISD::MSTORE: |
1153 | 1.25k | Action = TLI.getOperationAction(Node->getOpcode(), |
1154 | 1.25k | cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); |
1155 | 1.25k | break; |
1156 | 74 | case ISD::VECREDUCE_FADD: |
1157 | 0 | case ISD::VECREDUCE_FMUL: |
1158 | 0 | case ISD::VECREDUCE_ADD: |
1159 | 0 | case ISD::VECREDUCE_MUL: |
1160 | 0 | case ISD::VECREDUCE_AND: |
1161 | 0 | case ISD::VECREDUCE_OR: |
1162 | 0 | case ISD::VECREDUCE_XOR: |
1163 | 0 | case ISD::VECREDUCE_SMAX: |
1164 | 0 | case ISD::VECREDUCE_SMIN: |
1165 | 0 | case ISD::VECREDUCE_UMAX: |
1166 | 0 | case ISD::VECREDUCE_UMIN: |
1167 | 0 | case ISD::VECREDUCE_FMAX: |
1168 | 0 | case ISD::VECREDUCE_FMIN: |
1169 | 0 | Action = TLI.getOperationAction( |
1170 | 0 | Node->getOpcode(), Node->getOperand(0).getValueType()); |
1171 | 0 | break; |
1172 | 39.7M | default: |
1173 | 39.7M | if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { |
1174 | 5.73M | Action = TargetLowering::Legal; |
1175 | 33.9M | } else { |
1176 | 33.9M | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1177 | 33.9M | } |
1178 | 39.7M | break; |
1179 | 46.6M | } |
1180 | 46.6M | |
1181 | 46.6M | if (SimpleFinishLegalizing) { |
1182 | 41.2M | SDNode *NewNode = Node; |
1183 | 41.2M | switch (Node->getOpcode()) { |
1184 | 41.2M | default: break40.6M ; |
1185 | 41.2M | case ISD::SHL: |
1186 | 538k | case ISD::SRL: |
1187 | 538k | case ISD::SRA: |
1188 | 538k | case ISD::ROTL: |
1189 | 538k | case ISD::ROTR: { |
1190 | 538k | // Legalizing shifts/rotates requires adjusting the shift amount |
1191 | 538k | // to the appropriate width. |
1192 | 538k | SDValue Op0 = Node->getOperand(0); |
1193 | 538k | SDValue Op1 = Node->getOperand(1); |
1194 | 538k | if (!Op1.getValueType().isVector()) { |
1195 | 530k | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); |
1196 | 530k | // The getShiftAmountOperand() may create a new operand node or |
1197 | 530k | // return the existing one. If new operand is created we need |
1198 | 530k | // to update the parent node. |
1199 | 530k | // Do not try to legalize SAO here! It will be automatically legalized |
1200 | 530k | // in the next round. |
1201 | 530k | if (SAO != Op1) |
1202 | 28.3k | NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); |
1203 | 530k | } |
1204 | 538k | } |
1205 | 538k | break; |
1206 | 538k | case ISD::FSHL: |
1207 | 3.49k | case ISD::FSHR: |
1208 | 3.49k | case ISD::SRL_PARTS: |
1209 | 3.49k | case ISD::SRA_PARTS: |
1210 | 3.49k | case ISD::SHL_PARTS: { |
1211 | 3.49k | // Legalizing shifts/rotates requires adjusting the shift amount |
1212 | 3.49k | // to the appropriate width. |
1213 | 3.49k | SDValue Op0 = Node->getOperand(0); |
1214 | 3.49k | SDValue Op1 = Node->getOperand(1); |
1215 | 3.49k | SDValue Op2 = Node->getOperand(2); |
1216 | 3.49k | if (!Op2.getValueType().isVector()) { |
1217 | 3.49k | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); |
1218 | 3.49k | // The getShiftAmountOperand() may create a new operand node or |
1219 | 3.49k | // return the existing one. If new operand is created we need |
1220 | 3.49k | // to update the parent node. |
1221 | 3.49k | if (SAO != Op2) |
1222 | 108 | NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); |
1223 | 3.49k | } |
1224 | 3.49k | break; |
1225 | 41.2M | } |
1226 | 41.2M | } |
1227 | 41.2M | |
1228 | 41.2M | if (NewNode != Node) { |
1229 | 329 | ReplaceNode(Node, NewNode); |
1230 | 329 | Node = NewNode; |
1231 | 329 | } |
1232 | 41.2M | switch (Action) { |
1233 | 41.2M | case TargetLowering::Legal: |
1234 | 39.0M | LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n"); |
1235 | 39.0M | return; |
1236 | 41.2M | case TargetLowering::Custom: |
1237 | 2.09M | LLVM_DEBUG(dbgs() << "Trying custom legalization\n"); |
1238 | 2.09M | // FIXME: The handling for custom lowering with multiple results is |
1239 | 2.09M | // a complete mess. |
1240 | 2.09M | if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { |
1241 | 1.99M | if (!(Res.getNode() != Node || Res.getResNo() != 0525k )) |
1242 | 525k | return; |
1243 | 1.47M | |
1244 | 1.47M | if (Node->getNumValues() == 1) { |
1245 | 1.44M | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); |
1246 | 1.44M | // We can just directly replace this node with the lowered value. |
1247 | 1.44M | ReplaceNode(SDValue(Node, 0), Res); |
1248 | 1.44M | return; |
1249 | 1.44M | } |
1250 | 27.4k | |
1251 | 27.4k | SmallVector<SDValue, 8> ResultVals; |
1252 | 83.5k | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i56.0k ) |
1253 | 56.0k | ResultVals.push_back(Res.getValue(i)); |
1254 | 27.4k | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); |
1255 | 27.4k | ReplaceNode(Node, ResultVals.data()); |
1256 | 27.4k | return; |
1257 | 27.4k | } |
1258 | 96.9k | LLVM_DEBUG(dbgs() << "Could not custom legalize node\n"); |
1259 | 96.9k | LLVM_FALLTHROUGH; |
1260 | 210k | case TargetLowering::Expand: |
1261 | 210k | if (ExpandNode(Node)) |
1262 | 145k | return; |
1263 | 64.7k | LLVM_FALLTHROUGH; |
1264 | 64.8k | case TargetLowering::LibCall: |
1265 | 64.8k | ConvertNodeToLibcall(Node); |
1266 | 64.8k | return; |
1267 | 64.7k | case TargetLowering::Promote: |
1268 | 17.7k | PromoteNode(Node); |
1269 | 17.7k | return; |
1270 | 5.42M | } |
1271 | 5.42M | } |
1272 | 5.42M | |
1273 | 5.42M | switch (Node->getOpcode()) { |
1274 | 5.42M | default: |
1275 | | #ifndef NDEBUG |
1276 | | dbgs() << "NODE: "; |
1277 | | Node->dump( &DAG); |
1278 | | dbgs() << "\n"; |
1279 | | #endif |
1280 | 0 | llvm_unreachable("Do not know how to legalize this operator!"); |
1281 | 5.42M | |
1282 | 5.42M | case ISD::CALLSEQ_START: |
1283 | 1.58M | case ISD::CALLSEQ_END: |
1284 | 1.58M | break; |
1285 | 1.83M | case ISD::LOAD: |
1286 | 1.83M | return LegalizeLoadOps(Node); |
1287 | 2.00M | case ISD::STORE: |
1288 | 2.00M | return LegalizeStoreOps(Node); |
1289 | 5.42M | } |
1290 | 5.42M | } |
1291 | | |
1292 | 1.51k | SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { |
1293 | 1.51k | SDValue Vec = Op.getOperand(0); |
1294 | 1.51k | SDValue Idx = Op.getOperand(1); |
1295 | 1.51k | SDLoc dl(Op); |
1296 | 1.51k | |
1297 | 1.51k | // Before we generate a new store to a temporary stack slot, see if there is |
1298 | 1.51k | // already one that we can use. There often is because when we scalarize |
1299 | 1.51k | // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole |
1300 | 1.51k | // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in |
1301 | 1.51k | // the vector. If all are expanded here, we don't want one store per vector |
1302 | 1.51k | // element. |
1303 | 1.51k | |
1304 | 1.51k | // Caches for hasPredecessorHelper |
1305 | 1.51k | SmallPtrSet<const SDNode *, 32> Visited; |
1306 | 1.51k | SmallVector<const SDNode *, 16> Worklist; |
1307 | 1.51k | Visited.insert(Op.getNode()); |
1308 | 1.51k | Worklist.push_back(Idx.getNode()); |
1309 | 1.51k | SDValue StackPtr, Ch; |
1310 | 1.51k | for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), |
1311 | 3.06k | UE = Vec.getNode()->use_end(); UI != UE; ++UI1.55k ) { |
1312 | 2.60k | SDNode *User = *UI; |
1313 | 2.60k | if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) { |
1314 | 1.05k | if (ST->isIndexed() || ST->isTruncatingStore() || |
1315 | 1.05k | ST->getValue() != Vec) |
1316 | 0 | continue; |
1317 | 1.05k | |
1318 | 1.05k | // Make sure that nothing else could have stored into the destination of |
1319 | 1.05k | // this store. |
1320 | 1.05k | if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) |
1321 | 0 | continue; |
1322 | 1.05k | |
1323 | 1.05k | // If the index is dependent on the store we will introduce a cycle when |
1324 | 1.05k | // creating the load (the load uses the index, and by replacing the chain |
1325 | 1.05k | // we will make the index dependent on the load). Also, the store might be |
1326 | 1.05k | // dependent on the extractelement and introduce a cycle when creating |
1327 | 1.05k | // the load. |
1328 | 1.05k | if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) || |
1329 | 1.05k | ST->hasPredecessor(Op.getNode())1.05k ) |
1330 | 2 | continue; |
1331 | 1.04k | |
1332 | 1.04k | StackPtr = ST->getBasePtr(); |
1333 | 1.04k | Ch = SDValue(ST, 0); |
1334 | 1.04k | break; |
1335 | 1.04k | } |
1336 | 2.60k | } |
1337 | 1.51k | |
1338 | 1.51k | EVT VecVT = Vec.getValueType(); |
1339 | 1.51k | |
1340 | 1.51k | if (!Ch.getNode()) { |
1341 | 468 | // Store the value to a temporary stack slot, then LOAD the returned part. |
1342 | 468 | StackPtr = DAG.CreateStackTemporary(VecVT); |
1343 | 468 | Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, |
1344 | 468 | MachinePointerInfo()); |
1345 | 468 | } |
1346 | 1.51k | |
1347 | 1.51k | StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); |
1348 | 1.51k | |
1349 | 1.51k | SDValue NewLoad; |
1350 | 1.51k | |
1351 | 1.51k | if (Op.getValueType().isVector()) |
1352 | 0 | NewLoad = |
1353 | 0 | DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo()); |
1354 | 1.51k | else |
1355 | 1.51k | NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, |
1356 | 1.51k | MachinePointerInfo(), |
1357 | 1.51k | VecVT.getVectorElementType()); |
1358 | 1.51k | |
1359 | 1.51k | // Replace the chain going out of the store, by the one out of the load. |
1360 | 1.51k | DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); |
1361 | 1.51k | |
1362 | 1.51k | // We introduced a cycle though, so update the loads operands, making sure |
1363 | 1.51k | // to use the original store's chain as an incoming chain. |
1364 | 1.51k | SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), |
1365 | 1.51k | NewLoad->op_end()); |
1366 | 1.51k | NewLoadOperands[0] = Ch; |
1367 | 1.51k | NewLoad = |
1368 | 1.51k | SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); |
1369 | 1.51k | return NewLoad; |
1370 | 1.51k | } |
1371 | | |
1372 | 0 | SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { |
1373 | 0 | assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); |
1374 | 0 |
|
1375 | 0 | SDValue Vec = Op.getOperand(0); |
1376 | 0 | SDValue Part = Op.getOperand(1); |
1377 | 0 | SDValue Idx = Op.getOperand(2); |
1378 | 0 | SDLoc dl(Op); |
1379 | 0 |
|
1380 | 0 | // Store the value to a temporary stack slot, then LOAD the returned part. |
1381 | 0 | EVT VecVT = Vec.getValueType(); |
1382 | 0 | SDValue StackPtr = DAG.CreateStackTemporary(VecVT); |
1383 | 0 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
1384 | 0 | MachinePointerInfo PtrInfo = |
1385 | 0 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); |
1386 | 0 |
|
1387 | 0 | // First store the whole vector. |
1388 | 0 | SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); |
1389 | 0 |
|
1390 | 0 | // Then store the inserted part. |
1391 | 0 | SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); |
1392 | 0 |
|
1393 | 0 | // Store the subvector. |
1394 | 0 | Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo()); |
1395 | 0 |
|
1396 | 0 | // Finally, load the updated vector. |
1397 | 0 | return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo); |
1398 | 0 | } |
1399 | | |
1400 | 177 | SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { |
1401 | 177 | // We can't handle this case efficiently. Allocate a sufficiently |
1402 | 177 | // aligned object on the stack, store each element into it, then load |
1403 | 177 | // the result as a vector. |
1404 | 177 | // Create the stack frame object. |
1405 | 177 | EVT VT = Node->getValueType(0); |
1406 | 177 | EVT EltVT = VT.getVectorElementType(); |
1407 | 177 | SDLoc dl(Node); |
1408 | 177 | SDValue FIPtr = DAG.CreateStackTemporary(VT); |
1409 | 177 | int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); |
1410 | 177 | MachinePointerInfo PtrInfo = |
1411 | 177 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); |
1412 | 177 | |
1413 | 177 | // Emit a store of each element to the stack slot. |
1414 | 177 | SmallVector<SDValue, 8> Stores; |
1415 | 177 | unsigned TypeByteSize = EltVT.getSizeInBits() / 8; |
1416 | 177 | assert(TypeByteSize > 0 && "Vector element type too small for stack store!"); |
1417 | 177 | // Store (in the right endianness) the elements to memory. |
1418 | 873 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i696 ) { |
1419 | 696 | // Ignore undef elements. |
1420 | 696 | if (Node->getOperand(i).isUndef()) continue257 ; |
1421 | 439 | |
1422 | 439 | unsigned Offset = TypeByteSize*i; |
1423 | 439 | |
1424 | 439 | SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType()); |
1425 | 439 | Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); |
1426 | 439 | |
1427 | 439 | // If the destination vector element type is narrower than the source |
1428 | 439 | // element type, only store the bits necessary. |
1429 | 439 | if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { |
1430 | 132 | Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, |
1431 | 132 | Node->getOperand(i), Idx, |
1432 | 132 | PtrInfo.getWithOffset(Offset), EltVT)); |
1433 | 132 | } else |
1434 | 307 | Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), |
1435 | 307 | Idx, PtrInfo.getWithOffset(Offset))); |
1436 | 439 | } |
1437 | 177 | |
1438 | 177 | SDValue StoreChain; |
1439 | 177 | if (!Stores.empty()) // Not all undef elements? |
1440 | 177 | StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); |
1441 | 0 | else |
1442 | 0 | StoreChain = DAG.getEntryNode(); |
1443 | 177 | |
1444 | 177 | // Result is a load from the stack slot. |
1445 | 177 | return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo); |
1446 | 177 | } |
1447 | | |
1448 | | /// Bitcast a floating-point value to an integer value. Only bitcast the part |
1449 | | /// containing the sign bit if the target has no integer value capable of |
1450 | | /// holding all bits of the floating-point value. |
1451 | | void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State, |
1452 | | const SDLoc &DL, |
1453 | 368 | SDValue Value) const { |
1454 | 368 | EVT FloatVT = Value.getValueType(); |
1455 | 368 | unsigned NumBits = FloatVT.getSizeInBits(); |
1456 | 368 | State.FloatVT = FloatVT; |
1457 | 368 | EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits); |
1458 | 368 | // Convert to an integer of the same size. |
1459 | 368 | if (TLI.isTypeLegal(IVT)) { |
1460 | 119 | State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); |
1461 | 119 | State.SignMask = APInt::getSignMask(NumBits); |
1462 | 119 | State.SignBit = NumBits - 1; |
1463 | 119 | return; |
1464 | 119 | } |
1465 | 249 | |
1466 | 249 | auto &DataLayout = DAG.getDataLayout(); |
1467 | 249 | // Store the float to memory, then load the sign part out as an integer. |
1468 | 249 | MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8); |
1469 | 249 | // First create a temporary that is aligned for both the load and store. |
1470 | 249 | SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); |
1471 | 249 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
1472 | 249 | // Then store the float to it. |
1473 | 249 | State.FloatPtr = StackPtr; |
1474 | 249 | MachineFunction &MF = DAG.getMachineFunction(); |
1475 | 249 | State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI); |
1476 | 249 | State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr, |
1477 | 249 | State.FloatPointerInfo); |
1478 | 249 | |
1479 | 249 | SDValue IntPtr; |
1480 | 249 | if (DataLayout.isBigEndian()) { |
1481 | 1 | assert(FloatVT.isByteSized() && "Unsupported floating point type!"); |
1482 | 1 | // Load out a legal integer with the same sign bit as the float. |
1483 | 1 | IntPtr = StackPtr; |
1484 | 1 | State.IntPointerInfo = State.FloatPointerInfo; |
1485 | 248 | } else { |
1486 | 248 | // Advance the pointer so that the loaded byte will contain the sign bit. |
1487 | 248 | unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1; |
1488 | 248 | IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr, |
1489 | 248 | DAG.getConstant(ByteOffset, DL, StackPtr.getValueType())); |
1490 | 248 | State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI, |
1491 | 248 | ByteOffset); |
1492 | 248 | } |
1493 | 249 | |
1494 | 249 | State.IntPtr = IntPtr; |
1495 | 249 | State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, |
1496 | 249 | State.IntPointerInfo, MVT::i8); |
1497 | 249 | State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7); |
1498 | 249 | State.SignBit = 7; |
1499 | 249 | } |
1500 | | |
1501 | | /// Replace the integer value produced by getSignAsIntValue() with a new value |
1502 | | /// and cast the result back to a floating-point type. |
1503 | | SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State, |
1504 | | const SDLoc &DL, |
1505 | 43 | SDValue NewIntValue) const { |
1506 | 43 | if (!State.Chain) |
1507 | 40 | return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); |
1508 | 3 | |
1509 | 3 | // Override the part containing the sign bit in the value stored on the stack. |
1510 | 3 | SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, |
1511 | 3 | State.IntPointerInfo, MVT::i8); |
1512 | 3 | return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr, |
1513 | 3 | State.FloatPointerInfo); |
1514 | 3 | } |
1515 | | |
1516 | 325 | SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { |
1517 | 325 | SDLoc DL(Node); |
1518 | 325 | SDValue Mag = Node->getOperand(0); |
1519 | 325 | SDValue Sign = Node->getOperand(1); |
1520 | 325 | |
1521 | 325 | // Get sign bit into an integer value. |
1522 | 325 | FloatSignAsInt SignAsInt; |
1523 | 325 | getSignAsIntValue(SignAsInt, DL, Sign); |
1524 | 325 | |
1525 | 325 | EVT IntVT = SignAsInt.IntValue.getValueType(); |
1526 | 325 | SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); |
1527 | 325 | SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, |
1528 | 325 | SignMask); |
1529 | 325 | |
1530 | 325 | // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X) |
1531 | 325 | EVT FloatVT = Mag.getValueType(); |
1532 | 325 | if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && |
1533 | 325 | TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)283 ) { |
1534 | 283 | SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); |
1535 | 283 | SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); |
1536 | 283 | SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit, |
1537 | 283 | DAG.getConstant(0, DL, IntVT), ISD::SETNE); |
1538 | 283 | return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue); |
1539 | 283 | } |
1540 | 42 | |
1541 | 42 | // Transform Mag value to integer, and clear the sign bit. |
1542 | 42 | FloatSignAsInt MagAsInt; |
1543 | 42 | getSignAsIntValue(MagAsInt, DL, Mag); |
1544 | 42 | EVT MagVT = MagAsInt.IntValue.getValueType(); |
1545 | 42 | SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT); |
1546 | 42 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, |
1547 | 42 | ClearSignMask); |
1548 | 42 | |
1549 | 42 | // Get the signbit at the right position for MagAsInt. |
1550 | 42 | int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; |
1551 | 42 | EVT ShiftVT = IntVT; |
1552 | 42 | if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) { |
1553 | 0 | SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); |
1554 | 0 | ShiftVT = MagVT; |
1555 | 0 | } |
1556 | 42 | if (ShiftAmount > 0) { |
1557 | 22 | SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT); |
1558 | 22 | SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); |
1559 | 22 | } else if (20 ShiftAmount < 020 ) { |
1560 | 0 | SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT); |
1561 | 0 | SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); |
1562 | 0 | } |
1563 | 42 | if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) { |
1564 | 21 | SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); |
1565 | 21 | } |
1566 | 42 | |
1567 | 42 | // Store the part with the modified sign and convert back to float. |
1568 | 42 | SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); |
1569 | 42 | return modifySignAsInt(MagAsInt, DL, CopiedSign); |
1570 | 42 | } |
1571 | | |
1572 | 37 | SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { |
1573 | 37 | SDLoc DL(Node); |
1574 | 37 | SDValue Value = Node->getOperand(0); |
1575 | 37 | |
1576 | 37 | // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal. |
1577 | 37 | EVT FloatVT = Value.getValueType(); |
1578 | 37 | if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { |
1579 | 36 | SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT); |
1580 | 36 | return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); |
1581 | 36 | } |
1582 | 1 | |
1583 | 1 | // Transform value to integer, clear the sign bit and transform back. |
1584 | 1 | FloatSignAsInt ValueAsInt; |
1585 | 1 | getSignAsIntValue(ValueAsInt, DL, Value); |
1586 | 1 | EVT IntVT = ValueAsInt.IntValue.getValueType(); |
1587 | 1 | SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT); |
1588 | 1 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, |
1589 | 1 | ClearSignMask); |
1590 | 1 | return modifySignAsInt(ValueAsInt, DL, ClearedSign); |
1591 | 1 | } |
1592 | | |
1593 | | void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, |
1594 | 350 | SmallVectorImpl<SDValue> &Results) { |
1595 | 350 | unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); |
1596 | 350 | assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" |
1597 | 350 | " not tell us which reg is the stack pointer!"); |
1598 | 350 | SDLoc dl(Node); |
1599 | 350 | EVT VT = Node->getValueType(0); |
1600 | 350 | SDValue Tmp1 = SDValue(Node, 0); |
1601 | 350 | SDValue Tmp2 = SDValue(Node, 1); |
1602 | 350 | SDValue Tmp3 = Node->getOperand(2); |
1603 | 350 | SDValue Chain = Tmp1.getOperand(0); |
1604 | 350 | |
1605 | 350 | // Chain the dynamic stack allocation so that it doesn't modify the stack |
1606 | 350 | // pointer when other instructions are using the stack. |
1607 | 350 | Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); |
1608 | 350 | |
1609 | 350 | SDValue Size = Tmp2.getOperand(1); |
1610 | 350 | SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); |
1611 | 350 | Chain = SP.getValue(1); |
1612 | 350 | unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); |
1613 | 350 | unsigned StackAlign = |
1614 | 350 | DAG.getSubtarget().getFrameLowering()->getStackAlignment(); |
1615 | 350 | Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value |
1616 | 350 | if (Align > StackAlign) |
1617 | 63 | Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, |
1618 | 63 | DAG.getConstant(-(uint64_t)Align, dl, VT)); |
1619 | 350 | Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain |
1620 | 350 | |
1621 | 350 | Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true), |
1622 | 350 | DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); |
1623 | 350 | |
1624 | 350 | Results.push_back(Tmp1); |
1625 | 350 | Results.push_back(Tmp2); |
1626 | 350 | } |
1627 | | |
1628 | | /// Legalize a SETCC with given LHS and RHS and condition code CC on the current |
1629 | | /// target. |
1630 | | /// |
1631 | | /// If the SETCC has been legalized using AND / OR, then the legalized node |
1632 | | /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert |
1633 | | /// will be set to false. |
1634 | | /// |
1635 | | /// If the SETCC has been legalized by using getSetCCSwappedOperands(), |
1636 | | /// then the values of LHS and RHS will be swapped, CC will be set to the |
1637 | | /// new condition, and NeedInvert will be set to false. |
1638 | | /// |
1639 | | /// If the SETCC has been legalized using the inverse condcode, then LHS and |
1640 | | /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert |
1641 | | /// will be set to true. The caller must invert the result of the SETCC with |
1642 | | /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect |
1643 | | /// of a true/false result. |
1644 | | /// |
1645 | | /// \returns true if the SetCC has been legalized, false if it hasn't. |
1646 | | bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS, |
1647 | | SDValue &RHS, SDValue &CC, |
1648 | | bool &NeedInvert, |
1649 | 4.57k | const SDLoc &dl) { |
1650 | 4.57k | MVT OpVT = LHS.getSimpleValueType(); |
1651 | 4.57k | ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); |
1652 | 4.57k | NeedInvert = false; |
1653 | 4.57k | bool NeedSwap = false; |
1654 | 4.57k | switch (TLI.getCondCodeAction(CCCode, OpVT)) { |
1655 | 4.57k | default: 0 llvm_unreachable0 ("Unknown condition code action!"); |
1656 | 4.57k | case TargetLowering::Legal: |
1657 | 3.77k | // Nothing to do. |
1658 | 3.77k | break; |
1659 | 4.57k | case TargetLowering::Expand: { |
1660 | 797 | ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); |
1661 | 797 | if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { |
1662 | 200 | std::swap(LHS, RHS); |
1663 | 200 | CC = DAG.getCondCode(InvCC); |
1664 | 200 | return true; |
1665 | 200 | } |
1666 | 597 | // Swapping operands didn't work. Try inverting the condition. |
1667 | 597 | InvCC = getSetCCInverse(CCCode, OpVT.isInteger()); |
1668 | 597 | if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { |
1669 | 414 | // If inverting the condition is not enough, try swapping operands |
1670 | 414 | // on top of it. |
1671 | 414 | InvCC = ISD::getSetCCSwappedOperands(InvCC); |
1672 | 414 | NeedSwap = true; |
1673 | 414 | } |
1674 | 597 | if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { |
1675 | 222 | CC = DAG.getCondCode(InvCC); |
1676 | 222 | NeedInvert = true; |
1677 | 222 | if (NeedSwap) |
1678 | 39 | std::swap(LHS, RHS); |
1679 | 222 | return true; |
1680 | 222 | } |
1681 | 375 | |
1682 | 375 | ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; |
1683 | 375 | unsigned Opc = 0; |
1684 | 375 | switch (CCCode) { |
1685 | 375 | default: 0 llvm_unreachable0 ("Don't know how to expand this condition!"); |
1686 | 375 | case ISD::SETO: |
1687 | 86 | assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) |
1688 | 86 | && "If SETO is expanded, SETOEQ must be legal!"); |
1689 | 86 | CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; |
1690 | 375 | case ISD::SETUO: |
1691 | 88 | assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT) |
1692 | 88 | && "If SETUO is expanded, SETUNE must be legal!"); |
1693 | 88 | CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; |
1694 | 375 | case ISD::SETOEQ: |
1695 | 201 | case ISD::SETOGT: |
1696 | 201 | case ISD::SETOGE: |
1697 | 201 | case ISD::SETOLT: |
1698 | 201 | case ISD::SETOLE: |
1699 | 201 | case ISD::SETONE: |
1700 | 201 | case ISD::SETUEQ: |
1701 | 201 | case ISD::SETUNE: |
1702 | 201 | case ISD::SETUGT: |
1703 | 201 | case ISD::SETUGE: |
1704 | 201 | case ISD::SETULT: |
1705 | 201 | case ISD::SETULE: |
1706 | 201 | // If we are floating point, assign and break, otherwise fall through. |
1707 | 201 | if (!OpVT.isInteger()) { |
1708 | 201 | // We can use the 4th bit to tell if we are the unordered |
1709 | 201 | // or ordered version of the opcode. |
1710 | 201 | CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO90 : ISD::SETO111 ; |
1711 | 201 | Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR90 : ISD::AND111 ; |
1712 | 201 | CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); |
1713 | 201 | break; |
1714 | 201 | } |
1715 | 0 | // Fallthrough if we are unsigned integer. |
1716 | 0 | LLVM_FALLTHROUGH; |
1717 | 0 | case ISD::SETLE: |
1718 | 0 | case ISD::SETGT: |
1719 | 0 | case ISD::SETGE: |
1720 | 0 | case ISD::SETLT: |
1721 | 0 | case ISD::SETNE: |
1722 | 0 | case ISD::SETEQ: |
1723 | 0 | // If all combinations of inverting the condition and swapping operands |
1724 | 0 | // didn't work then we have no means to expand the condition. |
1725 | 0 | llvm_unreachable("Don't know how to expand this condition!"); |
1726 | 375 | } |
1727 | 375 | |
1728 | 375 | SDValue SetCC1, SetCC2; |
1729 | 375 | if (CCCode != ISD::SETO && CCCode != ISD::SETUO289 ) { |
1730 | 201 | // If we aren't the ordered or unorder operation, |
1731 | 201 | // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). |
1732 | 201 | SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); |
1733 | 201 | SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); |
1734 | 201 | } else { |
1735 | 174 | // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) |
1736 | 174 | SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); |
1737 | 174 | SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); |
1738 | 174 | } |
1739 | 375 | LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); |
1740 | 375 | RHS = SDValue(); |
1741 | 375 | CC = SDValue(); |
1742 | 375 | return true; |
1743 | 375 | } |
1744 | 3.77k | } |
1745 | 3.77k | return false; |
1746 | 3.77k | } |
1747 | | |
1748 | | /// Emit a store/load combination to the stack. This stores |
1749 | | /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does |
1750 | | /// a load from the stack slot to DestVT, extending it if needed. |
1751 | | /// The resultant code need not be legal. |
1752 | | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1753 | 841 | EVT DestVT, const SDLoc &dl) { |
1754 | 841 | return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode()); |
1755 | 841 | } |
1756 | | |
1757 | | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1758 | | EVT DestVT, const SDLoc &dl, |
1759 | 841 | SDValue Chain) { |
1760 | 841 | // Create the stack frame object. |
1761 | 841 | unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment( |
1762 | 841 | SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); |
1763 | 841 | SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); |
1764 | 841 | |
1765 | 841 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); |
1766 | 841 | int SPFI = StackPtrFI->getIndex(); |
1767 | 841 | MachinePointerInfo PtrInfo = |
1768 | 841 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); |
1769 | 841 | |
1770 | 841 | unsigned SrcSize = SrcOp.getValueSizeInBits(); |
1771 | 841 | unsigned SlotSize = SlotVT.getSizeInBits(); |
1772 | 841 | unsigned DestSize = DestVT.getSizeInBits(); |
1773 | 841 | Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); |
1774 | 841 | unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType); |
1775 | 841 | |
1776 | 841 | // Emit a store to the stack slot. Use a truncstore if the input value is |
1777 | 841 | // later than DestVT. |
1778 | 841 | SDValue Store; |
1779 | 841 | |
1780 | 841 | if (SrcSize > SlotSize) |
1781 | 0 | Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo, |
1782 | 0 | SlotVT, SrcAlign); |
1783 | 841 | else { |
1784 | 841 | assert(SrcSize == SlotSize && "Invalid store"); |
1785 | 841 | Store = |
1786 | 841 | DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign); |
1787 | 841 | } |
1788 | 841 | |
1789 | 841 | // Result is a load from the stack slot. |
1790 | 841 | if (SlotSize == DestSize) |
1791 | 841 | return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); |
1792 | 0 | |
1793 | 0 | assert(SlotSize < DestSize && "Unknown extension!"); |
1794 | 0 | return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, |
1795 | 0 | DestAlign); |
1796 | 0 | } |
1797 | | |
1798 | 28 | SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { |
1799 | 28 | SDLoc dl(Node); |
1800 | 28 | // Create a vector sized/aligned stack slot, store the value to element #0, |
1801 | 28 | // then load the whole vector back out. |
1802 | 28 | SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); |
1803 | 28 | |
1804 | 28 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); |
1805 | 28 | int SPFI = StackPtrFI->getIndex(); |
1806 | 28 | |
1807 | 28 | SDValue Ch = DAG.getTruncStore( |
1808 | 28 | DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, |
1809 | 28 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI), |
1810 | 28 | Node->getValueType(0).getVectorElementType()); |
1811 | 28 | return DAG.getLoad( |
1812 | 28 | Node->getValueType(0), dl, Ch, StackPtr, |
1813 | 28 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); |
1814 | 28 | } |
1815 | | |
1816 | | static bool |
1817 | | ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, |
1818 | 158 | const TargetLowering &TLI, SDValue &Res) { |
1819 | 158 | unsigned NumElems = Node->getNumOperands(); |
1820 | 158 | SDLoc dl(Node); |
1821 | 158 | EVT VT = Node->getValueType(0); |
1822 | 158 | |
1823 | 158 | // Try to group the scalars into pairs, shuffle the pairs together, then |
1824 | 158 | // shuffle the pairs of pairs together, etc. until the vector has |
1825 | 158 | // been built. This will work only if all of the necessary shuffle masks |
1826 | 158 | // are legal. |
1827 | 158 | |
1828 | 158 | // We do this in two phases; first to check the legality of the shuffles, |
1829 | 158 | // and next, assuming that all shuffles are legal, to create the new nodes. |
1830 | 474 | for (int Phase = 0; Phase < 2; ++Phase316 ) { |
1831 | 316 | SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals, |
1832 | 316 | NewIntermedVals; |
1833 | 3.00k | for (unsigned i = 0; i < NumElems; ++i2.68k ) { |
1834 | 2.68k | SDValue V = Node->getOperand(i); |
1835 | 2.68k | if (V.isUndef()) |
1836 | 688 | continue; |
1837 | 2.00k | |
1838 | 2.00k | SDValue Vec; |
1839 | 2.00k | if (Phase) |
1840 | 1.00k | Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); |
1841 | 2.00k | IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); |
1842 | 2.00k | } |
1843 | 316 | |
1844 | 808 | while (IntermedVals.size() > 2) { |
1845 | 492 | NewIntermedVals.clear(); |
1846 | 1.86k | for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 21.36k ) { |
1847 | 1.36k | // This vector and the next vector are shuffled together (simply to |
1848 | 1.36k | // append the one to the other). |
1849 | 1.36k | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1850 | 1.36k | |
1851 | 1.36k | SmallVector<int, 16> FinalIndices; |
1852 | 1.36k | FinalIndices.reserve(IntermedVals[i].second.size() + |
1853 | 1.36k | IntermedVals[i+1].second.size()); |
1854 | 1.36k | |
1855 | 1.36k | int k = 0; |
1856 | 3.29k | for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; |
1857 | 1.92k | ++j, ++k) { |
1858 | 1.92k | ShuffleVec[k] = j; |
1859 | 1.92k | FinalIndices.push_back(IntermedVals[i].second[j]); |
1860 | 1.92k | } |
1861 | 3.28k | for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; |
1862 | 1.92k | ++j, ++k) { |
1863 | 1.92k | ShuffleVec[k] = NumElems + j; |
1864 | 1.92k | FinalIndices.push_back(IntermedVals[i+1].second[j]); |
1865 | 1.92k | } |
1866 | 1.36k | |
1867 | 1.36k | SDValue Shuffle; |
1868 | 1.36k | if (Phase) |
1869 | 684 | Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, |
1870 | 684 | IntermedVals[i+1].first, |
1871 | 684 | ShuffleVec); |
1872 | 684 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) |
1873 | 0 | return false; |
1874 | 1.36k | NewIntermedVals.push_back( |
1875 | 1.36k | std::make_pair(Shuffle, std::move(FinalIndices))); |
1876 | 1.36k | } |
1877 | 492 | |
1878 | 492 | // If we had an odd number of defined values, then append the last |
1879 | 492 | // element to the array of new vectors. |
1880 | 492 | if ((IntermedVals.size() & 1) != 0) |
1881 | 104 | NewIntermedVals.push_back(IntermedVals.back()); |
1882 | 492 | |
1883 | 492 | IntermedVals.swap(NewIntermedVals); |
1884 | 492 | } |
1885 | 316 | |
1886 | 316 | assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && |
1887 | 316 | "Invalid number of intermediate vectors"); |
1888 | 316 | SDValue Vec1 = IntermedVals[0].first; |
1889 | 316 | SDValue Vec2; |
1890 | 316 | if (IntermedVals.size() > 1) |
1891 | 316 | Vec2 = IntermedVals[1].first; |
1892 | 0 | else if (Phase) |
1893 | 0 | Vec2 = DAG.getUNDEF(VT); |
1894 | 316 | |
1895 | 316 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1896 | 1.37k | for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i1.05k ) |
1897 | 1.05k | ShuffleVec[IntermedVals[0].second[i]] = i; |
1898 | 1.26k | for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i944 ) |
1899 | 944 | ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; |
1900 | 316 | |
1901 | 316 | if (Phase) |
1902 | 158 | Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); |
1903 | 158 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) |
1904 | 0 | return false; |
1905 | 316 | } |
1906 | 158 | |
1907 | 158 | return true; |
1908 | 158 | } |
1909 | | |
1910 | | /// Expand a BUILD_VECTOR node on targets that don't |
1911 | | /// support the operation, but do support the resultant vector type. |
1912 | 55.7k | SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { |
1913 | 55.7k | unsigned NumElems = Node->getNumOperands(); |
1914 | 55.7k | SDValue Value1, Value2; |
1915 | 55.7k | SDLoc dl(Node); |
1916 | 55.7k | EVT VT = Node->getValueType(0); |
1917 | 55.7k | EVT OpVT = Node->getOperand(0).getValueType(); |
1918 | 55.7k | EVT EltVT = VT.getVectorElementType(); |
1919 | 55.7k | |
1920 | 55.7k | // If the only non-undef value is the low element, turn this into a |
1921 | 55.7k | // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. |
1922 | 55.7k | bool isOnlyLowElement = true; |
1923 | 55.7k | bool MoreThanTwoValues = false; |
1924 | 55.7k | bool isConstant = true; |
1925 | 520k | for (unsigned i = 0; i < NumElems; ++i464k ) { |
1926 | 464k | SDValue V = Node->getOperand(i); |
1927 | 464k | if (V.isUndef()) |
1928 | 21.0k | continue; |
1929 | 443k | if (i > 0) |
1930 | 388k | isOnlyLowElement = false; |
1931 | 443k | if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)432k ) |
1932 | 12.2k | isConstant = false; |
1933 | 443k | |
1934 | 443k | if (!Value1.getNode()) { |
1935 | 55.7k | Value1 = V; |
1936 | 387k | } else if (!Value2.getNode()) { |
1937 | 175k | if (V != Value1) |
1938 | 43.7k | Value2 = V; |
1939 | 212k | } else if (V != Value1 && V != Value2185k ) { |
1940 | 148k | MoreThanTwoValues = true; |
1941 | 148k | } |
1942 | 443k | } |
1943 | 55.7k | |
1944 | 55.7k | if (!Value1.getNode()) |
1945 | 0 | return DAG.getUNDEF(VT); |
1946 | 55.7k | |
1947 | 55.7k | if (isOnlyLowElement) |
1948 | 61 | return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); |
1949 | 55.6k | |
1950 | 55.6k | // If all elements are constants, create a load from the constant pool. |
1951 | 55.6k | if (isConstant) { |
1952 | 52.8k | SmallVector<Constant*, 16> CV; |
1953 | 503k | for (unsigned i = 0, e = NumElems; i != e; ++i450k ) { |
1954 | 450k | if (ConstantFPSDNode *V = |
1955 | 10.6k | dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { |
1956 | 10.6k | CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); |
1957 | 440k | } else if (ConstantSDNode *V = |
1958 | 420k | dyn_cast<ConstantSDNode>(Node->getOperand(i))) { |
1959 | 420k | if (OpVT==EltVT) |
1960 | 385k | CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); |
1961 | 35.5k | else { |
1962 | 35.5k | // If OpVT and EltVT don't match, EltVT is not legal and the |
1963 | 35.5k | // element values have been promoted/truncated earlier. Undo this; |
1964 | 35.5k | // we don't want a v16i8 to become a v16i32 for example. |
1965 | 35.5k | const ConstantInt *CI = V->getConstantIntValue(); |
1966 | 35.5k | CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), |
1967 | 35.5k | CI->getZExtValue())); |
1968 | 35.5k | } |
1969 | 420k | } else { |
1970 | 19.7k | assert(Node->getOperand(i).isUndef()); |
1971 | 19.7k | Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); |
1972 | 19.7k | CV.push_back(UndefValue::get(OpNTy)); |
1973 | 19.7k | } |
1974 | 450k | } |
1975 | 52.8k | Constant *CP = ConstantVector::get(CV); |
1976 | 52.8k | SDValue CPIdx = |
1977 | 52.8k | DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout())); |
1978 | 52.8k | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); |
1979 | 52.8k | return DAG.getLoad( |
1980 | 52.8k | VT, dl, DAG.getEntryNode(), CPIdx, |
1981 | 52.8k | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
1982 | 52.8k | Alignment); |
1983 | 52.8k | } |
1984 | 2.86k | |
1985 | 2.86k | SmallSet<SDValue, 16> DefinedValues; |
1986 | 16.3k | for (unsigned i = 0; i < NumElems; ++i13.4k ) { |
1987 | 13.4k | if (Node->getOperand(i).isUndef()) |
1988 | 1.15k | continue; |
1989 | 12.2k | DefinedValues.insert(Node->getOperand(i)); |
1990 | 12.2k | } |
1991 | 2.86k | |
1992 | 2.86k | if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) { |
1993 | 2.79k | if (!MoreThanTwoValues) { |
1994 | 2.63k | SmallVector<int, 8> ShuffleVec(NumElems, -1); |
1995 | 14.5k | for (unsigned i = 0; i < NumElems; ++i11.8k ) { |
1996 | 11.8k | SDValue V = Node->getOperand(i); |
1997 | 11.8k | if (V.isUndef()) |
1998 | 809 | continue; |
1999 | 11.0k | ShuffleVec[i] = V == Value1 ? 09.46k : NumElems1.62k ; |
2000 | 11.0k | } |
2001 | 2.63k | if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { |
2002 | 2.53k | // Get the splatted value into the low element of a vector register. |
2003 | 2.53k | SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); |
2004 | 2.53k | SDValue Vec2; |
2005 | 2.53k | if (Value2.getNode()) |
2006 | 1.51k | Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); |
2007 | 1.01k | else |
2008 | 1.01k | Vec2 = DAG.getUNDEF(VT); |
2009 | 2.53k | |
2010 | 2.53k | // Return shuffle(LowValVec, undef, <0,0,0,0>) |
2011 | 2.53k | return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); |
2012 | 2.53k | } |
2013 | 158 | } else { |
2014 | 158 | SDValue Res; |
2015 | 158 | if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) |
2016 | 158 | return Res; |
2017 | 177 | } |
2018 | 2.79k | } |
2019 | 177 | |
2020 | 177 | // Otherwise, we can't handle this case efficiently. |
2021 | 177 | return ExpandVectorBuildThroughStack(Node); |
2022 | 177 | } |
2023 | | |
2024 | | // Expand a node into a call to a libcall. If the result value |
2025 | | // does not fit into a register, return the lo part and set the hi part to the |
2026 | | // by-reg argument. If it does fit into a single register, return the result |
2027 | | // and leave the Hi part unset. |
2028 | | SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, |
2029 | 5.64k | bool isSigned) { |
2030 | 5.64k | TargetLowering::ArgListTy Args; |
2031 | 5.64k | TargetLowering::ArgListEntry Entry; |
2032 | 7.94k | for (const SDValue &Op : Node->op_values()) { |
2033 | 7.94k | EVT ArgVT = Op.getValueType(); |
2034 | 7.94k | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
2035 | 7.94k | Entry.Node = Op; |
2036 | 7.94k | Entry.Ty = ArgTy; |
2037 | 7.94k | Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); |
2038 | 7.94k | Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); |
2039 | 7.94k | Args.push_back(Entry); |
2040 | 7.94k | } |
2041 | 5.64k | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2042 | 5.64k | TLI.getPointerTy(DAG.getDataLayout())); |
2043 | 5.64k | |
2044 | 5.64k | EVT RetVT = Node->getValueType(0); |
2045 | 5.64k | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2046 | 5.64k | |
2047 | 5.64k | // By default, the input chain to this libcall is the entry node of the |
2048 | 5.64k | // function. If the libcall is going to be emitted as a tail call then |
2049 | 5.64k | // TLI.isUsedByReturnOnly will change it to the right chain if the return |
2050 | 5.64k | // node which is being folded has a non-entry input chain. |
2051 | 5.64k | SDValue InChain = DAG.getEntryNode(); |
2052 | 5.64k | |
2053 | 5.64k | // isTailCall may be true since the callee does not reference caller stack |
2054 | 5.64k | // frame. Check if it's in the right position and that the return types match. |
2055 | 5.64k | SDValue TCChain = InChain; |
2056 | 5.64k | const Function &F = DAG.getMachineFunction().getFunction(); |
2057 | 5.64k | bool isTailCall = |
2058 | 5.64k | TLI.isInTailCallPosition(DAG, Node, TCChain) && |
2059 | 5.64k | (495 RetTy == F.getReturnType()495 || F.getReturnType()->isVoidTy()42 ); |
2060 | 5.64k | if (isTailCall) |
2061 | 453 | InChain = TCChain; |
2062 | 5.64k | |
2063 | 5.64k | TargetLowering::CallLoweringInfo CLI(DAG); |
2064 | 5.64k | bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned); |
2065 | 5.64k | CLI.setDebugLoc(SDLoc(Node)) |
2066 | 5.64k | .setChain(InChain) |
2067 | 5.64k | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2068 | 5.64k | std::move(Args)) |
2069 | 5.64k | .setTailCall(isTailCall) |
2070 | 5.64k | .setSExtResult(signExtend) |
2071 | 5.64k | .setZExtResult(!signExtend) |
2072 | 5.64k | .setIsPostTypeLegalization(true); |
2073 | 5.64k | |
2074 | 5.64k | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2075 | 5.64k | |
2076 | 5.64k | if (!CallInfo.second.getNode()) { |
2077 | 404 | LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump()); |
2078 | 404 | // It's a tailcall, return the chain (which is the DAG root). |
2079 | 404 | return DAG.getRoot(); |
2080 | 404 | } |
2081 | 5.24k | |
2082 | 5.24k | LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump()); |
2083 | 5.24k | return CallInfo.first; |
2084 | 5.24k | } |
2085 | | |
2086 | | // Expand a node into a call to a libcall. Similar to |
2087 | | // ExpandLibCall except that the first operand is the in-chain. |
2088 | | std::pair<SDValue, SDValue> |
2089 | | SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, |
2090 | | SDNode *Node, |
2091 | 86 | bool isSigned) { |
2092 | 86 | SDValue InChain = Node->getOperand(0); |
2093 | 86 | |
2094 | 86 | TargetLowering::ArgListTy Args; |
2095 | 86 | TargetLowering::ArgListEntry Entry; |
2096 | 275 | for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i189 ) { |
2097 | 189 | EVT ArgVT = Node->getOperand(i).getValueType(); |
2098 | 189 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
2099 | 189 | Entry.Node = Node->getOperand(i); |
2100 | 189 | Entry.Ty = ArgTy; |
2101 | 189 | Entry.IsSExt = isSigned; |
2102 | 189 | Entry.IsZExt = !isSigned; |
2103 | 189 | Args.push_back(Entry); |
2104 | 189 | } |
2105 | 86 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2106 | 86 | TLI.getPointerTy(DAG.getDataLayout())); |
2107 | 86 | |
2108 | 86 | Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); |
2109 | 86 | |
2110 | 86 | TargetLowering::CallLoweringInfo CLI(DAG); |
2111 | 86 | CLI.setDebugLoc(SDLoc(Node)) |
2112 | 86 | .setChain(InChain) |
2113 | 86 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2114 | 86 | std::move(Args)) |
2115 | 86 | .setSExtResult(isSigned) |
2116 | 86 | .setZExtResult(!isSigned); |
2117 | 86 | |
2118 | 86 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2119 | 86 | |
2120 | 86 | return CallInfo; |
2121 | 86 | } |
2122 | | |
2123 | | SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, |
2124 | | RTLIB::Libcall Call_F32, |
2125 | | RTLIB::Libcall Call_F64, |
2126 | | RTLIB::Libcall Call_F80, |
2127 | | RTLIB::Libcall Call_F128, |
2128 | 4.14k | RTLIB::Libcall Call_PPCF128) { |
2129 | 4.14k | if (Node->isStrictFPOpcode()) |
2130 | 1.09k | Node = DAG.mutateStrictFPToFP(Node); |
2131 | 4.14k | |
2132 | 4.14k | RTLIB::Libcall LC; |
2133 | 4.14k | switch (Node->getSimpleValueType(0).SimpleTy) { |
2134 | 4.14k | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2135 | 4.14k | case MVT::f32: LC = Call_F32; break2.04k ; |
2136 | 4.14k | case MVT::f64: LC = Call_F64; break1.94k ; |
2137 | 4.14k | case MVT::f80: LC = Call_F80; break84 ; |
2138 | 4.14k | case MVT::f128: LC = Call_F128; break69 ; |
2139 | 4.14k | case MVT::ppcf128: LC = Call_PPCF128; break0 ; |
2140 | 4.14k | } |
2141 | 4.14k | return ExpandLibCall(LC, Node, false); |
2142 | 4.14k | } |
2143 | | |
2144 | | SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, |
2145 | | RTLIB::Libcall Call_I8, |
2146 | | RTLIB::Libcall Call_I16, |
2147 | | RTLIB::Libcall Call_I32, |
2148 | | RTLIB::Libcall Call_I64, |
2149 | 268 | RTLIB::Libcall Call_I128) { |
2150 | 268 | RTLIB::Libcall LC; |
2151 | 268 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2152 | 268 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2153 | 268 | case MVT::i8: LC = Call_I8; break0 ; |
2154 | 268 | case MVT::i16: LC = Call_I16; break13 ; |
2155 | 268 | case MVT::i32: LC = Call_I32; break237 ; |
2156 | 268 | case MVT::i64: LC = Call_I64; break18 ; |
2157 | 268 | case MVT::i128: LC = Call_I128; break0 ; |
2158 | 268 | } |
2159 | 268 | return ExpandLibCall(LC, Node, isSigned); |
2160 | 268 | } |
2161 | | |
2162 | | /// Expand the node to a libcall based on first argument type (for instance |
2163 | | /// lround and its variant). |
2164 | | SDValue SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, |
2165 | | RTLIB::Libcall Call_F32, |
2166 | | RTLIB::Libcall Call_F64, |
2167 | | RTLIB::Libcall Call_F80, |
2168 | | RTLIB::Libcall Call_F128, |
2169 | 59 | RTLIB::Libcall Call_PPCF128) { |
2170 | 59 | RTLIB::Libcall LC; |
2171 | 59 | switch (Node->getOperand(0).getValueType().getSimpleVT().SimpleTy) { |
2172 | 59 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2173 | 59 | case MVT::f32: LC = Call_F32; break21 ; |
2174 | 59 | case MVT::f64: LC = Call_F64; break22 ; |
2175 | 59 | case MVT::f80: LC = Call_F80; break8 ; |
2176 | 59 | case MVT::f128: LC = Call_F128; break8 ; |
2177 | 59 | case MVT::ppcf128: LC = Call_PPCF128; break0 ; |
2178 | 59 | } |
2179 | 59 | |
2180 | 59 | return ExpandLibCall(LC, Node, false); |
2181 | 59 | } |
2182 | | |
2183 | | /// Issue libcalls to __{u}divmod to compute div / rem pairs. |
2184 | | void |
2185 | | SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, |
2186 | 14 | SmallVectorImpl<SDValue> &Results) { |
2187 | 14 | unsigned Opcode = Node->getOpcode(); |
2188 | 14 | bool isSigned = Opcode == ISD::SDIVREM; |
2189 | 14 | |
2190 | 14 | RTLIB::Libcall LC; |
2191 | 14 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2192 | 14 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2193 | 14 | case MVT::i8: LC= isSigned 0 ? RTLIB::SDIVREM_I80 : RTLIB::UDIVREM_I80 ; break; |
2194 | 14 | case MVT::i16: LC= isSigned 0 ? RTLIB::SDIVREM_I160 : RTLIB::UDIVREM_I160 ; break; |
2195 | 14 | case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I329 : RTLIB::UDIVREM_I325 ; break; |
2196 | 14 | case MVT::i64: LC= isSigned 0 ? RTLIB::SDIVREM_I640 : RTLIB::UDIVREM_I640 ; break; |
2197 | 14 | case MVT::i128: LC= isSigned 0 ? RTLIB::SDIVREM_I1280 :RTLIB::UDIVREM_I1280 ; break; |
2198 | 14 | } |
2199 | 14 | |
2200 | 14 | // The input chain to this libcall is the entry node of the function. |
2201 | 14 | // Legalizing the call will automatically add the previous call to the |
2202 | 14 | // dependence. |
2203 | 14 | SDValue InChain = DAG.getEntryNode(); |
2204 | 14 | |
2205 | 14 | EVT RetVT = Node->getValueType(0); |
2206 | 14 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2207 | 14 | |
2208 | 14 | TargetLowering::ArgListTy Args; |
2209 | 14 | TargetLowering::ArgListEntry Entry; |
2210 | 28 | for (const SDValue &Op : Node->op_values()) { |
2211 | 28 | EVT ArgVT = Op.getValueType(); |
2212 | 28 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
2213 | 28 | Entry.Node = Op; |
2214 | 28 | Entry.Ty = ArgTy; |
2215 | 28 | Entry.IsSExt = isSigned; |
2216 | 28 | Entry.IsZExt = !isSigned; |
2217 | 28 | Args.push_back(Entry); |
2218 | 28 | } |
2219 | 14 | |
2220 | 14 | // Also pass the return address of the remainder. |
2221 | 14 | SDValue FIPtr = DAG.CreateStackTemporary(RetVT); |
2222 | 14 | Entry.Node = FIPtr; |
2223 | 14 | Entry.Ty = RetTy->getPointerTo(); |
2224 | 14 | Entry.IsSExt = isSigned; |
2225 | 14 | Entry.IsZExt = !isSigned; |
2226 | 14 | Args.push_back(Entry); |
2227 | 14 | |
2228 | 14 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2229 | 14 | TLI.getPointerTy(DAG.getDataLayout())); |
2230 | 14 | |
2231 | 14 | SDLoc dl(Node); |
2232 | 14 | TargetLowering::CallLoweringInfo CLI(DAG); |
2233 | 14 | CLI.setDebugLoc(dl) |
2234 | 14 | .setChain(InChain) |
2235 | 14 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2236 | 14 | std::move(Args)) |
2237 | 14 | .setSExtResult(isSigned) |
2238 | 14 | .setZExtResult(!isSigned); |
2239 | 14 | |
2240 | 14 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2241 | 14 | |
2242 | 14 | // Remainder is loaded back from the stack frame. |
2243 | 14 | SDValue Rem = |
2244 | 14 | DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo()); |
2245 | 14 | Results.push_back(CallInfo.first); |
2246 | 14 | Results.push_back(Rem); |
2247 | 14 | } |
2248 | | |
2249 | | /// Return true if sincos libcall is available. |
2250 | 476 | static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { |
2251 | 476 | RTLIB::Libcall LC; |
2252 | 476 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2253 | 476 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2254 | 476 | case MVT::f32: LC = RTLIB::SINCOS_F32; break320 ; |
2255 | 476 | case MVT::f64: LC = RTLIB::SINCOS_F64; break123 ; |
2256 | 476 | case MVT::f80: LC = RTLIB::SINCOS_F80; break19 ; |
2257 | 476 | case MVT::f128: LC = RTLIB::SINCOS_F128; break14 ; |
2258 | 476 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break0 ; |
2259 | 476 | } |
2260 | 476 | return TLI.getLibcallName(LC) != nullptr; |
2261 | 476 | } |
2262 | | |
2263 | | /// Only issue sincos libcall if both sin and cos are needed. |
2264 | 213 | static bool useSinCos(SDNode *Node) { |
2265 | 213 | unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN |
2266 | 213 | ? ISD::FCOS106 : ISD::FSIN107 ; |
2267 | 213 | |
2268 | 213 | SDValue Op0 = Node->getOperand(0); |
2269 | 213 | for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), |
2270 | 284 | UE = Op0.getNode()->use_end(); UI != UE; ++UI71 ) { |
2271 | 213 | SDNode *User = *UI; |
2272 | 213 | if (User == Node) |
2273 | 71 | continue; |
2274 | 142 | // The other user might have been turned into sincos already. |
2275 | 142 | if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS71 ) |
2276 | 142 | return true; |
2277 | 142 | } |
2278 | 213 | return false71 ; |
2279 | 213 | } |
2280 | | |
2281 | | /// Issue libcalls to sincos to compute sin / cos pairs. |
2282 | | void |
2283 | | SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, |
2284 | 34 | SmallVectorImpl<SDValue> &Results) { |
2285 | 34 | RTLIB::Libcall LC; |
2286 | 34 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2287 | 34 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2288 | 34 | case MVT::f32: LC = RTLIB::SINCOS_F32; break13 ; |
2289 | 34 | case MVT::f64: LC = RTLIB::SINCOS_F64; break13 ; |
2290 | 34 | case MVT::f80: LC = RTLIB::SINCOS_F80; break5 ; |
2291 | 34 | case MVT::f128: LC = RTLIB::SINCOS_F128; break3 ; |
2292 | 34 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break0 ; |
2293 | 34 | } |
2294 | 34 | |
2295 | 34 | // The input chain to this libcall is the entry node of the function. |
2296 | 34 | // Legalizing the call will automatically add the previous call to the |
2297 | 34 | // dependence. |
2298 | 34 | SDValue InChain = DAG.getEntryNode(); |
2299 | 34 | |
2300 | 34 | EVT RetVT = Node->getValueType(0); |
2301 | 34 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2302 | 34 | |
2303 | 34 | TargetLowering::ArgListTy Args; |
2304 | 34 | TargetLowering::ArgListEntry Entry; |
2305 | 34 | |
2306 | 34 | // Pass the argument. |
2307 | 34 | Entry.Node = Node->getOperand(0); |
2308 | 34 | Entry.Ty = RetTy; |
2309 | 34 | Entry.IsSExt = false; |
2310 | 34 | Entry.IsZExt = false; |
2311 | 34 | Args.push_back(Entry); |
2312 | 34 | |
2313 | 34 | // Pass the return address of sin. |
2314 | 34 | SDValue SinPtr = DAG.CreateStackTemporary(RetVT); |
2315 | 34 | Entry.Node = SinPtr; |
2316 | 34 | Entry.Ty = RetTy->getPointerTo(); |
2317 | 34 | Entry.IsSExt = false; |
2318 | 34 | Entry.IsZExt = false; |
2319 | 34 | Args.push_back(Entry); |
2320 | 34 | |
2321 | 34 | // Also pass the return address of the cos. |
2322 | 34 | SDValue CosPtr = DAG.CreateStackTemporary(RetVT); |
2323 | 34 | Entry.Node = CosPtr; |
2324 | 34 | Entry.Ty = RetTy->getPointerTo(); |
2325 | 34 | Entry.IsSExt = false; |
2326 | 34 | Entry.IsZExt = false; |
2327 | 34 | Args.push_back(Entry); |
2328 | 34 | |
2329 | 34 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2330 | 34 | TLI.getPointerTy(DAG.getDataLayout())); |
2331 | 34 | |
2332 | 34 | SDLoc dl(Node); |
2333 | 34 | TargetLowering::CallLoweringInfo CLI(DAG); |
2334 | 34 | CLI.setDebugLoc(dl).setChain(InChain).setLibCallee( |
2335 | 34 | TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee, |
2336 | 34 | std::move(Args)); |
2337 | 34 | |
2338 | 34 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2339 | 34 | |
2340 | 34 | Results.push_back( |
2341 | 34 | DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo())); |
2342 | 34 | Results.push_back( |
2343 | 34 | DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo())); |
2344 | 34 | } |
2345 | | |
2346 | | /// This function is responsible for legalizing a |
2347 | | /// INT_TO_FP operation of the specified operand when the target requests that |
2348 | | /// we expand it. At this point, we know that the result and operand types are |
2349 | | /// legal for the target. |
2350 | | SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, |
2351 | | EVT DestVT, |
2352 | 56 | const SDLoc &dl) { |
2353 | 56 | EVT SrcVT = Op0.getValueType(); |
2354 | 56 | |
2355 | 56 | // TODO: Should any fast-math-flags be set for the created nodes? |
2356 | 56 | LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n"); |
2357 | 56 | if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)55 ) { |
2358 | 54 | LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double " |
2359 | 54 | "expansion\n"); |
2360 | 54 | |
2361 | 54 | // Get the stack frame index of a 8 byte buffer. |
2362 | 54 | SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); |
2363 | 54 | |
2364 | 54 | // word offset constant for Hi/Lo address computation |
2365 | 54 | SDValue WordOff = DAG.getConstant(sizeof(int), dl, |
2366 | 54 | StackSlot.getValueType()); |
2367 | 54 | // set up Hi and Lo (into buffer) address based on endian |
2368 | 54 | SDValue Hi = StackSlot; |
2369 | 54 | SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(), |
2370 | 54 | StackSlot, WordOff); |
2371 | 54 | if (DAG.getDataLayout().isLittleEndian()) |
2372 | 22 | std::swap(Hi, Lo); |
2373 | 54 | |
2374 | 54 | // if signed map to unsigned space |
2375 | 54 | SDValue Op0Mapped; |
2376 | 54 | if (isSigned) { |
2377 | 4 | // constant used to invert sign bit (signed to unsigned mapping) |
2378 | 4 | SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32); |
2379 | 4 | Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); |
2380 | 50 | } else { |
2381 | 50 | Op0Mapped = Op0; |
2382 | 50 | } |
2383 | 54 | // store the lo of the constructed double - based on integer input |
2384 | 54 | SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo, |
2385 | 54 | MachinePointerInfo()); |
2386 | 54 | // initial hi portion of constructed double |
2387 | 54 | SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32); |
2388 | 54 | // store the hi of the constructed double - biased exponent |
2389 | 54 | SDValue Store2 = |
2390 | 54 | DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo()); |
2391 | 54 | // load the constructed double |
2392 | 54 | SDValue Load = |
2393 | 54 | DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo()); |
2394 | 54 | // FP constant to bias correct the final result |
2395 | 54 | SDValue Bias = DAG.getConstantFP(isSigned ? |
2396 | 4 | BitsToDouble(0x4330000080000000ULL) : |
2397 | 54 | BitsToDouble(0x4330000000000000ULL)50 , |
2398 | 54 | dl, MVT::f64); |
2399 | 54 | // subtract the bias |
2400 | 54 | SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); |
2401 | 54 | // final result |
2402 | 54 | SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT); |
2403 | 54 | return Result; |
2404 | 54 | } |
2405 | 2 | assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); |
2406 | 2 | // Code below here assumes !isSigned without checking again. |
2407 | 2 | |
2408 | 2 | SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); |
2409 | 2 | |
2410 | 2 | SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0, |
2411 | 2 | DAG.getConstant(0, dl, SrcVT), ISD::SETLT); |
2412 | 2 | SDValue Zero = DAG.getIntPtrConstant(0, dl), |
2413 | 2 | Four = DAG.getIntPtrConstant(4, dl); |
2414 | 2 | SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), |
2415 | 2 | SignSet, Four, Zero); |
2416 | 2 | |
2417 | 2 | // If the sign bit of the integer is set, the large number will be treated |
2418 | 2 | // as a negative number. To counteract this, the dynamic code adds an |
2419 | 2 | // offset depending on the data type. |
2420 | 2 | uint64_t FF; |
2421 | 2 | switch (SrcVT.getSimpleVT().SimpleTy) { |
2422 | 2 | default: 0 llvm_unreachable0 ("Unsupported integer type!"); |
2423 | 2 | case MVT::i8 : FF = 0x43800000ULL; break0 ; // 2^8 (as a float) |
2424 | 2 | case MVT::i16: FF = 0x47800000ULL; break0 ; // 2^16 (as a float) |
2425 | 2 | case MVT::i32: FF = 0x4F800000ULL; break1 ; // 2^32 (as a float) |
2426 | 2 | case MVT::i64: FF = 0x5F800000ULL; break1 ; // 2^64 (as a float) |
2427 | 2 | } |
2428 | 2 | if (DAG.getDataLayout().isLittleEndian()) |
2429 | 0 | FF <<= 32; |
2430 | 2 | Constant *FudgeFactor = ConstantInt::get( |
2431 | 2 | Type::getInt64Ty(*DAG.getContext()), FF); |
2432 | 2 | |
2433 | 2 | SDValue CPIdx = |
2434 | 2 | DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout())); |
2435 | 2 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); |
2436 | 2 | CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); |
2437 | 2 | Alignment = std::min(Alignment, 4u); |
2438 | 2 | SDValue FudgeInReg; |
2439 | 2 | if (DestVT == MVT::f32) |
2440 | 1 | FudgeInReg = DAG.getLoad( |
2441 | 1 | MVT::f32, dl, DAG.getEntryNode(), CPIdx, |
2442 | 1 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
2443 | 1 | Alignment); |
2444 | 1 | else { |
2445 | 1 | SDValue Load = DAG.getExtLoad( |
2446 | 1 | ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, |
2447 | 1 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, |
2448 | 1 | Alignment); |
2449 | 1 | HandleSDNode Handle(Load); |
2450 | 1 | LegalizeOp(Load.getNode()); |
2451 | 1 | FudgeInReg = Handle.getValue(); |
2452 | 1 | } |
2453 | 2 | |
2454 | 2 | return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); |
2455 | 2 | } |
2456 | | |
2457 | | /// This function is responsible for legalizing a |
2458 | | /// *INT_TO_FP operation of the specified operand when the target requests that |
2459 | | /// we promote it. At this point, we know that the result and operand types are |
2460 | | /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP |
2461 | | /// operation that takes a larger input. |
2462 | | SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, |
2463 | | bool isSigned, |
2464 | 177 | const SDLoc &dl) { |
2465 | 177 | // First step, figure out the appropriate *INT_TO_FP operation to use. |
2466 | 177 | EVT NewInTy = LegalOp.getValueType(); |
2467 | 177 | |
2468 | 177 | unsigned OpToUse = 0; |
2469 | 177 | |
2470 | 177 | // Scan for the appropriate larger type to use. |
2471 | 187 | while (true) { |
2472 | 187 | NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); |
2473 | 187 | assert(NewInTy.isInteger() && "Ran out of possibilities!"); |
2474 | 187 | |
2475 | 187 | // If the target supports SINT_TO_FP of this type, use it. |
2476 | 187 | if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { |
2477 | 177 | OpToUse = ISD::SINT_TO_FP; |
2478 | 177 | break; |
2479 | 177 | } |
2480 | 10 | if (isSigned) continue9 ; |
2481 | 1 | |
2482 | 1 | // If the target supports UINT_TO_FP of this type, use it. |
2483 | 1 | if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { |
2484 | 0 | OpToUse = ISD::UINT_TO_FP; |
2485 | 0 | break; |
2486 | 0 | } |
2487 | 1 | |
2488 | 1 | // Otherwise, try a larger type. |
2489 | 1 | } |
2490 | 177 | |
2491 | 177 | // Okay, we found the operation and type to use. Zero extend our input to the |
2492 | 177 | // desired type then run the operation on it. |
2493 | 177 | return DAG.getNode(OpToUse, dl, DestVT, |
2494 | 177 | DAG.getNode(isSigned ? ISD::SIGN_EXTEND39 : ISD::ZERO_EXTEND138 , |
2495 | 177 | dl, NewInTy, LegalOp)); |
2496 | 177 | } |
2497 | | |
2498 | | /// This function is responsible for legalizing a |
2499 | | /// FP_TO_*INT operation of the specified operand when the target requests that |
2500 | | /// we promote it. At this point, we know that the result and operand types are |
2501 | | /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT |
2502 | | /// operation that returns a larger result. |
2503 | | SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, |
2504 | | bool isSigned, |
2505 | 286 | const SDLoc &dl) { |
2506 | 286 | // First step, figure out the appropriate FP_TO*INT operation to use. |
2507 | 286 | EVT NewOutTy = DestVT; |
2508 | 286 | |
2509 | 286 | unsigned OpToUse = 0; |
2510 | 286 | |
2511 | 286 | // Scan for the appropriate larger type to use. |
2512 | 286 | while (true) { |
2513 | 286 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); |
2514 | 286 | assert(NewOutTy.isInteger() && "Ran out of possibilities!"); |
2515 | 286 | |
2516 | 286 | // A larger signed type can hold all unsigned values of the requested type, |
2517 | 286 | // so using FP_TO_SINT is valid |
2518 | 286 | if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { |
2519 | 286 | OpToUse = ISD::FP_TO_SINT; |
2520 | 286 | break; |
2521 | 286 | } |
2522 | 0 | |
2523 | 0 | // However, if the value may be < 0.0, we *must* use some FP_TO_SINT. |
2524 | 0 | if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { |
2525 | 0 | OpToUse = ISD::FP_TO_UINT; |
2526 | 0 | break; |
2527 | 0 | } |
2528 | 0 |
|
2529 | 0 | // Otherwise, try a larger type. |
2530 | 0 | } |
2531 | 286 | |
2532 | 286 | // Okay, we found the operation and type to use. |
2533 | 286 | SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); |
2534 | 286 | |
2535 | 286 | // Truncate the result of the extended FP_TO_*INT operation to the desired |
2536 | 286 | // size. |
2537 | 286 | return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); |
2538 | 286 | } |
2539 | | |
2540 | | /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts. |
2541 | 119 | SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) { |
2542 | 119 | EVT VT = Op.getValueType(); |
2543 | 119 | EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2544 | 119 | unsigned Sz = VT.getScalarSizeInBits(); |
2545 | 119 | |
2546 | 119 | SDValue Tmp, Tmp2, Tmp3; |
2547 | 119 | |
2548 | 119 | // If we can, perform BSWAP first and then the mask+swap the i4, then i2 |
2549 | 119 | // and finally the i1 pairs. |
2550 | 119 | // TODO: We can easily support i4/i2 legal types if any target ever does. |
2551 | 119 | if (Sz >= 8 && isPowerOf2_32(Sz)) { |
2552 | 119 | // Create the masks - repeating the pattern every byte. |
2553 | 119 | APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0)); |
2554 | 119 | APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC)); |
2555 | 119 | APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA)); |
2556 | 119 | APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F)); |
2557 | 119 | APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33)); |
2558 | 119 | APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55)); |
2559 | 119 | |
2560 | 119 | // BSWAP if the type is wider than a single byte. |
2561 | 119 | Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op)78 : Op41 ); |
2562 | 119 | |
2563 | 119 | // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4) |
2564 | 119 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT)); |
2565 | 119 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT)); |
2566 | 119 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT)); |
2567 | 119 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); |
2568 | 119 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); |
2569 | 119 | |
2570 | 119 | // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2) |
2571 | 119 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT)); |
2572 | 119 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT)); |
2573 | 119 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT)); |
2574 | 119 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); |
2575 | 119 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); |
2576 | 119 | |
2577 | 119 | // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1) |
2578 | 119 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT)); |
2579 | 119 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT)); |
2580 | 119 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT)); |
2581 | 119 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); |
2582 | 119 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); |
2583 | 119 | return Tmp; |
2584 | 119 | } |
2585 | 0 | |
2586 | 0 | Tmp = DAG.getConstant(0, dl, VT); |
2587 | 0 | for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) { |
2588 | 0 | if (I < J) |
2589 | 0 | Tmp2 = |
2590 | 0 | DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); |
2591 | 0 | else |
2592 | 0 | Tmp2 = |
2593 | 0 | DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); |
2594 | 0 |
|
2595 | 0 | APInt Shift(Sz, 1); |
2596 | 0 | Shift <<= J; |
2597 | 0 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); |
2598 | 0 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); |
2599 | 0 | } |
2600 | 0 |
|
2601 | 0 | return Tmp; |
2602 | 0 | } |
2603 | | |
2604 | | /// Open code the operations for BSWAP of the specified operation. |
2605 | 176 | SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) { |
2606 | 176 | EVT VT = Op.getValueType(); |
2607 | 176 | EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2608 | 176 | SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; |
2609 | 176 | switch (VT.getSimpleVT().getScalarType().SimpleTy) { |
2610 | 176 | default: 0 llvm_unreachable0 ("Unhandled Expand type in BSWAP!"); |
2611 | 176 | case MVT::i16: |
2612 | 105 | // Use a rotate by 8. This can be further expanded if necessary. |
2613 | 105 | return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2614 | 176 | case MVT::i32: |
2615 | 31 | Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); |
2616 | 31 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2617 | 31 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2618 | 31 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); |
2619 | 31 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, |
2620 | 31 | DAG.getConstant(0xFF0000, dl, VT)); |
2621 | 31 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); |
2622 | 31 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); |
2623 | 31 | Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); |
2624 | 31 | return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); |
2625 | 176 | case MVT::i64: |
2626 | 40 | Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); |
2627 | 40 | Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); |
2628 | 40 | Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); |
2629 | 40 | Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2630 | 40 | Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2631 | 40 | Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); |
2632 | 40 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); |
2633 | 40 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); |
2634 | 40 | Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, |
2635 | 40 | DAG.getConstant(255ULL<<48, dl, VT)); |
2636 | 40 | Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, |
2637 | 40 | DAG.getConstant(255ULL<<40, dl, VT)); |
2638 | 40 | Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, |
2639 | 40 | DAG.getConstant(255ULL<<32, dl, VT)); |
2640 | 40 | Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, |
2641 | 40 | DAG.getConstant(255ULL<<24, dl, VT)); |
2642 | 40 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, |
2643 | 40 | DAG.getConstant(255ULL<<16, dl, VT)); |
2644 | 40 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, |
2645 | 40 | DAG.getConstant(255ULL<<8 , dl, VT)); |
2646 | 40 | Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); |
2647 | 40 | Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); |
2648 | 40 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); |
2649 | 40 | Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); |
2650 | 40 | Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); |
2651 | 40 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); |
2652 | 40 | return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); |
2653 | 176 | } |
2654 | 176 | } |
2655 | | |
2656 | 210k | bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { |
2657 | 210k | LLVM_DEBUG(dbgs() << "Trying to expand node\n"); |
2658 | 210k | SmallVector<SDValue, 8> Results; |
2659 | 210k | SDLoc dl(Node); |
2660 | 210k | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
2661 | 210k | bool NeedInvert; |
2662 | 210k | switch (Node->getOpcode()) { |
2663 | 210k | case ISD::ABS: |
2664 | 413 | if (TLI.expandABS(Node, Tmp1, DAG)) |
2665 | 413 | Results.push_back(Tmp1); |
2666 | 413 | break; |
2667 | 210k | case ISD::CTPOP: |
2668 | 205 | if (TLI.expandCTPOP(Node, Tmp1, DAG)) |
2669 | 205 | Results.push_back(Tmp1); |
2670 | 205 | break; |
2671 | 210k | case ISD::CTLZ: |
2672 | 1.81k | case ISD::CTLZ_ZERO_UNDEF: |
2673 | 1.81k | if (TLI.expandCTLZ(Node, Tmp1, DAG)) |
2674 | 1.81k | Results.push_back(Tmp1); |
2675 | 1.81k | break; |
2676 | 1.81k | case ISD::CTTZ: |
2677 | 273 | case ISD::CTTZ_ZERO_UNDEF: |
2678 | 273 | if (TLI.expandCTTZ(Node, Tmp1, DAG)) |
2679 | 273 | Results.push_back(Tmp1); |
2680 | 273 | break; |
2681 | 273 | case ISD::BITREVERSE: |
2682 | 119 | Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl)); |
2683 | 119 | break; |
2684 | 273 | case ISD::BSWAP: |
2685 | 176 | Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); |
2686 | 176 | break; |
2687 | 273 | case ISD::FRAMEADDR: |
2688 | 5 | case ISD::RETURNADDR: |
2689 | 5 | case ISD::FRAME_TO_ARGS_OFFSET: |
2690 | 5 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); |
2691 | 5 | break; |
2692 | 6 | case ISD::EH_DWARF_CFA: { |
2693 | 6 | SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, |
2694 | 6 | TLI.getPointerTy(DAG.getDataLayout())); |
2695 | 6 | SDValue Offset = DAG.getNode(ISD::ADD, dl, |
2696 | 6 | CfaArg.getValueType(), |
2697 | 6 | DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, |
2698 | 6 | CfaArg.getValueType()), |
2699 | 6 | CfaArg); |
2700 | 6 | SDValue FA = DAG.getNode( |
2701 | 6 | ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), |
2702 | 6 | DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
2703 | 6 | Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), |
2704 | 6 | FA, Offset)); |
2705 | 6 | break; |
2706 | 5 | } |
2707 | 5 | case ISD::FLT_ROUNDS_: |
2708 | 3 | Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); |
2709 | 3 | break; |
2710 | 830 | case ISD::EH_RETURN: |
2711 | 830 | case ISD::EH_LABEL: |
2712 | 830 | case ISD::PREFETCH: |
2713 | 830 | case ISD::VAEND: |
2714 | 830 | case ISD::EH_SJLJ_LONGJMP: |
2715 | 830 | // If the target didn't expand these, there's nothing to do, so just |
2716 | 830 | // preserve the chain and be done. |
2717 | 830 | Results.push_back(Node->getOperand(0)); |
2718 | 830 | break; |
2719 | 830 | case ISD::READCYCLECOUNTER: |
2720 | 3 | // If the target didn't expand this, just return 'zero' and preserve the |
2721 | 3 | // chain. |
2722 | 3 | Results.append(Node->getNumValues() - 1, |
2723 | 3 | DAG.getConstant(0, dl, Node->getValueType(0))); |
2724 | 3 | Results.push_back(Node->getOperand(0)); |
2725 | 3 | break; |
2726 | 830 | case ISD::EH_SJLJ_SETJMP: |
2727 | 0 | // If the target didn't expand this, just return 'zero' and preserve the |
2728 | 0 | // chain. |
2729 | 0 | Results.push_back(DAG.getConstant(0, dl, MVT::i32)); |
2730 | 0 | Results.push_back(Node->getOperand(0)); |
2731 | 0 | break; |
2732 | 830 | case ISD::ATOMIC_LOAD: { |
2733 | 6 | // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. |
2734 | 6 | SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); |
2735 | 6 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
2736 | 6 | SDValue Swap = DAG.getAtomicCmpSwap( |
2737 | 6 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, |
2738 | 6 | Node->getOperand(0), Node->getOperand(1), Zero, Zero, |
2739 | 6 | cast<AtomicSDNode>(Node)->getMemOperand()); |
2740 | 6 | Results.push_back(Swap.getValue(0)); |
2741 | 6 | Results.push_back(Swap.getValue(1)); |
2742 | 6 | break; |
2743 | 830 | } |
2744 | 830 | case ISD::ATOMIC_STORE: { |
2745 | 14 | // There is no libcall for atomic store; fake it with ATOMIC_SWAP. |
2746 | 14 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
2747 | 14 | cast<AtomicSDNode>(Node)->getMemoryVT(), |
2748 | 14 | Node->getOperand(0), |
2749 | 14 | Node->getOperand(1), Node->getOperand(2), |
2750 | 14 | cast<AtomicSDNode>(Node)->getMemOperand()); |
2751 | 14 | Results.push_back(Swap.getValue(1)); |
2752 | 14 | break; |
2753 | 830 | } |
2754 | 1.25k | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
2755 | 1.25k | // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and |
2756 | 1.25k | // splits out the success value as a comparison. Expanding the resulting |
2757 | 1.25k | // ATOMIC_CMP_SWAP will produce a libcall. |
2758 | 1.25k | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
2759 | 1.25k | SDValue Res = DAG.getAtomicCmpSwap( |
2760 | 1.25k | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, |
2761 | 1.25k | Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), |
2762 | 1.25k | Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); |
2763 | 1.25k | |
2764 | 1.25k | SDValue ExtRes = Res; |
2765 | 1.25k | SDValue LHS = Res; |
2766 | 1.25k | SDValue RHS = Node->getOperand(1); |
2767 | 1.25k | |
2768 | 1.25k | EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); |
2769 | 1.25k | EVT OuterType = Node->getValueType(0); |
2770 | 1.25k | switch (TLI.getExtendForAtomicOps()) { |
2771 | 1.25k | case ISD::SIGN_EXTEND: |
2772 | 120 | LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, |
2773 | 120 | DAG.getValueType(AtomicType)); |
2774 | 120 | RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, |
2775 | 120 | Node->getOperand(2), DAG.getValueType(AtomicType)); |
2776 | 120 | ExtRes = LHS; |
2777 | 120 | break; |
2778 | 1.25k | case ISD::ZERO_EXTEND: |
2779 | 1.08k | LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, |
2780 | 1.08k | DAG.getValueType(AtomicType)); |
2781 | 1.08k | RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); |
2782 | 1.08k | ExtRes = LHS; |
2783 | 1.08k | break; |
2784 | 1.25k | case ISD::ANY_EXTEND: |
2785 | 51 | LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType); |
2786 | 51 | RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); |
2787 | 51 | break; |
2788 | 1.25k | default: |
2789 | 0 | llvm_unreachable("Invalid atomic op extension"); |
2790 | 1.25k | } |
2791 | 1.25k | |
2792 | 1.25k | SDValue Success = |
2793 | 1.25k | DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); |
2794 | 1.25k | |
2795 | 1.25k | Results.push_back(ExtRes.getValue(0)); |
2796 | 1.25k | Results.push_back(Success); |
2797 | 1.25k | Results.push_back(Res.getValue(1)); |
2798 | 1.25k | break; |
2799 | 1.25k | } |
2800 | 1.25k | case ISD::DYNAMIC_STACKALLOC: |
2801 | 350 | ExpandDYNAMIC_STACKALLOC(Node, Results); |
2802 | 350 | break; |
2803 | 35.6k | case ISD::MERGE_VALUES: |
2804 | 108k | for (unsigned i = 0; i < Node->getNumValues(); i++72.5k ) |
2805 | 72.5k | Results.push_back(Node->getOperand(i)); |
2806 | 35.6k | break; |
2807 | 1.25k | case ISD::UNDEF: { |
2808 | 170 | EVT VT = Node->getValueType(0); |
2809 | 170 | if (VT.isInteger()) |
2810 | 157 | Results.push_back(DAG.getConstant(0, dl, VT)); |
2811 | 13 | else { |
2812 | 13 | assert(VT.isFloatingPoint() && "Unknown value type!"); |
2813 | 13 | Results.push_back(DAG.getConstantFP(0, dl, VT)); |
2814 | 13 | } |
2815 | 170 | break; |
2816 | 1.25k | } |
2817 | 1.25k | case ISD::STRICT_FP_ROUND: |
2818 | 0 | Tmp1 = EmitStackConvert(Node->getOperand(1), |
2819 | 0 | Node->getValueType(0), |
2820 | 0 | Node->getValueType(0), dl, Node->getOperand(0)); |
2821 | 0 | ReplaceNode(Node, Tmp1.getNode()); |
2822 | 0 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n"); |
2823 | 0 | return true; |
2824 | 1.25k | case ISD::FP_ROUND: |
2825 | 841 | case ISD::BITCAST: |
2826 | 841 | Tmp1 = EmitStackConvert(Node->getOperand(0), |
2827 | 841 | Node->getValueType(0), |
2828 | 841 | Node->getValueType(0), dl); |
2829 | 841 | Results.push_back(Tmp1); |
2830 | 841 | break; |
2831 | 841 | case ISD::STRICT_FP_EXTEND: |
2832 | 0 | Tmp1 = EmitStackConvert(Node->getOperand(1), |
2833 | 0 | Node->getOperand(1).getValueType(), |
2834 | 0 | Node->getValueType(0), dl, Node->getOperand(0)); |
2835 | 0 | ReplaceNode(Node, Tmp1.getNode()); |
2836 | 0 | LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n"); |
2837 | 0 | return true; |
2838 | 841 | case ISD::FP_EXTEND: |
2839 | 0 | Tmp1 = EmitStackConvert(Node->getOperand(0), |
2840 | 0 | Node->getOperand(0).getValueType(), |
2841 | 0 | Node->getValueType(0), dl); |
2842 | 0 | Results.push_back(Tmp1); |
2843 | 0 | break; |
2844 | 2.64k | case ISD::SIGN_EXTEND_INREG: { |
2845 | 2.64k | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
2846 | 2.64k | EVT VT = Node->getValueType(0); |
2847 | 2.64k | |
2848 | 2.64k | // An in-register sign-extend of a boolean is a negation: |
2849 | 2.64k | // 'true' (1) sign-extended is -1. |
2850 | 2.64k | // 'false' (0) sign-extended is 0. |
2851 | 2.64k | // However, we must mask the high bits of the source operand because the |
2852 | 2.64k | // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero. |
2853 | 2.64k | |
2854 | 2.64k | // TODO: Do this for vectors too? |
2855 | 2.64k | if (ExtraVT.getSizeInBits() == 1) { |
2856 | 827 | SDValue One = DAG.getConstant(1, dl, VT); |
2857 | 827 | SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); |
2858 | 827 | SDValue Zero = DAG.getConstant(0, dl, VT); |
2859 | 827 | SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); |
2860 | 827 | Results.push_back(Neg); |
2861 | 827 | break; |
2862 | 827 | } |
2863 | 1.82k | |
2864 | 1.82k | // NOTE: we could fall back on load/store here too for targets without |
2865 | 1.82k | // SRA. However, it is doubtful that any exist. |
2866 | 1.82k | EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2867 | 1.82k | unsigned BitsDiff = VT.getScalarSizeInBits() - |
2868 | 1.82k | ExtraVT.getScalarSizeInBits(); |
2869 | 1.82k | SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); |
2870 | 1.82k | Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), |
2871 | 1.82k | Node->getOperand(0), ShiftCst); |
2872 | 1.82k | Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); |
2873 | 1.82k | Results.push_back(Tmp1); |
2874 | 1.82k | break; |
2875 | 1.82k | } |
2876 | 1.82k | case ISD::FP_ROUND_INREG: { |
2877 | 0 | // The only way we can lower this is to turn it into a TRUNCSTORE, |
2878 | 0 | // EXTLOAD pair, targeting a temporary location (a stack slot). |
2879 | 0 |
|
2880 | 0 | // NOTE: there is a choice here between constantly creating new stack |
2881 | 0 | // slots and always reusing the same one. We currently always create |
2882 | 0 | // new ones, as reuse may inhibit scheduling. |
2883 | 0 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
2884 | 0 | Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, |
2885 | 0 | Node->getValueType(0), dl); |
2886 | 0 | Results.push_back(Tmp1); |
2887 | 0 | break; |
2888 | 1.82k | } |
2889 | 1.82k | case ISD::UINT_TO_FP: |
2890 | 301 | if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) { |
2891 | 249 | Results.push_back(Tmp1); |
2892 | 249 | break; |
2893 | 249 | } |
2894 | 52 | LLVM_FALLTHROUGH; |
2895 | 56 | case ISD::SINT_TO_FP: |
2896 | 56 | Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, |
2897 | 56 | Node->getOperand(0), Node->getValueType(0), dl); |
2898 | 56 | Results.push_back(Tmp1); |
2899 | 56 | break; |
2900 | 61 | case ISD::FP_TO_SINT: |
2901 | 61 | if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) |
2902 | 61 | Results.push_back(Tmp1); |
2903 | 61 | break; |
2904 | 345 | case ISD::FP_TO_UINT: |
2905 | 345 | if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG)) |
2906 | 345 | Results.push_back(Tmp1); |
2907 | 345 | break; |
2908 | 52 | case ISD::LROUND: |
2909 | 17 | Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, |
2910 | 17 | RTLIB::LROUND_F64, RTLIB::LROUND_F80, |
2911 | 17 | RTLIB::LROUND_F128, |
2912 | 17 | RTLIB::LROUND_PPCF128)); |
2913 | 17 | break; |
2914 | 52 | case ISD::LLROUND: |
2915 | 14 | Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, |
2916 | 14 | RTLIB::LLROUND_F64, RTLIB::LLROUND_F80, |
2917 | 14 | RTLIB::LLROUND_F128, |
2918 | 14 | RTLIB::LLROUND_PPCF128)); |
2919 | 14 | break; |
2920 | 52 | case ISD::LRINT: |
2921 | 16 | Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, |
2922 | 16 | RTLIB::LRINT_F64, RTLIB::LRINT_F80, |
2923 | 16 | RTLIB::LRINT_F128, |
2924 | 16 | RTLIB::LRINT_PPCF128)); |
2925 | 16 | break; |
2926 | 52 | case ISD::LLRINT: |
2927 | 12 | Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, |
2928 | 12 | RTLIB::LLRINT_F64, RTLIB::LLRINT_F80, |
2929 | 12 | RTLIB::LLRINT_F128, |
2930 | 12 | RTLIB::LLRINT_PPCF128)); |
2931 | 12 | break; |
2932 | 118 | case ISD::VAARG: |
2933 | 118 | Results.push_back(DAG.expandVAArg(Node)); |
2934 | 118 | Results.push_back(Results[0].getValue(1)); |
2935 | 118 | break; |
2936 | 100 | case ISD::VACOPY: |
2937 | 100 | Results.push_back(DAG.expandVACopy(Node)); |
2938 | 100 | break; |
2939 | 1.62k | case ISD::EXTRACT_VECTOR_ELT: |
2940 | 1.62k | if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) |
2941 | 112 | // This must be an access of the only element. Return it. |
2942 | 112 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), |
2943 | 112 | Node->getOperand(0)); |
2944 | 1.51k | else |
2945 | 1.51k | Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); |
2946 | 1.62k | Results.push_back(Tmp1); |
2947 | 1.62k | break; |
2948 | 52 | case ISD::EXTRACT_SUBVECTOR: |
2949 | 0 | Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); |
2950 | 0 | break; |
2951 | 52 | case ISD::INSERT_SUBVECTOR: |
2952 | 0 | Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); |
2953 | 0 | break; |
2954 | 52 | case ISD::CONCAT_VECTORS: |
2955 | 0 | Results.push_back(ExpandVectorBuildThroughStack(Node)); |
2956 | 0 | break; |
2957 | 52 | case ISD::SCALAR_TO_VECTOR: |
2958 | 28 | Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); |
2959 | 28 | break; |
2960 | 1.88k | case ISD::INSERT_VECTOR_ELT: |
2961 | 1.88k | Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), |
2962 | 1.88k | Node->getOperand(1), |
2963 | 1.88k | Node->getOperand(2), dl)); |
2964 | 1.88k | break; |
2965 | 453 | case ISD::VECTOR_SHUFFLE: { |
2966 | 453 | SmallVector<int, 32> NewMask; |
2967 | 453 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); |
2968 | 453 | |
2969 | 453 | EVT VT = Node->getValueType(0); |
2970 | 453 | EVT EltVT = VT.getVectorElementType(); |
2971 | 453 | SDValue Op0 = Node->getOperand(0); |
2972 | 453 | SDValue Op1 = Node->getOperand(1); |
2973 | 453 | if (!TLI.isTypeLegal(EltVT)) { |
2974 | 12 | EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); |
2975 | 12 | |
2976 | 12 | // BUILD_VECTOR operands are allowed to be wider than the element type. |
2977 | 12 | // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept |
2978 | 12 | // it. |
2979 | 12 | if (NewEltVT.bitsLT(EltVT)) { |
2980 | 0 | // Convert shuffle node. |
2981 | 0 | // If original node was v4i64 and the new EltVT is i32, |
2982 | 0 | // cast operands to v8i32 and re-build the mask. |
2983 | 0 |
|
2984 | 0 | // Calculate new VT, the size of the new VT should be equal to original. |
2985 | 0 | EVT NewVT = |
2986 | 0 | EVT::getVectorVT(*DAG.getContext(), NewEltVT, |
2987 | 0 | VT.getSizeInBits() / NewEltVT.getSizeInBits()); |
2988 | 0 | assert(NewVT.bitsEq(VT)); |
2989 | 0 |
|
2990 | 0 | // cast operands to new VT |
2991 | 0 | Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); |
2992 | 0 | Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); |
2993 | 0 |
|
2994 | 0 | // Convert the shuffle mask |
2995 | 0 | unsigned int factor = |
2996 | 0 | NewVT.getVectorNumElements()/VT.getVectorNumElements(); |
2997 | 0 |
|
2998 | 0 | // EltVT gets smaller |
2999 | 0 | assert(factor > 0); |
3000 | 0 |
|
3001 | 0 | for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { |
3002 | 0 | if (Mask[i] < 0) { |
3003 | 0 | for (unsigned fi = 0; fi < factor; ++fi) |
3004 | 0 | NewMask.push_back(Mask[i]); |
3005 | 0 | } |
3006 | 0 | else { |
3007 | 0 | for (unsigned fi = 0; fi < factor; ++fi) |
3008 | 0 | NewMask.push_back(Mask[i]*factor+fi); |
3009 | 0 | } |
3010 | 0 | } |
3011 | 0 | Mask = NewMask; |
3012 | 0 | VT = NewVT; |
3013 | 0 | } |
3014 | 12 | EltVT = NewEltVT; |
3015 | 12 | } |
3016 | 453 | unsigned NumElems = VT.getVectorNumElements(); |
3017 | 453 | SmallVector<SDValue, 16> Ops; |
3018 | 1.76k | for (unsigned i = 0; i != NumElems; ++i1.31k ) { |
3019 | 1.31k | if (Mask[i] < 0) { |
3020 | 22 | Ops.push_back(DAG.getUNDEF(EltVT)); |
3021 | 22 | continue; |
3022 | 22 | } |
3023 | 1.29k | unsigned Idx = Mask[i]; |
3024 | 1.29k | if (Idx < NumElems) |
3025 | 848 | Ops.push_back(DAG.getNode( |
3026 | 848 | ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, |
3027 | 848 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); |
3028 | 442 | else |
3029 | 442 | Ops.push_back(DAG.getNode( |
3030 | 442 | ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, |
3031 | 442 | DAG.getConstant(Idx - NumElems, dl, |
3032 | 442 | TLI.getVectorIdxTy(DAG.getDataLayout())))); |
3033 | 1.29k | } |
3034 | 453 | |
3035 | 453 | Tmp1 = DAG.getBuildVector(VT, dl, Ops); |
3036 | 453 | // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. |
3037 | 453 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); |
3038 | 453 | Results.push_back(Tmp1); |
3039 | 453 | break; |
3040 | 52 | } |
3041 | 712 | case ISD::EXTRACT_ELEMENT: { |
3042 | 712 | EVT OpTy = Node->getOperand(0).getValueType(); |
3043 | 712 | if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { |
3044 | 336 | // 1 -> Hi |
3045 | 336 | Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), |
3046 | 336 | DAG.getConstant(OpTy.getSizeInBits() / 2, dl, |
3047 | 336 | TLI.getShiftAmountTy( |
3048 | 336 | Node->getOperand(0).getValueType(), |
3049 | 336 | DAG.getDataLayout()))); |
3050 | 336 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); |
3051 | 376 | } else { |
3052 | 376 | // 0 -> Lo |
3053 | 376 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), |
3054 | 376 | Node->getOperand(0)); |
3055 | 376 | } |
3056 | 712 | Results.push_back(Tmp1); |
3057 | 712 | break; |
3058 | 52 | } |
3059 | 221 | case ISD::STACKSAVE: |
3060 | 221 | // Expand to CopyFromReg if the target set |
3061 | 221 | // StackPointerRegisterToSaveRestore. |
3062 | 221 | if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { |
3063 | 221 | Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, |
3064 | 221 | Node->getValueType(0))); |
3065 | 221 | Results.push_back(Results[0].getValue(1)); |
3066 | 221 | } else { |
3067 | 0 | Results.push_back(DAG.getUNDEF(Node->getValueType(0))); |
3068 | 0 | Results.push_back(Node->getOperand(0)); |
3069 | 0 | } |
3070 | 221 | break; |
3071 | 82 | case ISD::STACKRESTORE: |
3072 | 82 | // Expand to CopyToReg if the target set |
3073 | 82 | // StackPointerRegisterToSaveRestore. |
3074 | 82 | if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { |
3075 | 82 | Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, |
3076 | 82 | Node->getOperand(1))); |
3077 | 82 | } else { |
3078 | 0 | Results.push_back(Node->getOperand(0)); |
3079 | 0 | } |
3080 | 82 | break; |
3081 | 52 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
3082 | 0 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); |
3083 | 0 | Results.push_back(Results[0].getValue(0)); |
3084 | 0 | break; |
3085 | 325 | case ISD::FCOPYSIGN: |
3086 | 325 | Results.push_back(ExpandFCOPYSIGN(Node)); |
3087 | 325 | break; |
3088 | 52 | case ISD::FNEG: |
3089 | 33 | // Expand Y = FNEG(X) -> Y = SUB -0.0, X |
3090 | 33 | Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0)); |
3091 | 33 | // TODO: If FNEG has fast-math-flags, propagate them to the FSUB. |
3092 | 33 | Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, |
3093 | 33 | Node->getOperand(0)); |
3094 | 33 | Results.push_back(Tmp1); |
3095 | 33 | break; |
3096 | 52 | case ISD::FABS: |
3097 | 37 | Results.push_back(ExpandFABS(Node)); |
3098 | 37 | break; |
3099 | 52 | case ISD::SMIN: |
3100 | 2 | case ISD::SMAX: |
3101 | 2 | case ISD::UMIN: |
3102 | 2 | case ISD::UMAX: { |
3103 | 2 | // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B |
3104 | 2 | ISD::CondCode Pred; |
3105 | 2 | switch (Node->getOpcode()) { |
3106 | 2 | default: 0 llvm_unreachable0 ("How did we get here?"); |
3107 | 2 | case ISD::SMAX: Pred = ISD::SETGT; break0 ; |
3108 | 2 | case ISD::SMIN: Pred = ISD::SETLT; break0 ; |
3109 | 2 | case ISD::UMAX: Pred = ISD::SETUGT; break; |
3110 | 2 | case ISD::UMIN: Pred = ISD::SETULT; break0 ; |
3111 | 2 | } |
3112 | 2 | Tmp1 = Node->getOperand(0); |
3113 | 2 | Tmp2 = Node->getOperand(1); |
3114 | 2 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred); |
3115 | 2 | Results.push_back(Tmp1); |
3116 | 2 | break; |
3117 | 2 | } |
3118 | 157 | case ISD::FMINNUM: |
3119 | 157 | case ISD::FMAXNUM: { |
3120 | 157 | if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) |
3121 | 4 | Results.push_back(Expanded); |
3122 | 157 | break; |
3123 | 157 | } |
3124 | 577 | case ISD::FSIN: |
3125 | 577 | case ISD::FCOS: { |
3126 | 577 | EVT VT = Node->getValueType(0); |
3127 | 577 | // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / |
3128 | 577 | // fcos which share the same operand and both are used. |
3129 | 577 | if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || |
3130 | 577 | isSinCosLibcallAvailable(Node, TLI)476 ) |
3131 | 577 | && useSinCos(Node)213 ) { |
3132 | 142 | SDVTList VTs = DAG.getVTList(VT, VT); |
3133 | 142 | Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); |
3134 | 142 | if (Node->getOpcode() == ISD::FCOS) |
3135 | 71 | Tmp1 = Tmp1.getValue(1); |
3136 | 142 | Results.push_back(Tmp1); |
3137 | 142 | } |
3138 | 577 | break; |
3139 | 577 | } |
3140 | 577 | case ISD::FMAD: |
3141 | 0 | llvm_unreachable("Illegal fmad should never be formed"); |
3142 | 577 | |
3143 | 577 | case ISD::FP16_TO_FP: |
3144 | 565 | if (Node->getValueType(0) != MVT::f32) { |
3145 | 0 | // We can extend to types bigger than f32 in two steps without changing |
3146 | 0 | // the result. Since "f16 -> f32" is much more commonly available, give |
3147 | 0 | // CodeGen the option of emitting that before resorting to a libcall. |
3148 | 0 | SDValue Res = |
3149 | 0 | DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); |
3150 | 0 | Results.push_back( |
3151 | 0 | DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); |
3152 | 0 | } |
3153 | 565 | break; |
3154 | 577 | case ISD::FP_TO_FP16: |
3155 | 564 | LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n"); |
3156 | 564 | if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) { |
3157 | 15 | SDValue Op = Node->getOperand(0); |
3158 | 15 | MVT SVT = Op.getSimpleValueType(); |
3159 | 15 | if ((SVT == MVT::f64 || SVT == MVT::f802 ) && |
3160 | 15 | TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { |
3161 | 13 | // Under fastmath, we can expand this node into a fround followed by |
3162 | 13 | // a float-half conversion. |
3163 | 13 | SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, |
3164 | 13 | DAG.getIntPtrConstant(0, dl)); |
3165 | 13 | Results.push_back( |
3166 | 13 | DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); |
3167 | 13 | } |
3168 | 15 | } |
3169 | 564 | break; |
3170 | 28.7k | case ISD::ConstantFP: { |
3171 | 28.7k | ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); |
3172 | 28.7k | // Check to see if this FP immediate is already legal. |
3173 | 28.7k | // If this is a legal constant, turn it into a TargetConstantFP node. |
3174 | 28.7k | if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), |
3175 | 28.7k | DAG.getMachineFunction().getFunction().hasOptSize())) |
3176 | 12.4k | Results.push_back(ExpandConstantFP(CFP, true)); |
3177 | 28.7k | break; |
3178 | 577 | } |
3179 | 577 | case ISD::Constant: { |
3180 | 0 | ConstantSDNode *CP = cast<ConstantSDNode>(Node); |
3181 | 0 | Results.push_back(ExpandConstant(CP)); |
3182 | 0 | break; |
3183 | 577 | } |
3184 | 577 | case ISD::FSUB: { |
3185 | 299 | EVT VT = Node->getValueType(0); |
3186 | 299 | if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && |
3187 | 299 | TLI.isOperationLegalOrCustom(ISD::FNEG, VT)181 ) { |
3188 | 181 | const SDNodeFlags Flags = Node->getFlags(); |
3189 | 181 | Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); |
3190 | 181 | Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); |
3191 | 181 | Results.push_back(Tmp1); |
3192 | 181 | } |
3193 | 299 | break; |
3194 | 577 | } |
3195 | 577 | case ISD::SUB: { |
3196 | 0 | EVT VT = Node->getValueType(0); |
3197 | 0 | assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && |
3198 | 0 | TLI.isOperationLegalOrCustom(ISD::XOR, VT) && |
3199 | 0 | "Don't know how to expand this subtraction!"); |
3200 | 0 | Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), |
3201 | 0 | DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, |
3202 | 0 | VT)); |
3203 | 0 | Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); |
3204 | 0 | Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); |
3205 | 0 | break; |
3206 | 577 | } |
3207 | 3.40k | case ISD::UREM: |
3208 | 3.40k | case ISD::SREM: { |
3209 | 3.40k | EVT VT = Node->getValueType(0); |
3210 | 3.40k | bool isSigned = Node->getOpcode() == ISD::SREM; |
3211 | 3.40k | unsigned DivOpc = isSigned ? ISD::SDIV1.96k : ISD::UDIV1.43k ; |
3212 | 3.40k | unsigned DivRemOpc = isSigned ? ISD::SDIVREM1.96k : ISD::UDIVREM1.43k ; |
3213 | 3.40k | Tmp2 = Node->getOperand(0); |
3214 | 3.40k | Tmp3 = Node->getOperand(1); |
3215 | 3.40k | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { |
3216 | 1.44k | SDVTList VTs = DAG.getVTList(VT, VT); |
3217 | 1.44k | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); |
3218 | 1.44k | Results.push_back(Tmp1); |
3219 | 1.95k | } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { |
3220 | 1.86k | // X % Y -> X-X/Y*Y |
3221 | 1.86k | Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); |
3222 | 1.86k | Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); |
3223 | 1.86k | Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); |
3224 | 1.86k | Results.push_back(Tmp1); |
3225 | 1.86k | } |
3226 | 3.40k | break; |
3227 | 3.40k | } |
3228 | 3.40k | case ISD::UDIV: |
3229 | 1.10k | case ISD::SDIV: { |
3230 | 1.10k | bool isSigned = Node->getOpcode() == ISD::SDIV; |
3231 | 1.10k | unsigned DivRemOpc = isSigned ? ISD::SDIVREM310 : ISD::UDIVREM792 ; |
3232 | 1.10k | EVT VT = Node->getValueType(0); |
3233 | 1.10k | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { |
3234 | 1.08k | SDVTList VTs = DAG.getVTList(VT, VT); |
3235 | 1.08k | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), |
3236 | 1.08k | Node->getOperand(1)); |
3237 | 1.08k | Results.push_back(Tmp1); |
3238 | 1.08k | } |
3239 | 1.10k | break; |
3240 | 1.10k | } |
3241 | 1.10k | case ISD::MULHU: |
3242 | 784 | case ISD::MULHS: { |
3243 | 784 | unsigned ExpandOpcode = |
3244 | 784 | Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI653 : ISD::SMUL_LOHI131 ; |
3245 | 784 | EVT VT = Node->getValueType(0); |
3246 | 784 | SDVTList VTs = DAG.getVTList(VT, VT); |
3247 | 784 | |
3248 | 784 | Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), |
3249 | 784 | Node->getOperand(1)); |
3250 | 784 | Results.push_back(Tmp1.getValue(1)); |
3251 | 784 | break; |
3252 | 784 | } |
3253 | 784 | case ISD::UMUL_LOHI: |
3254 | 690 | case ISD::SMUL_LOHI: { |
3255 | 690 | SDValue LHS = Node->getOperand(0); |
3256 | 690 | SDValue RHS = Node->getOperand(1); |
3257 | 690 | MVT VT = LHS.getSimpleValueType(); |
3258 | 690 | unsigned MULHOpcode = |
3259 | 690 | Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS0 ; |
3260 | 690 | |
3261 | 690 | if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) { |
3262 | 552 | Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); |
3263 | 552 | Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); |
3264 | 552 | break; |
3265 | 552 | } |
3266 | 138 | |
3267 | 138 | SmallVector<SDValue, 4> Halves; |
3268 | 138 | EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); |
3269 | 138 | assert(TLI.isTypeLegal(HalfType)); |
3270 | 138 | if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves, |
3271 | 138 | HalfType, DAG, |
3272 | 138 | TargetLowering::MulExpansionKind::Always)) { |
3273 | 414 | for (unsigned i = 0; i < 2; ++i276 ) { |
3274 | 276 | SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); |
3275 | 276 | SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); |
3276 | 276 | SDValue Shift = DAG.getConstant( |
3277 | 276 | HalfType.getScalarSizeInBits(), dl, |
3278 | 276 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); |
3279 | 276 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); |
3280 | 276 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); |
3281 | 276 | } |
3282 | 138 | break; |
3283 | 138 | } |
3284 | 0 | break; |
3285 | 0 | } |
3286 | 706 | case ISD::MUL: { |
3287 | 706 | EVT VT = Node->getValueType(0); |
3288 | 706 | SDVTList VTs = DAG.getVTList(VT, VT); |
3289 | 706 | // See if multiply or divide can be lowered using two-result operations. |
3290 | 706 | // We just need the low half of the multiply; try both the signed |
3291 | 706 | // and unsigned forms. If the target supports both SMUL_LOHI and |
3292 | 706 | // UMUL_LOHI, form a preference by checking which forms of plain |
3293 | 706 | // MULH it supports. |
3294 | 706 | bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); |
3295 | 706 | bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); |
3296 | 706 | bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); |
3297 | 706 | bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); |
3298 | 706 | unsigned OpToUse = 0; |
3299 | 706 | if (HasSMUL_LOHI && !HasMULHS7 ) { |
3300 | 7 | OpToUse = ISD::SMUL_LOHI; |
3301 | 699 | } else if (HasUMUL_LOHI && !HasMULHU0 ) { |
3302 | 0 | OpToUse = ISD::UMUL_LOHI; |
3303 | 699 | } else if (HasSMUL_LOHI) { |
3304 | 0 | OpToUse = ISD::SMUL_LOHI; |
3305 | 699 | } else if (HasUMUL_LOHI) { |
3306 | 0 | OpToUse = ISD::UMUL_LOHI; |
3307 | 0 | } |
3308 | 706 | if (OpToUse) { |
3309 | 7 | Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), |
3310 | 7 | Node->getOperand(1))); |
3311 | 7 | break; |
3312 | 7 | } |
3313 | 699 | |
3314 | 699 | SDValue Lo, Hi; |
3315 | 699 | EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); |
3316 | 699 | if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && |
3317 | 699 | TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && |
3318 | 699 | TLI.isOperationLegalOrCustom(ISD::SHL, VT) && |
3319 | 699 | TLI.isOperationLegalOrCustom(ISD::OR, VT) && |
3320 | 699 | TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, |
3321 | 699 | TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { |
3322 | 671 | Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); |
3323 | 671 | Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); |
3324 | 671 | SDValue Shift = |
3325 | 671 | DAG.getConstant(HalfType.getSizeInBits(), dl, |
3326 | 671 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); |
3327 | 671 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); |
3328 | 671 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); |
3329 | 671 | } |
3330 | 699 | break; |
3331 | 699 | } |
3332 | 699 | case ISD::FSHL: |
3333 | 30 | case ISD::FSHR: |
3334 | 30 | if (TLI.expandFunnelShift(Node, Tmp1, DAG)) |
3335 | 30 | Results.push_back(Tmp1); |
3336 | 30 | break; |
3337 | 30 | case ISD::ROTL: |
3338 | 0 | case ISD::ROTR: |
3339 | 0 | if (TLI.expandROT(Node, Tmp1, DAG)) |
3340 | 0 | Results.push_back(Tmp1); |
3341 | 0 | break; |
3342 | 212 | case ISD::SADDSAT: |
3343 | 212 | case ISD::UADDSAT: |
3344 | 212 | case ISD::SSUBSAT: |
3345 | 212 | case ISD::USUBSAT: |
3346 | 212 | Results.push_back(TLI.expandAddSubSat(Node, DAG)); |
3347 | 212 | break; |
3348 | 212 | case ISD::SMULFIX: |
3349 | 74 | case ISD::SMULFIXSAT: |
3350 | 74 | case ISD::UMULFIX: |
3351 | 74 | Results.push_back(TLI.expandFixedPointMul(Node, DAG)); |
3352 | 74 | break; |
3353 | 74 | case ISD::ADDCARRY: |
3354 | 4 | case ISD::SUBCARRY: { |
3355 | 4 | SDValue LHS = Node->getOperand(0); |
3356 | 4 | SDValue RHS = Node->getOperand(1); |
3357 | 4 | SDValue Carry = Node->getOperand(2); |
3358 | 4 | |
3359 | 4 | bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; |
3360 | 4 | |
3361 | 4 | // Initial add of the 2 operands. |
3362 | 4 | unsigned Op = IsAdd ? ISD::ADD : ISD::SUB0 ; |
3363 | 4 | EVT VT = LHS.getValueType(); |
3364 | 4 | SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS); |
3365 | 4 | |
3366 | 4 | // Initial check for overflow. |
3367 | 4 | EVT CarryType = Node->getValueType(1); |
3368 | 4 | EVT SetCCType = getSetCCResultType(Node->getValueType(0)); |
3369 | 4 | ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT0 ; |
3370 | 4 | SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); |
3371 | 4 | |
3372 | 4 | // Add of the sum and the carry. |
3373 | 4 | SDValue CarryExt = |
3374 | 4 | DAG.getZeroExtendInReg(DAG.getZExtOrTrunc(Carry, dl, VT), dl, MVT::i1); |
3375 | 4 | SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt); |
3376 | 4 | |
3377 | 4 | // Second check for overflow. If we are adding, we can only overflow if the |
3378 | 4 | // initial sum is all 1s ang the carry is set, resulting in a new sum of 0. |
3379 | 4 | // If we are subtracting, we can only overflow if the initial sum is 0 and |
3380 | 4 | // the carry is set, resulting in a new sum of all 1s. |
3381 | 4 | SDValue Zero = DAG.getConstant(0, dl, VT); |
3382 | 4 | SDValue Overflow2 = |
3383 | 4 | IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) |
3384 | 4 | : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ)0 ; |
3385 | 4 | Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, |
3386 | 4 | DAG.getZExtOrTrunc(Carry, dl, SetCCType)); |
3387 | 4 | |
3388 | 4 | SDValue ResultCarry = |
3389 | 4 | DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); |
3390 | 4 | |
3391 | 4 | Results.push_back(Sum2); |
3392 | 4 | Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT)); |
3393 | 4 | break; |
3394 | 4 | } |
3395 | 44 | case ISD::SADDO: |
3396 | 44 | case ISD::SSUBO: { |
3397 | 44 | SDValue Result, Overflow; |
3398 | 44 | TLI.expandSADDSUBO(Node, Result, Overflow, DAG); |
3399 | 44 | Results.push_back(Result); |
3400 | 44 | Results.push_back(Overflow); |
3401 | 44 | break; |
3402 | 44 | } |
3403 | 44 | case ISD::UADDO: |
3404 | 39 | case ISD::USUBO: { |
3405 | 39 | SDValue Result, Overflow; |
3406 | 39 | TLI.expandUADDSUBO(Node, Result, Overflow, DAG); |
3407 | 39 | Results.push_back(Result); |
3408 | 39 | Results.push_back(Overflow); |
3409 | 39 | break; |
3410 | 39 | } |
3411 | 60 | case ISD::UMULO: |
3412 | 60 | case ISD::SMULO: { |
3413 | 60 | SDValue Result, Overflow; |
3414 | 60 | if (TLI.expandMULO(Node, Result, Overflow, DAG)) { |
3415 | 60 | Results.push_back(Result); |
3416 | 60 | Results.push_back(Overflow); |
3417 | 60 | } |
3418 | 60 | break; |
3419 | 60 | } |
3420 | 60 | case ISD::BUILD_PAIR: { |
3421 | 24 | EVT PairTy = Node->getValueType(0); |
3422 | 24 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); |
3423 | 24 | Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); |
3424 | 24 | Tmp2 = DAG.getNode( |
3425 | 24 | ISD::SHL, dl, PairTy, Tmp2, |
3426 | 24 | DAG.getConstant(PairTy.getSizeInBits() / 2, dl, |
3427 | 24 | TLI.getShiftAmountTy(PairTy, DAG.getDataLayout()))); |
3428 | 24 | Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); |
3429 | 24 | break; |
3430 | 60 | } |
3431 | 212 | case ISD::SELECT: |
3432 | 212 | Tmp1 = Node->getOperand(0); |
3433 | 212 | Tmp2 = Node->getOperand(1); |
3434 | 212 | Tmp3 = Node->getOperand(2); |
3435 | 212 | if (Tmp1.getOpcode() == ISD::SETCC) { |
3436 | 155 | Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), |
3437 | 155 | Tmp2, Tmp3, |
3438 | 155 | cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); |
3439 | 155 | } else { |
3440 | 57 | Tmp1 = DAG.getSelectCC(dl, Tmp1, |
3441 | 57 | DAG.getConstant(0, dl, Tmp1.getValueType()), |
3442 | 57 | Tmp2, Tmp3, ISD::SETNE); |
3443 | 57 | } |
3444 | 212 | Tmp1->setFlags(Node->getFlags()); |
3445 | 212 | Results.push_back(Tmp1); |
3446 | 212 | break; |
3447 | 374 | case ISD::BR_JT: { |
3448 | 374 | SDValue Chain = Node->getOperand(0); |
3449 | 374 | SDValue Table = Node->getOperand(1); |
3450 | 374 | SDValue Index = Node->getOperand(2); |
3451 | 374 | |
3452 | 374 | const DataLayout &TD = DAG.getDataLayout(); |
3453 | 374 | EVT PTy = TLI.getPointerTy(TD); |
3454 | 374 | |
3455 | 374 | unsigned EntrySize = |
3456 | 374 | DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); |
3457 | 374 | |
3458 | 374 | // For power-of-two jumptable entry sizes convert multiplication to a shift. |
3459 | 374 | // This transformation needs to be done here since otherwise the MIPS |
3460 | 374 | // backend will end up emitting a three instruction multiply sequence |
3461 | 374 | // instead of a single shift and MSP430 will call a runtime function. |
3462 | 374 | if (llvm::isPowerOf2_32(EntrySize)) |
3463 | 374 | Index = DAG.getNode( |
3464 | 374 | ISD::SHL, dl, Index.getValueType(), Index, |
3465 | 374 | DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType())); |
3466 | 0 | else |
3467 | 0 | Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, |
3468 | 0 | DAG.getConstant(EntrySize, dl, Index.getValueType())); |
3469 | 374 | SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), |
3470 | 374 | Index, Table); |
3471 | 374 | |
3472 | 374 | EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); |
3473 | 374 | SDValue LD = DAG.getExtLoad( |
3474 | 374 | ISD::SEXTLOAD, dl, PTy, Chain, Addr, |
3475 | 374 | MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); |
3476 | 374 | Addr = LD; |
3477 | 374 | if (TLI.isJumpTableRelative()) { |
3478 | 280 | // For PIC, the sequence is: |
3479 | 280 | // BRIND(load(Jumptable + index) + RelocBase) |
3480 | 280 | // RelocBase can be JumpTable, GOT or some sort of global base. |
3481 | 280 | Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, |
3482 | 280 | TLI.getPICJumpTableRelocBase(Table, DAG)); |
3483 | 280 | } |
3484 | 374 | |
3485 | 374 | Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG); |
3486 | 374 | Results.push_back(Tmp1); |
3487 | 374 | break; |
3488 | 60 | } |
3489 | 11.4k | case ISD::BRCOND: |
3490 | 11.4k | // Expand brcond's setcc into its constituent parts and create a BR_CC |
3491 | 11.4k | // Node. |
3492 | 11.4k | Tmp1 = Node->getOperand(0); |
3493 | 11.4k | Tmp2 = Node->getOperand(1); |
3494 | 11.4k | if (Tmp2.getOpcode() == ISD::SETCC) { |
3495 | 1 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, |
3496 | 1 | Tmp1, Tmp2.getOperand(2), |
3497 | 1 | Tmp2.getOperand(0), Tmp2.getOperand(1), |
3498 | 1 | Node->getOperand(2)); |
3499 | 11.4k | } else { |
3500 | 11.4k | // We test only the i1 bit. Skip the AND if UNDEF or another AND. |
3501 | 11.4k | if (Tmp2.isUndef() || |
3502 | 11.4k | (Tmp2.getOpcode() == ISD::AND && |
3503 | 11.4k | isa<ConstantSDNode>(Tmp2.getOperand(1))1.57k && |
3504 | 11.4k | cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 11.35k )) |
3505 | 1.35k | Tmp3 = Tmp2; |
3506 | 10.0k | else |
3507 | 10.0k | Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, |
3508 | 10.0k | DAG.getConstant(1, dl, Tmp2.getValueType())); |
3509 | 11.4k | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, |
3510 | 11.4k | DAG.getCondCode(ISD::SETNE), Tmp3, |
3511 | 11.4k | DAG.getConstant(0, dl, Tmp3.getValueType()), |
3512 | 11.4k | Node->getOperand(2)); |
3513 | 11.4k | } |
3514 | 11.4k | Results.push_back(Tmp1); |
3515 | 11.4k | break; |
3516 | 4.52k | case ISD::SETCC: { |
3517 | 4.52k | Tmp1 = Node->getOperand(0); |
3518 | 4.52k | Tmp2 = Node->getOperand(1); |
3519 | 4.52k | Tmp3 = Node->getOperand(2); |
3520 | 4.52k | bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, |
3521 | 4.52k | Tmp3, NeedInvert, dl); |
3522 | 4.52k | |
3523 | 4.52k | if (Legalized) { |
3524 | 748 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the |
3525 | 748 | // condition code, create a new SETCC node. |
3526 | 748 | if (Tmp3.getNode()) |
3527 | 409 | Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), |
3528 | 409 | Tmp1, Tmp2, Tmp3, Node->getFlags()); |
3529 | 748 | |
3530 | 748 | // If we expanded the SETCC by inverting the condition code, then wrap |
3531 | 748 | // the existing SETCC in a NOT to restore the intended condition. |
3532 | 748 | if (NeedInvert) |
3533 | 222 | Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); |
3534 | 748 | |
3535 | 748 | Results.push_back(Tmp1); |
3536 | 748 | break; |
3537 | 748 | } |
3538 | 3.77k | |
3539 | 3.77k | // Otherwise, SETCC for the given comparison type must be completely |
3540 | 3.77k | // illegal; expand it into a SELECT_CC. |
3541 | 3.77k | EVT VT = Node->getValueType(0); |
3542 | 3.77k | int TrueValue; |
3543 | 3.77k | switch (TLI.getBooleanContents(Tmp1.getValueType())) { |
3544 | 3.77k | case TargetLowering::ZeroOrOneBooleanContent: |
3545 | 2.41k | case TargetLowering::UndefinedBooleanContent: |
3546 | 2.41k | TrueValue = 1; |
3547 | 2.41k | break; |
3548 | 2.41k | case TargetLowering::ZeroOrNegativeOneBooleanContent: |
3549 | 1.36k | TrueValue = -1; |
3550 | 1.36k | break; |
3551 | 3.77k | } |
3552 | 3.77k | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, |
3553 | 3.77k | DAG.getConstant(TrueValue, dl, VT), |
3554 | 3.77k | DAG.getConstant(0, dl, VT), |
3555 | 3.77k | Tmp3); |
3556 | 3.77k | Tmp1->setFlags(Node->getFlags()); |
3557 | 3.77k | Results.push_back(Tmp1); |
3558 | 3.77k | break; |
3559 | 3.77k | } |
3560 | 3.77k | case ISD::SELECT_CC: { |
3561 | 2.12k | Tmp1 = Node->getOperand(0); // LHS |
3562 | 2.12k | Tmp2 = Node->getOperand(1); // RHS |
3563 | 2.12k | Tmp3 = Node->getOperand(2); // True |
3564 | 2.12k | Tmp4 = Node->getOperand(3); // False |
3565 | 2.12k | EVT VT = Node->getValueType(0); |
3566 | 2.12k | SDValue CC = Node->getOperand(4); |
3567 | 2.12k | ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); |
3568 | 2.12k | |
3569 | 2.12k | if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) { |
3570 | 1.81k | // If the condition code is legal, then we need to expand this |
3571 | 1.81k | // node using SETCC and SELECT. |
3572 | 1.81k | EVT CmpVT = Tmp1.getValueType(); |
3573 | 1.81k | assert(!TLI.isOperationExpand(ISD::SELECT, VT) && |
3574 | 1.81k | "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " |
3575 | 1.81k | "expanded."); |
3576 | 1.81k | EVT CCVT = getSetCCResultType(CmpVT); |
3577 | 1.81k | SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); |
3578 | 1.81k | Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); |
3579 | 1.81k | break; |
3580 | 1.81k | } |
3581 | 309 | |
3582 | 309 | // SELECT_CC is legal, so the condition code must not be. |
3583 | 309 | bool Legalized = false; |
3584 | 309 | // Try to legalize by inverting the condition. This is for targets that |
3585 | 309 | // might support an ordered version of a condition, but not the unordered |
3586 | 309 | // version (or vice versa). |
3587 | 309 | ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, |
3588 | 309 | Tmp1.getValueType().isInteger()); |
3589 | 309 | if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) { |
3590 | 257 | // Use the new condition code and swap true and false |
3591 | 257 | Legalized = true; |
3592 | 257 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); |
3593 | 257 | Tmp1->setFlags(Node->getFlags()); |
3594 | 257 | } else { |
3595 | 52 | // If The inverse is not legal, then try to swap the arguments using |
3596 | 52 | // the inverse condition code. |
3597 | 52 | ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); |
3598 | 52 | if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) { |
3599 | 13 | // The swapped inverse condition is legal, so swap true and false, |
3600 | 13 | // lhs and rhs. |
3601 | 13 | Legalized = true; |
3602 | 13 | Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); |
3603 | 13 | Tmp1->setFlags(Node->getFlags()); |
3604 | 13 | } |
3605 | 52 | } |
3606 | 309 | |
3607 | 309 | if (!Legalized) { |
3608 | 39 | Legalized = LegalizeSetCCCondCode( |
3609 | 39 | getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert, |
3610 | 39 | dl); |
3611 | 39 | |
3612 | 39 | assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); |
3613 | 39 | |
3614 | 39 | // If we expanded the SETCC by inverting the condition code, then swap |
3615 | 39 | // the True/False operands to match. |
3616 | 39 | if (NeedInvert) |
3617 | 0 | std::swap(Tmp3, Tmp4); |
3618 | 39 | |
3619 | 39 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the |
3620 | 39 | // condition code, create a new SELECT_CC node. |
3621 | 39 | if (CC.getNode()) { |
3622 | 13 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), |
3623 | 13 | Tmp1, Tmp2, Tmp3, Tmp4, CC); |
3624 | 26 | } else { |
3625 | 26 | Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); |
3626 | 26 | CC = DAG.getCondCode(ISD::SETNE); |
3627 | 26 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, |
3628 | 26 | Tmp2, Tmp3, Tmp4, CC); |
3629 | 26 | } |
3630 | 39 | Tmp1->setFlags(Node->getFlags()); |
3631 | 39 | } |
3632 | 309 | Results.push_back(Tmp1); |
3633 | 309 | break; |
3634 | 309 | } |
3635 | 309 | case ISD::BR_CC: { |
3636 | 10 | Tmp1 = Node->getOperand(0); // Chain |
3637 | 10 | Tmp2 = Node->getOperand(2); // LHS |
3638 | 10 | Tmp3 = Node->getOperand(3); // RHS |
3639 | 10 | Tmp4 = Node->getOperand(1); // CC |
3640 | 10 | |
3641 | 10 | bool Legalized = LegalizeSetCCCondCode(getSetCCResultType( |
3642 | 10 | Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl); |
3643 | 10 | (void)Legalized; |
3644 | 10 | assert(Legalized && "Can't legalize BR_CC with legal condition!"); |
3645 | 10 | |
3646 | 10 | assert(!NeedInvert && "Don't know how to invert BR_CC!"); |
3647 | 10 | |
3648 | 10 | // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC |
3649 | 10 | // node. |
3650 | 10 | if (Tmp4.getNode()) { |
3651 | 0 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, |
3652 | 0 | Tmp4, Tmp2, Tmp3, Node->getOperand(4)); |
3653 | 10 | } else { |
3654 | 10 | Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType()); |
3655 | 10 | Tmp4 = DAG.getCondCode(ISD::SETNE); |
3656 | 10 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, |
3657 | 10 | Tmp2, Tmp3, Node->getOperand(4)); |
3658 | 10 | } |
3659 | 10 | Results.push_back(Tmp1); |
3660 | 10 | break; |
3661 | 309 | } |
3662 | 55.7k | case ISD::BUILD_VECTOR: |
3663 | 55.7k | Results.push_back(ExpandBUILD_VECTOR(Node)); |
3664 | 55.7k | break; |
3665 | 309 | case ISD::SRA: |
3666 | 38 | case ISD::SRL: |
3667 | 38 | case ISD::SHL: { |
3668 | 38 | // Scalarize vector SRA/SRL/SHL. |
3669 | 38 | EVT VT = Node->getValueType(0); |
3670 | 38 | assert(VT.isVector() && "Unable to legalize non-vector shift"); |
3671 | 38 | assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); |
3672 | 38 | unsigned NumElem = VT.getVectorNumElements(); |
3673 | 38 | |
3674 | 38 | SmallVector<SDValue, 8> Scalars; |
3675 | 138 | for (unsigned Idx = 0; Idx < NumElem; Idx++100 ) { |
3676 | 100 | SDValue Ex = DAG.getNode( |
3677 | 100 | ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0), |
3678 | 100 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
3679 | 100 | SDValue Sh = DAG.getNode( |
3680 | 100 | ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1), |
3681 | 100 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
3682 | 100 | Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, |
3683 | 100 | VT.getScalarType(), Ex, Sh)); |
3684 | 100 | } |
3685 | 38 | |
3686 | 38 | SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); |
3687 | 38 | ReplaceNode(SDValue(Node, 0), Result); |
3688 | 38 | break; |
3689 | 38 | } |
3690 | 38 | case ISD::VECREDUCE_FADD: |
3691 | 0 | case ISD::VECREDUCE_FMUL: |
3692 | 0 | case ISD::VECREDUCE_ADD: |
3693 | 0 | case ISD::VECREDUCE_MUL: |
3694 | 0 | case ISD::VECREDUCE_AND: |
3695 | 0 | case ISD::VECREDUCE_OR: |
3696 | 0 | case ISD::VECREDUCE_XOR: |
3697 | 0 | case ISD::VECREDUCE_SMAX: |
3698 | 0 | case ISD::VECREDUCE_SMIN: |
3699 | 0 | case ISD::VECREDUCE_UMAX: |
3700 | 0 | case ISD::VECREDUCE_UMIN: |
3701 | 0 | case ISD::VECREDUCE_FMAX: |
3702 | 0 | case ISD::VECREDUCE_FMIN: |
3703 | 0 | Results.push_back(TLI.expandVecReduce(Node, DAG)); |
3704 | 0 | break; |
3705 | 25.3k | case ISD::GLOBAL_OFFSET_TABLE: |
3706 | 25.3k | case ISD::GlobalAddress: |
3707 | 25.3k | case ISD::GlobalTLSAddress: |
3708 | 25.3k | case ISD::ExternalSymbol: |
3709 | 25.3k | case ISD::ConstantPool: |
3710 | 25.3k | case ISD::JumpTable: |
3711 | 25.3k | case ISD::INTRINSIC_W_CHAIN: |
3712 | 25.3k | case ISD::INTRINSIC_WO_CHAIN: |
3713 | 25.3k | case ISD::INTRINSIC_VOID: |
3714 | 25.3k | // FIXME: Custom lowering for these operations shouldn't return null! |
3715 | 25.3k | break; |
3716 | 210k | } |
3717 | 210k | |
3718 | 210k | // Replace the original node with the legalized result. |
3719 | 210k | if (Results.empty()) { |
3720 | 64.6k | LLVM_DEBUG(dbgs() << "Cannot expand node\n"); |
3721 | 64.6k | return false; |
3722 | 64.6k | } |
3723 | 145k | |
3724 | 145k | LLVM_DEBUG(dbgs() << "Successfully expanded node\n"); |
3725 | 145k | ReplaceNode(Node, Results.data()); |
3726 | 145k | return true; |
3727 | 145k | } |
3728 | | |
3729 | 64.8k | void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { |
3730 | 64.8k | LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n"); |
3731 | 64.8k | SmallVector<SDValue, 8> Results; |
3732 | 64.8k | SDLoc dl(Node); |
3733 | 64.8k | // FIXME: Check flags on the node to see if we can use a finite call. |
3734 | 64.8k | bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath110 ; |
3735 | 64.8k | unsigned Opc = Node->getOpcode(); |
3736 | 64.8k | switch (Opc) { |
3737 | 64.8k | case ISD::ATOMIC_FENCE: { |
3738 | 13 | // If the target didn't lower this, lower it to '__sync_synchronize()' call |
3739 | 13 | // FIXME: handle "fence singlethread" more efficiently. |
3740 | 13 | TargetLowering::ArgListTy Args; |
3741 | 13 | |
3742 | 13 | TargetLowering::CallLoweringInfo CLI(DAG); |
3743 | 13 | CLI.setDebugLoc(dl) |
3744 | 13 | .setChain(Node->getOperand(0)) |
3745 | 13 | .setLibCallee( |
3746 | 13 | CallingConv::C, Type::getVoidTy(*DAG.getContext()), |
3747 | 13 | DAG.getExternalSymbol("__sync_synchronize", |
3748 | 13 | TLI.getPointerTy(DAG.getDataLayout())), |
3749 | 13 | std::move(Args)); |
3750 | 13 | |
3751 | 13 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3752 | 13 | |
3753 | 13 | Results.push_back(CallResult.second); |
3754 | 13 | break; |
3755 | 64.8k | } |
3756 | 64.8k | // By default, atomic intrinsics are marked Legal and lowered. Targets |
3757 | 64.8k | // which don't support them directly, however, may want libcalls, in which |
3758 | 64.8k | // case they mark them Expand, and we get here. |
3759 | 64.8k | case ISD::ATOMIC_SWAP: |
3760 | 86 | case ISD::ATOMIC_LOAD_ADD: |
3761 | 86 | case ISD::ATOMIC_LOAD_SUB: |
3762 | 86 | case ISD::ATOMIC_LOAD_AND: |
3763 | 86 | case ISD::ATOMIC_LOAD_CLR: |
3764 | 86 | case ISD::ATOMIC_LOAD_OR: |
3765 | 86 | case ISD::ATOMIC_LOAD_XOR: |
3766 | 86 | case ISD::ATOMIC_LOAD_NAND: |
3767 | 86 | case ISD::ATOMIC_LOAD_MIN: |
3768 | 86 | case ISD::ATOMIC_LOAD_MAX: |
3769 | 86 | case ISD::ATOMIC_LOAD_UMIN: |
3770 | 86 | case ISD::ATOMIC_LOAD_UMAX: |
3771 | 86 | case ISD::ATOMIC_CMP_SWAP: { |
3772 | 86 | MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); |
3773 | 86 | RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); |
3774 | 86 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); |
3775 | 86 | |
3776 | 86 | std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false); |
3777 | 86 | Results.push_back(Tmp.first); |
3778 | 86 | Results.push_back(Tmp.second); |
3779 | 86 | break; |
3780 | 86 | } |
3781 | 86 | case ISD::TRAP: { |
3782 | 0 | // If this operation is not supported, lower it to 'abort()' call |
3783 | 0 | TargetLowering::ArgListTy Args; |
3784 | 0 | TargetLowering::CallLoweringInfo CLI(DAG); |
3785 | 0 | CLI.setDebugLoc(dl) |
3786 | 0 | .setChain(Node->getOperand(0)) |
3787 | 0 | .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), |
3788 | 0 | DAG.getExternalSymbol( |
3789 | 0 | "abort", TLI.getPointerTy(DAG.getDataLayout())), |
3790 | 0 | std::move(Args)); |
3791 | 0 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3792 | 0 |
|
3793 | 0 | Results.push_back(CallResult.second); |
3794 | 0 | break; |
3795 | 86 | } |
3796 | 117 | case ISD::FMINNUM: |
3797 | 117 | case ISD::STRICT_FMINNUM: |
3798 | 117 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, |
3799 | 117 | RTLIB::FMIN_F80, RTLIB::FMIN_F128, |
3800 | 117 | RTLIB::FMIN_PPCF128)); |
3801 | 117 | break; |
3802 | 160 | case ISD::FMAXNUM: |
3803 | 160 | case ISD::STRICT_FMAXNUM: |
3804 | 160 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, |
3805 | 160 | RTLIB::FMAX_F80, RTLIB::FMAX_F128, |
3806 | 160 | RTLIB::FMAX_PPCF128)); |
3807 | 160 | break; |
3808 | 160 | case ISD::FSQRT: |
3809 | 7 | case ISD::STRICT_FSQRT: |
3810 | 7 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, |
3811 | 7 | RTLIB::SQRT_F80, RTLIB::SQRT_F128, |
3812 | 7 | RTLIB::SQRT_PPCF128)); |
3813 | 7 | break; |
3814 | 7 | case ISD::FCBRT: |
3815 | 2 | Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, |
3816 | 2 | RTLIB::CBRT_F80, RTLIB::CBRT_F128, |
3817 | 2 | RTLIB::CBRT_PPCF128)); |
3818 | 2 | break; |
3819 | 330 | case ISD::FSIN: |
3820 | 330 | case ISD::STRICT_FSIN: |
3821 | 330 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, |
3822 | 330 | RTLIB::SIN_F80, RTLIB::SIN_F128, |
3823 | 330 | RTLIB::SIN_PPCF128)); |
3824 | 330 | break; |
3825 | 330 | case ISD::FCOS: |
3826 | 265 | case ISD::STRICT_FCOS: |
3827 | 265 | Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, |
3828 | 265 | RTLIB::COS_F80, RTLIB::COS_F128, |
3829 | 265 | RTLIB::COS_PPCF128)); |
3830 | 265 | break; |
3831 | 265 | case ISD::FSINCOS: |
3832 | 34 | // Expand into sincos libcall. |
3833 | 34 | ExpandSinCosLibCall(Node, Results); |
3834 | 34 | break; |
3835 | 265 | case ISD::FLOG: |
3836 | 229 | case ISD::STRICT_FLOG: |
3837 | 229 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite)11 ) |
3838 | 4 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32, |
3839 | 4 | RTLIB::LOG_FINITE_F64, |
3840 | 4 | RTLIB::LOG_FINITE_F80, |
3841 | 4 | RTLIB::LOG_FINITE_F128, |
3842 | 4 | RTLIB::LOG_FINITE_PPCF128)); |
3843 | 225 | else |
3844 | 225 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, |
3845 | 225 | RTLIB::LOG_F80, RTLIB::LOG_F128, |
3846 | 225 | RTLIB::LOG_PPCF128)); |
3847 | 229 | break; |
3848 | 229 | case ISD::FLOG2: |
3849 | 223 | case ISD::STRICT_FLOG2: |
3850 | 223 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite)11 ) |
3851 | 4 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32, |
3852 | 4 | RTLIB::LOG2_FINITE_F64, |
3853 | 4 | RTLIB::LOG2_FINITE_F80, |
3854 | 4 | RTLIB::LOG2_FINITE_F128, |
3855 | 4 | RTLIB::LOG2_FINITE_PPCF128)); |
3856 | 219 | else |
3857 | 219 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, |
3858 | 219 | RTLIB::LOG2_F80, RTLIB::LOG2_F128, |
3859 | 219 | RTLIB::LOG2_PPCF128)); |
3860 | 223 | break; |
3861 | 255 | case ISD::FLOG10: |
3862 | 255 | case ISD::STRICT_FLOG10: |
3863 | 255 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite)11 ) |
3864 | 4 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32, |
3865 | 4 | RTLIB::LOG10_FINITE_F64, |
3866 | 4 | RTLIB::LOG10_FINITE_F80, |
3867 | 4 | RTLIB::LOG10_FINITE_F128, |
3868 | 4 | RTLIB::LOG10_FINITE_PPCF128)); |
3869 | 251 | else |
3870 | 251 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, |
3871 | 251 | RTLIB::LOG10_F80, RTLIB::LOG10_F128, |
3872 | 251 | RTLIB::LOG10_PPCF128)); |
3873 | 255 | break; |
3874 | 336 | case ISD::FEXP: |
3875 | 336 | case ISD::STRICT_FEXP: |
3876 | 336 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite)11 ) |
3877 | 4 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32, |
3878 | 4 | RTLIB::EXP_FINITE_F64, |
3879 | 4 | RTLIB::EXP_FINITE_F80, |
3880 | 4 | RTLIB::EXP_FINITE_F128, |
3881 | 4 | RTLIB::EXP_FINITE_PPCF128)); |
3882 | 332 | else |
3883 | 332 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, |
3884 | 332 | RTLIB::EXP_F80, RTLIB::EXP_F128, |
3885 | 332 | RTLIB::EXP_PPCF128)); |
3886 | 336 | break; |
3887 | 336 | case ISD::FEXP2: |
3888 | 233 | case ISD::STRICT_FEXP2: |
3889 | 233 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite)11 ) |
3890 | 4 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32, |
3891 | 4 | RTLIB::EXP2_FINITE_F64, |
3892 | 4 | RTLIB::EXP2_FINITE_F80, |
3893 | 4 | RTLIB::EXP2_FINITE_F128, |
3894 | 4 | RTLIB::EXP2_FINITE_PPCF128)); |
3895 | 229 | else |
3896 | 229 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, |
3897 | 229 | RTLIB::EXP2_F80, RTLIB::EXP2_F128, |
3898 | 229 | RTLIB::EXP2_PPCF128)); |
3899 | 233 | break; |
3900 | 233 | case ISD::FTRUNC: |
3901 | 68 | case ISD::STRICT_FTRUNC: |
3902 | 68 | Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, |
3903 | 68 | RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, |
3904 | 68 | RTLIB::TRUNC_PPCF128)); |
3905 | 68 | break; |
3906 | 101 | case ISD::FFLOOR: |
3907 | 101 | case ISD::STRICT_FFLOOR: |
3908 | 101 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, |
3909 | 101 | RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, |
3910 | 101 | RTLIB::FLOOR_PPCF128)); |
3911 | 101 | break; |
3912 | 101 | case ISD::FCEIL: |
3913 | 58 | case ISD::STRICT_FCEIL: |
3914 | 58 | Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, |
3915 | 58 | RTLIB::CEIL_F80, RTLIB::CEIL_F128, |
3916 | 58 | RTLIB::CEIL_PPCF128)); |
3917 | 58 | break; |
3918 | 80 | case ISD::FRINT: |
3919 | 80 | case ISD::STRICT_FRINT: |
3920 | 80 | Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, |
3921 | 80 | RTLIB::RINT_F80, RTLIB::RINT_F128, |
3922 | 80 | RTLIB::RINT_PPCF128)); |
3923 | 80 | break; |
3924 | 98 | case ISD::FNEARBYINT: |
3925 | 98 | case ISD::STRICT_FNEARBYINT: |
3926 | 98 | Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, |
3927 | 98 | RTLIB::NEARBYINT_F64, |
3928 | 98 | RTLIB::NEARBYINT_F80, |
3929 | 98 | RTLIB::NEARBYINT_F128, |
3930 | 98 | RTLIB::NEARBYINT_PPCF128)); |
3931 | 98 | break; |
3932 | 98 | case ISD::FROUND: |
3933 | 70 | case ISD::STRICT_FROUND: |
3934 | 70 | Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32, |
3935 | 70 | RTLIB::ROUND_F64, |
3936 | 70 | RTLIB::ROUND_F80, |
3937 | 70 | RTLIB::ROUND_F128, |
3938 | 70 | RTLIB::ROUND_PPCF128)); |
3939 | 70 | break; |
3940 | 269 | case ISD::FPOWI: |
3941 | 269 | case ISD::STRICT_FPOWI: |
3942 | 269 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, |
3943 | 269 | RTLIB::POWI_F80, RTLIB::POWI_F128, |
3944 | 269 | RTLIB::POWI_PPCF128)); |
3945 | 269 | break; |
3946 | 300 | case ISD::FPOW: |
3947 | 300 | case ISD::STRICT_FPOW: |
3948 | 300 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite)11 ) |
3949 | 4 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32, |
3950 | 4 | RTLIB::POW_FINITE_F64, |
3951 | 4 | RTLIB::POW_FINITE_F80, |
3952 | 4 | RTLIB::POW_FINITE_F128, |
3953 | 4 | RTLIB::POW_FINITE_PPCF128)); |
3954 | 296 | else |
3955 | 296 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, |
3956 | 296 | RTLIB::POW_F80, RTLIB::POW_F128, |
3957 | 296 | RTLIB::POW_PPCF128)); |
3958 | 300 | break; |
3959 | 300 | case ISD::FDIV: |
3960 | 23 | Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, |
3961 | 23 | RTLIB::DIV_F80, RTLIB::DIV_F128, |
3962 | 23 | RTLIB::DIV_PPCF128)); |
3963 | 23 | break; |
3964 | 300 | case ISD::FREM: |
3965 | 219 | case ISD::STRICT_FREM: |
3966 | 219 | Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, |
3967 | 219 | RTLIB::REM_F80, RTLIB::REM_F128, |
3968 | 219 | RTLIB::REM_PPCF128)); |
3969 | 219 | break; |
3970 | 241 | case ISD::FMA: |
3971 | 241 | case ISD::STRICT_FMA: |
3972 | 241 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, |
3973 | 241 | RTLIB::FMA_F80, RTLIB::FMA_F128, |
3974 | 241 | RTLIB::FMA_PPCF128)); |
3975 | 241 | break; |
3976 | 241 | case ISD::FADD: |
3977 | 116 | Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, |
3978 | 116 | RTLIB::ADD_F80, RTLIB::ADD_F128, |
3979 | 116 | RTLIB::ADD_PPCF128)); |
3980 | 116 | break; |
3981 | 241 | case ISD::FMUL: |
3982 | 224 | Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, |
3983 | 224 | RTLIB::MUL_F80, RTLIB::MUL_F128, |
3984 | 224 | RTLIB::MUL_PPCF128)); |
3985 | 224 | break; |
3986 | 565 | case ISD::FP16_TO_FP: |
3987 | 565 | if (Node->getValueType(0) == MVT::f32) { |
3988 | 565 | Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); |
3989 | 565 | } |
3990 | 565 | break; |
3991 | 551 | case ISD::FP_TO_FP16: { |
3992 | 551 | RTLIB::Libcall LC = |
3993 | 551 | RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); |
3994 | 551 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"); |
3995 | 551 | Results.push_back(ExpandLibCall(LC, Node, false)); |
3996 | 551 | break; |
3997 | 241 | } |
3998 | 241 | case ISD::FSUB: |
3999 | 118 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, |
4000 | 118 | RTLIB::SUB_F80, RTLIB::SUB_F128, |
4001 | 118 | RTLIB::SUB_PPCF128)); |
4002 | 118 | break; |
4003 | 241 | case ISD::SREM: |
4004 | 24 | Results.push_back(ExpandIntLibCall(Node, true, |
4005 | 24 | RTLIB::SREM_I8, |
4006 | 24 | RTLIB::SREM_I16, RTLIB::SREM_I32, |
4007 | 24 | RTLIB::SREM_I64, RTLIB::SREM_I128)); |
4008 | 24 | break; |
4009 | 241 | case ISD::UREM: |
4010 | 68 | Results.push_back(ExpandIntLibCall(Node, false, |
4011 | 68 | RTLIB::UREM_I8, |
4012 | 68 | RTLIB::UREM_I16, RTLIB::UREM_I32, |
4013 | 68 | RTLIB::UREM_I64, RTLIB::UREM_I128)); |
4014 | 68 | break; |
4015 | 241 | case ISD::SDIV: |
4016 | 64 | Results.push_back(ExpandIntLibCall(Node, true, |
4017 | 64 | RTLIB::SDIV_I8, |
4018 | 64 | RTLIB::SDIV_I16, RTLIB::SDIV_I32, |
4019 | 64 | RTLIB::SDIV_I64, RTLIB::SDIV_I128)); |
4020 | 64 | break; |
4021 | 241 | case ISD::UDIV: |
4022 | 75 | Results.push_back(ExpandIntLibCall(Node, false, |
4023 | 75 | RTLIB::UDIV_I8, |
4024 | 75 | RTLIB::UDIV_I16, RTLIB::UDIV_I32, |
4025 | 75 | RTLIB::UDIV_I64, RTLIB::UDIV_I128)); |
4026 | 75 | break; |
4027 | 241 | case ISD::SDIVREM: |
4028 | 14 | case ISD::UDIVREM: |
4029 | 14 | // Expand into divrem libcall |
4030 | 14 | ExpandDivRemLibCall(Node, Results); |
4031 | 14 | break; |
4032 | 37 | case ISD::MUL: |
4033 | 37 | Results.push_back(ExpandIntLibCall(Node, false, |
4034 | 37 | RTLIB::MUL_I8, |
4035 | 37 | RTLIB::MUL_I16, RTLIB::MUL_I32, |
4036 | 37 | RTLIB::MUL_I64, RTLIB::MUL_I128)); |
4037 | 37 | break; |
4038 | 61 | case ISD::CTLZ_ZERO_UNDEF: |
4039 | 61 | switch (Node->getSimpleValueType(0).SimpleTy) { |
4040 | 61 | default: |
4041 | 0 | llvm_unreachable("LibCall explicitly requested, but not available"); |
4042 | 61 | case MVT::i32: |
4043 | 61 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false)); |
4044 | 61 | break; |
4045 | 61 | case MVT::i64: |
4046 | 0 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false)); |
4047 | 0 | break; |
4048 | 61 | case MVT::i128: |
4049 | 0 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false)); |
4050 | 0 | break; |
4051 | 61 | } |
4052 | 61 | break; |
4053 | 64.8k | } |
4054 | 64.8k | |
4055 | 64.8k | // Replace the original node with the legalized result. |
4056 | 64.8k | if (!Results.empty()) { |
4057 | 5.73k | LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n"); |
4058 | 5.73k | ReplaceNode(Node, Results.data()); |
4059 | 5.73k | } else |
4060 | 64.8k | LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n"); |
4061 | 64.8k | } |
4062 | | |
4063 | | // Determine the vector type to use in place of an original scalar element when |
4064 | | // promoting equally sized vectors. |
4065 | | static MVT getPromotedVectorElementType(const TargetLowering &TLI, |
4066 | 13.7k | MVT EltVT, MVT NewEltVT) { |
4067 | 13.7k | unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits(); |
4068 | 13.7k | MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt); |
4069 | 13.7k | assert(TLI.isTypeLegal(MidVT) && "unexpected"); |
4070 | 13.7k | return MidVT; |
4071 | 13.7k | } |
4072 | | |
4073 | 17.7k | void SelectionDAGLegalize::PromoteNode(SDNode *Node) { |
4074 | 17.7k | LLVM_DEBUG(dbgs() << "Trying to promote node\n"); |
4075 | 17.7k | SmallVector<SDValue, 8> Results; |
4076 | 17.7k | MVT OVT = Node->getSimpleValueType(0); |
4077 | 17.7k | if (Node->getOpcode() == ISD::UINT_TO_FP || |
4078 | 17.7k | Node->getOpcode() == ISD::SINT_TO_FP17.5k || |
4079 | 17.7k | Node->getOpcode() == ISD::SETCC17.5k || |
4080 | 17.7k | Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT17.2k || |
4081 | 17.7k | Node->getOpcode() == ISD::INSERT_VECTOR_ELT5.48k ) { |
4082 | 12.2k | OVT = Node->getOperand(0).getSimpleValueType(); |
4083 | 12.2k | } |
4084 | 17.7k | if (Node->getOpcode() == ISD::BR_CC) |
4085 | 1 | OVT = Node->getOperand(2).getSimpleValueType(); |
4086 | 17.7k | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); |
4087 | 17.7k | SDLoc dl(Node); |
4088 | 17.7k | SDValue Tmp1, Tmp2, Tmp3; |
4089 | 17.7k | switch (Node->getOpcode()) { |
4090 | 17.7k | case ISD::CTTZ: |
4091 | 90 | case ISD::CTTZ_ZERO_UNDEF: |
4092 | 90 | case ISD::CTLZ: |
4093 | 90 | case ISD::CTLZ_ZERO_UNDEF: |
4094 | 90 | case ISD::CTPOP: |
4095 | 90 | // Zero extend the argument. |
4096 | 90 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); |
4097 | 90 | if (Node->getOpcode() == ISD::CTTZ) { |
4098 | 3 | // The count is the same in the promoted type except if the original |
4099 | 3 | // value was zero. This can be handled by setting the bit just off |
4100 | 3 | // the top of the original type. |
4101 | 3 | auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(), |
4102 | 3 | OVT.getSizeInBits()); |
4103 | 3 | Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, |
4104 | 3 | DAG.getConstant(TopBit, dl, NVT)); |
4105 | 3 | } |
4106 | 90 | // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is |
4107 | 90 | // already the correct result. |
4108 | 90 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4109 | 90 | if (Node->getOpcode() == ISD::CTLZ || |
4110 | 90 | Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF80 ) { |
4111 | 21 | // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) |
4112 | 21 | Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, |
4113 | 21 | DAG.getConstant(NVT.getSizeInBits() - |
4114 | 21 | OVT.getSizeInBits(), dl, NVT)); |
4115 | 21 | } |
4116 | 90 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4117 | 90 | break; |
4118 | 90 | case ISD::BITREVERSE: |
4119 | 3 | case ISD::BSWAP: { |
4120 | 3 | unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); |
4121 | 3 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); |
4122 | 3 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4123 | 3 | Tmp1 = DAG.getNode( |
4124 | 3 | ISD::SRL, dl, NVT, Tmp1, |
4125 | 3 | DAG.getConstant(DiffBits, dl, |
4126 | 3 | TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); |
4127 | 3 | |
4128 | 3 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4129 | 3 | break; |
4130 | 3 | } |
4131 | 286 | case ISD::FP_TO_UINT: |
4132 | 286 | case ISD::FP_TO_SINT: |
4133 | 286 | Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), |
4134 | 286 | Node->getOpcode() == ISD::FP_TO_SINT, dl); |
4135 | 286 | Results.push_back(Tmp1); |
4136 | 286 | break; |
4137 | 286 | case ISD::UINT_TO_FP: |
4138 | 177 | case ISD::SINT_TO_FP: |
4139 | 177 | Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), |
4140 | 177 | Node->getOpcode() == ISD::SINT_TO_FP, dl); |
4141 | 177 | Results.push_back(Tmp1); |
4142 | 177 | break; |
4143 | 177 | case ISD::VAARG: { |
4144 | 1 | SDValue Chain = Node->getOperand(0); // Get the chain. |
4145 | 1 | SDValue Ptr = Node->getOperand(1); // Get the pointer. |
4146 | 1 | |
4147 | 1 | unsigned TruncOp; |
4148 | 1 | if (OVT.isVector()) { |
4149 | 0 | TruncOp = ISD::BITCAST; |
4150 | 1 | } else { |
4151 | 1 | assert(OVT.isInteger() |
4152 | 1 | && "VAARG promotion is supported only for vectors or integer types"); |
4153 | 1 | TruncOp = ISD::TRUNCATE; |
4154 | 1 | } |
4155 | 1 | |
4156 | 1 | // Perform the larger operation, then convert back |
4157 | 1 | Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), |
4158 | 1 | Node->getConstantOperandVal(3)); |
4159 | 1 | Chain = Tmp1.getValue(1); |
4160 | 1 | |
4161 | 1 | Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); |
4162 | 1 | |
4163 | 1 | // Modified the chain result - switch anything that used the old chain to |
4164 | 1 | // use the new one. |
4165 | 1 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); |
4166 | 1 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); |
4167 | 1 | if (UpdatedNodes) { |
4168 | 0 | UpdatedNodes->insert(Tmp2.getNode()); |
4169 | 0 | UpdatedNodes->insert(Chain.getNode()); |
4170 | 0 | } |
4171 | 1 | ReplacedNode(Node); |
4172 | 1 | break; |
4173 | 177 | } |
4174 | 445 | case ISD::MUL: |
4175 | 445 | case ISD::SDIV: |
4176 | 445 | case ISD::SREM: |
4177 | 445 | case ISD::UDIV: |
4178 | 445 | case ISD::UREM: |
4179 | 445 | case ISD::AND: |
4180 | 445 | case ISD::OR: |
4181 | 445 | case ISD::XOR: { |
4182 | 445 | unsigned ExtOp, TruncOp; |
4183 | 445 | if (OVT.isVector()) { |
4184 | 444 | ExtOp = ISD::BITCAST; |
4185 | 444 | TruncOp = ISD::BITCAST; |
4186 | 444 | } else { |
4187 | 1 | assert(OVT.isInteger() && "Cannot promote logic operation"); |
4188 | 1 | |
4189 | 1 | switch (Node->getOpcode()) { |
4190 | 1 | default: |
4191 | 1 | ExtOp = ISD::ANY_EXTEND; |
4192 | 1 | break; |
4193 | 1 | case ISD::SDIV: |
4194 | 0 | case ISD::SREM: |
4195 | 0 | ExtOp = ISD::SIGN_EXTEND; |
4196 | 0 | break; |
4197 | 0 | case ISD::UDIV: |
4198 | 0 | case ISD::UREM: |
4199 | 0 | ExtOp = ISD::ZERO_EXTEND; |
4200 | 0 | break; |
4201 | 1 | } |
4202 | 1 | TruncOp = ISD::TRUNCATE; |
4203 | 1 | } |
4204 | 445 | // Promote each of the values to the new type. |
4205 | 445 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4206 | 445 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4207 | 445 | // Perform the larger operation, then convert back |
4208 | 445 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); |
4209 | 445 | Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); |
4210 | 445 | break; |
4211 | 445 | } |
4212 | 445 | case ISD::UMUL_LOHI: |
4213 | 0 | case ISD::SMUL_LOHI: { |
4214 | 0 | // Promote to a multiply in a wider integer type. |
4215 | 0 | unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND |
4216 | 0 | : ISD::SIGN_EXTEND; |
4217 | 0 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4218 | 0 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4219 | 0 | Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); |
4220 | 0 |
|
4221 | 0 | auto &DL = DAG.getDataLayout(); |
4222 | 0 | unsigned OriginalSize = OVT.getScalarSizeInBits(); |
4223 | 0 | Tmp2 = DAG.getNode( |
4224 | 0 | ISD::SRL, dl, NVT, Tmp1, |
4225 | 0 | DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); |
4226 | 0 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4227 | 0 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); |
4228 | 0 | break; |
4229 | 0 | } |
4230 | 467 | case ISD::SELECT: { |
4231 | 467 | unsigned ExtOp, TruncOp; |
4232 | 467 | if (Node->getValueType(0).isVector() || |
4233 | 467 | Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { |
4234 | 203 | ExtOp = ISD::BITCAST; |
4235 | 203 | TruncOp = ISD::BITCAST; |
4236 | 264 | } else if (Node->getValueType(0).isInteger()) { |
4237 | 259 | ExtOp = ISD::ANY_EXTEND; |
4238 | 259 | TruncOp = ISD::TRUNCATE; |
4239 | 259 | } else { |
4240 | 5 | ExtOp = ISD::FP_EXTEND; |
4241 | 5 | TruncOp = ISD::FP_ROUND; |
4242 | 5 | } |
4243 | 467 | Tmp1 = Node->getOperand(0); |
4244 | 467 | // Promote each of the values to the new type. |
4245 | 467 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4246 | 467 | Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4247 | 467 | // Perform the larger operation, then round down. |
4248 | 467 | Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); |
4249 | 467 | Tmp1->setFlags(Node->getFlags()); |
4250 | 467 | if (TruncOp != ISD::FP_ROUND) |
4251 | 462 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); |
4252 | 5 | else |
4253 | 5 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, |
4254 | 5 | DAG.getIntPtrConstant(0, dl)); |
4255 | 467 | Results.push_back(Tmp1); |
4256 | 467 | break; |
4257 | 0 | } |
4258 | 1.19k | case ISD::VECTOR_SHUFFLE: { |
4259 | 1.19k | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); |
4260 | 1.19k | |
4261 | 1.19k | // Cast the two input vectors. |
4262 | 1.19k | Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); |
4263 | 1.19k | Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); |
4264 | 1.19k | |
4265 | 1.19k | // Convert the shuffle mask to the right # elements. |
4266 | 1.19k | Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); |
4267 | 1.19k | Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); |
4268 | 1.19k | Results.push_back(Tmp1); |
4269 | 1.19k | break; |
4270 | 0 | } |
4271 | 241 | case ISD::SETCC: { |
4272 | 241 | unsigned ExtOp = ISD::FP_EXTEND; |
4273 | 241 | if (NVT.isInteger()) { |
4274 | 124 | ISD::CondCode CCCode = |
4275 | 124 | cast<CondCodeSDNode>(Node->getOperand(2))->get(); |
4276 | 124 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND0 : ISD::ZERO_EXTEND; |
4277 | 124 | } |
4278 | 241 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4279 | 241 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4280 | 241 | Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, |
4281 | 241 | Tmp2, Node->getOperand(2), Node->getFlags())); |
4282 | 241 | break; |
4283 | 0 | } |
4284 | 1 | case ISD::BR_CC: { |
4285 | 1 | unsigned ExtOp = ISD::FP_EXTEND; |
4286 | 1 | if (NVT.isInteger()) { |
4287 | 0 | ISD::CondCode CCCode = |
4288 | 0 | cast<CondCodeSDNode>(Node->getOperand(1))->get(); |
4289 | 0 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; |
4290 | 0 | } |
4291 | 1 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4292 | 1 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); |
4293 | 1 | Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), |
4294 | 1 | Node->getOperand(0), Node->getOperand(1), |
4295 | 1 | Tmp1, Tmp2, Node->getOperand(4))); |
4296 | 1 | break; |
4297 | 0 | } |
4298 | 283 | case ISD::FADD: |
4299 | 283 | case ISD::FSUB: |
4300 | 283 | case ISD::FMUL: |
4301 | 283 | case ISD::FDIV: |
4302 | 283 | case ISD::FREM: |
4303 | 283 | case ISD::FMINNUM: |
4304 | 283 | case ISD::FMAXNUM: |
4305 | 283 | case ISD::FPOW: |
4306 | 283 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4307 | 283 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); |
4308 | 283 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, |
4309 | 283 | Node->getFlags()); |
4310 | 283 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4311 | 283 | Tmp3, DAG.getIntPtrConstant(0, dl))); |
4312 | 283 | break; |
4313 | 283 | case ISD::FMA: |
4314 | 25 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4315 | 25 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); |
4316 | 25 | Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); |
4317 | 25 | Results.push_back( |
4318 | 25 | DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4319 | 25 | DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), |
4320 | 25 | DAG.getIntPtrConstant(0, dl))); |
4321 | 25 | break; |
4322 | 283 | case ISD::FCOPYSIGN: |
4323 | 67 | case ISD::FPOWI: { |
4324 | 67 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4325 | 67 | Tmp2 = Node->getOperand(1); |
4326 | 67 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); |
4327 | 67 | |
4328 | 67 | // fcopysign doesn't change anything but the sign bit, so |
4329 | 67 | // (fp_round (fcopysign (fpext a), b)) |
4330 | 67 | // is as precise as |
4331 | 67 | // (fp_round (fpext a)) |
4332 | 67 | // which is a no-op. Mark it as a TRUNCating FP_ROUND. |
4333 | 67 | const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); |
4334 | 67 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4335 | 67 | Tmp3, DAG.getIntPtrConstant(isTrunc, dl))); |
4336 | 67 | break; |
4337 | 67 | } |
4338 | 484 | case ISD::FFLOOR: |
4339 | 484 | case ISD::FCEIL: |
4340 | 484 | case ISD::FRINT: |
4341 | 484 | case ISD::FNEARBYINT: |
4342 | 484 | case ISD::FROUND: |
4343 | 484 | case ISD::FTRUNC: |
4344 | 484 | case ISD::FNEG: |
4345 | 484 | case ISD::FSQRT: |
4346 | 484 | case ISD::FSIN: |
4347 | 484 | case ISD::FCOS: |
4348 | 484 | case ISD::FLOG: |
4349 | 484 | case ISD::FLOG2: |
4350 | 484 | case ISD::FLOG10: |
4351 | 484 | case ISD::FABS: |
4352 | 484 | case ISD::FEXP: |
4353 | 484 | case ISD::FEXP2: |
4354 | 484 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4355 | 484 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4356 | 484 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4357 | 484 | Tmp2, DAG.getIntPtrConstant(0, dl))); |
4358 | 484 | break; |
4359 | 484 | case ISD::BUILD_VECTOR: { |
4360 | 395 | MVT EltVT = OVT.getVectorElementType(); |
4361 | 395 | MVT NewEltVT = NVT.getVectorElementType(); |
4362 | 395 | |
4363 | 395 | // Handle bitcasts to a different vector type with the same total bit size |
4364 | 395 | // |
4365 | 395 | // e.g. v2i64 = build_vector i64:x, i64:y => v4i32 |
4366 | 395 | // => |
4367 | 395 | // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y)) |
4368 | 395 | |
4369 | 395 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4370 | 395 | "Invalid promote type for build_vector"); |
4371 | 395 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4372 | 395 | |
4373 | 395 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4374 | 395 | |
4375 | 395 | SmallVector<SDValue, 8> NewOps; |
4376 | 1.18k | for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I790 ) { |
4377 | 790 | SDValue Op = Node->getOperand(I); |
4378 | 790 | NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); |
4379 | 790 | } |
4380 | 395 | |
4381 | 395 | SDLoc SL(Node); |
4382 | 395 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps); |
4383 | 395 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); |
4384 | 395 | Results.push_back(CvtVec); |
4385 | 395 | break; |
4386 | 484 | } |
4387 | 11.8k | case ISD::EXTRACT_VECTOR_ELT: { |
4388 | 11.8k | MVT EltVT = OVT.getVectorElementType(); |
4389 | 11.8k | MVT NewEltVT = NVT.getVectorElementType(); |
4390 | 11.8k | |
4391 | 11.8k | // Handle bitcasts to a different vector type with the same total bit size. |
4392 | 11.8k | // |
4393 | 11.8k | // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32 |
4394 | 11.8k | // => |
4395 | 11.8k | // v4i32:castx = bitcast x:v2i64 |
4396 | 11.8k | // |
4397 | 11.8k | // i64 = bitcast |
4398 | 11.8k | // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), |
4399 | 11.8k | // (i32 (extract_vector_elt castx, (2 * y + 1))) |
4400 | 11.8k | // |
4401 | 11.8k | |
4402 | 11.8k | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4403 | 11.8k | "Invalid promote type for extract_vector_elt"); |
4404 | 11.8k | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4405 | 11.8k | |
4406 | 11.8k | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4407 | 11.8k | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
4408 | 11.8k | |
4409 | 11.8k | SDValue Idx = Node->getOperand(1); |
4410 | 11.8k | EVT IdxVT = Idx.getValueType(); |
4411 | 11.8k | SDLoc SL(Node); |
4412 | 11.8k | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT); |
4413 | 11.8k | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); |
4414 | 11.8k | |
4415 | 11.8k | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); |
4416 | 11.8k | |
4417 | 11.8k | SmallVector<SDValue, 8> NewOps; |
4418 | 35.4k | for (unsigned I = 0; I < NewEltsPerOldElt; ++I23.6k ) { |
4419 | 23.6k | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); |
4420 | 23.6k | SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); |
4421 | 23.6k | |
4422 | 23.6k | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, |
4423 | 23.6k | CastVec, TmpIdx); |
4424 | 23.6k | NewOps.push_back(Elt); |
4425 | 23.6k | } |
4426 | 11.8k | |
4427 | 11.8k | SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps); |
4428 | 11.8k | Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); |
4429 | 11.8k | break; |
4430 | 484 | } |
4431 | 484 | case ISD::INSERT_VECTOR_ELT: { |
4432 | 0 | MVT EltVT = OVT.getVectorElementType(); |
4433 | 0 | MVT NewEltVT = NVT.getVectorElementType(); |
4434 | 0 |
|
4435 | 0 | // Handle bitcasts to a different vector type with the same total bit size |
4436 | 0 | // |
4437 | 0 | // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32 |
4438 | 0 | // => |
4439 | 0 | // v4i32:castx = bitcast x:v2i64 |
4440 | 0 | // v2i32:casty = bitcast y:i64 |
4441 | 0 | // |
4442 | 0 | // v2i64 = bitcast |
4443 | 0 | // (v4i32 insert_vector_elt |
4444 | 0 | // (v4i32 insert_vector_elt v4i32:castx, |
4445 | 0 | // (extract_vector_elt casty, 0), 2 * z), |
4446 | 0 | // (extract_vector_elt casty, 1), (2 * z + 1)) |
4447 | 0 |
|
4448 | 0 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4449 | 0 | "Invalid promote type for insert_vector_elt"); |
4450 | 0 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4451 | 0 |
|
4452 | 0 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4453 | 0 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
4454 | 0 |
|
4455 | 0 | SDValue Val = Node->getOperand(1); |
4456 | 0 | SDValue Idx = Node->getOperand(2); |
4457 | 0 | EVT IdxVT = Idx.getValueType(); |
4458 | 0 | SDLoc SL(Node); |
4459 | 0 |
|
4460 | 0 | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT); |
4461 | 0 | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); |
4462 | 0 |
|
4463 | 0 | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); |
4464 | 0 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); |
4465 | 0 |
|
4466 | 0 | SDValue NewVec = CastVec; |
4467 | 0 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { |
4468 | 0 | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); |
4469 | 0 | SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); |
4470 | 0 |
|
4471 | 0 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, |
4472 | 0 | CastVal, IdxOffset); |
4473 | 0 |
|
4474 | 0 | NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, |
4475 | 0 | NewVec, Elt, InEltIdx); |
4476 | 0 | } |
4477 | 0 |
|
4478 | 0 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); |
4479 | 0 | break; |
4480 | 484 | } |
4481 | 1.56k | case ISD::SCALAR_TO_VECTOR: { |
4482 | 1.56k | MVT EltVT = OVT.getVectorElementType(); |
4483 | 1.56k | MVT NewEltVT = NVT.getVectorElementType(); |
4484 | 1.56k | |
4485 | 1.56k | // Handle bitcasts to different vector type with the same total bit size. |
4486 | 1.56k | // |
4487 | 1.56k | // e.g. v2i64 = scalar_to_vector x:i64 |
4488 | 1.56k | // => |
4489 | 1.56k | // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef) |
4490 | 1.56k | // |
4491 | 1.56k | |
4492 | 1.56k | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4493 | 1.56k | SDValue Val = Node->getOperand(0); |
4494 | 1.56k | SDLoc SL(Node); |
4495 | 1.56k | |
4496 | 1.56k | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); |
4497 | 1.56k | SDValue Undef = DAG.getUNDEF(MidVT); |
4498 | 1.56k | |
4499 | 1.56k | SmallVector<SDValue, 8> NewElts; |
4500 | 1.56k | NewElts.push_back(CastVal); |
4501 | 3.12k | for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I1.56k ) |
4502 | 1.56k | NewElts.push_back(Undef); |
4503 | 1.56k | |
4504 | 1.56k | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); |
4505 | 1.56k | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); |
4506 | 1.56k | Results.push_back(CvtVec); |
4507 | 1.56k | break; |
4508 | 484 | } |
4509 | 484 | case ISD::ATOMIC_SWAP: { |
4510 | 23 | AtomicSDNode *AM = cast<AtomicSDNode>(Node); |
4511 | 23 | SDLoc SL(Node); |
4512 | 23 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); |
4513 | 23 | assert(NVT.getSizeInBits() == OVT.getSizeInBits() && |
4514 | 23 | "unexpected promotion type"); |
4515 | 23 | assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && |
4516 | 23 | "unexpected atomic_swap with illegal type"); |
4517 | 23 | |
4518 | 23 | SDValue NewAtomic |
4519 | 23 | = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, |
4520 | 23 | DAG.getVTList(NVT, MVT::Other), |
4521 | 23 | { AM->getChain(), AM->getBasePtr(), CastVal }, |
4522 | 23 | AM->getMemOperand()); |
4523 | 23 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); |
4524 | 23 | Results.push_back(NewAtomic.getValue(1)); |
4525 | 23 | break; |
4526 | 17.7k | } |
4527 | 17.7k | } |
4528 | 17.7k | |
4529 | 17.7k | // Replace the original node with the legalized result. |
4530 | 17.7k | if (!Results.empty()) { |
4531 | 17.5k | LLVM_DEBUG(dbgs() << "Successfully promoted node\n"); |
4532 | 17.5k | ReplaceNode(Node, Results.data()); |
4533 | 17.5k | } else |
4534 | 17.7k | LLVM_DEBUG(dbgs() << "Could not promote node\n"); |
4535 | 17.7k | } |
4536 | | |
4537 | | /// This is the entry point for the file. |
4538 | 1.24M | void SelectionDAG::Legalize() { |
4539 | 1.24M | AssignTopologicalOrder(); |
4540 | 1.24M | |
4541 | 1.24M | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
4542 | 1.24M | // Use a delete listener to remove nodes which were deleted during |
4543 | 1.24M | // legalization from LegalizeNodes. This is needed to handle the situation |
4544 | 1.24M | // where a new node is allocated by the object pool to the same address of a |
4545 | 1.24M | // previously deleted node. |
4546 | 1.24M | DAGNodeDeletedListener DeleteListener( |
4547 | 1.24M | *this, |
4548 | 1.24M | [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); }4.29k ); |
4549 | 1.24M | |
4550 | 1.24M | SelectionDAGLegalize Legalizer(*this, LegalizedNodes); |
4551 | 1.24M | |
4552 | 1.24M | // Visit all the nodes. We start in topological order, so that we see |
4553 | 1.24M | // nodes with their original operands intact. Legalization can produce |
4554 | 1.24M | // new nodes which may themselves need to be legalized. Iterate until all |
4555 | 1.24M | // nodes have been legalized. |
4556 | 3.48M | while (true) { |
4557 | 3.48M | bool AnyLegalized = false; |
4558 | 79.9M | for (auto NI = allnodes_end(); NI != allnodes_begin();) { |
4559 | 76.4M | --NI; |
4560 | 76.4M | |
4561 | 76.4M | SDNode *N = &*NI; |
4562 | 76.4M | if (N->use_empty() && N != getRoot().getNode()5.12M ) { |
4563 | 1.64M | ++NI; |
4564 | 1.64M | DeleteNode(N); |
4565 | 1.64M | continue; |
4566 | 1.64M | } |
4567 | 74.7M | |
4568 | 74.7M | if (LegalizedNodes.insert(N).second) { |
4569 | 28.2M | AnyLegalized = true; |
4570 | 28.2M | Legalizer.LegalizeOp(N); |
4571 | 28.2M | |
4572 | 28.2M | if (N->use_empty() && N != getRoot().getNode()3.11M ) { |
4573 | 1.86M | ++NI; |
4574 | 1.86M | DeleteNode(N); |
4575 | 1.86M | } |
4576 | 28.2M | } |
4577 | 74.7M | } |
4578 | 3.48M | if (!AnyLegalized) |
4579 | 1.24M | break; |
4580 | 3.48M | |
4581 | 3.48M | } |
4582 | 1.24M | |
4583 | 1.24M | // Remove dead nodes now. |
4584 | 1.24M | RemoveDeadNodes(); |
4585 | 1.24M | } |
4586 | | |
4587 | | bool SelectionDAG::LegalizeOp(SDNode *N, |
4588 | 27.4M | SmallSetVector<SDNode *, 16> &UpdatedNodes) { |
4589 | 27.4M | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
4590 | 27.4M | SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); |
4591 | 27.4M | |
4592 | 27.4M | // Directly insert the node in question, and legalize it. This will recurse |
4593 | 27.4M | // as needed through operands. |
4594 | 27.4M | LegalizedNodes.insert(N); |
4595 | 27.4M | Legalizer.LegalizeOp(N); |
4596 | 27.4M | |
4597 | 27.4M | return LegalizedNodes.count(N); |
4598 | 27.4M | } |