/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | // |
10 | | // This file implements the SelectionDAG::Legalize method. |
11 | | // |
12 | | //===----------------------------------------------------------------------===// |
13 | | |
14 | | #include "llvm/ADT/APFloat.h" |
15 | | #include "llvm/ADT/APInt.h" |
16 | | #include "llvm/ADT/ArrayRef.h" |
17 | | #include "llvm/ADT/SetVector.h" |
18 | | #include "llvm/ADT/SmallPtrSet.h" |
19 | | #include "llvm/ADT/SmallSet.h" |
20 | | #include "llvm/ADT/SmallVector.h" |
21 | | #include "llvm/CodeGen/ISDOpcodes.h" |
22 | | #include "llvm/CodeGen/MachineFunction.h" |
23 | | #include "llvm/CodeGen/MachineJumpTableInfo.h" |
24 | | #include "llvm/CodeGen/MachineMemOperand.h" |
25 | | #include "llvm/CodeGen/MachineValueType.h" |
26 | | #include "llvm/CodeGen/RuntimeLibcalls.h" |
27 | | #include "llvm/CodeGen/SelectionDAG.h" |
28 | | #include "llvm/CodeGen/SelectionDAGNodes.h" |
29 | | #include "llvm/CodeGen/ValueTypes.h" |
30 | | #include "llvm/IR/CallingConv.h" |
31 | | #include "llvm/IR/Constants.h" |
32 | | #include "llvm/IR/DataLayout.h" |
33 | | #include "llvm/IR/DerivedTypes.h" |
34 | | #include "llvm/IR/Function.h" |
35 | | #include "llvm/IR/Metadata.h" |
36 | | #include "llvm/IR/Type.h" |
37 | | #include "llvm/Support/Casting.h" |
38 | | #include "llvm/Support/Compiler.h" |
39 | | #include "llvm/Support/Debug.h" |
40 | | #include "llvm/Support/ErrorHandling.h" |
41 | | #include "llvm/Support/MathExtras.h" |
42 | | #include "llvm/Support/raw_ostream.h" |
43 | | #include "llvm/Target/TargetFrameLowering.h" |
44 | | #include "llvm/Target/TargetLowering.h" |
45 | | #include "llvm/Target/TargetMachine.h" |
46 | | #include "llvm/Target/TargetOptions.h" |
47 | | #include "llvm/Target/TargetSubtargetInfo.h" |
48 | | #include <algorithm> |
49 | | #include <cassert> |
50 | | #include <cstdint> |
51 | | #include <tuple> |
52 | | #include <utility> |
53 | | |
54 | | using namespace llvm; |
55 | | |
56 | | #define DEBUG_TYPE "legalizedag" |
57 | | |
58 | | namespace { |
59 | | |
60 | | /// Keeps track of state when getting the sign of a floating-point value as an |
61 | | /// integer. |
62 | | struct FloatSignAsInt { |
63 | | EVT FloatVT; |
64 | | SDValue Chain; |
65 | | SDValue FloatPtr; |
66 | | SDValue IntPtr; |
67 | | MachinePointerInfo IntPointerInfo; |
68 | | MachinePointerInfo FloatPointerInfo; |
69 | | SDValue IntValue; |
70 | | APInt SignMask; |
71 | | uint8_t SignBit; |
72 | | }; |
73 | | |
74 | | //===----------------------------------------------------------------------===// |
75 | | /// This takes an arbitrary SelectionDAG as input and |
76 | | /// hacks on it until the target machine can handle it. This involves |
77 | | /// eliminating value sizes the machine cannot handle (promoting small sizes to |
78 | | /// large sizes or splitting up large values into small values) as well as |
79 | | /// eliminating operations the machine cannot handle. |
80 | | /// |
81 | | /// This code also does a small amount of optimization and recognition of idioms |
82 | | /// as part of its processing. For example, if a target does not support a |
83 | | /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this |
84 | | /// will attempt merge setcc and brc instructions into brcc's. |
85 | | class SelectionDAGLegalize { |
86 | | const TargetMachine &TM; |
87 | | const TargetLowering &TLI; |
88 | | SelectionDAG &DAG; |
89 | | |
90 | | /// \brief The set of nodes which have already been legalized. We hold a |
91 | | /// reference to it in order to update as necessary on node deletion. |
92 | | SmallPtrSetImpl<SDNode *> &LegalizedNodes; |
93 | | |
94 | | /// \brief A set of all the nodes updated during legalization. |
95 | | SmallSetVector<SDNode *, 16> *UpdatedNodes; |
96 | | |
97 | 664 | EVT getSetCCResultType(EVT VT) const { |
98 | 664 | return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); |
99 | 664 | } |
100 | | |
101 | | // Libcall insertion helpers. |
102 | | |
103 | | public: |
104 | | SelectionDAGLegalize(SelectionDAG &DAG, |
105 | | SmallPtrSetImpl<SDNode *> &LegalizedNodes, |
106 | | SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) |
107 | | : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), |
108 | 76.2M | LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} |
109 | | |
110 | | /// \brief Legalizes the given operation. |
111 | | void LegalizeOp(SDNode *Node); |
112 | | |
113 | | private: |
114 | | SDValue OptimizeFloatStore(StoreSDNode *ST); |
115 | | |
116 | | void LegalizeLoadOps(SDNode *Node); |
117 | | void LegalizeStoreOps(SDNode *Node); |
118 | | |
119 | | /// Some targets cannot handle a variable |
120 | | /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it |
121 | | /// is necessary to spill the vector being inserted into to memory, perform |
122 | | /// the insert there, and then read the result back. |
123 | | SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, |
124 | | const SDLoc &dl); |
125 | | SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, |
126 | | const SDLoc &dl); |
127 | | |
128 | | /// Return a vector shuffle operation which |
129 | | /// performs the same shuffe in terms of order or result bytes, but on a type |
130 | | /// whose vector element type is narrower than the original shuffle type. |
131 | | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> |
132 | | SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl, |
133 | | SDValue N1, SDValue N2, |
134 | | ArrayRef<int> Mask) const; |
135 | | |
136 | | bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, |
137 | | bool &NeedInvert, const SDLoc &dl); |
138 | | |
139 | | SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); |
140 | | SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, |
141 | | unsigned NumOps, bool isSigned, const SDLoc &dl); |
142 | | |
143 | | std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, |
144 | | SDNode *Node, bool isSigned); |
145 | | SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, |
146 | | RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, |
147 | | RTLIB::Libcall Call_F128, |
148 | | RTLIB::Libcall Call_PPCF128); |
149 | | SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, |
150 | | RTLIB::Libcall Call_I8, |
151 | | RTLIB::Libcall Call_I16, |
152 | | RTLIB::Libcall Call_I32, |
153 | | RTLIB::Libcall Call_I64, |
154 | | RTLIB::Libcall Call_I128); |
155 | | void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
156 | | void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); |
157 | | |
158 | | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, |
159 | | const SDLoc &dl); |
160 | | SDValue ExpandBUILD_VECTOR(SDNode *Node); |
161 | | SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); |
162 | | void ExpandDYNAMIC_STACKALLOC(SDNode *Node, |
163 | | SmallVectorImpl<SDValue> &Results); |
164 | | void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, |
165 | | SDValue Value) const; |
166 | | SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL, |
167 | | SDValue NewIntValue) const; |
168 | | SDValue ExpandFCOPYSIGN(SDNode *Node) const; |
169 | | SDValue ExpandFABS(SDNode *Node) const; |
170 | | SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT, |
171 | | const SDLoc &dl); |
172 | | SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, |
173 | | const SDLoc &dl); |
174 | | SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, |
175 | | const SDLoc &dl); |
176 | | |
177 | | SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl); |
178 | | SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl); |
179 | | SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl); |
180 | | |
181 | | SDValue ExpandExtractFromVectorThroughStack(SDValue Op); |
182 | | SDValue ExpandInsertToVectorThroughStack(SDValue Op); |
183 | | SDValue ExpandVectorBuildThroughStack(SDNode* Node); |
184 | | |
185 | | SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); |
186 | | SDValue ExpandConstant(ConstantSDNode *CP); |
187 | | |
188 | | // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall |
189 | | bool ExpandNode(SDNode *Node); |
190 | | void ConvertNodeToLibcall(SDNode *Node); |
191 | | void PromoteNode(SDNode *Node); |
192 | | |
193 | | public: |
194 | | // Node replacement helpers |
195 | | |
196 | 3.99M | void ReplacedNode(SDNode *N) { |
197 | 3.99M | LegalizedNodes.erase(N); |
198 | 3.99M | if (UpdatedNodes) |
199 | 7.46k | UpdatedNodes->insert(N); |
200 | 3.99M | } |
201 | | |
202 | 157 | void ReplaceNode(SDNode *Old, SDNode *New) { |
203 | 157 | DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
204 | 157 | dbgs() << " with: "; New->dump(&DAG)); |
205 | 157 | |
206 | 157 | assert(Old->getNumValues() == New->getNumValues() && |
207 | 157 | "Replacing one node with another that produces a different number " |
208 | 157 | "of values!"); |
209 | 157 | DAG.ReplaceAllUsesWith(Old, New); |
210 | 157 | if (UpdatedNodes) |
211 | 1 | UpdatedNodes->insert(New); |
212 | 157 | ReplacedNode(Old); |
213 | 157 | } |
214 | | |
215 | 3.69M | void ReplaceNode(SDValue Old, SDValue New) { |
216 | 3.69M | DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG); |
217 | 3.69M | dbgs() << " with: "; New->dump(&DAG)); |
218 | 3.69M | |
219 | 3.69M | DAG.ReplaceAllUsesWith(Old, New); |
220 | 3.69M | if (UpdatedNodes) |
221 | 3.73k | UpdatedNodes->insert(New.getNode()); |
222 | 3.69M | ReplacedNode(Old.getNode()); |
223 | 3.69M | } |
224 | | |
225 | 213k | void ReplaceNode(SDNode *Old, const SDValue *New) { |
226 | 213k | DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG)); |
227 | 213k | |
228 | 213k | DAG.ReplaceAllUsesWith(Old, New); |
229 | 474k | for (unsigned i = 0, e = Old->getNumValues(); i != e474k ; ++i261k ) { |
230 | 261k | DEBUG(dbgs() << (i == 0 ? " with: " |
231 | 261k | : " and: "); |
232 | 261k | New[i]->dump(&DAG)); |
233 | 261k | if (UpdatedNodes) |
234 | 1.97k | UpdatedNodes->insert(New[i].getNode()); |
235 | 261k | } |
236 | 213k | ReplacedNode(Old); |
237 | 213k | } |
238 | | }; |
239 | | |
240 | | } // end anonymous namespace |
241 | | |
242 | | /// Return a vector shuffle operation which |
243 | | /// performs the same shuffe in terms of order or result bytes, but on a type |
244 | | /// whose vector element type is narrower than the original shuffle type. |
245 | | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> |
246 | | SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType( |
247 | | EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, |
248 | 402 | ArrayRef<int> Mask) const { |
249 | 402 | unsigned NumMaskElts = VT.getVectorNumElements(); |
250 | 402 | unsigned NumDestElts = NVT.getVectorNumElements(); |
251 | 402 | unsigned NumEltsGrowth = NumDestElts / NumMaskElts; |
252 | 402 | |
253 | 402 | assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!"); |
254 | 402 | |
255 | 402 | if (NumEltsGrowth == 1) |
256 | 0 | return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); |
257 | 402 | |
258 | 402 | SmallVector<int, 8> NewMask; |
259 | 2.35k | for (unsigned i = 0; i != NumMaskElts2.35k ; ++i1.95k ) { |
260 | 1.95k | int Idx = Mask[i]; |
261 | 8.38k | for (unsigned j = 0; j != NumEltsGrowth8.38k ; ++j6.43k ) { |
262 | 6.43k | if (Idx < 0) |
263 | 712 | NewMask.push_back(-1); |
264 | 6.43k | else |
265 | 5.72k | NewMask.push_back(Idx * NumEltsGrowth + j); |
266 | 6.43k | } |
267 | 1.95k | } |
268 | 402 | assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"); |
269 | 402 | assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"); |
270 | 402 | return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask); |
271 | 402 | } |
272 | | |
273 | | /// Expands the ConstantFP node to an integer constant or |
274 | | /// a load from the constant pool. |
275 | | SDValue |
276 | 34.1k | SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { |
277 | 34.1k | bool Extend = false; |
278 | 34.1k | SDLoc dl(CFP); |
279 | 34.1k | |
280 | 34.1k | // If a FP immediate is precise when represented as a float and if the |
281 | 34.1k | // target can do an extending load from float to double, we put it into |
282 | 34.1k | // the constant pool as a float, even if it's is statically typed as a |
283 | 34.1k | // double. This shrinks FP constants and canonicalizes them for targets where |
284 | 34.1k | // an FP extending load is the same cost as a normal load (such as on the x87 |
285 | 34.1k | // fp stack or PPC FP unit). |
286 | 34.1k | EVT VT = CFP->getValueType(0); |
287 | 34.1k | ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); |
288 | 34.1k | if (!UseCP34.1k ) { |
289 | 0 | assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"); |
290 | 0 | return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, |
291 | 0 | (VT == MVT::f64) ? MVT::i640 : MVT::i320 ); |
292 | 0 | } |
293 | 34.1k | |
294 | 34.1k | APFloat APF = CFP->getValueAPF(); |
295 | 34.1k | EVT OrigVT = VT; |
296 | 34.1k | EVT SVT = VT; |
297 | 34.1k | |
298 | 34.1k | // We don't want to shrink SNaNs. Converting the SNaN back to its real type |
299 | 34.1k | // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ). |
300 | 34.1k | if (!APF.isSignaling()34.1k ) { |
301 | 55.4k | while (SVT != MVT::f32 && 55.4k SVT != MVT::f1621.3k ) { |
302 | 21.3k | SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); |
303 | 21.3k | if (ConstantFPSDNode::isValueValidForType(SVT, APF) && |
304 | 21.3k | // Only do this if the target has a native EXTLOAD instruction from |
305 | 21.3k | // smaller type. |
306 | 9.99k | TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && |
307 | 21.3k | TLI.ShouldShrinkFPConstant(OrigVT)1.78k ) { |
308 | 1.07k | Type *SType = SVT.getTypeForEVT(*DAG.getContext()); |
309 | 1.07k | LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); |
310 | 1.07k | VT = SVT; |
311 | 1.07k | Extend = true; |
312 | 1.07k | } |
313 | 21.3k | } |
314 | 34.1k | } |
315 | 34.1k | |
316 | 34.1k | SDValue CPIdx = |
317 | 34.1k | DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout())); |
318 | 34.1k | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); |
319 | 34.1k | if (Extend34.1k ) { |
320 | 733 | SDValue Result = DAG.getExtLoad( |
321 | 733 | ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, |
322 | 733 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT, |
323 | 733 | Alignment); |
324 | 733 | return Result; |
325 | 733 | } |
326 | 33.3k | SDValue Result = DAG.getLoad( |
327 | 33.3k | OrigVT, dl, DAG.getEntryNode(), CPIdx, |
328 | 33.3k | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); |
329 | 33.3k | return Result; |
330 | 33.3k | } |
331 | | |
332 | | /// Expands the Constant node to a load from the constant pool. |
333 | 0 | SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) { |
334 | 0 | SDLoc dl(CP); |
335 | 0 | EVT VT = CP->getValueType(0); |
336 | 0 | SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), |
337 | 0 | TLI.getPointerTy(DAG.getDataLayout())); |
338 | 0 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); |
339 | 0 | SDValue Result = DAG.getLoad( |
340 | 0 | VT, dl, DAG.getEntryNode(), CPIdx, |
341 | 0 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); |
342 | 0 | return Result; |
343 | 0 | } |
344 | | |
345 | | /// Some target cannot handle a variable insertion index for the |
346 | | /// INSERT_VECTOR_ELT instruction. In this case, it |
347 | | /// is necessary to spill the vector being inserted into to memory, perform |
348 | | /// the insert there, and then read the result back. |
349 | | SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, |
350 | | SDValue Val, |
351 | | SDValue Idx, |
352 | 16 | const SDLoc &dl) { |
353 | 16 | SDValue Tmp1 = Vec; |
354 | 16 | SDValue Tmp2 = Val; |
355 | 16 | SDValue Tmp3 = Idx; |
356 | 16 | |
357 | 16 | // If the target doesn't support this, we have to spill the input vector |
358 | 16 | // to a temporary stack slot, update the element, then reload it. This is |
359 | 16 | // badness. We could also load the value into a vector register (either |
360 | 16 | // with a "move to register" or "extload into register" instruction, then |
361 | 16 | // permute it into place, if the idx is a constant and if the idx is |
362 | 16 | // supported by the target. |
363 | 16 | EVT VT = Tmp1.getValueType(); |
364 | 16 | EVT EltVT = VT.getVectorElementType(); |
365 | 16 | SDValue StackPtr = DAG.CreateStackTemporary(VT); |
366 | 16 | |
367 | 16 | int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
368 | 16 | |
369 | 16 | // Store the vector. |
370 | 16 | SDValue Ch = DAG.getStore( |
371 | 16 | DAG.getEntryNode(), dl, Tmp1, StackPtr, |
372 | 16 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); |
373 | 16 | |
374 | 16 | SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3); |
375 | 16 | |
376 | 16 | // Store the scalar value. |
377 | 16 | Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT); |
378 | 16 | // Load the updated vector. |
379 | 16 | return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack( |
380 | 16 | DAG.getMachineFunction(), SPFI)); |
381 | 16 | } |
382 | | |
383 | | SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, |
384 | | SDValue Idx, |
385 | 404 | const SDLoc &dl) { |
386 | 404 | if (ConstantSDNode *InsertPos404 = dyn_cast<ConstantSDNode>(Idx)) { |
387 | 388 | // SCALAR_TO_VECTOR requires that the type of the value being inserted |
388 | 388 | // match the element type of the vector being created, except for |
389 | 388 | // integers in which case the inserted value can be over width. |
390 | 388 | EVT EltVT = Vec.getValueType().getVectorElementType(); |
391 | 388 | if (Val.getValueType() == EltVT || |
392 | 388 | (EltVT.isInteger() && 1 Val.getValueType().bitsGE(EltVT)1 )) { |
393 | 388 | SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, |
394 | 388 | Vec.getValueType(), Val); |
395 | 388 | |
396 | 388 | unsigned NumElts = Vec.getValueType().getVectorNumElements(); |
397 | 388 | // We generate a shuffle of InVec and ScVec, so the shuffle mask |
398 | 388 | // should be 0,1,2,3,4,5... with the appropriate element replaced with |
399 | 388 | // elt 0 of the RHS. |
400 | 388 | SmallVector<int, 8> ShufOps; |
401 | 1.96k | for (unsigned i = 0; i != NumElts1.96k ; ++i1.58k ) |
402 | 1.58k | ShufOps.push_back(i != InsertPos->getZExtValue() ? 1.58k i1.19k : NumElts388 ); |
403 | 388 | |
404 | 388 | return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps); |
405 | 388 | } |
406 | 16 | } |
407 | 16 | return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); |
408 | 16 | } |
409 | | |
410 | 3.07M | SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { |
411 | 3.07M | // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' |
412 | 3.07M | // FIXME: We shouldn't do this for TargetConstantFP's. |
413 | 3.07M | // FIXME: move this to the DAG Combiner! Note that we can't regress due |
414 | 3.07M | // to phase ordering between legalized code and the dag combiner. This |
415 | 3.07M | // probably means that we need to integrate dag combiner and legalizer |
416 | 3.07M | // together. |
417 | 3.07M | // We generally can't do this one for long doubles. |
418 | 3.07M | SDValue Chain = ST->getChain(); |
419 | 3.07M | SDValue Ptr = ST->getBasePtr(); |
420 | 3.07M | unsigned Alignment = ST->getAlignment(); |
421 | 3.07M | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
422 | 3.07M | AAMDNodes AAInfo = ST->getAAInfo(); |
423 | 3.07M | SDLoc dl(ST); |
424 | 3.07M | if (ConstantFPSDNode *CFP3.07M = dyn_cast<ConstantFPSDNode>(ST->getValue())) { |
425 | 170 | if (CFP->getValueType(0) == MVT::f32 && |
426 | 170 | TLI.isTypeLegal(MVT::i32)1 ) { |
427 | 1 | SDValue Con = DAG.getConstant(CFP->getValueAPF(). |
428 | 1 | bitcastToAPInt().zextOrTrunc(32), |
429 | 1 | SDLoc(CFP), MVT::i32); |
430 | 1 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment, |
431 | 1 | MMOFlags, AAInfo); |
432 | 1 | } |
433 | 169 | |
434 | 169 | if (169 CFP->getValueType(0) == MVT::f64169 ) { |
435 | 8 | // If this target supports 64-bit registers, do a single 64-bit store. |
436 | 8 | if (TLI.isTypeLegal(MVT::i64)8 ) { |
437 | 0 | SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). |
438 | 0 | zextOrTrunc(64), SDLoc(CFP), MVT::i64); |
439 | 0 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), |
440 | 0 | Alignment, MMOFlags, AAInfo); |
441 | 0 | } |
442 | 8 | |
443 | 8 | if (8 TLI.isTypeLegal(MVT::i32) && 8 !ST->isVolatile()8 ) { |
444 | 0 | // Otherwise, if the target supports 32-bit registers, use 2 32-bit |
445 | 0 | // stores. If the target supports neither 32- nor 64-bits, this |
446 | 0 | // xform is certainly not worth it. |
447 | 0 | const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); |
448 | 0 | SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32); |
449 | 0 | SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32); |
450 | 0 | if (DAG.getDataLayout().isBigEndian()) |
451 | 0 | std::swap(Lo, Hi); |
452 | 0 |
|
453 | 0 | Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment, |
454 | 0 | MMOFlags, AAInfo); |
455 | 0 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
456 | 0 | DAG.getConstant(4, dl, Ptr.getValueType())); |
457 | 0 | Hi = DAG.getStore(Chain, dl, Hi, Ptr, |
458 | 0 | ST->getPointerInfo().getWithOffset(4), |
459 | 0 | MinAlign(Alignment, 4U), MMOFlags, AAInfo); |
460 | 0 |
|
461 | 0 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
462 | 0 | } |
463 | 3.07M | } |
464 | 170 | } |
465 | 3.07M | return SDValue(nullptr, 0); |
466 | 3.07M | } |
467 | | |
468 | 3.81M | void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { |
469 | 3.81M | StoreSDNode *ST = cast<StoreSDNode>(Node); |
470 | 3.81M | SDValue Chain = ST->getChain(); |
471 | 3.81M | SDValue Ptr = ST->getBasePtr(); |
472 | 3.81M | SDLoc dl(Node); |
473 | 3.81M | |
474 | 3.81M | unsigned Alignment = ST->getAlignment(); |
475 | 3.81M | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); |
476 | 3.81M | AAMDNodes AAInfo = ST->getAAInfo(); |
477 | 3.81M | |
478 | 3.81M | if (!ST->isTruncatingStore()3.81M ) { |
479 | 3.07M | if (SDNode *OptStore3.07M = OptimizeFloatStore(ST).getNode()) { |
480 | 1 | ReplaceNode(ST, OptStore); |
481 | 1 | return; |
482 | 1 | } |
483 | 3.07M | |
484 | 3.07M | { |
485 | 3.07M | SDValue Value = ST->getValue(); |
486 | 3.07M | MVT VT = Value.getSimpleValueType(); |
487 | 3.07M | switch (TLI.getOperationAction(ISD::STORE, VT)) { |
488 | 0 | default: 0 llvm_unreachable0 ("This action is not supported yet!"); |
489 | 2.83M | case TargetLowering::Legal: { |
490 | 2.83M | // If this is an unaligned store and the target doesn't support it, |
491 | 2.83M | // expand it. |
492 | 2.83M | EVT MemVT = ST->getMemoryVT(); |
493 | 2.83M | unsigned AS = ST->getAddressSpace(); |
494 | 2.83M | unsigned Align = ST->getAlignment(); |
495 | 2.83M | const DataLayout &DL = DAG.getDataLayout(); |
496 | 2.83M | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)2.83M ) { |
497 | 218 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
498 | 218 | ReplaceNode(SDValue(ST, 0), Result); |
499 | 218 | } |
500 | 2.83M | break; |
501 | 3.07M | } |
502 | 100k | case TargetLowering::Custom: { |
503 | 100k | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); |
504 | 100k | if (Res && 100k Res != SDValue(Node, 0)33.5k ) |
505 | 12.7k | ReplaceNode(SDValue(Node, 0), Res); |
506 | 100k | return; |
507 | 3.07M | } |
508 | 143k | case TargetLowering::Promote: { |
509 | 143k | MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); |
510 | 143k | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
511 | 143k | "Can only promote stores to same size type"); |
512 | 143k | Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); |
513 | 143k | SDValue Result = |
514 | 143k | DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
515 | 143k | Alignment, MMOFlags, AAInfo); |
516 | 143k | ReplaceNode(SDValue(Node, 0), Result); |
517 | 143k | break; |
518 | 2.97M | } |
519 | 2.97M | } |
520 | 2.97M | return; |
521 | 2.97M | } |
522 | 0 | } else { |
523 | 739k | SDValue Value = ST->getValue(); |
524 | 739k | |
525 | 739k | EVT StVT = ST->getMemoryVT(); |
526 | 739k | unsigned StWidth = StVT.getSizeInBits(); |
527 | 739k | auto &DL = DAG.getDataLayout(); |
528 | 739k | |
529 | 739k | if (StWidth != StVT.getStoreSizeInBits()739k ) { |
530 | 28.2k | // Promote to a byte-sized store with upper bits zero if not |
531 | 28.2k | // storing an integral number of bytes. For example, promote |
532 | 28.2k | // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) |
533 | 28.2k | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), |
534 | 28.2k | StVT.getStoreSizeInBits()); |
535 | 28.2k | Value = DAG.getZeroExtendInReg(Value, dl, StVT); |
536 | 28.2k | SDValue Result = |
537 | 28.2k | DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, |
538 | 28.2k | Alignment, MMOFlags, AAInfo); |
539 | 28.2k | ReplaceNode(SDValue(Node, 0), Result); |
540 | 739k | } else if (710k StWidth & (StWidth - 1)710k ) { |
541 | 907 | // If not storing a power-of-2 number of bits, expand as two stores. |
542 | 907 | assert(!StVT.isVector() && "Unsupported truncstore!"); |
543 | 907 | unsigned RoundWidth = 1 << Log2_32(StWidth); |
544 | 907 | assert(RoundWidth < StWidth); |
545 | 907 | unsigned ExtraWidth = StWidth - RoundWidth; |
546 | 907 | assert(ExtraWidth < RoundWidth); |
547 | 907 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
548 | 907 | "Store size not an integral number of bytes!"); |
549 | 907 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); |
550 | 907 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); |
551 | 907 | SDValue Lo, Hi; |
552 | 907 | unsigned IncrementSize; |
553 | 907 | |
554 | 907 | if (DL.isLittleEndian()907 ) { |
555 | 897 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) |
556 | 897 | // Store the bottom RoundWidth bits. |
557 | 897 | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
558 | 897 | RoundVT, Alignment, MMOFlags, AAInfo); |
559 | 897 | |
560 | 897 | // Store the remaining ExtraWidth bits. |
561 | 897 | IncrementSize = RoundWidth / 8; |
562 | 897 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
563 | 897 | DAG.getConstant(IncrementSize, dl, |
564 | 897 | Ptr.getValueType())); |
565 | 897 | Hi = DAG.getNode( |
566 | 897 | ISD::SRL, dl, Value.getValueType(), Value, |
567 | 897 | DAG.getConstant(RoundWidth, dl, |
568 | 897 | TLI.getShiftAmountTy(Value.getValueType(), DL))); |
569 | 897 | Hi = DAG.getTruncStore( |
570 | 897 | Chain, dl, Hi, Ptr, |
571 | 897 | ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT, |
572 | 897 | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); |
573 | 907 | } else { |
574 | 10 | // Big endian - avoid unaligned stores. |
575 | 10 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X |
576 | 10 | // Store the top RoundWidth bits. |
577 | 10 | Hi = DAG.getNode( |
578 | 10 | ISD::SRL, dl, Value.getValueType(), Value, |
579 | 10 | DAG.getConstant(ExtraWidth, dl, |
580 | 10 | TLI.getShiftAmountTy(Value.getValueType(), DL))); |
581 | 10 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), |
582 | 10 | RoundVT, Alignment, MMOFlags, AAInfo); |
583 | 10 | |
584 | 10 | // Store the remaining ExtraWidth bits. |
585 | 10 | IncrementSize = RoundWidth / 8; |
586 | 10 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
587 | 10 | DAG.getConstant(IncrementSize, dl, |
588 | 10 | Ptr.getValueType())); |
589 | 10 | Lo = DAG.getTruncStore( |
590 | 10 | Chain, dl, Value, Ptr, |
591 | 10 | ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT, |
592 | 10 | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); |
593 | 10 | } |
594 | 907 | |
595 | 907 | // The order of the stores doesn't matter. |
596 | 907 | SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); |
597 | 907 | ReplaceNode(SDValue(Node, 0), Result); |
598 | 710k | } else { |
599 | 710k | switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { |
600 | 0 | default: 0 llvm_unreachable0 ("This action is not supported yet!"); |
601 | 706k | case TargetLowering::Legal: { |
602 | 706k | EVT MemVT = ST->getMemoryVT(); |
603 | 706k | unsigned AS = ST->getAddressSpace(); |
604 | 706k | unsigned Align = ST->getAlignment(); |
605 | 706k | // If this is an unaligned store and the target doesn't support it, |
606 | 706k | // expand it. |
607 | 706k | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)706k ) { |
608 | 415 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); |
609 | 415 | ReplaceNode(SDValue(ST, 0), Result); |
610 | 415 | } |
611 | 706k | break; |
612 | 710k | } |
613 | 1.63k | case TargetLowering::Custom: { |
614 | 1.63k | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); |
615 | 1.63k | if (Res && 1.63k Res != SDValue(Node, 0)1.49k ) |
616 | 1.49k | ReplaceNode(SDValue(Node, 0), Res); |
617 | 1.63k | return; |
618 | 710k | } |
619 | 1.66k | case TargetLowering::Expand: |
620 | 1.66k | assert(!StVT.isVector() && |
621 | 1.66k | "Vector Stores are handled in LegalizeVectorOps"); |
622 | 1.66k | |
623 | 1.66k | // TRUNCSTORE:i16 i32 -> STORE i16 |
624 | 1.66k | assert(TLI.isTypeLegal(StVT) && |
625 | 1.66k | "Do not know how to expand this store!"); |
626 | 1.66k | Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); |
627 | 1.66k | SDValue Result = |
628 | 1.66k | DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), |
629 | 1.66k | Alignment, MMOFlags, AAInfo); |
630 | 1.66k | ReplaceNode(SDValue(Node, 0), Result); |
631 | 1.66k | break; |
632 | 737k | } |
633 | 737k | } |
634 | 739k | } |
635 | 3.81M | } |
636 | | |
637 | 4.54M | void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { |
638 | 4.54M | LoadSDNode *LD = cast<LoadSDNode>(Node); |
639 | 4.54M | SDValue Chain = LD->getChain(); // The chain. |
640 | 4.54M | SDValue Ptr = LD->getBasePtr(); // The base pointer. |
641 | 4.54M | SDValue Value; // The value returned by the load op. |
642 | 4.54M | SDLoc dl(Node); |
643 | 4.54M | |
644 | 4.54M | ISD::LoadExtType ExtType = LD->getExtensionType(); |
645 | 4.54M | if (ExtType == ISD::NON_EXTLOAD4.54M ) { |
646 | 3.57M | MVT VT = Node->getSimpleValueType(0); |
647 | 3.57M | SDValue RVal = SDValue(Node, 0); |
648 | 3.57M | SDValue RChain = SDValue(Node, 1); |
649 | 3.57M | |
650 | 3.57M | switch (TLI.getOperationAction(Node->getOpcode(), VT)) { |
651 | 0 | default: 0 llvm_unreachable0 ("This action is not supported yet!"); |
652 | 3.38M | case TargetLowering::Legal: { |
653 | 3.38M | EVT MemVT = LD->getMemoryVT(); |
654 | 3.38M | unsigned AS = LD->getAddressSpace(); |
655 | 3.38M | unsigned Align = LD->getAlignment(); |
656 | 3.38M | const DataLayout &DL = DAG.getDataLayout(); |
657 | 3.38M | // If this is an unaligned load and the target doesn't support it, |
658 | 3.38M | // expand it. |
659 | 3.38M | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)3.38M ) { |
660 | 224 | std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); |
661 | 224 | } |
662 | 3.38M | break; |
663 | 3.57M | } |
664 | 124k | case TargetLowering::Custom: |
665 | 124k | if (SDValue Res124k = TLI.LowerOperation(RVal, DAG)) { |
666 | 15.8k | RVal = Res; |
667 | 15.8k | RChain = Res.getValue(1); |
668 | 15.8k | } |
669 | 124k | break; |
670 | 3.57M | |
671 | 62.0k | case TargetLowering::Promote: { |
672 | 62.0k | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); |
673 | 62.0k | assert(NVT.getSizeInBits() == VT.getSizeInBits() && |
674 | 62.0k | "Can only promote loads to same size type"); |
675 | 62.0k | |
676 | 62.0k | SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); |
677 | 62.0k | RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); |
678 | 62.0k | RChain = Res.getValue(1); |
679 | 62.0k | break; |
680 | 3.57M | } |
681 | 3.57M | } |
682 | 3.57M | if (3.57M RChain.getNode() != Node3.57M ) { |
683 | 75.6k | assert(RVal.getNode() != Node && "Load must be completely replaced"); |
684 | 75.6k | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); |
685 | 75.6k | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); |
686 | 75.6k | if (UpdatedNodes75.6k ) { |
687 | 1.75k | UpdatedNodes->insert(RVal.getNode()); |
688 | 1.75k | UpdatedNodes->insert(RChain.getNode()); |
689 | 1.75k | } |
690 | 75.6k | ReplacedNode(Node); |
691 | 75.6k | } |
692 | 3.57M | return; |
693 | 3.57M | } |
694 | 973k | |
695 | 973k | EVT SrcVT = LD->getMemoryVT(); |
696 | 973k | unsigned SrcWidth = SrcVT.getSizeInBits(); |
697 | 973k | unsigned Alignment = LD->getAlignment(); |
698 | 973k | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); |
699 | 973k | AAMDNodes AAInfo = LD->getAAInfo(); |
700 | 973k | |
701 | 973k | if (SrcWidth != SrcVT.getStoreSizeInBits() && |
702 | 973k | // Some targets pretend to have an i1 loading operation, and actually |
703 | 973k | // load an i8. This trick is correct for ZEXTLOAD because the top 7 |
704 | 973k | // bits are guaranteed to be zero; it helps the optimizers understand |
705 | 973k | // that these bits are zero. It is also useful for EXTLOAD, since it |
706 | 973k | // tells the optimizers that those bits are undefined. It would be |
707 | 973k | // nice to have an effective generic way of getting these benefits... |
708 | 973k | // Until such a way is found, don't insist on promoting i1 here. |
709 | 8.04k | (SrcVT != MVT::i1 || |
710 | 7.69k | TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == |
711 | 973k | TargetLowering::Promote)) { |
712 | 611 | // Promote to a byte-sized load if not loading an integral number of |
713 | 611 | // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. |
714 | 611 | unsigned NewWidth = SrcVT.getStoreSizeInBits(); |
715 | 611 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); |
716 | 611 | SDValue Ch; |
717 | 611 | |
718 | 611 | // The extra bits are guaranteed to be zero, since we stored them that |
719 | 611 | // way. A zext load from NVT thus automatically gives zext from SrcVT. |
720 | 611 | |
721 | 611 | ISD::LoadExtType NewExtType = |
722 | 611 | ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD262 : ISD::EXTLOAD349 ; |
723 | 611 | |
724 | 611 | SDValue Result = |
725 | 611 | DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr, |
726 | 611 | LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo); |
727 | 611 | |
728 | 611 | Ch = Result.getValue(1); // The chain. |
729 | 611 | |
730 | 611 | if (ExtType == ISD::SEXTLOAD) |
731 | 611 | // Having the top bits zero doesn't help when sign extending. |
732 | 81 | Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, |
733 | 81 | Result.getValueType(), |
734 | 81 | Result, DAG.getValueType(SrcVT)); |
735 | 530 | else if (530 ExtType == ISD::ZEXTLOAD || 530 NVT == Result.getValueType()268 ) |
736 | 530 | // All the top bits are guaranteed to be zero - inform the optimizers. |
737 | 335 | Result = DAG.getNode(ISD::AssertZext, dl, |
738 | 335 | Result.getValueType(), Result, |
739 | 335 | DAG.getValueType(SrcVT)); |
740 | 611 | |
741 | 611 | Value = Result; |
742 | 611 | Chain = Ch; |
743 | 973k | } else if (973k SrcWidth & (SrcWidth - 1)973k ) { |
744 | 647 | // If not loading a power-of-2 number of bits, expand as two loads. |
745 | 647 | assert(!SrcVT.isVector() && "Unsupported extload!"); |
746 | 647 | unsigned RoundWidth = 1 << Log2_32(SrcWidth); |
747 | 647 | assert(RoundWidth < SrcWidth); |
748 | 647 | unsigned ExtraWidth = SrcWidth - RoundWidth; |
749 | 647 | assert(ExtraWidth < RoundWidth); |
750 | 647 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) && |
751 | 647 | "Load size not an integral number of bytes!"); |
752 | 647 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); |
753 | 647 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); |
754 | 647 | SDValue Lo, Hi, Ch; |
755 | 647 | unsigned IncrementSize; |
756 | 647 | auto &DL = DAG.getDataLayout(); |
757 | 647 | |
758 | 647 | if (DL.isLittleEndian()647 ) { |
759 | 622 | // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) |
760 | 622 | // Load the bottom RoundWidth bits. |
761 | 622 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, |
762 | 622 | LD->getPointerInfo(), RoundVT, Alignment, MMOFlags, |
763 | 622 | AAInfo); |
764 | 622 | |
765 | 622 | // Load the remaining ExtraWidth bits. |
766 | 622 | IncrementSize = RoundWidth / 8; |
767 | 622 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
768 | 622 | DAG.getConstant(IncrementSize, dl, |
769 | 622 | Ptr.getValueType())); |
770 | 622 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
771 | 622 | LD->getPointerInfo().getWithOffset(IncrementSize), |
772 | 622 | ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags, |
773 | 622 | AAInfo); |
774 | 622 | |
775 | 622 | // Build a factor node to remember that this load is independent of |
776 | 622 | // the other one. |
777 | 622 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
778 | 622 | Hi.getValue(1)); |
779 | 622 | |
780 | 622 | // Move the top bits to the right place. |
781 | 622 | Hi = DAG.getNode( |
782 | 622 | ISD::SHL, dl, Hi.getValueType(), Hi, |
783 | 622 | DAG.getConstant(RoundWidth, dl, |
784 | 622 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); |
785 | 622 | |
786 | 622 | // Join the hi and lo parts. |
787 | 622 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); |
788 | 647 | } else { |
789 | 25 | // Big endian - avoid unaligned loads. |
790 | 25 | // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 |
791 | 25 | // Load the top RoundWidth bits. |
792 | 25 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, |
793 | 25 | LD->getPointerInfo(), RoundVT, Alignment, MMOFlags, |
794 | 25 | AAInfo); |
795 | 25 | |
796 | 25 | // Load the remaining ExtraWidth bits. |
797 | 25 | IncrementSize = RoundWidth / 8; |
798 | 25 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, |
799 | 25 | DAG.getConstant(IncrementSize, dl, |
800 | 25 | Ptr.getValueType())); |
801 | 25 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, |
802 | 25 | LD->getPointerInfo().getWithOffset(IncrementSize), |
803 | 25 | ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags, |
804 | 25 | AAInfo); |
805 | 25 | |
806 | 25 | // Build a factor node to remember that this load is independent of |
807 | 25 | // the other one. |
808 | 25 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), |
809 | 25 | Hi.getValue(1)); |
810 | 25 | |
811 | 25 | // Move the top bits to the right place. |
812 | 25 | Hi = DAG.getNode( |
813 | 25 | ISD::SHL, dl, Hi.getValueType(), Hi, |
814 | 25 | DAG.getConstant(ExtraWidth, dl, |
815 | 25 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); |
816 | 25 | |
817 | 25 | // Join the hi and lo parts. |
818 | 25 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); |
819 | 25 | } |
820 | 647 | |
821 | 647 | Chain = Ch; |
822 | 973k | } else { |
823 | 972k | bool isCustom = false; |
824 | 972k | switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), |
825 | 972k | SrcVT.getSimpleVT())) { |
826 | 0 | default: 0 llvm_unreachable0 ("This action is not supported yet!"); |
827 | 5.92k | case TargetLowering::Custom: |
828 | 5.92k | isCustom = true; |
829 | 5.92k | LLVM_FALLTHROUGH; |
830 | 972k | case TargetLowering::Legal: |
831 | 972k | Value = SDValue(Node, 0); |
832 | 972k | Chain = SDValue(Node, 1); |
833 | 972k | |
834 | 972k | if (isCustom972k ) { |
835 | 5.92k | if (SDValue Res5.92k = TLI.LowerOperation(SDValue(Node, 0), DAG)) { |
836 | 4.34k | Value = Res; |
837 | 4.34k | Chain = Res.getValue(1); |
838 | 4.34k | } |
839 | 972k | } else { |
840 | 966k | // If this is an unaligned load and the target doesn't support it, |
841 | 966k | // expand it. |
842 | 966k | EVT MemVT = LD->getMemoryVT(); |
843 | 966k | unsigned AS = LD->getAddressSpace(); |
844 | 966k | unsigned Align = LD->getAlignment(); |
845 | 966k | const DataLayout &DL = DAG.getDataLayout(); |
846 | 966k | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)966k ) { |
847 | 448 | std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); |
848 | 448 | } |
849 | 966k | } |
850 | 972k | break; |
851 | 5.92k | |
852 | 323 | case TargetLowering::Expand: { |
853 | 323 | EVT DestVT = Node->getValueType(0); |
854 | 323 | if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)323 ) { |
855 | 308 | // If the source type is not legal, see if there is a legal extload to |
856 | 308 | // an intermediate type that we can then extend further. |
857 | 308 | EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); |
858 | 308 | if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT? |
859 | 308 | TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)56 ) { |
860 | 308 | // If we are loading a legal type, this is a non-extload followed by a |
861 | 308 | // full extend. |
862 | 308 | ISD::LoadExtType MidExtType = |
863 | 308 | (LoadVT == SrcVT) ? ISD::NON_EXTLOAD252 : ExtType56 ; |
864 | 308 | |
865 | 308 | SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, |
866 | 308 | SrcVT, LD->getMemOperand()); |
867 | 308 | unsigned ExtendOp = |
868 | 308 | ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); |
869 | 308 | Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); |
870 | 308 | Chain = Load.getValue(1); |
871 | 308 | break; |
872 | 308 | } |
873 | 0 |
|
874 | 0 | // Handle the special case of fp16 extloads. EXTLOAD doesn't have the |
875 | 0 | // normal undefined upper bits behavior to allow using an in-reg extend |
876 | 0 | // with the illegal FP type, so load as an integer and do the |
877 | 0 | // from-integer conversion. |
878 | 0 | if (0 SrcVT.getScalarType() == MVT::f160 ) { |
879 | 0 | EVT ISrcVT = SrcVT.changeTypeToInteger(); |
880 | 0 | EVT IDestVT = DestVT.changeTypeToInteger(); |
881 | 0 | EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT()); |
882 | 0 |
|
883 | 0 | SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT, |
884 | 0 | Chain, Ptr, ISrcVT, |
885 | 0 | LD->getMemOperand()); |
886 | 0 | Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result); |
887 | 0 | Chain = Result.getValue(1); |
888 | 0 | break; |
889 | 0 | } |
890 | 15 | } |
891 | 15 | |
892 | 0 | assert(!SrcVT.isVector() && |
893 | 15 | "Vector Loads are handled in LegalizeVectorOps"); |
894 | 15 | |
895 | 15 | // FIXME: This does not work for vectors on most targets. Sign- |
896 | 15 | // and zero-extend operations are currently folded into extending |
897 | 15 | // loads, whether they are legal or not, and then we end up here |
898 | 15 | // without any support for legalizing them. |
899 | 15 | assert(ExtType != ISD::EXTLOAD && |
900 | 15 | "EXTLOAD should always be supported!"); |
901 | 15 | // Turn the unsupported load into an EXTLOAD followed by an |
902 | 15 | // explicit zero/sign extend inreg. |
903 | 15 | SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, |
904 | 15 | Node->getValueType(0), |
905 | 15 | Chain, Ptr, SrcVT, |
906 | 15 | LD->getMemOperand()); |
907 | 15 | SDValue ValRes; |
908 | 15 | if (ExtType == ISD::SEXTLOAD) |
909 | 14 | ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, |
910 | 14 | Result.getValueType(), |
911 | 14 | Result, DAG.getValueType(SrcVT)); |
912 | 15 | else |
913 | 1 | ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); |
914 | 5.92k | Value = ValRes; |
915 | 5.92k | Chain = Result.getValue(1); |
916 | 5.92k | break; |
917 | 5.92k | } |
918 | 973k | } |
919 | 973k | } |
920 | 973k | |
921 | 973k | // Since loads produce two values, make sure to remember that we legalized |
922 | 973k | // both of them. |
923 | 973k | if (973k Chain.getNode() != Node973k ) { |
924 | 6.36k | assert(Value.getNode() != Node && "Load must be completely replaced"); |
925 | 6.36k | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); |
926 | 6.36k | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); |
927 | 6.36k | if (UpdatedNodes6.36k ) { |
928 | 1 | UpdatedNodes->insert(Value.getNode()); |
929 | 1 | UpdatedNodes->insert(Chain.getNode()); |
930 | 1 | } |
931 | 6.36k | ReplacedNode(Node); |
932 | 6.36k | } |
933 | 4.54M | } |
934 | | |
935 | | static TargetLowering::LegalizeAction |
936 | 34 | getStrictFPOpcodeAction(const TargetLowering &TLI, unsigned Opcode, EVT VT) { |
937 | 34 | unsigned EqOpc; |
938 | 34 | switch (Opcode) { |
939 | 0 | default: 0 llvm_unreachable0 ("Unexpected FP pseudo-opcode"); |
940 | 4 | case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break; |
941 | 2 | case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break; |
942 | 2 | case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break; |
943 | 6 | case ISD::STRICT_FMA: EqOpc = ISD::FMA; break; |
944 | 2 | case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break; |
945 | 2 | case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break; |
946 | 2 | case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break; |
947 | 2 | case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break; |
948 | 2 | case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break; |
949 | 2 | case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break; |
950 | 2 | case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break; |
951 | 3 | case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break; |
952 | 3 | case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break; |
953 | 34 | } |
954 | 34 | |
955 | 34 | auto Action = TLI.getOperationAction(EqOpc, VT); |
956 | 34 | |
957 | 34 | // We don't currently handle Custom or Promote for strict FP pseudo-ops. |
958 | 34 | // For now, we just expand for those cases. |
959 | 34 | if (Action != TargetLowering::Legal) |
960 | 22 | Action = TargetLowering::Expand; |
961 | 34 | |
962 | 34 | return Action; |
963 | 34 | } |
964 | | |
965 | | /// Return a legal replacement for the given operation, with all legal operands. |
966 | 148M | void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { |
967 | 148M | DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG)); |
968 | 148M | |
969 | 148M | if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. |
970 | 2.74M | return; |
971 | 145M | |
972 | | #ifndef NDEBUG |
973 | | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) |
974 | | assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == |
975 | | TargetLowering::TypeLegal || |
976 | | TLI.isTypeLegal(Node->getValueType(i))) && |
977 | | "Unexpected illegal type!"); |
978 | | |
979 | | for (const SDValue &Op : Node->op_values()) |
980 | | assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == |
981 | | TargetLowering::TypeLegal || |
982 | | TLI.isTypeLegal(Op.getValueType()) || |
983 | | Op.getOpcode() == ISD::TargetConstant) && |
984 | | "Unexpected illegal type!"); |
985 | | #endif |
986 | | |
987 | 145M | // Figure out the correct action; the way to query this varies by opcode |
988 | 145M | TargetLowering::LegalizeAction Action = TargetLowering::Legal; |
989 | 145M | bool SimpleFinishLegalizing = true; |
990 | 145M | switch (Node->getOpcode()) { |
991 | 324k | case ISD::INTRINSIC_W_CHAIN: |
992 | 324k | case ISD::INTRINSIC_WO_CHAIN: |
993 | 324k | case ISD::INTRINSIC_VOID: |
994 | 324k | case ISD::STACKSAVE: |
995 | 324k | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); |
996 | 324k | break; |
997 | 4 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
998 | 4 | Action = TLI.getOperationAction(Node->getOpcode(), |
999 | 4 | Node->getValueType(0)); |
1000 | 4 | break; |
1001 | 368 | case ISD::VAARG: |
1002 | 368 | Action = TLI.getOperationAction(Node->getOpcode(), |
1003 | 368 | Node->getValueType(0)); |
1004 | 368 | if (Action != TargetLowering::Promote) |
1005 | 367 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); |
1006 | 368 | break; |
1007 | 499k | case ISD::FP_TO_FP16: |
1008 | 499k | case ISD::SINT_TO_FP: |
1009 | 499k | case ISD::UINT_TO_FP: |
1010 | 499k | case ISD::EXTRACT_VECTOR_ELT: |
1011 | 499k | Action = TLI.getOperationAction(Node->getOpcode(), |
1012 | 499k | Node->getOperand(0).getValueType()); |
1013 | 499k | break; |
1014 | 65.2k | case ISD::FP_ROUND_INREG: |
1015 | 65.2k | case ISD::SIGN_EXTEND_INREG: { |
1016 | 65.2k | EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
1017 | 65.2k | Action = TLI.getOperationAction(Node->getOpcode(), InnerType); |
1018 | 65.2k | break; |
1019 | 65.2k | } |
1020 | 24.1k | case ISD::ATOMIC_STORE: |
1021 | 24.1k | Action = TLI.getOperationAction(Node->getOpcode(), |
1022 | 24.1k | Node->getOperand(2).getValueType()); |
1023 | 24.1k | break; |
1024 | 1.93M | case ISD::SELECT_CC: |
1025 | 1.93M | case ISD::SETCC: |
1026 | 1.93M | case ISD::BR_CC: { |
1027 | 124k | unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : |
1028 | 1.81M | Node->getOpcode() == ISD::SETCC ? 1.81M 272.6k : |
1029 | 1.73M | Node->getOpcode() == ISD::SETCCE ? 1.73M 30 : 11.73M ; |
1030 | 1.93M | unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 21.73M : 0196k ; |
1031 | 1.93M | MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); |
1032 | 1.93M | ISD::CondCode CCCode = |
1033 | 1.93M | cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); |
1034 | 1.93M | Action = TLI.getCondCodeAction(CCCode, OpVT); |
1035 | 1.93M | if (Action == TargetLowering::Legal1.93M ) { |
1036 | 1.93M | if (Node->getOpcode() == ISD::SELECT_CC) |
1037 | 123k | Action = TLI.getOperationAction(Node->getOpcode(), |
1038 | 123k | Node->getValueType(0)); |
1039 | 1.93M | else |
1040 | 1.81M | Action = TLI.getOperationAction(Node->getOpcode(), OpVT); |
1041 | 1.93M | } |
1042 | 1.93M | break; |
1043 | 1.93M | } |
1044 | 8.35M | case ISD::LOAD: |
1045 | 8.35M | case ISD::STORE: |
1046 | 8.35M | // FIXME: Model these properly. LOAD and STORE are complicated, and |
1047 | 8.35M | // STORE expects the unlegalized operand in some cases. |
1048 | 8.35M | SimpleFinishLegalizing = false; |
1049 | 8.35M | break; |
1050 | 6.27M | case ISD::CALLSEQ_START: |
1051 | 6.27M | case ISD::CALLSEQ_END: |
1052 | 6.27M | // FIXME: This shouldn't be necessary. These nodes have special properties |
1053 | 6.27M | // dealing with the recursive nature of legalization. Removing this |
1054 | 6.27M | // special case should be done as part of making LegalizeDAG non-recursive. |
1055 | 6.27M | SimpleFinishLegalizing = false; |
1056 | 6.27M | break; |
1057 | 26.4k | case ISD::EXTRACT_ELEMENT: |
1058 | 26.4k | case ISD::FLT_ROUNDS_: |
1059 | 26.4k | case ISD::MERGE_VALUES: |
1060 | 26.4k | case ISD::EH_RETURN: |
1061 | 26.4k | case ISD::FRAME_TO_ARGS_OFFSET: |
1062 | 26.4k | case ISD::EH_DWARF_CFA: |
1063 | 26.4k | case ISD::EH_SJLJ_SETJMP: |
1064 | 26.4k | case ISD::EH_SJLJ_LONGJMP: |
1065 | 26.4k | case ISD::EH_SJLJ_SETUP_DISPATCH: |
1066 | 26.4k | // These operations lie about being legal: when they claim to be legal, |
1067 | 26.4k | // they should actually be expanded. |
1068 | 26.4k | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1069 | 26.4k | if (Action == TargetLowering::Legal) |
1070 | 24.7k | Action = TargetLowering::Expand; |
1071 | 26.4k | break; |
1072 | 207 | case ISD::INIT_TRAMPOLINE: |
1073 | 207 | case ISD::ADJUST_TRAMPOLINE: |
1074 | 207 | case ISD::FRAMEADDR: |
1075 | 207 | case ISD::RETURNADDR: |
1076 | 207 | case ISD::ADDROFRETURNADDR: |
1077 | 207 | // These operations lie about being legal: when they claim to be legal, |
1078 | 207 | // they should actually be custom-lowered. |
1079 | 207 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1080 | 207 | if (Action == TargetLowering::Legal) |
1081 | 203 | Action = TargetLowering::Custom; |
1082 | 207 | break; |
1083 | 25 | case ISD::READCYCLECOUNTER: |
1084 | 25 | // READCYCLECOUNTER returns an i64, even if type legalization might have |
1085 | 25 | // expanded that to several smaller types. |
1086 | 25 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); |
1087 | 25 | break; |
1088 | 752 | case ISD::READ_REGISTER: |
1089 | 752 | case ISD::WRITE_REGISTER: |
1090 | 752 | // Named register is legal in the DAG, but blocked by register name |
1091 | 752 | // selection if not implemented by target (to chose the correct register) |
1092 | 752 | // They'll be converted to Copy(To/From)Reg. |
1093 | 752 | Action = TargetLowering::Legal; |
1094 | 752 | break; |
1095 | 26 | case ISD::DEBUGTRAP: |
1096 | 26 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1097 | 26 | if (Action == TargetLowering::Expand26 ) { |
1098 | 11 | // replace ISD::DEBUGTRAP with ISD::TRAP |
1099 | 11 | SDValue NewVal; |
1100 | 11 | NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), |
1101 | 11 | Node->getOperand(0)); |
1102 | 11 | ReplaceNode(Node, NewVal.getNode()); |
1103 | 11 | LegalizeOp(NewVal.getNode()); |
1104 | 11 | return; |
1105 | 11 | } |
1106 | 15 | break; |
1107 | 34 | case ISD::STRICT_FSQRT: |
1108 | 34 | case ISD::STRICT_FMA: |
1109 | 34 | case ISD::STRICT_FPOW: |
1110 | 34 | case ISD::STRICT_FPOWI: |
1111 | 34 | case ISD::STRICT_FSIN: |
1112 | 34 | case ISD::STRICT_FCOS: |
1113 | 34 | case ISD::STRICT_FEXP: |
1114 | 34 | case ISD::STRICT_FEXP2: |
1115 | 34 | case ISD::STRICT_FLOG: |
1116 | 34 | case ISD::STRICT_FLOG10: |
1117 | 34 | case ISD::STRICT_FLOG2: |
1118 | 34 | case ISD::STRICT_FRINT: |
1119 | 34 | case ISD::STRICT_FNEARBYINT: |
1120 | 34 | // These pseudo-ops get legalized as if they were their non-strict |
1121 | 34 | // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT |
1122 | 34 | // is also legal, but if ISD::FSQRT requires expansion then so does |
1123 | 34 | // ISD::STRICT_FSQRT. |
1124 | 34 | Action = getStrictFPOpcodeAction(TLI, Node->getOpcode(), |
1125 | 34 | Node->getValueType(0)); |
1126 | 34 | break; |
1127 | 128M | default: |
1128 | 128M | if (Node->getOpcode() >= ISD::BUILTIN_OP_END128M ) { |
1129 | 15.2M | Action = TargetLowering::Legal; |
1130 | 128M | } else { |
1131 | 113M | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); |
1132 | 113M | } |
1133 | 324k | break; |
1134 | 145M | } |
1135 | 145M | |
1136 | 145M | if (145M SimpleFinishLegalizing145M ) { |
1137 | 131M | SDNode *NewNode = Node; |
1138 | 131M | switch (Node->getOpcode()) { |
1139 | 130M | default: break; |
1140 | 945k | case ISD::SHL: |
1141 | 945k | case ISD::SRL: |
1142 | 945k | case ISD::SRA: |
1143 | 945k | case ISD::ROTL: |
1144 | 945k | case ISD::ROTR: { |
1145 | 945k | // Legalizing shifts/rotates requires adjusting the shift amount |
1146 | 945k | // to the appropriate width. |
1147 | 945k | SDValue Op0 = Node->getOperand(0); |
1148 | 945k | SDValue Op1 = Node->getOperand(1); |
1149 | 945k | if (!Op1.getValueType().isVector()945k ) { |
1150 | 940k | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); |
1151 | 940k | // The getShiftAmountOperand() may create a new operand node or |
1152 | 940k | // return the existing one. If new operand is created we need |
1153 | 940k | // to update the parent node. |
1154 | 940k | // Do not try to legalize SAO here! It will be automatically legalized |
1155 | 940k | // in the next round. |
1156 | 940k | if (SAO != Op1) |
1157 | 15.0k | NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); |
1158 | 940k | } |
1159 | 945k | } |
1160 | 945k | break; |
1161 | 1.28k | case ISD::SRL_PARTS: |
1162 | 1.28k | case ISD::SRA_PARTS: |
1163 | 1.28k | case ISD::SHL_PARTS: { |
1164 | 1.28k | // Legalizing shifts/rotates requires adjusting the shift amount |
1165 | 1.28k | // to the appropriate width. |
1166 | 1.28k | SDValue Op0 = Node->getOperand(0); |
1167 | 1.28k | SDValue Op1 = Node->getOperand(1); |
1168 | 1.28k | SDValue Op2 = Node->getOperand(2); |
1169 | 1.28k | if (!Op2.getValueType().isVector()1.28k ) { |
1170 | 1.28k | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); |
1171 | 1.28k | // The getShiftAmountOperand() may create a new operand node or |
1172 | 1.28k | // return the existing one. If new operand is created we need |
1173 | 1.28k | // to update the parent node. |
1174 | 1.28k | if (SAO != Op2) |
1175 | 0 | NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); |
1176 | 1.28k | } |
1177 | 1.28k | break; |
1178 | 131M | } |
1179 | 131M | } |
1180 | 131M | |
1181 | 131M | if (131M NewNode != Node131M ) { |
1182 | 145 | ReplaceNode(Node, NewNode); |
1183 | 145 | Node = NewNode; |
1184 | 145 | } |
1185 | 131M | switch (Action) { |
1186 | 126M | case TargetLowering::Legal: |
1187 | 126M | return; |
1188 | 4.30M | case TargetLowering::Custom: |
1189 | 4.30M | // FIXME: The handling for custom lowering with multiple results is |
1190 | 4.30M | // a complete mess. |
1191 | 4.30M | if (SDValue Res4.30M = TLI.LowerOperation(SDValue(Node, 0), DAG)) { |
1192 | 4.20M | if (!(Res.getNode() != Node || 4.20M Res.getResNo() != 0683k )) |
1193 | 683k | return; |
1194 | 3.52M | |
1195 | 3.52M | if (3.52M Node->getNumValues() == 13.52M ) { |
1196 | 3.50M | // We can just directly replace this node with the lowered value. |
1197 | 3.50M | ReplaceNode(SDValue(Node, 0), Res); |
1198 | 3.50M | return; |
1199 | 3.50M | } |
1200 | 16.6k | |
1201 | 16.6k | SmallVector<SDValue, 8> ResultVals; |
1202 | 49.8k | for (unsigned i = 0, e = Node->getNumValues(); i != e49.8k ; ++i33.2k ) |
1203 | 33.2k | ResultVals.push_back(Res.getValue(i)); |
1204 | 4.20M | ReplaceNode(Node, ResultVals.data()); |
1205 | 4.20M | return; |
1206 | 4.20M | } |
1207 | 98.2k | LLVM_FALLTHROUGH98.2k ; |
1208 | 334k | case TargetLowering::Expand: |
1209 | 334k | if (ExpandNode(Node)) |
1210 | 186k | return; |
1211 | 147k | LLVM_FALLTHROUGH147k ; |
1212 | 147k | case TargetLowering::LibCall: |
1213 | 147k | ConvertNodeToLibcall(Node); |
1214 | 147k | return; |
1215 | 6.94k | case TargetLowering::Promote: |
1216 | 6.94k | PromoteNode(Node); |
1217 | 6.94k | return; |
1218 | 14.6M | } |
1219 | 14.6M | } |
1220 | 14.6M | |
1221 | 14.6M | switch (Node->getOpcode()) { |
1222 | 0 | default: |
1223 | | #ifndef NDEBUG |
1224 | | dbgs() << "NODE: "; |
1225 | | Node->dump( &DAG); |
1226 | | dbgs() << "\n"; |
1227 | | #endif |
1228 | 0 | llvm_unreachable("Do not know how to legalize this operator!"); |
1229 | 14.6M | |
1230 | 6.27M | case ISD::CALLSEQ_START: |
1231 | 6.27M | case ISD::CALLSEQ_END: |
1232 | 6.27M | break; |
1233 | 4.54M | case ISD::LOAD: |
1234 | 4.54M | return LegalizeLoadOps(Node); |
1235 | 3.81M | case ISD::STORE: |
1236 | 3.81M | return LegalizeStoreOps(Node); |
1237 | 6.27M | } |
1238 | 6.27M | } |
1239 | | |
1240 | 2.06k | SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { |
1241 | 2.06k | SDValue Vec = Op.getOperand(0); |
1242 | 2.06k | SDValue Idx = Op.getOperand(1); |
1243 | 2.06k | SDLoc dl(Op); |
1244 | 2.06k | |
1245 | 2.06k | // Before we generate a new store to a temporary stack slot, see if there is |
1246 | 2.06k | // already one that we can use. There often is because when we scalarize |
1247 | 2.06k | // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole |
1248 | 2.06k | // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in |
1249 | 2.06k | // the vector. If all are expanded here, we don't want one store per vector |
1250 | 2.06k | // element. |
1251 | 2.06k | |
1252 | 2.06k | // Caches for hasPredecessorHelper |
1253 | 2.06k | SmallPtrSet<const SDNode *, 32> Visited; |
1254 | 2.06k | SmallVector<const SDNode *, 16> Worklist; |
1255 | 2.06k | Worklist.push_back(Idx.getNode()); |
1256 | 2.06k | SDValue StackPtr, Ch; |
1257 | 2.06k | for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), |
1258 | 4.15k | UE = Vec.getNode()->use_end(); UI != UE4.15k ; ++UI2.09k ) { |
1259 | 3.73k | SDNode *User = *UI; |
1260 | 3.73k | if (StoreSDNode *ST3.73k = dyn_cast<StoreSDNode>(User)) { |
1261 | 1.64k | if (ST->isIndexed() || 1.64k ST->isTruncatingStore()1.64k || |
1262 | 1.64k | ST->getValue() != Vec) |
1263 | 0 | continue; |
1264 | 1.64k | |
1265 | 1.64k | // Make sure that nothing else could have stored into the destination of |
1266 | 1.64k | // this store. |
1267 | 1.64k | if (1.64k !ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())1.64k ) |
1268 | 0 | continue; |
1269 | 1.64k | |
1270 | 1.64k | // If the index is dependent on the store we will introduce a cycle when |
1271 | 1.64k | // creating the load (the load uses the index, and by replacing the chain |
1272 | 1.64k | // we will make the index dependent on the load). Also, the store might be |
1273 | 1.64k | // dependent on the extractelement and introduce a cycle when creating |
1274 | 1.64k | // the load. |
1275 | 1.64k | if (1.64k SDNode::hasPredecessorHelper(ST, Visited, Worklist) || |
1276 | 1.64k | ST->hasPredecessor(Op.getNode())) |
1277 | 2 | continue; |
1278 | 1.64k | |
1279 | 1.64k | StackPtr = ST->getBasePtr(); |
1280 | 1.64k | Ch = SDValue(ST, 0); |
1281 | 1.64k | break; |
1282 | 1.64k | } |
1283 | 3.73k | } |
1284 | 2.06k | |
1285 | 2.06k | EVT VecVT = Vec.getValueType(); |
1286 | 2.06k | |
1287 | 2.06k | if (!Ch.getNode()2.06k ) { |
1288 | 420 | // Store the value to a temporary stack slot, then LOAD the returned part. |
1289 | 420 | StackPtr = DAG.CreateStackTemporary(VecVT); |
1290 | 420 | Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, |
1291 | 420 | MachinePointerInfo()); |
1292 | 420 | } |
1293 | 2.06k | |
1294 | 2.06k | StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); |
1295 | 2.06k | |
1296 | 2.06k | SDValue NewLoad; |
1297 | 2.06k | |
1298 | 2.06k | if (Op.getValueType().isVector()) |
1299 | 0 | NewLoad = |
1300 | 0 | DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo()); |
1301 | 2.06k | else |
1302 | 2.06k | NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, |
1303 | 2.06k | MachinePointerInfo(), |
1304 | 2.06k | VecVT.getVectorElementType()); |
1305 | 2.06k | |
1306 | 2.06k | // Replace the chain going out of the store, by the one out of the load. |
1307 | 2.06k | DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); |
1308 | 2.06k | |
1309 | 2.06k | // We introduced a cycle though, so update the loads operands, making sure |
1310 | 2.06k | // to use the original store's chain as an incoming chain. |
1311 | 2.06k | SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), |
1312 | 2.06k | NewLoad->op_end()); |
1313 | 2.06k | NewLoadOperands[0] = Ch; |
1314 | 2.06k | NewLoad = |
1315 | 2.06k | SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); |
1316 | 2.06k | return NewLoad; |
1317 | 2.06k | } |
1318 | | |
1319 | 0 | SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { |
1320 | 0 | assert(Op.getValueType().isVector() && "Non-vector insert subvector!"); |
1321 | 0 |
|
1322 | 0 | SDValue Vec = Op.getOperand(0); |
1323 | 0 | SDValue Part = Op.getOperand(1); |
1324 | 0 | SDValue Idx = Op.getOperand(2); |
1325 | 0 | SDLoc dl(Op); |
1326 | 0 |
|
1327 | 0 | // Store the value to a temporary stack slot, then LOAD the returned part. |
1328 | 0 | EVT VecVT = Vec.getValueType(); |
1329 | 0 | SDValue StackPtr = DAG.CreateStackTemporary(VecVT); |
1330 | 0 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
1331 | 0 | MachinePointerInfo PtrInfo = |
1332 | 0 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); |
1333 | 0 |
|
1334 | 0 | // First store the whole vector. |
1335 | 0 | SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); |
1336 | 0 |
|
1337 | 0 | // Then store the inserted part. |
1338 | 0 | SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); |
1339 | 0 |
|
1340 | 0 | // Store the subvector. |
1341 | 0 | Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo()); |
1342 | 0 |
|
1343 | 0 | // Finally, load the updated vector. |
1344 | 0 | return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo); |
1345 | 0 | } |
1346 | | |
1347 | 144 | SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { |
1348 | 144 | // We can't handle this case efficiently. Allocate a sufficiently |
1349 | 144 | // aligned object on the stack, store each element into it, then load |
1350 | 144 | // the result as a vector. |
1351 | 144 | // Create the stack frame object. |
1352 | 144 | EVT VT = Node->getValueType(0); |
1353 | 144 | EVT EltVT = VT.getVectorElementType(); |
1354 | 144 | SDLoc dl(Node); |
1355 | 144 | SDValue FIPtr = DAG.CreateStackTemporary(VT); |
1356 | 144 | int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); |
1357 | 144 | MachinePointerInfo PtrInfo = |
1358 | 144 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); |
1359 | 144 | |
1360 | 144 | // Emit a store of each element to the stack slot. |
1361 | 144 | SmallVector<SDValue, 8> Stores; |
1362 | 144 | unsigned TypeByteSize = EltVT.getSizeInBits() / 8; |
1363 | 144 | // Store (in the right endianness) the elements to memory. |
1364 | 1.16k | for (unsigned i = 0, e = Node->getNumOperands(); i != e1.16k ; ++i1.01k ) { |
1365 | 1.01k | // Ignore undef elements. |
1366 | 1.01k | if (Node->getOperand(i).isUndef()1.01k ) continue125 ; |
1367 | 891 | |
1368 | 891 | unsigned Offset = TypeByteSize*i; |
1369 | 891 | |
1370 | 891 | SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType()); |
1371 | 891 | Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); |
1372 | 891 | |
1373 | 891 | // If the destination vector element type is narrower than the source |
1374 | 891 | // element type, only store the bits necessary. |
1375 | 891 | if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())891 ) { |
1376 | 606 | Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, |
1377 | 606 | Node->getOperand(i), Idx, |
1378 | 606 | PtrInfo.getWithOffset(Offset), EltVT)); |
1379 | 606 | } else |
1380 | 285 | Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), |
1381 | 285 | Idx, PtrInfo.getWithOffset(Offset))); |
1382 | 1.01k | } |
1383 | 144 | |
1384 | 144 | SDValue StoreChain; |
1385 | 144 | if (!Stores.empty()) // Not all undef elements? |
1386 | 144 | StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); |
1387 | 144 | else |
1388 | 0 | StoreChain = DAG.getEntryNode(); |
1389 | 144 | |
1390 | 144 | // Result is a load from the stack slot. |
1391 | 144 | return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo); |
1392 | 144 | } |
1393 | | |
1394 | | /// Bitcast a floating-point value to an integer value. Only bitcast the part |
1395 | | /// containing the sign bit if the target has no integer value capable of |
1396 | | /// holding all bits of the floating-point value. |
1397 | | void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State, |
1398 | | const SDLoc &DL, |
1399 | 333 | SDValue Value) const { |
1400 | 333 | EVT FloatVT = Value.getValueType(); |
1401 | 333 | unsigned NumBits = FloatVT.getSizeInBits(); |
1402 | 333 | State.FloatVT = FloatVT; |
1403 | 333 | EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits); |
1404 | 333 | // Convert to an integer of the same size. |
1405 | 333 | if (TLI.isTypeLegal(IVT)333 ) { |
1406 | 109 | State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); |
1407 | 109 | State.SignMask = APInt::getSignMask(NumBits); |
1408 | 109 | State.SignBit = NumBits - 1; |
1409 | 109 | return; |
1410 | 109 | } |
1411 | 224 | |
1412 | 224 | auto &DataLayout = DAG.getDataLayout(); |
1413 | 224 | // Store the float to memory, then load the sign part out as an integer. |
1414 | 224 | MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8); |
1415 | 224 | // First create a temporary that is aligned for both the load and store. |
1416 | 224 | SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); |
1417 | 224 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); |
1418 | 224 | // Then store the float to it. |
1419 | 224 | State.FloatPtr = StackPtr; |
1420 | 224 | MachineFunction &MF = DAG.getMachineFunction(); |
1421 | 224 | State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI); |
1422 | 224 | State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr, |
1423 | 224 | State.FloatPointerInfo); |
1424 | 224 | |
1425 | 224 | SDValue IntPtr; |
1426 | 224 | if (DataLayout.isBigEndian()224 ) { |
1427 | 1 | assert(FloatVT.isByteSized() && "Unsupported floating point type!"); |
1428 | 1 | // Load out a legal integer with the same sign bit as the float. |
1429 | 1 | IntPtr = StackPtr; |
1430 | 1 | State.IntPointerInfo = State.FloatPointerInfo; |
1431 | 224 | } else { |
1432 | 223 | // Advance the pointer so that the loaded byte will contain the sign bit. |
1433 | 223 | unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1; |
1434 | 223 | IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr, |
1435 | 223 | DAG.getConstant(ByteOffset, DL, StackPtr.getValueType())); |
1436 | 223 | State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI, |
1437 | 223 | ByteOffset); |
1438 | 223 | } |
1439 | 333 | |
1440 | 333 | State.IntPtr = IntPtr; |
1441 | 333 | State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, |
1442 | 333 | State.IntPointerInfo, MVT::i8); |
1443 | 333 | State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7); |
1444 | 333 | State.SignBit = 7; |
1445 | 333 | } |
1446 | | |
1447 | | /// Replace the integer value produced by getSignAsIntValue() with a new value |
1448 | | /// and cast the result back to a floating-point type. |
1449 | | SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State, |
1450 | | const SDLoc &DL, |
1451 | 37 | SDValue NewIntValue) const { |
1452 | 37 | if (!State.Chain) |
1453 | 36 | return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); |
1454 | 1 | |
1455 | 1 | // Override the part containing the sign bit in the value stored on the stack. |
1456 | 1 | SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, |
1457 | 1 | State.IntPointerInfo, MVT::i8); |
1458 | 1 | return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr, |
1459 | 1 | State.FloatPointerInfo); |
1460 | 1 | } |
1461 | | |
1462 | 296 | SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { |
1463 | 296 | SDLoc DL(Node); |
1464 | 296 | SDValue Mag = Node->getOperand(0); |
1465 | 296 | SDValue Sign = Node->getOperand(1); |
1466 | 296 | |
1467 | 296 | // Get sign bit into an integer value. |
1468 | 296 | FloatSignAsInt SignAsInt; |
1469 | 296 | getSignAsIntValue(SignAsInt, DL, Sign); |
1470 | 296 | |
1471 | 296 | EVT IntVT = SignAsInt.IntValue.getValueType(); |
1472 | 296 | SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); |
1473 | 296 | SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, |
1474 | 296 | SignMask); |
1475 | 296 | |
1476 | 296 | // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X) |
1477 | 296 | EVT FloatVT = Mag.getValueType(); |
1478 | 296 | if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && |
1479 | 296 | TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)259 ) { |
1480 | 259 | SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); |
1481 | 259 | SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); |
1482 | 259 | SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit, |
1483 | 259 | DAG.getConstant(0, DL, IntVT), ISD::SETNE); |
1484 | 259 | return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue); |
1485 | 259 | } |
1486 | 37 | |
1487 | 37 | // Transform Mag value to integer, and clear the sign bit. |
1488 | 37 | FloatSignAsInt MagAsInt; |
1489 | 37 | getSignAsIntValue(MagAsInt, DL, Mag); |
1490 | 37 | EVT MagVT = MagAsInt.IntValue.getValueType(); |
1491 | 37 | SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT); |
1492 | 37 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, |
1493 | 37 | ClearSignMask); |
1494 | 37 | |
1495 | 37 | // Get the signbit at the right position for MagAsInt. |
1496 | 37 | int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; |
1497 | 37 | if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()37 ) { |
1498 | 19 | if (ShiftAmount > 019 ) { |
1499 | 19 | SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT); |
1500 | 19 | SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst); |
1501 | 19 | } else if (0 ShiftAmount < 00 ) { |
1502 | 0 | SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT); |
1503 | 0 | SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst); |
1504 | 0 | } |
1505 | 19 | SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); |
1506 | 37 | } else if (18 SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()18 ) { |
1507 | 0 | SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); |
1508 | 0 | if (ShiftAmount > 00 ) { |
1509 | 0 | SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT); |
1510 | 0 | SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst); |
1511 | 0 | } else if (0 ShiftAmount < 00 ) { |
1512 | 0 | SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT); |
1513 | 0 | SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst); |
1514 | 0 | } |
1515 | 18 | } |
1516 | 296 | |
1517 | 296 | // Store the part with the modified sign and convert back to float. |
1518 | 296 | SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); |
1519 | 296 | return modifySignAsInt(MagAsInt, DL, CopiedSign); |
1520 | 296 | } |
1521 | | |
1522 | 32 | SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { |
1523 | 32 | SDLoc DL(Node); |
1524 | 32 | SDValue Value = Node->getOperand(0); |
1525 | 32 | |
1526 | 32 | // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal. |
1527 | 32 | EVT FloatVT = Value.getValueType(); |
1528 | 32 | if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)32 ) { |
1529 | 32 | SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT); |
1530 | 32 | return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); |
1531 | 32 | } |
1532 | 0 |
|
1533 | 0 | // Transform value to integer, clear the sign bit and transform back. |
1534 | 0 | FloatSignAsInt ValueAsInt; |
1535 | 0 | getSignAsIntValue(ValueAsInt, DL, Value); |
1536 | 0 | EVT IntVT = ValueAsInt.IntValue.getValueType(); |
1537 | 0 | SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT); |
1538 | 0 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, |
1539 | 0 | ClearSignMask); |
1540 | 0 | return modifySignAsInt(ValueAsInt, DL, ClearedSign); |
1541 | 0 | } |
1542 | | |
1543 | | void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, |
1544 | 315 | SmallVectorImpl<SDValue> &Results) { |
1545 | 315 | unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); |
1546 | 315 | assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" |
1547 | 315 | " not tell us which reg is the stack pointer!"); |
1548 | 315 | SDLoc dl(Node); |
1549 | 315 | EVT VT = Node->getValueType(0); |
1550 | 315 | SDValue Tmp1 = SDValue(Node, 0); |
1551 | 315 | SDValue Tmp2 = SDValue(Node, 1); |
1552 | 315 | SDValue Tmp3 = Node->getOperand(2); |
1553 | 315 | SDValue Chain = Tmp1.getOperand(0); |
1554 | 315 | |
1555 | 315 | // Chain the dynamic stack allocation so that it doesn't modify the stack |
1556 | 315 | // pointer when other instructions are using the stack. |
1557 | 315 | Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); |
1558 | 315 | |
1559 | 315 | SDValue Size = Tmp2.getOperand(1); |
1560 | 315 | SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); |
1561 | 315 | Chain = SP.getValue(1); |
1562 | 315 | unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); |
1563 | 315 | unsigned StackAlign = |
1564 | 315 | DAG.getSubtarget().getFrameLowering()->getStackAlignment(); |
1565 | 315 | Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value |
1566 | 315 | if (Align > StackAlign) |
1567 | 51 | Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, |
1568 | 51 | DAG.getConstant(-(uint64_t)Align, dl, VT)); |
1569 | 315 | Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain |
1570 | 315 | |
1571 | 315 | Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true), |
1572 | 315 | DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); |
1573 | 315 | |
1574 | 315 | Results.push_back(Tmp1); |
1575 | 315 | Results.push_back(Tmp2); |
1576 | 315 | } |
1577 | | |
1578 | | /// Legalize a SETCC with given LHS and RHS and condition code CC on the current |
1579 | | /// target. |
1580 | | /// |
1581 | | /// If the SETCC has been legalized using AND / OR, then the legalized node |
1582 | | /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert |
1583 | | /// will be set to false. |
1584 | | /// |
1585 | | /// If the SETCC has been legalized by using getSetCCSwappedOperands(), |
1586 | | /// then the values of LHS and RHS will be swapped, CC will be set to the |
1587 | | /// new condition, and NeedInvert will be set to false. |
1588 | | /// |
1589 | | /// If the SETCC has been legalized using the inverse condcode, then LHS and |
1590 | | /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert |
1591 | | /// will be set to true. The caller must invert the result of the SETCC with |
1592 | | /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect |
1593 | | /// of a true/false result. |
1594 | | /// |
1595 | | /// \returns true if the SetCC has been legalized, false if it hasn't. |
1596 | | bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS, |
1597 | | SDValue &RHS, SDValue &CC, |
1598 | | bool &NeedInvert, |
1599 | 3.27k | const SDLoc &dl) { |
1600 | 3.27k | MVT OpVT = LHS.getSimpleValueType(); |
1601 | 3.27k | ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); |
1602 | 3.27k | NeedInvert = false; |
1603 | 3.27k | switch (TLI.getCondCodeAction(CCCode, OpVT)) { |
1604 | 0 | default: 0 llvm_unreachable0 ("Unknown condition code action!"); |
1605 | 2.92k | case TargetLowering::Legal: |
1606 | 2.92k | // Nothing to do. |
1607 | 2.92k | break; |
1608 | 355 | case TargetLowering::Expand: { |
1609 | 355 | ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); |
1610 | 355 | if (TLI.isCondCodeLegal(InvCC, OpVT)355 ) { |
1611 | 231 | std::swap(LHS, RHS); |
1612 | 231 | CC = DAG.getCondCode(InvCC); |
1613 | 231 | return true; |
1614 | 231 | } |
1615 | 124 | ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; |
1616 | 124 | unsigned Opc = 0; |
1617 | 124 | switch (CCCode) { |
1618 | 0 | default: 0 llvm_unreachable0 ("Don't know how to expand this condition!"); |
1619 | 6 | case ISD::SETO: |
1620 | 6 | assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT) |
1621 | 6 | == TargetLowering::Legal |
1622 | 6 | && "If SETO is expanded, SETOEQ must be legal!"); |
1623 | 6 | CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; |
1624 | 15 | case ISD::SETUO: |
1625 | 15 | assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT) |
1626 | 15 | == TargetLowering::Legal |
1627 | 15 | && "If SETUO is expanded, SETUNE must be legal!"); |
1628 | 15 | CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; |
1629 | 91 | case ISD::SETOEQ: |
1630 | 91 | case ISD::SETOGT: |
1631 | 91 | case ISD::SETOGE: |
1632 | 91 | case ISD::SETOLT: |
1633 | 91 | case ISD::SETOLE: |
1634 | 91 | case ISD::SETONE: |
1635 | 91 | case ISD::SETUEQ: |
1636 | 91 | case ISD::SETUNE: |
1637 | 91 | case ISD::SETUGT: |
1638 | 91 | case ISD::SETUGE: |
1639 | 91 | case ISD::SETULT: |
1640 | 91 | case ISD::SETULE: |
1641 | 91 | // If we are floating point, assign and break, otherwise fall through. |
1642 | 91 | if (!OpVT.isInteger()91 ) { |
1643 | 91 | // We can use the 4th bit to tell if we are the unordered |
1644 | 91 | // or ordered version of the opcode. |
1645 | 91 | CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO39 : ISD::SETO52 ; |
1646 | 91 | Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR39 : ISD::AND52 ; |
1647 | 91 | CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); |
1648 | 91 | break; |
1649 | 91 | } |
1650 | 0 | // Fallthrough if we are unsigned integer. |
1651 | 0 | LLVM_FALLTHROUGH0 ; |
1652 | 0 | case ISD::SETLE: |
1653 | 0 | case ISD::SETGT: |
1654 | 0 | case ISD::SETGE: |
1655 | 0 | case ISD::SETLT: |
1656 | 0 | // We only support using the inverted operation, which is computed above |
1657 | 0 | // and not a different manner of supporting expanding these cases. |
1658 | 0 | llvm_unreachable("Don't know how to expand this condition!"); |
1659 | 12 | case ISD::SETNE: |
1660 | 12 | case ISD::SETEQ: |
1661 | 12 | // Try inverting the result of the inverse condition. |
1662 | 12 | InvCC = CCCode == ISD::SETEQ ? ISD::SETNE0 : ISD::SETEQ12 ; |
1663 | 12 | if (TLI.isCondCodeLegal(InvCC, OpVT)12 ) { |
1664 | 12 | CC = DAG.getCondCode(InvCC); |
1665 | 12 | NeedInvert = true; |
1666 | 12 | return true; |
1667 | 12 | } |
1668 | 0 | // If inverting the condition didn't work then we have no means to expand |
1669 | 0 | // the condition. |
1670 | 0 | llvm_unreachable0 ("Don't know how to expand this condition!"); |
1671 | 112 | } |
1672 | 112 | |
1673 | 112 | SDValue SetCC1, SetCC2; |
1674 | 112 | if (CCCode != ISD::SETO && 112 CCCode != ISD::SETUO106 ) { |
1675 | 91 | // If we aren't the ordered or unorder operation, |
1676 | 91 | // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). |
1677 | 91 | SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); |
1678 | 91 | SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); |
1679 | 112 | } else { |
1680 | 21 | // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) |
1681 | 21 | SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); |
1682 | 21 | SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); |
1683 | 21 | } |
1684 | 355 | LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); |
1685 | 355 | RHS = SDValue(); |
1686 | 355 | CC = SDValue(); |
1687 | 355 | return true; |
1688 | 355 | } |
1689 | 2.92k | } |
1690 | 2.92k | return false; |
1691 | 2.92k | } |
1692 | | |
1693 | | /// Emit a store/load combination to the stack. This stores |
1694 | | /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does |
1695 | | /// a load from the stack slot to DestVT, extending it if needed. |
1696 | | /// The resultant code need not be legal. |
1697 | | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, |
1698 | 520 | EVT DestVT, const SDLoc &dl) { |
1699 | 520 | // Create the stack frame object. |
1700 | 520 | unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment( |
1701 | 520 | SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); |
1702 | 520 | SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); |
1703 | 520 | |
1704 | 520 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); |
1705 | 520 | int SPFI = StackPtrFI->getIndex(); |
1706 | 520 | MachinePointerInfo PtrInfo = |
1707 | 520 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); |
1708 | 520 | |
1709 | 520 | unsigned SrcSize = SrcOp.getValueSizeInBits(); |
1710 | 520 | unsigned SlotSize = SlotVT.getSizeInBits(); |
1711 | 520 | unsigned DestSize = DestVT.getSizeInBits(); |
1712 | 520 | Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); |
1713 | 520 | unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType); |
1714 | 520 | |
1715 | 520 | // Emit a store to the stack slot. Use a truncstore if the input value is |
1716 | 520 | // later than DestVT. |
1717 | 520 | SDValue Store; |
1718 | 520 | |
1719 | 520 | if (SrcSize > SlotSize) |
1720 | 0 | Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, |
1721 | 0 | SlotVT, SrcAlign); |
1722 | 520 | else { |
1723 | 520 | assert(SrcSize == SlotSize && "Invalid store"); |
1724 | 520 | Store = |
1725 | 520 | DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign); |
1726 | 520 | } |
1727 | 520 | |
1728 | 520 | // Result is a load from the stack slot. |
1729 | 520 | if (SlotSize == DestSize) |
1730 | 520 | return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); |
1731 | 0 |
|
1732 | 520 | assert(SlotSize < DestSize && "Unknown extension!"); |
1733 | 0 | return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, |
1734 | 0 | DestAlign); |
1735 | 0 | } |
1736 | | |
1737 | 18 | SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { |
1738 | 18 | SDLoc dl(Node); |
1739 | 18 | // Create a vector sized/aligned stack slot, store the value to element #0, |
1740 | 18 | // then load the whole vector back out. |
1741 | 18 | SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); |
1742 | 18 | |
1743 | 18 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); |
1744 | 18 | int SPFI = StackPtrFI->getIndex(); |
1745 | 18 | |
1746 | 18 | SDValue Ch = DAG.getTruncStore( |
1747 | 18 | DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, |
1748 | 18 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI), |
1749 | 18 | Node->getValueType(0).getVectorElementType()); |
1750 | 18 | return DAG.getLoad( |
1751 | 18 | Node->getValueType(0), dl, Ch, StackPtr, |
1752 | 18 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); |
1753 | 18 | } |
1754 | | |
1755 | | static bool |
1756 | | ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, |
1757 | 27 | const TargetLowering &TLI, SDValue &Res) { |
1758 | 27 | unsigned NumElems = Node->getNumOperands(); |
1759 | 27 | SDLoc dl(Node); |
1760 | 27 | EVT VT = Node->getValueType(0); |
1761 | 27 | |
1762 | 27 | // Try to group the scalars into pairs, shuffle the pairs together, then |
1763 | 27 | // shuffle the pairs of pairs together, etc. until the vector has |
1764 | 27 | // been built. This will work only if all of the necessary shuffle masks |
1765 | 27 | // are legal. |
1766 | 27 | |
1767 | 27 | // We do this in two phases; first to check the legality of the shuffles, |
1768 | 27 | // and next, assuming that all shuffles are legal, to create the new nodes. |
1769 | 81 | for (int Phase = 0; Phase < 281 ; ++Phase54 ) { |
1770 | 54 | SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals, |
1771 | 54 | NewIntermedVals; |
1772 | 398 | for (unsigned i = 0; i < NumElems398 ; ++i344 ) { |
1773 | 344 | SDValue V = Node->getOperand(i); |
1774 | 344 | if (V.isUndef()) |
1775 | 16 | continue; |
1776 | 328 | |
1777 | 328 | SDValue Vec; |
1778 | 328 | if (Phase) |
1779 | 164 | Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); |
1780 | 344 | IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); |
1781 | 344 | } |
1782 | 54 | |
1783 | 130 | while (IntermedVals.size() > 2130 ) { |
1784 | 76 | NewIntermedVals.clear(); |
1785 | 296 | for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e296 ; i += 2220 ) { |
1786 | 220 | // This vector and the next vector are shuffled together (simply to |
1787 | 220 | // append the one to the other). |
1788 | 220 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1789 | 220 | |
1790 | 220 | SmallVector<int, 16> FinalIndices; |
1791 | 220 | FinalIndices.reserve(IntermedVals[i].second.size() + |
1792 | 220 | IntermedVals[i+1].second.size()); |
1793 | 220 | |
1794 | 220 | int k = 0; |
1795 | 520 | for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; |
1796 | 300 | ++j, ++k300 ) { |
1797 | 300 | ShuffleVec[k] = j; |
1798 | 300 | FinalIndices.push_back(IntermedVals[i].second[j]); |
1799 | 300 | } |
1800 | 520 | for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; |
1801 | 300 | ++j, ++k300 ) { |
1802 | 300 | ShuffleVec[k] = NumElems + j; |
1803 | 300 | FinalIndices.push_back(IntermedVals[i+1].second[j]); |
1804 | 300 | } |
1805 | 220 | |
1806 | 220 | SDValue Shuffle; |
1807 | 220 | if (Phase) |
1808 | 110 | Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, |
1809 | 110 | IntermedVals[i+1].first, |
1810 | 110 | ShuffleVec); |
1811 | 110 | else if (110 !TLI.isShuffleMaskLegal(ShuffleVec, VT)110 ) |
1812 | 0 | return false; |
1813 | 220 | NewIntermedVals.push_back( |
1814 | 220 | std::make_pair(Shuffle, std::move(FinalIndices))); |
1815 | 220 | } |
1816 | 76 | |
1817 | 76 | // If we had an odd number of defined values, then append the last |
1818 | 76 | // element to the array of new vectors. |
1819 | 76 | if (76 (IntermedVals.size() & 1) != 076 ) |
1820 | 0 | NewIntermedVals.push_back(IntermedVals.back()); |
1821 | 76 | |
1822 | 76 | IntermedVals.swap(NewIntermedVals); |
1823 | 76 | } |
1824 | 54 | |
1825 | 54 | assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 && |
1826 | 54 | "Invalid number of intermediate vectors"); |
1827 | 54 | SDValue Vec1 = IntermedVals[0].first; |
1828 | 54 | SDValue Vec2; |
1829 | 54 | if (IntermedVals.size() > 1) |
1830 | 54 | Vec2 = IntermedVals[1].first; |
1831 | 0 | else if (0 Phase0 ) |
1832 | 0 | Vec2 = DAG.getUNDEF(VT); |
1833 | 54 | |
1834 | 54 | SmallVector<int, 16> ShuffleVec(NumElems, -1); |
1835 | 218 | for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e218 ; ++i164 ) |
1836 | 164 | ShuffleVec[IntermedVals[0].second[i]] = i; |
1837 | 218 | for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e218 ; ++i164 ) |
1838 | 164 | ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; |
1839 | 54 | |
1840 | 54 | if (Phase) |
1841 | 27 | Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); |
1842 | 27 | else if (27 !TLI.isShuffleMaskLegal(ShuffleVec, VT)27 ) |
1843 | 0 | return false; |
1844 | 54 | } |
1845 | 27 | |
1846 | 27 | return true; |
1847 | 27 | } |
1848 | | |
1849 | | /// Expand a BUILD_VECTOR node on targets that don't |
1850 | | /// support the operation, but do support the resultant vector type. |
1851 | 63.0k | SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { |
1852 | 63.0k | unsigned NumElems = Node->getNumOperands(); |
1853 | 63.0k | SDValue Value1, Value2; |
1854 | 63.0k | SDLoc dl(Node); |
1855 | 63.0k | EVT VT = Node->getValueType(0); |
1856 | 63.0k | EVT OpVT = Node->getOperand(0).getValueType(); |
1857 | 63.0k | EVT EltVT = VT.getVectorElementType(); |
1858 | 63.0k | |
1859 | 63.0k | // If the only non-undef value is the low element, turn this into a |
1860 | 63.0k | // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. |
1861 | 63.0k | bool isOnlyLowElement = true; |
1862 | 63.0k | bool MoreThanTwoValues = false; |
1863 | 63.0k | bool isConstant = true; |
1864 | 413k | for (unsigned i = 0; i < NumElems413k ; ++i350k ) { |
1865 | 350k | SDValue V = Node->getOperand(i); |
1866 | 350k | if (V.isUndef()) |
1867 | 11.3k | continue; |
1868 | 338k | if (338k i > 0338k ) |
1869 | 276k | isOnlyLowElement = false; |
1870 | 338k | if (!isa<ConstantFPSDNode>(V) && 338k !isa<ConstantSDNode>(V)314k ) |
1871 | 9.06k | isConstant = false; |
1872 | 338k | |
1873 | 338k | if (!Value1.getNode()338k ) { |
1874 | 63.0k | Value1 = V; |
1875 | 338k | } else if (275k !Value2.getNode()275k ) { |
1876 | 113k | if (V != Value1) |
1877 | 58.8k | Value2 = V; |
1878 | 275k | } else if (161k V != Value1 && 161k V != Value2141k ) { |
1879 | 123k | MoreThanTwoValues = true; |
1880 | 123k | } |
1881 | 350k | } |
1882 | 63.0k | |
1883 | 63.0k | if (!Value1.getNode()) |
1884 | 0 | return DAG.getUNDEF(VT); |
1885 | 63.0k | |
1886 | 63.0k | if (63.0k isOnlyLowElement63.0k ) |
1887 | 15 | return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); |
1888 | 63.0k | |
1889 | 63.0k | // If all elements are constants, create a load from the constant pool. |
1890 | 63.0k | if (63.0k isConstant63.0k ) { |
1891 | 61.1k | SmallVector<Constant*, 16> CV; |
1892 | 401k | for (unsigned i = 0, e = NumElems; i != e401k ; ++i340k ) { |
1893 | 340k | if (ConstantFPSDNode *V = |
1894 | 24.2k | dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { |
1895 | 24.2k | CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); |
1896 | 340k | } else if (ConstantSDNode *316k V316k = |
1897 | 305k | dyn_cast<ConstantSDNode>(Node->getOperand(i))) { |
1898 | 305k | if (OpVT==EltVT) |
1899 | 248k | CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); |
1900 | 56.5k | else { |
1901 | 56.5k | // If OpVT and EltVT don't match, EltVT is not legal and the |
1902 | 56.5k | // element values have been promoted/truncated earlier. Undo this; |
1903 | 56.5k | // we don't want a v16i8 to become a v16i32 for example. |
1904 | 56.5k | const ConstantInt *CI = V->getConstantIntValue(); |
1905 | 56.5k | CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), |
1906 | 56.5k | CI->getZExtValue())); |
1907 | 56.5k | } |
1908 | 316k | } else { |
1909 | 11.0k | assert(Node->getOperand(i).isUndef()); |
1910 | 11.0k | Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); |
1911 | 11.0k | CV.push_back(UndefValue::get(OpNTy)); |
1912 | 11.0k | } |
1913 | 340k | } |
1914 | 61.1k | Constant *CP = ConstantVector::get(CV); |
1915 | 61.1k | SDValue CPIdx = |
1916 | 61.1k | DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout())); |
1917 | 61.1k | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); |
1918 | 61.1k | return DAG.getLoad( |
1919 | 61.1k | VT, dl, DAG.getEntryNode(), CPIdx, |
1920 | 61.1k | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
1921 | 61.1k | Alignment); |
1922 | 61.1k | } |
1923 | 1.89k | |
1924 | 1.89k | SmallSet<SDValue, 16> DefinedValues; |
1925 | 11.2k | for (unsigned i = 0; i < NumElems11.2k ; ++i9.35k ) { |
1926 | 9.35k | if (Node->getOperand(i).isUndef()) |
1927 | 261 | continue; |
1928 | 9.09k | DefinedValues.insert(Node->getOperand(i)); |
1929 | 9.09k | } |
1930 | 1.89k | |
1931 | 1.89k | if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())1.89k ) { |
1932 | 1.82k | if (!MoreThanTwoValues1.82k ) { |
1933 | 1.79k | SmallVector<int, 8> ShuffleVec(NumElems, -1); |
1934 | 10.2k | for (unsigned i = 0; i < NumElems10.2k ; ++i8.44k ) { |
1935 | 8.44k | SDValue V = Node->getOperand(i); |
1936 | 8.44k | if (V.isUndef()) |
1937 | 228 | continue; |
1938 | 8.21k | ShuffleVec[i] = V == Value1 ? 8.21k 07.01k : NumElems1.20k ; |
1939 | 8.44k | } |
1940 | 1.79k | if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))1.79k ) { |
1941 | 1.72k | // Get the splatted value into the low element of a vector register. |
1942 | 1.72k | SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); |
1943 | 1.72k | SDValue Vec2; |
1944 | 1.72k | if (Value2.getNode()) |
1945 | 1.12k | Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); |
1946 | 1.72k | else |
1947 | 600 | Vec2 = DAG.getUNDEF(VT); |
1948 | 1.72k | |
1949 | 1.72k | // Return shuffle(LowValVec, undef, <0,0,0,0>) |
1950 | 1.72k | return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); |
1951 | 1.72k | } |
1952 | 27 | } else { |
1953 | 27 | SDValue Res; |
1954 | 27 | if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) |
1955 | 27 | return Res; |
1956 | 144 | } |
1957 | 1.82k | } |
1958 | 144 | |
1959 | 144 | // Otherwise, we can't handle this case efficiently. |
1960 | 144 | return ExpandVectorBuildThroughStack(Node); |
1961 | 144 | } |
1962 | | |
1963 | | // Expand a node into a call to a libcall. If the result value |
1964 | | // does not fit into a register, return the lo part and set the hi part to the |
1965 | | // by-reg argument. If it does fit into a single register, return the result |
1966 | | // and leave the Hi part unset. |
1967 | | SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, |
1968 | 2.52k | bool isSigned) { |
1969 | 2.52k | TargetLowering::ArgListTy Args; |
1970 | 2.52k | TargetLowering::ArgListEntry Entry; |
1971 | 3.67k | for (const SDValue &Op : Node->op_values()) { |
1972 | 3.67k | EVT ArgVT = Op.getValueType(); |
1973 | 3.67k | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
1974 | 3.67k | Entry.Node = Op; |
1975 | 3.67k | Entry.Ty = ArgTy; |
1976 | 3.67k | Entry.IsSExt = isSigned; |
1977 | 3.67k | Entry.IsZExt = !isSigned; |
1978 | 3.67k | Args.push_back(Entry); |
1979 | 3.67k | } |
1980 | 2.52k | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
1981 | 2.52k | TLI.getPointerTy(DAG.getDataLayout())); |
1982 | 2.52k | |
1983 | 2.52k | Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); |
1984 | 2.52k | |
1985 | 2.52k | // By default, the input chain to this libcall is the entry node of the |
1986 | 2.52k | // function. If the libcall is going to be emitted as a tail call then |
1987 | 2.52k | // TLI.isUsedByReturnOnly will change it to the right chain if the return |
1988 | 2.52k | // node which is being folded has a non-entry input chain. |
1989 | 2.52k | SDValue InChain = DAG.getEntryNode(); |
1990 | 2.52k | |
1991 | 2.52k | // isTailCall may be true since the callee does not reference caller stack |
1992 | 2.52k | // frame. Check if it's in the right position and that the return types match. |
1993 | 2.52k | SDValue TCChain = InChain; |
1994 | 2.52k | const Function *F = DAG.getMachineFunction().getFunction(); |
1995 | 2.52k | bool isTailCall = |
1996 | 2.52k | TLI.isInTailCallPosition(DAG, Node, TCChain) && |
1997 | 285 | (RetTy == F->getReturnType() || 285 F->getReturnType()->isVoidTy()6 ); |
1998 | 2.52k | if (isTailCall) |
1999 | 279 | InChain = TCChain; |
2000 | 2.52k | |
2001 | 2.52k | TargetLowering::CallLoweringInfo CLI(DAG); |
2002 | 2.52k | CLI.setDebugLoc(SDLoc(Node)) |
2003 | 2.52k | .setChain(InChain) |
2004 | 2.52k | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2005 | 2.52k | std::move(Args)) |
2006 | 2.52k | .setTailCall(isTailCall) |
2007 | 2.52k | .setSExtResult(isSigned) |
2008 | 2.52k | .setZExtResult(!isSigned) |
2009 | 2.52k | .setIsPostTypeLegalization(true); |
2010 | 2.52k | |
2011 | 2.52k | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2012 | 2.52k | |
2013 | 2.52k | if (!CallInfo.second.getNode()) |
2014 | 2.52k | // It's a tailcall, return the chain (which is the DAG root). |
2015 | 233 | return DAG.getRoot(); |
2016 | 2.29k | |
2017 | 2.29k | return CallInfo.first; |
2018 | 2.29k | } |
2019 | | |
2020 | | /// Generate a libcall taking the given operands as arguments |
2021 | | /// and returning a result of type RetVT. |
2022 | | SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, |
2023 | | const SDValue *Ops, unsigned NumOps, |
2024 | 5 | bool isSigned, const SDLoc &dl) { |
2025 | 5 | TargetLowering::ArgListTy Args; |
2026 | 5 | Args.reserve(NumOps); |
2027 | 5 | |
2028 | 5 | TargetLowering::ArgListEntry Entry; |
2029 | 25 | for (unsigned i = 0; i != NumOps25 ; ++i20 ) { |
2030 | 20 | Entry.Node = Ops[i]; |
2031 | 20 | Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); |
2032 | 20 | Entry.IsSExt = isSigned; |
2033 | 20 | Entry.IsZExt = !isSigned; |
2034 | 20 | Args.push_back(Entry); |
2035 | 20 | } |
2036 | 5 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2037 | 5 | TLI.getPointerTy(DAG.getDataLayout())); |
2038 | 5 | |
2039 | 5 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2040 | 5 | |
2041 | 5 | TargetLowering::CallLoweringInfo CLI(DAG); |
2042 | 5 | CLI.setDebugLoc(dl) |
2043 | 5 | .setChain(DAG.getEntryNode()) |
2044 | 5 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2045 | 5 | std::move(Args)) |
2046 | 5 | .setSExtResult(isSigned) |
2047 | 5 | .setZExtResult(!isSigned) |
2048 | 5 | .setIsPostTypeLegalization(true); |
2049 | 5 | |
2050 | 5 | std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2051 | 5 | |
2052 | 5 | return CallInfo.first; |
2053 | 5 | } |
2054 | | |
2055 | | // Expand a node into a call to a libcall. Similar to |
2056 | | // ExpandLibCall except that the first operand is the in-chain. |
2057 | | std::pair<SDValue, SDValue> |
2058 | | SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, |
2059 | | SDNode *Node, |
2060 | 328 | bool isSigned) { |
2061 | 328 | SDValue InChain = Node->getOperand(0); |
2062 | 328 | |
2063 | 328 | TargetLowering::ArgListTy Args; |
2064 | 328 | TargetLowering::ArgListEntry Entry; |
2065 | 1.00k | for (unsigned i = 1, e = Node->getNumOperands(); i != e1.00k ; ++i673 ) { |
2066 | 673 | EVT ArgVT = Node->getOperand(i).getValueType(); |
2067 | 673 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
2068 | 673 | Entry.Node = Node->getOperand(i); |
2069 | 673 | Entry.Ty = ArgTy; |
2070 | 673 | Entry.IsSExt = isSigned; |
2071 | 673 | Entry.IsZExt = !isSigned; |
2072 | 673 | Args.push_back(Entry); |
2073 | 673 | } |
2074 | 328 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2075 | 328 | TLI.getPointerTy(DAG.getDataLayout())); |
2076 | 328 | |
2077 | 328 | Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); |
2078 | 328 | |
2079 | 328 | TargetLowering::CallLoweringInfo CLI(DAG); |
2080 | 328 | CLI.setDebugLoc(SDLoc(Node)) |
2081 | 328 | .setChain(InChain) |
2082 | 328 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2083 | 328 | std::move(Args)) |
2084 | 328 | .setSExtResult(isSigned) |
2085 | 328 | .setZExtResult(!isSigned); |
2086 | 328 | |
2087 | 328 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2088 | 328 | |
2089 | 328 | return CallInfo; |
2090 | 328 | } |
2091 | | |
2092 | | SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, |
2093 | | RTLIB::Libcall Call_F32, |
2094 | | RTLIB::Libcall Call_F64, |
2095 | | RTLIB::Libcall Call_F80, |
2096 | | RTLIB::Libcall Call_F128, |
2097 | 1.86k | RTLIB::Libcall Call_PPCF128) { |
2098 | 1.86k | if (Node->isStrictFPOpcode()) |
2099 | 22 | Node = DAG.mutateStrictFPToFP(Node); |
2100 | 1.86k | |
2101 | 1.86k | RTLIB::Libcall LC; |
2102 | 1.86k | switch (Node->getSimpleValueType(0).SimpleTy) { |
2103 | 0 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2104 | 783 | case MVT::f32: LC = Call_F32; break; |
2105 | 1.00k | case MVT::f64: LC = Call_F64; break; |
2106 | 43 | case MVT::f80: LC = Call_F80; break; |
2107 | 35 | case MVT::f128: LC = Call_F128; break; |
2108 | 0 | case MVT::ppcf128: LC = Call_PPCF128; break; |
2109 | 1.86k | } |
2110 | 1.86k | return ExpandLibCall(LC, Node, false); |
2111 | 1.86k | } |
2112 | | |
2113 | | SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, |
2114 | | RTLIB::Libcall Call_I8, |
2115 | | RTLIB::Libcall Call_I16, |
2116 | | RTLIB::Libcall Call_I32, |
2117 | | RTLIB::Libcall Call_I64, |
2118 | 173 | RTLIB::Libcall Call_I128) { |
2119 | 173 | RTLIB::Libcall LC; |
2120 | 173 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2121 | 0 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2122 | 0 | case MVT::i8: LC = Call_I8; break; |
2123 | 14 | case MVT::i16: LC = Call_I16; break; |
2124 | 159 | case MVT::i32: LC = Call_I32; break; |
2125 | 0 | case MVT::i64: LC = Call_I64; break; |
2126 | 0 | case MVT::i128: LC = Call_I128; break; |
2127 | 173 | } |
2128 | 173 | return ExpandLibCall(LC, Node, isSigned); |
2129 | 173 | } |
2130 | | |
2131 | | /// Issue libcalls to __{u}divmod to compute div / rem pairs. |
2132 | | void |
2133 | | SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, |
2134 | 14 | SmallVectorImpl<SDValue> &Results) { |
2135 | 14 | unsigned Opcode = Node->getOpcode(); |
2136 | 14 | bool isSigned = Opcode == ISD::SDIVREM; |
2137 | 14 | |
2138 | 14 | RTLIB::Libcall LC; |
2139 | 14 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2140 | 0 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2141 | 0 | case MVT::i8: LC= isSigned ? 0 RTLIB::SDIVREM_I80 : RTLIB::UDIVREM_I80 ; break; |
2142 | 0 | case MVT::i16: LC= isSigned ? 0 RTLIB::SDIVREM_I160 : RTLIB::UDIVREM_I160 ; break; |
2143 | 14 | case MVT::i32: LC= isSigned ? 14 RTLIB::SDIVREM_I329 : RTLIB::UDIVREM_I325 ; break; |
2144 | 0 | case MVT::i64: LC= isSigned ? 0 RTLIB::SDIVREM_I640 : RTLIB::UDIVREM_I640 ; break; |
2145 | 0 | case MVT::i128: LC= isSigned ? 0 RTLIB::SDIVREM_I1280 :RTLIB::UDIVREM_I1280 ; break; |
2146 | 14 | } |
2147 | 14 | |
2148 | 14 | // The input chain to this libcall is the entry node of the function. |
2149 | 14 | // Legalizing the call will automatically add the previous call to the |
2150 | 14 | // dependence. |
2151 | 14 | SDValue InChain = DAG.getEntryNode(); |
2152 | 14 | |
2153 | 14 | EVT RetVT = Node->getValueType(0); |
2154 | 14 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2155 | 14 | |
2156 | 14 | TargetLowering::ArgListTy Args; |
2157 | 14 | TargetLowering::ArgListEntry Entry; |
2158 | 28 | for (const SDValue &Op : Node->op_values()) { |
2159 | 28 | EVT ArgVT = Op.getValueType(); |
2160 | 28 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); |
2161 | 28 | Entry.Node = Op; |
2162 | 28 | Entry.Ty = ArgTy; |
2163 | 28 | Entry.IsSExt = isSigned; |
2164 | 28 | Entry.IsZExt = !isSigned; |
2165 | 28 | Args.push_back(Entry); |
2166 | 28 | } |
2167 | 14 | |
2168 | 14 | // Also pass the return address of the remainder. |
2169 | 14 | SDValue FIPtr = DAG.CreateStackTemporary(RetVT); |
2170 | 14 | Entry.Node = FIPtr; |
2171 | 14 | Entry.Ty = RetTy->getPointerTo(); |
2172 | 14 | Entry.IsSExt = isSigned; |
2173 | 14 | Entry.IsZExt = !isSigned; |
2174 | 14 | Args.push_back(Entry); |
2175 | 14 | |
2176 | 14 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2177 | 14 | TLI.getPointerTy(DAG.getDataLayout())); |
2178 | 14 | |
2179 | 14 | SDLoc dl(Node); |
2180 | 14 | TargetLowering::CallLoweringInfo CLI(DAG); |
2181 | 14 | CLI.setDebugLoc(dl) |
2182 | 14 | .setChain(InChain) |
2183 | 14 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, |
2184 | 14 | std::move(Args)) |
2185 | 14 | .setSExtResult(isSigned) |
2186 | 14 | .setZExtResult(!isSigned); |
2187 | 14 | |
2188 | 14 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2189 | 14 | |
2190 | 14 | // Remainder is loaded back from the stack frame. |
2191 | 14 | SDValue Rem = |
2192 | 14 | DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo()); |
2193 | 14 | Results.push_back(CallInfo.first); |
2194 | 14 | Results.push_back(Rem); |
2195 | 14 | } |
2196 | | |
2197 | | /// Return true if sincos libcall is available. |
2198 | 253 | static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { |
2199 | 253 | RTLIB::Libcall LC; |
2200 | 253 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2201 | 0 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2202 | 136 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; |
2203 | 86 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; |
2204 | 19 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; |
2205 | 12 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; |
2206 | 0 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; |
2207 | 253 | } |
2208 | 253 | return TLI.getLibcallName(LC) != nullptr; |
2209 | 253 | } |
2210 | | |
2211 | | /// Only issue sincos libcall if both sin and cos are needed. |
2212 | 704 | static bool useSinCos(SDNode *Node) { |
2213 | 704 | unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN |
2214 | 704 | ? ISD::FCOS302 : ISD::FSIN402 ; |
2215 | 704 | |
2216 | 704 | SDValue Op0 = Node->getOperand(0); |
2217 | 704 | for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), |
2218 | 1.06k | UE = Op0.getNode()->use_end(); UI != UE1.06k ; ++UI360 ) { |
2219 | 770 | SDNode *User = *UI; |
2220 | 770 | if (User == Node) |
2221 | 294 | continue; |
2222 | 476 | // The other user might have been turned into sincos already. |
2223 | 476 | if (476 User->getOpcode() == OtherOpcode || 476 User->getOpcode() == ISD::FSINCOS271 ) |
2224 | 410 | return true; |
2225 | 770 | } |
2226 | 294 | return false; |
2227 | 704 | } |
2228 | | |
2229 | | /// Issue libcalls to sincos to compute sin / cos pairs. |
2230 | | void |
2231 | | SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, |
2232 | 30 | SmallVectorImpl<SDValue> &Results) { |
2233 | 30 | RTLIB::Libcall LC; |
2234 | 30 | switch (Node->getSimpleValueType(0).SimpleTy) { |
2235 | 0 | default: 0 llvm_unreachable0 ("Unexpected request for libcall!"); |
2236 | 11 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; |
2237 | 11 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; |
2238 | 5 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; |
2239 | 3 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; |
2240 | 0 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; |
2241 | 30 | } |
2242 | 30 | |
2243 | 30 | // The input chain to this libcall is the entry node of the function. |
2244 | 30 | // Legalizing the call will automatically add the previous call to the |
2245 | 30 | // dependence. |
2246 | 30 | SDValue InChain = DAG.getEntryNode(); |
2247 | 30 | |
2248 | 30 | EVT RetVT = Node->getValueType(0); |
2249 | 30 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); |
2250 | 30 | |
2251 | 30 | TargetLowering::ArgListTy Args; |
2252 | 30 | TargetLowering::ArgListEntry Entry; |
2253 | 30 | |
2254 | 30 | // Pass the argument. |
2255 | 30 | Entry.Node = Node->getOperand(0); |
2256 | 30 | Entry.Ty = RetTy; |
2257 | 30 | Entry.IsSExt = false; |
2258 | 30 | Entry.IsZExt = false; |
2259 | 30 | Args.push_back(Entry); |
2260 | 30 | |
2261 | 30 | // Pass the return address of sin. |
2262 | 30 | SDValue SinPtr = DAG.CreateStackTemporary(RetVT); |
2263 | 30 | Entry.Node = SinPtr; |
2264 | 30 | Entry.Ty = RetTy->getPointerTo(); |
2265 | 30 | Entry.IsSExt = false; |
2266 | 30 | Entry.IsZExt = false; |
2267 | 30 | Args.push_back(Entry); |
2268 | 30 | |
2269 | 30 | // Also pass the return address of the cos. |
2270 | 30 | SDValue CosPtr = DAG.CreateStackTemporary(RetVT); |
2271 | 30 | Entry.Node = CosPtr; |
2272 | 30 | Entry.Ty = RetTy->getPointerTo(); |
2273 | 30 | Entry.IsSExt = false; |
2274 | 30 | Entry.IsZExt = false; |
2275 | 30 | Args.push_back(Entry); |
2276 | 30 | |
2277 | 30 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), |
2278 | 30 | TLI.getPointerTy(DAG.getDataLayout())); |
2279 | 30 | |
2280 | 30 | SDLoc dl(Node); |
2281 | 30 | TargetLowering::CallLoweringInfo CLI(DAG); |
2282 | 30 | CLI.setDebugLoc(dl).setChain(InChain).setLibCallee( |
2283 | 30 | TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee, |
2284 | 30 | std::move(Args)); |
2285 | 30 | |
2286 | 30 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); |
2287 | 30 | |
2288 | 30 | Results.push_back( |
2289 | 30 | DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo())); |
2290 | 30 | Results.push_back( |
2291 | 30 | DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo())); |
2292 | 30 | } |
2293 | | |
2294 | | /// This function is responsible for legalizing a |
2295 | | /// INT_TO_FP operation of the specified operand when the target requests that |
2296 | | /// we expand it. At this point, we know that the result and operand types are |
2297 | | /// legal for the target. |
2298 | | SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, |
2299 | | EVT DestVT, |
2300 | 138 | const SDLoc &dl) { |
2301 | 138 | // TODO: Should any fast-math-flags be set for the created nodes? |
2302 | 138 | |
2303 | 138 | if (Op0.getValueType() == MVT::i32 && 138 TLI.isTypeLegal(MVT::f64)51 ) { |
2304 | 50 | // simple 32-bit [signed|unsigned] integer to float/double expansion |
2305 | 50 | |
2306 | 50 | // Get the stack frame index of a 8 byte buffer. |
2307 | 50 | SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); |
2308 | 50 | |
2309 | 50 | // word offset constant for Hi/Lo address computation |
2310 | 50 | SDValue WordOff = DAG.getConstant(sizeof(int), dl, |
2311 | 50 | StackSlot.getValueType()); |
2312 | 50 | // set up Hi and Lo (into buffer) address based on endian |
2313 | 50 | SDValue Hi = StackSlot; |
2314 | 50 | SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(), |
2315 | 50 | StackSlot, WordOff); |
2316 | 50 | if (DAG.getDataLayout().isLittleEndian()) |
2317 | 20 | std::swap(Hi, Lo); |
2318 | 50 | |
2319 | 50 | // if signed map to unsigned space |
2320 | 50 | SDValue Op0Mapped; |
2321 | 50 | if (isSigned50 ) { |
2322 | 4 | // constant used to invert sign bit (signed to unsigned mapping) |
2323 | 4 | SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32); |
2324 | 4 | Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); |
2325 | 50 | } else { |
2326 | 46 | Op0Mapped = Op0; |
2327 | 46 | } |
2328 | 50 | // store the lo of the constructed double - based on integer input |
2329 | 50 | SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo, |
2330 | 50 | MachinePointerInfo()); |
2331 | 50 | // initial hi portion of constructed double |
2332 | 50 | SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32); |
2333 | 50 | // store the hi of the constructed double - biased exponent |
2334 | 50 | SDValue Store2 = |
2335 | 50 | DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo()); |
2336 | 50 | // load the constructed double |
2337 | 50 | SDValue Load = |
2338 | 50 | DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo()); |
2339 | 50 | // FP constant to bias correct the final result |
2340 | 50 | SDValue Bias = DAG.getConstantFP(isSigned ? |
2341 | 4 | BitsToDouble(0x4330000080000000ULL) : |
2342 | 46 | BitsToDouble(0x4330000000000000ULL), |
2343 | 50 | dl, MVT::f64); |
2344 | 50 | // subtract the bias |
2345 | 50 | SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); |
2346 | 50 | // final result |
2347 | 50 | SDValue Result; |
2348 | 50 | // handle final rounding |
2349 | 50 | if (DestVT == MVT::f6450 ) { |
2350 | 10 | // do nothing |
2351 | 10 | Result = Sub; |
2352 | 50 | } else if (40 DestVT.bitsLT(MVT::f64)40 ) { |
2353 | 36 | Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, |
2354 | 36 | DAG.getIntPtrConstant(0, dl)); |
2355 | 40 | } else if (4 DestVT.bitsGT(MVT::f64)4 ) { |
2356 | 4 | Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); |
2357 | 4 | } |
2358 | 50 | return Result; |
2359 | 50 | } |
2360 | 138 | assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"); |
2361 | 88 | // Code below here assumes !isSigned without checking again. |
2362 | 88 | |
2363 | 88 | // Implementation of unsigned i64 to f64 following the algorithm in |
2364 | 88 | // __floatundidf in compiler_rt. This implementation has the advantage |
2365 | 88 | // of performing rounding correctly, both in the default rounding mode |
2366 | 88 | // and in all alternate rounding modes. |
2367 | 88 | // TODO: Generalize this for use with other types. |
2368 | 88 | if (Op0.getValueType() == MVT::i64 && 88 DestVT == MVT::f6487 ) { |
2369 | 4 | SDValue TwoP52 = |
2370 | 4 | DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64); |
2371 | 4 | SDValue TwoP84PlusTwoP52 = |
2372 | 4 | DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl, |
2373 | 4 | MVT::f64); |
2374 | 4 | SDValue TwoP84 = |
2375 | 4 | DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64); |
2376 | 4 | |
2377 | 4 | SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32); |
2378 | 4 | SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, |
2379 | 4 | DAG.getConstant(32, dl, MVT::i64)); |
2380 | 4 | SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52); |
2381 | 4 | SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84); |
2382 | 4 | SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr); |
2383 | 4 | SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr); |
2384 | 4 | SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt, |
2385 | 4 | TwoP84PlusTwoP52); |
2386 | 4 | return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); |
2387 | 4 | } |
2388 | 84 | |
2389 | 84 | // Implementation of unsigned i64 to f32. |
2390 | 84 | // TODO: Generalize this for use with other types. |
2391 | 84 | if (84 Op0.getValueType() == MVT::i64 && 84 DestVT == MVT::f3283 ) { |
2392 | 82 | // For unsigned conversions, convert them to signed conversions using the |
2393 | 82 | // algorithm from the x86_64 __floatundidf in compiler_rt. |
2394 | 82 | if (!isSigned82 ) { |
2395 | 82 | SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0); |
2396 | 82 | |
2397 | 82 | SDValue ShiftConst = DAG.getConstant( |
2398 | 82 | 1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout())); |
2399 | 82 | SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst); |
2400 | 82 | SDValue AndConst = DAG.getConstant(1, dl, MVT::i64); |
2401 | 82 | SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst); |
2402 | 82 | SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr); |
2403 | 82 | |
2404 | 82 | SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or); |
2405 | 82 | SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt); |
2406 | 82 | |
2407 | 82 | // TODO: This really should be implemented using a branch rather than a |
2408 | 82 | // select. We happen to get lucky and machinesink does the right |
2409 | 82 | // thing most of the time. This would be a good candidate for a |
2410 | 82 | //pseudo-op, or, even better, for whole-function isel. |
2411 | 82 | SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), |
2412 | 82 | Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT); |
2413 | 82 | return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast); |
2414 | 82 | } |
2415 | 0 |
|
2416 | 0 | // Otherwise, implement the fully general conversion. |
2417 | 0 |
|
2418 | 0 | SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, |
2419 | 0 | DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64)); |
2420 | 0 | SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, |
2421 | 0 | DAG.getConstant(UINT64_C(0x800), dl, MVT::i64)); |
2422 | 0 | SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, |
2423 | 0 | DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64)); |
2424 | 0 | SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2, |
2425 | 0 | DAG.getConstant(UINT64_C(0), dl, MVT::i64), |
2426 | 0 | ISD::SETNE); |
2427 | 0 | SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0); |
2428 | 0 | SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0, |
2429 | 0 | DAG.getConstant(UINT64_C(0x0020000000000000), dl, |
2430 | 0 | MVT::i64), |
2431 | 0 | ISD::SETUGE); |
2432 | 0 | SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0); |
2433 | 0 | EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout()); |
2434 | 0 |
|
2435 | 0 | SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, |
2436 | 0 | DAG.getConstant(32, dl, SHVT)); |
2437 | 0 | SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); |
2438 | 0 | SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); |
2439 | 0 | SDValue TwoP32 = |
2440 | 0 | DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl, |
2441 | 0 | MVT::f64); |
2442 | 0 | SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); |
2443 | 0 | SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); |
2444 | 0 | SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); |
2445 | 0 | SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); |
2446 | 0 | return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, |
2447 | 0 | DAG.getIntPtrConstant(0, dl)); |
2448 | 0 | } |
2449 | 2 | |
2450 | 2 | SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); |
2451 | 2 | |
2452 | 2 | SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()), |
2453 | 2 | Op0, |
2454 | 2 | DAG.getConstant(0, dl, Op0.getValueType()), |
2455 | 2 | ISD::SETLT); |
2456 | 2 | SDValue Zero = DAG.getIntPtrConstant(0, dl), |
2457 | 2 | Four = DAG.getIntPtrConstant(4, dl); |
2458 | 2 | SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), |
2459 | 2 | SignSet, Four, Zero); |
2460 | 2 | |
2461 | 2 | // If the sign bit of the integer is set, the large number will be treated |
2462 | 2 | // as a negative number. To counteract this, the dynamic code adds an |
2463 | 2 | // offset depending on the data type. |
2464 | 2 | uint64_t FF; |
2465 | 2 | switch (Op0.getSimpleValueType().SimpleTy) { |
2466 | 0 | default: 0 llvm_unreachable0 ("Unsupported integer type!"); |
2467 | 0 | case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) |
2468 | 0 | case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) |
2469 | 1 | case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) |
2470 | 1 | case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) |
2471 | 2 | } |
2472 | 2 | if (2 DAG.getDataLayout().isLittleEndian()2 ) |
2473 | 0 | FF <<= 32; |
2474 | 2 | Constant *FudgeFactor = ConstantInt::get( |
2475 | 2 | Type::getInt64Ty(*DAG.getContext()), FF); |
2476 | 2 | |
2477 | 2 | SDValue CPIdx = |
2478 | 2 | DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout())); |
2479 | 2 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); |
2480 | 2 | CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); |
2481 | 2 | Alignment = std::min(Alignment, 4u); |
2482 | 2 | SDValue FudgeInReg; |
2483 | 2 | if (DestVT == MVT::f32) |
2484 | 1 | FudgeInReg = DAG.getLoad( |
2485 | 1 | MVT::f32, dl, DAG.getEntryNode(), CPIdx, |
2486 | 1 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), |
2487 | 1 | Alignment); |
2488 | 1 | else { |
2489 | 1 | SDValue Load = DAG.getExtLoad( |
2490 | 1 | ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, |
2491 | 1 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, |
2492 | 1 | Alignment); |
2493 | 1 | HandleSDNode Handle(Load); |
2494 | 1 | LegalizeOp(Load.getNode()); |
2495 | 1 | FudgeInReg = Handle.getValue(); |
2496 | 1 | } |
2497 | 138 | |
2498 | 138 | return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); |
2499 | 138 | } |
2500 | | |
2501 | | /// This function is responsible for legalizing a |
2502 | | /// *INT_TO_FP operation of the specified operand when the target requests that |
2503 | | /// we promote it. At this point, we know that the result and operand types are |
2504 | | /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP |
2505 | | /// operation that takes a larger input. |
2506 | | SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, |
2507 | | bool isSigned, |
2508 | 88 | const SDLoc &dl) { |
2509 | 88 | // First step, figure out the appropriate *INT_TO_FP operation to use. |
2510 | 88 | EVT NewInTy = LegalOp.getValueType(); |
2511 | 88 | |
2512 | 88 | unsigned OpToUse = 0; |
2513 | 88 | |
2514 | 88 | // Scan for the appropriate larger type to use. |
2515 | 103 | while (true103 ) { |
2516 | 103 | NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); |
2517 | 103 | assert(NewInTy.isInteger() && "Ran out of possibilities!"); |
2518 | 103 | |
2519 | 103 | // If the target supports SINT_TO_FP of this type, use it. |
2520 | 103 | if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)103 ) { |
2521 | 88 | OpToUse = ISD::SINT_TO_FP; |
2522 | 88 | break; |
2523 | 88 | } |
2524 | 15 | if (15 isSigned15 ) continue14 ; |
2525 | 1 | |
2526 | 1 | // If the target supports UINT_TO_FP of this type, use it. |
2527 | 1 | if (1 TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)1 ) { |
2528 | 0 | OpToUse = ISD::UINT_TO_FP; |
2529 | 0 | break; |
2530 | 0 | } |
2531 | 103 | |
2532 | 103 | // Otherwise, try a larger type. |
2533 | 103 | } |
2534 | 88 | |
2535 | 88 | // Okay, we found the operation and type to use. Zero extend our input to the |
2536 | 88 | // desired type then run the operation on it. |
2537 | 88 | return DAG.getNode(OpToUse, dl, DestVT, |
2538 | 88 | DAG.getNode(isSigned ? ISD::SIGN_EXTEND26 : ISD::ZERO_EXTEND62 , |
2539 | 88 | dl, NewInTy, LegalOp)); |
2540 | 88 | } |
2541 | | |
2542 | | /// This function is responsible for legalizing a |
2543 | | /// FP_TO_*INT operation of the specified operand when the target requests that |
2544 | | /// we promote it. At this point, we know that the result and operand types are |
2545 | | /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT |
2546 | | /// operation that returns a larger result. |
2547 | | SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, |
2548 | | bool isSigned, |
2549 | 136 | const SDLoc &dl) { |
2550 | 136 | // First step, figure out the appropriate FP_TO*INT operation to use. |
2551 | 136 | EVT NewOutTy = DestVT; |
2552 | 136 | |
2553 | 136 | unsigned OpToUse = 0; |
2554 | 136 | |
2555 | 136 | // Scan for the appropriate larger type to use. |
2556 | 156 | while (true156 ) { |
2557 | 156 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); |
2558 | 156 | assert(NewOutTy.isInteger() && "Ran out of possibilities!"); |
2559 | 156 | |
2560 | 156 | // A larger signed type can hold all unsigned values of the requested type, |
2561 | 156 | // so using FP_TO_SINT is valid |
2562 | 156 | if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)156 ) { |
2563 | 136 | OpToUse = ISD::FP_TO_SINT; |
2564 | 136 | break; |
2565 | 136 | } |
2566 | 20 | |
2567 | 20 | // However, if the value may be < 0.0, we *must* use some FP_TO_SINT. |
2568 | 20 | if (20 !isSigned && 20 TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)4 ) { |
2569 | 0 | OpToUse = ISD::FP_TO_UINT; |
2570 | 0 | break; |
2571 | 0 | } |
2572 | 156 | |
2573 | 156 | // Otherwise, try a larger type. |
2574 | 156 | } |
2575 | 136 | |
2576 | 136 | // Okay, we found the operation and type to use. |
2577 | 136 | SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); |
2578 | 136 | |
2579 | 136 | // Truncate the result of the extended FP_TO_*INT operation to the desired |
2580 | 136 | // size. |
2581 | 136 | return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); |
2582 | 136 | } |
2583 | | |
2584 | | /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts. |
2585 | 92 | SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) { |
2586 | 92 | EVT VT = Op.getValueType(); |
2587 | 92 | EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2588 | 92 | unsigned Sz = VT.getScalarSizeInBits(); |
2589 | 92 | |
2590 | 92 | SDValue Tmp, Tmp2, Tmp3; |
2591 | 92 | |
2592 | 92 | // If we can, perform BSWAP first and then the mask+swap the i4, then i2 |
2593 | 92 | // and finally the i1 pairs. |
2594 | 92 | // TODO: We can easily support i4/i2 legal types if any target ever does. |
2595 | 92 | if (Sz >= 8 && 92 isPowerOf2_32(Sz)92 ) { |
2596 | 92 | // Create the masks - repeating the pattern every byte. |
2597 | 92 | APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0); |
2598 | 92 | APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0); |
2599 | 424 | for (unsigned J = 0; J != Sz424 ; J += 8332 ) { |
2600 | 332 | MaskHi4 = MaskHi4 | (0xF0ull << J); |
2601 | 332 | MaskLo4 = MaskLo4 | (0x0Full << J); |
2602 | 332 | MaskHi2 = MaskHi2 | (0xCCull << J); |
2603 | 332 | MaskLo2 = MaskLo2 | (0x33ull << J); |
2604 | 332 | MaskHi1 = MaskHi1 | (0xAAull << J); |
2605 | 332 | MaskLo1 = MaskLo1 | (0x55ull << J); |
2606 | 332 | } |
2607 | 92 | |
2608 | 92 | // BSWAP if the type is wider than a single byte. |
2609 | 92 | Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op)52 : Op40 ); |
2610 | 92 | |
2611 | 92 | // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4) |
2612 | 92 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT)); |
2613 | 92 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT)); |
2614 | 92 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT)); |
2615 | 92 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT)); |
2616 | 92 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); |
2617 | 92 | |
2618 | 92 | // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2) |
2619 | 92 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT)); |
2620 | 92 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT)); |
2621 | 92 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT)); |
2622 | 92 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT)); |
2623 | 92 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); |
2624 | 92 | |
2625 | 92 | // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1) |
2626 | 92 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT)); |
2627 | 92 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT)); |
2628 | 92 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT)); |
2629 | 92 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT)); |
2630 | 92 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); |
2631 | 92 | return Tmp; |
2632 | 92 | } |
2633 | 0 |
|
2634 | 0 | Tmp = DAG.getConstant(0, dl, VT); |
2635 | 0 | for (unsigned I = 0, J = Sz-1; I < Sz0 ; ++I, --J0 ) { |
2636 | 0 | if (I < J) |
2637 | 0 | Tmp2 = |
2638 | 0 | DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); |
2639 | 0 | else |
2640 | 0 | Tmp2 = |
2641 | 0 | DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); |
2642 | 0 |
|
2643 | 0 | APInt Shift(Sz, 1); |
2644 | 0 | Shift <<= J; |
2645 | 0 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); |
2646 | 0 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); |
2647 | 0 | } |
2648 | 92 | |
2649 | 92 | return Tmp; |
2650 | 92 | } |
2651 | | |
2652 | | /// Open code the operations for BSWAP of the specified operation. |
2653 | 118 | SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) { |
2654 | 118 | EVT VT = Op.getValueType(); |
2655 | 118 | EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2656 | 118 | SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; |
2657 | 118 | switch (VT.getSimpleVT().getScalarType().SimpleTy) { |
2658 | 0 | default: 0 llvm_unreachable0 ("Unhandled Expand type in BSWAP!"); |
2659 | 55 | case MVT::i16: |
2660 | 55 | Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2661 | 55 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2662 | 55 | return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); |
2663 | 25 | case MVT::i32: |
2664 | 25 | Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); |
2665 | 25 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2666 | 25 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2667 | 25 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); |
2668 | 25 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, |
2669 | 25 | DAG.getConstant(0xFF0000, dl, VT)); |
2670 | 25 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); |
2671 | 25 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); |
2672 | 25 | Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); |
2673 | 25 | return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); |
2674 | 38 | case MVT::i64: |
2675 | 38 | Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); |
2676 | 38 | Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); |
2677 | 38 | Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); |
2678 | 38 | Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2679 | 38 | Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); |
2680 | 38 | Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); |
2681 | 38 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); |
2682 | 38 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); |
2683 | 38 | Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, |
2684 | 38 | DAG.getConstant(255ULL<<48, dl, VT)); |
2685 | 38 | Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, |
2686 | 38 | DAG.getConstant(255ULL<<40, dl, VT)); |
2687 | 38 | Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, |
2688 | 38 | DAG.getConstant(255ULL<<32, dl, VT)); |
2689 | 38 | Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, |
2690 | 38 | DAG.getConstant(255ULL<<24, dl, VT)); |
2691 | 38 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, |
2692 | 38 | DAG.getConstant(255ULL<<16, dl, VT)); |
2693 | 38 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, |
2694 | 38 | DAG.getConstant(255ULL<<8 , dl, VT)); |
2695 | 38 | Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); |
2696 | 38 | Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); |
2697 | 38 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); |
2698 | 38 | Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); |
2699 | 38 | Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); |
2700 | 38 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); |
2701 | 38 | return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); |
2702 | 0 | } |
2703 | 0 | } |
2704 | | |
2705 | | /// Expand the specified bitcount instruction into operations. |
2706 | | SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, |
2707 | 3.50k | const SDLoc &dl) { |
2708 | 3.50k | switch (Opc) { |
2709 | 0 | default: 0 llvm_unreachable0 ("Cannot expand this yet!"); |
2710 | 134 | case ISD::CTPOP: { |
2711 | 134 | EVT VT = Op.getValueType(); |
2712 | 134 | EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2713 | 134 | unsigned Len = VT.getSizeInBits(); |
2714 | 134 | |
2715 | 134 | assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 && |
2716 | 134 | "CTPOP not implemented for this type."); |
2717 | 134 | |
2718 | 134 | // This is the "best" algorithm from |
2719 | 134 | // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel |
2720 | 134 | |
2721 | 134 | SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), |
2722 | 134 | dl, VT); |
2723 | 134 | SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), |
2724 | 134 | dl, VT); |
2725 | 134 | SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), |
2726 | 134 | dl, VT); |
2727 | 134 | SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), |
2728 | 134 | dl, VT); |
2729 | 134 | |
2730 | 134 | // v = v - ((v >> 1) & 0x55555555...) |
2731 | 134 | Op = DAG.getNode(ISD::SUB, dl, VT, Op, |
2732 | 134 | DAG.getNode(ISD::AND, dl, VT, |
2733 | 134 | DAG.getNode(ISD::SRL, dl, VT, Op, |
2734 | 134 | DAG.getConstant(1, dl, ShVT)), |
2735 | 134 | Mask55)); |
2736 | 134 | // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...) |
2737 | 134 | Op = DAG.getNode(ISD::ADD, dl, VT, |
2738 | 134 | DAG.getNode(ISD::AND, dl, VT, Op, Mask33), |
2739 | 134 | DAG.getNode(ISD::AND, dl, VT, |
2740 | 134 | DAG.getNode(ISD::SRL, dl, VT, Op, |
2741 | 134 | DAG.getConstant(2, dl, ShVT)), |
2742 | 134 | Mask33)); |
2743 | 134 | // v = (v + (v >> 4)) & 0x0F0F0F0F... |
2744 | 134 | Op = DAG.getNode(ISD::AND, dl, VT, |
2745 | 134 | DAG.getNode(ISD::ADD, dl, VT, Op, |
2746 | 134 | DAG.getNode(ISD::SRL, dl, VT, Op, |
2747 | 134 | DAG.getConstant(4, dl, ShVT))), |
2748 | 134 | Mask0F); |
2749 | 134 | // v = (v * 0x01010101...) >> (Len - 8) |
2750 | 134 | Op = DAG.getNode(ISD::SRL, dl, VT, |
2751 | 134 | DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), |
2752 | 134 | DAG.getConstant(Len - 8, dl, ShVT)); |
2753 | 134 | |
2754 | 134 | return Op; |
2755 | 3.50k | } |
2756 | 3.08k | case ISD::CTLZ_ZERO_UNDEF: |
2757 | 3.08k | // This trivially expands to CTLZ. |
2758 | 3.08k | return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); |
2759 | 119 | case ISD::CTLZ: { |
2760 | 119 | EVT VT = Op.getValueType(); |
2761 | 119 | unsigned len = VT.getSizeInBits(); |
2762 | 119 | |
2763 | 119 | if (TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)119 ) { |
2764 | 40 | EVT SetCCVT = getSetCCResultType(VT); |
2765 | 40 | SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op); |
2766 | 40 | SDValue Zero = DAG.getConstant(0, dl, VT); |
2767 | 40 | SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ); |
2768 | 40 | return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero, |
2769 | 40 | DAG.getConstant(len, dl, VT), CTLZ); |
2770 | 40 | } |
2771 | 79 | |
2772 | 79 | // for now, we do this: |
2773 | 79 | // x = x | (x >> 1); |
2774 | 79 | // x = x | (x >> 2); |
2775 | 79 | // ... |
2776 | 79 | // x = x | (x >>16); |
2777 | 79 | // x = x | (x >>32); // for 64-bit input |
2778 | 79 | // return popcount(~x); |
2779 | 79 | // |
2780 | 79 | // Ref: "Hacker's Delight" by Henry Warren |
2781 | 79 | EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2782 | 480 | for (unsigned i = 0; (1U << i) <= (len / 2)480 ; ++i401 ) { |
2783 | 401 | SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT); |
2784 | 401 | Op = DAG.getNode(ISD::OR, dl, VT, Op, |
2785 | 401 | DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); |
2786 | 401 | } |
2787 | 79 | Op = DAG.getNOT(dl, Op, VT); |
2788 | 79 | return DAG.getNode(ISD::CTPOP, dl, VT, Op); |
2789 | 79 | } |
2790 | 140 | case ISD::CTTZ_ZERO_UNDEF: |
2791 | 140 | // This trivially expands to CTTZ. |
2792 | 140 | return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); |
2793 | 25 | case ISD::CTTZ: { |
2794 | 25 | // for now, we use: { return popcount(~x & (x - 1)); } |
2795 | 25 | // unless the target has ctlz but not ctpop, in which case we use: |
2796 | 25 | // { return 32 - nlz(~x & (x-1)); } |
2797 | 25 | // Ref: "Hacker's Delight" by Henry Warren |
2798 | 25 | EVT VT = Op.getValueType(); |
2799 | 25 | SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, |
2800 | 25 | DAG.getNOT(dl, Op, VT), |
2801 | 25 | DAG.getNode(ISD::SUB, dl, VT, Op, |
2802 | 25 | DAG.getConstant(1, dl, VT))); |
2803 | 25 | // If ISD::CTLZ is legal and CTPOP isn't, then do that instead. |
2804 | 25 | if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && |
2805 | 17 | TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) |
2806 | 5 | return DAG.getNode(ISD::SUB, dl, VT, |
2807 | 5 | DAG.getConstant(VT.getSizeInBits(), dl, VT), |
2808 | 5 | DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); |
2809 | 20 | return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); |
2810 | 20 | } |
2811 | 3.50k | } |
2812 | 3.50k | } |
2813 | | |
2814 | 334k | bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { |
2815 | 334k | SmallVector<SDValue, 8> Results; |
2816 | 334k | SDLoc dl(Node); |
2817 | 334k | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
2818 | 334k | bool NeedInvert; |
2819 | 334k | switch (Node->getOpcode()) { |
2820 | 3.50k | case ISD::CTPOP: |
2821 | 3.50k | case ISD::CTLZ: |
2822 | 3.50k | case ISD::CTLZ_ZERO_UNDEF: |
2823 | 3.50k | case ISD::CTTZ: |
2824 | 3.50k | case ISD::CTTZ_ZERO_UNDEF: |
2825 | 3.50k | Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl); |
2826 | 3.50k | Results.push_back(Tmp1); |
2827 | 3.50k | break; |
2828 | 92 | case ISD::BITREVERSE: |
2829 | 92 | Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl)); |
2830 | 92 | break; |
2831 | 118 | case ISD::BSWAP: |
2832 | 118 | Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); |
2833 | 118 | break; |
2834 | 0 | case ISD::FRAMEADDR: |
2835 | 0 | case ISD::RETURNADDR: |
2836 | 0 | case ISD::FRAME_TO_ARGS_OFFSET: |
2837 | 0 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); |
2838 | 0 | break; |
2839 | 6 | case ISD::EH_DWARF_CFA: { |
2840 | 6 | SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, |
2841 | 6 | TLI.getPointerTy(DAG.getDataLayout())); |
2842 | 6 | SDValue Offset = DAG.getNode(ISD::ADD, dl, |
2843 | 6 | CfaArg.getValueType(), |
2844 | 6 | DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, |
2845 | 6 | CfaArg.getValueType()), |
2846 | 6 | CfaArg); |
2847 | 6 | SDValue FA = DAG.getNode( |
2848 | 6 | ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), |
2849 | 6 | DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()))); |
2850 | 6 | Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), |
2851 | 6 | FA, Offset)); |
2852 | 6 | break; |
2853 | 0 | } |
2854 | 1 | case ISD::FLT_ROUNDS_: |
2855 | 1 | Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); |
2856 | 1 | break; |
2857 | 470 | case ISD::EH_RETURN: |
2858 | 470 | case ISD::EH_LABEL: |
2859 | 470 | case ISD::PREFETCH: |
2860 | 470 | case ISD::VAEND: |
2861 | 470 | case ISD::EH_SJLJ_LONGJMP: |
2862 | 470 | // If the target didn't expand these, there's nothing to do, so just |
2863 | 470 | // preserve the chain and be done. |
2864 | 470 | Results.push_back(Node->getOperand(0)); |
2865 | 470 | break; |
2866 | 3 | case ISD::READCYCLECOUNTER: |
2867 | 3 | // If the target didn't expand this, just return 'zero' and preserve the |
2868 | 3 | // chain. |
2869 | 3 | Results.append(Node->getNumValues() - 1, |
2870 | 3 | DAG.getConstant(0, dl, Node->getValueType(0))); |
2871 | 3 | Results.push_back(Node->getOperand(0)); |
2872 | 3 | break; |
2873 | 0 | case ISD::EH_SJLJ_SETJMP: |
2874 | 0 | // If the target didn't expand this, just return 'zero' and preserve the |
2875 | 0 | // chain. |
2876 | 0 | Results.push_back(DAG.getConstant(0, dl, MVT::i32)); |
2877 | 0 | Results.push_back(Node->getOperand(0)); |
2878 | 0 | break; |
2879 | 6 | case ISD::ATOMIC_LOAD: { |
2880 | 6 | // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. |
2881 | 6 | SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); |
2882 | 6 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
2883 | 6 | SDValue Swap = DAG.getAtomicCmpSwap( |
2884 | 6 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, |
2885 | 6 | Node->getOperand(0), Node->getOperand(1), Zero, Zero, |
2886 | 6 | cast<AtomicSDNode>(Node)->getMemOperand()); |
2887 | 6 | Results.push_back(Swap.getValue(0)); |
2888 | 6 | Results.push_back(Swap.getValue(1)); |
2889 | 6 | break; |
2890 | 470 | } |
2891 | 14 | case ISD::ATOMIC_STORE: { |
2892 | 14 | // There is no libcall for atomic store; fake it with ATOMIC_SWAP. |
2893 | 14 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, |
2894 | 14 | cast<AtomicSDNode>(Node)->getMemoryVT(), |
2895 | 14 | Node->getOperand(0), |
2896 | 14 | Node->getOperand(1), Node->getOperand(2), |
2897 | 14 | cast<AtomicSDNode>(Node)->getMemOperand()); |
2898 | 14 | Results.push_back(Swap.getValue(1)); |
2899 | 14 | break; |
2900 | 470 | } |
2901 | 1.99k | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { |
2902 | 1.99k | // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and |
2903 | 1.99k | // splits out the success value as a comparison. Expanding the resulting |
2904 | 1.99k | // ATOMIC_CMP_SWAP will produce a libcall. |
2905 | 1.99k | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); |
2906 | 1.99k | SDValue Res = DAG.getAtomicCmpSwap( |
2907 | 1.99k | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, |
2908 | 1.99k | Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), |
2909 | 1.99k | Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); |
2910 | 1.99k | |
2911 | 1.99k | SDValue ExtRes = Res; |
2912 | 1.99k | SDValue LHS = Res; |
2913 | 1.99k | SDValue RHS = Node->getOperand(1); |
2914 | 1.99k | |
2915 | 1.99k | EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); |
2916 | 1.99k | EVT OuterType = Node->getValueType(0); |
2917 | 1.99k | switch (TLI.getExtendForAtomicOps()) { |
2918 | 55 | case ISD::SIGN_EXTEND: |
2919 | 55 | LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, |
2920 | 55 | DAG.getValueType(AtomicType)); |
2921 | 55 | RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, |
2922 | 55 | Node->getOperand(2), DAG.getValueType(AtomicType)); |
2923 | 55 | ExtRes = LHS; |
2924 | 55 | break; |
2925 | 1.90k | case ISD::ZERO_EXTEND: |
2926 | 1.90k | LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, |
2927 | 1.90k | DAG.getValueType(AtomicType)); |
2928 | 1.90k | RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2)); |
2929 | 1.90k | ExtRes = LHS; |
2930 | 1.90k | break; |
2931 | 39 | case ISD::ANY_EXTEND: |
2932 | 39 | LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType); |
2933 | 39 | RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2)); |
2934 | 39 | break; |
2935 | 0 | default: |
2936 | 0 | llvm_unreachable("Invalid atomic op extension"); |
2937 | 1.99k | } |
2938 | 1.99k | |
2939 | 1.99k | SDValue Success = |
2940 | 1.99k | DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); |
2941 | 1.99k | |
2942 | 1.99k | Results.push_back(ExtRes.getValue(0)); |
2943 | 1.99k | Results.push_back(Success); |
2944 | 1.99k | Results.push_back(Res.getValue(1)); |
2945 | 1.99k | break; |
2946 | 1.99k | } |
2947 | 315 | case ISD::DYNAMIC_STACKALLOC: |
2948 | 315 | ExpandDYNAMIC_STACKALLOC(Node, Results); |
2949 | 315 | break; |
2950 | 26.2k | case ISD::MERGE_VALUES: |
2951 | 78.6k | for (unsigned i = 0; i < Node->getNumValues()78.6k ; i++52.4k ) |
2952 | 52.4k | Results.push_back(Node->getOperand(i)); |
2953 | 26.2k | break; |
2954 | 448 | case ISD::UNDEF: { |
2955 | 448 | EVT VT = Node->getValueType(0); |
2956 | 448 | if (VT.isInteger()) |
2957 | 330 | Results.push_back(DAG.getConstant(0, dl, VT)); |
2958 | 118 | else { |
2959 | 118 | assert(VT.isFloatingPoint() && "Unknown value type!"); |
2960 | 118 | Results.push_back(DAG.getConstantFP(0, dl, VT)); |
2961 | 118 | } |
2962 | 448 | break; |
2963 | 1.99k | } |
2964 | 520 | case ISD::FP_ROUND: |
2965 | 520 | case ISD::BITCAST: |
2966 | 520 | Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), |
2967 | 520 | Node->getValueType(0), dl); |
2968 | 520 | Results.push_back(Tmp1); |
2969 | 520 | break; |
2970 | 0 | case ISD::FP_EXTEND: |
2971 | 0 | Tmp1 = EmitStackConvert(Node->getOperand(0), |
2972 | 0 | Node->getOperand(0).getValueType(), |
2973 | 0 | Node->getValueType(0), dl); |
2974 | 0 | Results.push_back(Tmp1); |
2975 | 0 | break; |
2976 | 843 | case ISD::SIGN_EXTEND_INREG: { |
2977 | 843 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
2978 | 843 | EVT VT = Node->getValueType(0); |
2979 | 843 | |
2980 | 843 | // An in-register sign-extend of a boolean is a negation: |
2981 | 843 | // 'true' (1) sign-extended is -1. |
2982 | 843 | // 'false' (0) sign-extended is 0. |
2983 | 843 | // However, we must mask the high bits of the source operand because the |
2984 | 843 | // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero. |
2985 | 843 | |
2986 | 843 | // TODO: Do this for vectors too? |
2987 | 843 | if (ExtraVT.getSizeInBits() == 1843 ) { |
2988 | 232 | SDValue One = DAG.getConstant(1, dl, VT); |
2989 | 232 | SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); |
2990 | 232 | SDValue Zero = DAG.getConstant(0, dl, VT); |
2991 | 232 | SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); |
2992 | 232 | Results.push_back(Neg); |
2993 | 232 | break; |
2994 | 232 | } |
2995 | 611 | |
2996 | 611 | // NOTE: we could fall back on load/store here too for targets without |
2997 | 611 | // SRA. However, it is doubtful that any exist. |
2998 | 611 | EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); |
2999 | 611 | unsigned BitsDiff = VT.getScalarSizeInBits() - |
3000 | 611 | ExtraVT.getScalarSizeInBits(); |
3001 | 611 | SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); |
3002 | 611 | Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), |
3003 | 611 | Node->getOperand(0), ShiftCst); |
3004 | 611 | Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); |
3005 | 611 | Results.push_back(Tmp1); |
3006 | 611 | break; |
3007 | 611 | } |
3008 | 0 | case ISD::FP_ROUND_INREG: { |
3009 | 0 | // The only way we can lower this is to turn it into a TRUNCSTORE, |
3010 | 0 | // EXTLOAD pair, targeting a temporary location (a stack slot). |
3011 | 0 |
|
3012 | 0 | // NOTE: there is a choice here between constantly creating new stack |
3013 | 0 | // slots and always reusing the same one. We currently always create |
3014 | 0 | // new ones, as reuse may inhibit scheduling. |
3015 | 0 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); |
3016 | 0 | Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, |
3017 | 0 | Node->getValueType(0), dl); |
3018 | 0 | Results.push_back(Tmp1); |
3019 | 0 | break; |
3020 | 611 | } |
3021 | 138 | case ISD::SINT_TO_FP: |
3022 | 138 | case ISD::UINT_TO_FP: |
3023 | 138 | Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, |
3024 | 138 | Node->getOperand(0), Node->getValueType(0), dl); |
3025 | 138 | Results.push_back(Tmp1); |
3026 | 138 | break; |
3027 | 60 | case ISD::FP_TO_SINT: |
3028 | 60 | if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) |
3029 | 60 | Results.push_back(Tmp1); |
3030 | 60 | break; |
3031 | 190 | case ISD::FP_TO_UINT: { |
3032 | 190 | SDValue True, False; |
3033 | 190 | EVT VT = Node->getOperand(0).getValueType(); |
3034 | 190 | EVT NVT = Node->getValueType(0); |
3035 | 190 | APFloat apf(DAG.EVTToAPFloatSemantics(VT), |
3036 | 190 | APInt::getNullValue(VT.getSizeInBits())); |
3037 | 190 | APInt x = APInt::getSignMask(NVT.getSizeInBits()); |
3038 | 190 | (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven); |
3039 | 190 | Tmp1 = DAG.getConstantFP(apf, dl, VT); |
3040 | 190 | Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT), |
3041 | 190 | Node->getOperand(0), |
3042 | 190 | Tmp1, ISD::SETLT); |
3043 | 190 | True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); |
3044 | 190 | // TODO: Should any fast-math-flags be set for the FSUB? |
3045 | 190 | False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, |
3046 | 190 | DAG.getNode(ISD::FSUB, dl, VT, |
3047 | 190 | Node->getOperand(0), Tmp1)); |
3048 | 190 | False = DAG.getNode(ISD::XOR, dl, NVT, False, |
3049 | 190 | DAG.getConstant(x, dl, NVT)); |
3050 | 190 | Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False); |
3051 | 190 | Results.push_back(Tmp1); |
3052 | 190 | break; |
3053 | 138 | } |
3054 | 24 | case ISD::VAARG: |
3055 | 24 | Results.push_back(DAG.expandVAArg(Node)); |
3056 | 24 | Results.push_back(Results[0].getValue(1)); |
3057 | 24 | break; |
3058 | 31 | case ISD::VACOPY: |
3059 | 31 | Results.push_back(DAG.expandVACopy(Node)); |
3060 | 31 | break; |
3061 | 2.20k | case ISD::EXTRACT_VECTOR_ELT: |
3062 | 2.20k | if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) |
3063 | 2.20k | // This must be an access of the only element. Return it. |
3064 | 136 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), |
3065 | 136 | Node->getOperand(0)); |
3066 | 2.20k | else |
3067 | 2.06k | Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); |
3068 | 2.20k | Results.push_back(Tmp1); |
3069 | 2.20k | break; |
3070 | 0 | case ISD::EXTRACT_SUBVECTOR: |
3071 | 0 | Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); |
3072 | 0 | break; |
3073 | 0 | case ISD::INSERT_SUBVECTOR: |
3074 | 0 | Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); |
3075 | 0 | break; |
3076 | 0 | case ISD::CONCAT_VECTORS: |
3077 | 0 | Results.push_back(ExpandVectorBuildThroughStack(Node)); |
3078 | 0 | break; |
3079 | 18 | case ISD::SCALAR_TO_VECTOR: |
3080 | 18 | Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); |
3081 | 18 | break; |
3082 | 404 | case ISD::INSERT_VECTOR_ELT: |
3083 | 404 | Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), |
3084 | 404 | Node->getOperand(1), |
3085 | 404 | Node->getOperand(2), dl)); |
3086 | 404 | break; |
3087 | 183 | case ISD::VECTOR_SHUFFLE: { |
3088 | 183 | SmallVector<int, 32> NewMask; |
3089 | 183 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); |
3090 | 183 | |
3091 | 183 | EVT VT = Node->getValueType(0); |
3092 | 183 | EVT EltVT = VT.getVectorElementType(); |
3093 | 183 | SDValue Op0 = Node->getOperand(0); |
3094 | 183 | SDValue Op1 = Node->getOperand(1); |
3095 | 183 | if (!TLI.isTypeLegal(EltVT)183 ) { |
3096 | 11 | EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); |
3097 | 11 | |
3098 | 11 | // BUILD_VECTOR operands are allowed to be wider than the element type. |
3099 | 11 | // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept |
3100 | 11 | // it. |
3101 | 11 | if (NewEltVT.bitsLT(EltVT)11 ) { |
3102 | 0 | // Convert shuffle node. |
3103 | 0 | // If original node was v4i64 and the new EltVT is i32, |
3104 | 0 | // cast operands to v8i32 and re-build the mask. |
3105 | 0 |
|
3106 | 0 | // Calculate new VT, the size of the new VT should be equal to original. |
3107 | 0 | EVT NewVT = |
3108 | 0 | EVT::getVectorVT(*DAG.getContext(), NewEltVT, |
3109 | 0 | VT.getSizeInBits() / NewEltVT.getSizeInBits()); |
3110 | 0 | assert(NewVT.bitsEq(VT)); |
3111 | 0 |
|
3112 | 0 | // cast operands to new VT |
3113 | 0 | Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); |
3114 | 0 | Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); |
3115 | 0 |
|
3116 | 0 | // Convert the shuffle mask |
3117 | 0 | unsigned int factor = |
3118 | 0 | NewVT.getVectorNumElements()/VT.getVectorNumElements(); |
3119 | 0 |
|
3120 | 0 | // EltVT gets smaller |
3121 | 0 | assert(factor > 0); |
3122 | 0 |
|
3123 | 0 | for (unsigned i = 0; i < VT.getVectorNumElements()0 ; ++i0 ) { |
3124 | 0 | if (Mask[i] < 00 ) { |
3125 | 0 | for (unsigned fi = 0; fi < factor0 ; ++fi0 ) |
3126 | 0 | NewMask.push_back(Mask[i]); |
3127 | 0 | } |
3128 | 0 | else { |
3129 | 0 | for (unsigned fi = 0; fi < factor0 ; ++fi0 ) |
3130 | 0 | NewMask.push_back(Mask[i]*factor+fi); |
3131 | 0 | } |
3132 | 0 | } |
3133 | 0 | Mask = NewMask; |
3134 | 0 | VT = NewVT; |
3135 | 0 | } |
3136 | 11 | EltVT = NewEltVT; |
3137 | 11 | } |
3138 | 183 | unsigned NumElems = VT.getVectorNumElements(); |
3139 | 183 | SmallVector<SDValue, 16> Ops; |
3140 | 1.31k | for (unsigned i = 0; i != NumElems1.31k ; ++i1.12k ) { |
3141 | 1.12k | if (Mask[i] < 01.12k ) { |
3142 | 23 | Ops.push_back(DAG.getUNDEF(EltVT)); |
3143 | 23 | continue; |
3144 | 23 | } |
3145 | 1.10k | unsigned Idx = Mask[i]; |
3146 | 1.10k | if (Idx < NumElems) |
3147 | 597 | Ops.push_back(DAG.getNode( |
3148 | 597 | ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, |
3149 | 597 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); |
3150 | 1.10k | else |
3151 | 508 | Ops.push_back(DAG.getNode( |
3152 | 508 | ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, |
3153 | 508 | DAG.getConstant(Idx - NumElems, dl, |
3154 | 508 | TLI.getVectorIdxTy(DAG.getDataLayout())))); |
3155 | 1.12k | } |
3156 | 183 | |
3157 | 183 | Tmp1 = DAG.getBuildVector(VT, dl, Ops); |
3158 | 183 | // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. |
3159 | 183 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); |
3160 | 183 | Results.push_back(Tmp1); |
3161 | 183 | break; |
3162 | 138 | } |
3163 | 144 | case ISD::EXTRACT_ELEMENT: { |
3164 | 144 | EVT OpTy = Node->getOperand(0).getValueType(); |
3165 | 144 | if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()144 ) { |
3166 | 56 | // 1 -> Hi |
3167 | 56 | Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), |
3168 | 56 | DAG.getConstant(OpTy.getSizeInBits() / 2, dl, |
3169 | 56 | TLI.getShiftAmountTy( |
3170 | 56 | Node->getOperand(0).getValueType(), |
3171 | 56 | DAG.getDataLayout()))); |
3172 | 56 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); |
3173 | 144 | } else { |
3174 | 88 | // 0 -> Lo |
3175 | 88 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), |
3176 | 88 | Node->getOperand(0)); |
3177 | 88 | } |
3178 | 144 | Results.push_back(Tmp1); |
3179 | 144 | break; |
3180 | 138 | } |
3181 | 111 | case ISD::STACKSAVE: |
3182 | 111 | // Expand to CopyFromReg if the target set |
3183 | 111 | // StackPointerRegisterToSaveRestore. |
3184 | 111 | if (unsigned SP111 = TLI.getStackPointerRegisterToSaveRestore()) { |
3185 | 111 | Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, |
3186 | 111 | Node->getValueType(0))); |
3187 | 111 | Results.push_back(Results[0].getValue(1)); |
3188 | 111 | } else { |
3189 | 0 | Results.push_back(DAG.getUNDEF(Node->getValueType(0))); |
3190 | 0 | Results.push_back(Node->getOperand(0)); |
3191 | 0 | } |
3192 | 111 | break; |
3193 | 27 | case ISD::STACKRESTORE: |
3194 | 27 | // Expand to CopyToReg if the target set |
3195 | 27 | // StackPointerRegisterToSaveRestore. |
3196 | 27 | if (unsigned SP27 = TLI.getStackPointerRegisterToSaveRestore()) { |
3197 | 27 | Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, |
3198 | 27 | Node->getOperand(1))); |
3199 | 27 | } else { |
3200 | 0 | Results.push_back(Node->getOperand(0)); |
3201 | 0 | } |
3202 | 27 | break; |
3203 | 0 | case ISD::GET_DYNAMIC_AREA_OFFSET: |
3204 | 0 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); |
3205 | 0 | Results.push_back(Results[0].getValue(0)); |
3206 | 0 | break; |
3207 | 296 | case ISD::FCOPYSIGN: |
3208 | 296 | Results.push_back(ExpandFCOPYSIGN(Node)); |
3209 | 296 | break; |
3210 | 3 | case ISD::FNEG: |
3211 | 3 | // Expand Y = FNEG(X) -> Y = SUB -0.0, X |
3212 | 3 | Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0)); |
3213 | 3 | // TODO: If FNEG has fast-math-flags, propagate them to the FSUB. |
3214 | 3 | Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, |
3215 | 3 | Node->getOperand(0)); |
3216 | 3 | Results.push_back(Tmp1); |
3217 | 3 | break; |
3218 | 32 | case ISD::FABS: |
3219 | 32 | Results.push_back(ExpandFABS(Node)); |
3220 | 32 | break; |
3221 | 0 | case ISD::SMIN: |
3222 | 0 | case ISD::SMAX: |
3223 | 0 | case ISD::UMIN: |
3224 | 0 | case ISD::UMAX: { |
3225 | 0 | // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B |
3226 | 0 | ISD::CondCode Pred; |
3227 | 0 | switch (Node->getOpcode()) { |
3228 | 0 | default: 0 llvm_unreachable0 ("How did we get here?"); |
3229 | 0 | case ISD::SMAX: Pred = ISD::SETGT; break; |
3230 | 0 | case ISD::SMIN: Pred = ISD::SETLT; break; |
3231 | 0 | case ISD::UMAX: Pred = ISD::SETUGT; break; |
3232 | 0 | case ISD::UMIN: Pred = ISD::SETULT; break; |
3233 | 0 | } |
3234 | 0 | Tmp1 = Node->getOperand(0); |
3235 | 0 | Tmp2 = Node->getOperand(1); |
3236 | 0 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred); |
3237 | 0 | Results.push_back(Tmp1); |
3238 | 0 | break; |
3239 | 0 | } |
3240 | 0 |
|
3241 | 860 | case ISD::FSIN: |
3242 | 860 | case ISD::FCOS: { |
3243 | 860 | EVT VT = Node->getValueType(0); |
3244 | 860 | // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / |
3245 | 860 | // fcos which share the same operand and both are used. |
3246 | 860 | if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || |
3247 | 253 | isSinCosLibcallAvailable(Node, TLI)) |
3248 | 860 | && useSinCos(Node)704 ) { |
3249 | 410 | SDVTList VTs = DAG.getVTList(VT, VT); |
3250 | 410 | Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); |
3251 | 410 | if (Node->getOpcode() == ISD::FCOS) |
3252 | 205 | Tmp1 = Tmp1.getValue(1); |
3253 | 410 | Results.push_back(Tmp1); |
3254 | 410 | } |
3255 | 860 | break; |
3256 | 860 | } |
3257 | 0 | case ISD::FMAD: |
3258 | 0 | llvm_unreachable("Illegal fmad should never be formed"); |
3259 | 860 | |
3260 | 157 | case ISD::FP16_TO_FP: |
3261 | 157 | if (Node->getValueType(0) != MVT::f32157 ) { |
3262 | 0 | // We can extend to types bigger than f32 in two steps without changing |
3263 | 0 | // the result. Since "f16 -> f32" is much more commonly available, give |
3264 | 0 | // CodeGen the option of emitting that before resorting to a libcall. |
3265 | 0 | SDValue Res = |
3266 | 0 | DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); |
3267 | 0 | Results.push_back( |
3268 | 0 | DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); |
3269 | 0 | } |
3270 | 157 | break; |
3271 | 345 | case ISD::FP_TO_FP16: |
3272 | 345 | if (!TLI.useSoftFloat() && 345 TM.Options.UnsafeFPMath345 ) { |
3273 | 15 | SDValue Op = Node->getOperand(0); |
3274 | 15 | MVT SVT = Op.getSimpleValueType(); |
3275 | 15 | if ((SVT == MVT::f64 || 15 SVT == MVT::f802 ) && |
3276 | 15 | TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)15 ) { |
3277 | 13 | // Under fastmath, we can expand this node into a fround followed by |
3278 | 13 | // a float-half conversion. |
3279 | 13 | SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, |
3280 | 13 | DAG.getIntPtrConstant(0, dl)); |
3281 | 13 | Results.push_back( |
3282 | 13 | DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); |
3283 | 13 | } |
3284 | 15 | } |
3285 | 345 | break; |
3286 | 122k | case ISD::ConstantFP: { |
3287 | 122k | ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); |
3288 | 122k | // Check to see if this FP immediate is already legal. |
3289 | 122k | // If this is a legal constant, turn it into a TargetConstantFP node. |
3290 | 122k | if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0))) |
3291 | 34.1k | Results.push_back(ExpandConstantFP(CFP, true)); |
3292 | 122k | break; |
3293 | 860 | } |
3294 | 0 | case ISD::Constant: { |
3295 | 0 | ConstantSDNode *CP = cast<ConstantSDNode>(Node); |
3296 | 0 | Results.push_back(ExpandConstant(CP)); |
3297 | 0 | break; |
3298 | 860 | } |
3299 | 180 | case ISD::FSUB: { |
3300 | 180 | EVT VT = Node->getValueType(0); |
3301 | 180 | if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && |
3302 | 180 | TLI.isOperationLegalOrCustom(ISD::FNEG, VT)164 ) { |
3303 | 164 | const SDNodeFlags Flags = Node->getFlags(); |
3304 | 164 | Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); |
3305 | 164 | Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); |
3306 | 164 | Results.push_back(Tmp1); |
3307 | 164 | } |
3308 | 180 | break; |
3309 | 860 | } |
3310 | 0 | case ISD::SUB: { |
3311 | 0 | EVT VT = Node->getValueType(0); |
3312 | 0 | assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && |
3313 | 0 | TLI.isOperationLegalOrCustom(ISD::XOR, VT) && |
3314 | 0 | "Don't know how to expand this subtraction!"); |
3315 | 0 | Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), |
3316 | 0 | DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, |
3317 | 0 | VT)); |
3318 | 0 | Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); |
3319 | 0 | Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); |
3320 | 0 | break; |
3321 | 860 | } |
3322 | 4.95k | case ISD::UREM: |
3323 | 4.95k | case ISD::SREM: { |
3324 | 4.95k | EVT VT = Node->getValueType(0); |
3325 | 4.95k | bool isSigned = Node->getOpcode() == ISD::SREM; |
3326 | 4.95k | unsigned DivOpc = isSigned ? ISD::SDIV3.74k : ISD::UDIV1.21k ; |
3327 | 4.95k | unsigned DivRemOpc = isSigned ? ISD::SDIVREM3.74k : ISD::UDIVREM1.21k ; |
3328 | 4.95k | Tmp2 = Node->getOperand(0); |
3329 | 4.95k | Tmp3 = Node->getOperand(1); |
3330 | 4.95k | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)4.95k ) { |
3331 | 720 | SDVTList VTs = DAG.getVTList(VT, VT); |
3332 | 720 | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); |
3333 | 720 | Results.push_back(Tmp1); |
3334 | 4.95k | } else if (4.23k TLI.isOperationLegalOrCustom(DivOpc, VT)4.23k ) { |
3335 | 4.19k | // X % Y -> X-X/Y*Y |
3336 | 4.19k | Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); |
3337 | 4.19k | Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); |
3338 | 4.19k | Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); |
3339 | 4.19k | Results.push_back(Tmp1); |
3340 | 4.19k | } |
3341 | 4.95k | break; |
3342 | 4.95k | } |
3343 | 751 | case ISD::UDIV: |
3344 | 751 | case ISD::SDIV: { |
3345 | 751 | bool isSigned = Node->getOpcode() == ISD::SDIV; |
3346 | 751 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM331 : ISD::UDIVREM420 ; |
3347 | 751 | EVT VT = Node->getValueType(0); |
3348 | 751 | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)751 ) { |
3349 | 743 | SDVTList VTs = DAG.getVTList(VT, VT); |
3350 | 743 | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), |
3351 | 743 | Node->getOperand(1)); |
3352 | 743 | Results.push_back(Tmp1); |
3353 | 743 | } |
3354 | 751 | break; |
3355 | 751 | } |
3356 | 384 | case ISD::MULHU: |
3357 | 384 | case ISD::MULHS: { |
3358 | 384 | unsigned ExpandOpcode = |
3359 | 384 | Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI314 : ISD::SMUL_LOHI70 ; |
3360 | 384 | EVT VT = Node->getValueType(0); |
3361 | 384 | SDVTList VTs = DAG.getVTList(VT, VT); |
3362 | 384 | |
3363 | 384 | Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), |
3364 | 384 | Node->getOperand(1)); |
3365 | 384 | Results.push_back(Tmp1.getValue(1)); |
3366 | 384 | break; |
3367 | 384 | } |
3368 | 0 | case ISD::UMUL_LOHI: |
3369 | 0 | case ISD::SMUL_LOHI: { |
3370 | 0 | SDValue LHS = Node->getOperand(0); |
3371 | 0 | SDValue RHS = Node->getOperand(1); |
3372 | 0 | MVT VT = LHS.getSimpleValueType(); |
3373 | 0 | unsigned MULHOpcode = |
3374 | 0 | Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU0 : ISD::MULHS0 ; |
3375 | 0 |
|
3376 | 0 | if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)0 ) { |
3377 | 0 | Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); |
3378 | 0 | Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); |
3379 | 0 | break; |
3380 | 0 | } |
3381 | 0 |
|
3382 | 0 | SmallVector<SDValue, 4> Halves; |
3383 | 0 | EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); |
3384 | 0 | assert(TLI.isTypeLegal(HalfType)); |
3385 | 0 | if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves, |
3386 | 0 | HalfType, DAG, |
3387 | 0 | TargetLowering::MulExpansionKind::Always)) { |
3388 | 0 | for (unsigned i = 0; i < 20 ; ++i0 ) { |
3389 | 0 | SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); |
3390 | 0 | SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); |
3391 | 0 | SDValue Shift = DAG.getConstant( |
3392 | 0 | HalfType.getScalarSizeInBits(), dl, |
3393 | 0 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); |
3394 | 0 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); |
3395 | 0 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); |
3396 | 0 | } |
3397 | 0 | break; |
3398 | 0 | } |
3399 | 0 | break; |
3400 | 0 | } |
3401 | 50 | case ISD::MUL: { |
3402 | 50 | EVT VT = Node->getValueType(0); |
3403 | 50 | SDVTList VTs = DAG.getVTList(VT, VT); |
3404 | 50 | // See if multiply or divide can be lowered using two-result operations. |
3405 | 50 | // We just need the low half of the multiply; try both the signed |
3406 | 50 | // and unsigned forms. If the target supports both SMUL_LOHI and |
3407 | 50 | // UMUL_LOHI, form a preference by checking which forms of plain |
3408 | 50 | // MULH it supports. |
3409 | 50 | bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); |
3410 | 50 | bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); |
3411 | 50 | bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); |
3412 | 50 | bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); |
3413 | 50 | unsigned OpToUse = 0; |
3414 | 50 | if (HasSMUL_LOHI && 50 !HasMULHS7 ) { |
3415 | 7 | OpToUse = ISD::SMUL_LOHI; |
3416 | 50 | } else if (43 HasUMUL_LOHI && 43 !HasMULHU0 ) { |
3417 | 0 | OpToUse = ISD::UMUL_LOHI; |
3418 | 43 | } else if (43 HasSMUL_LOHI43 ) { |
3419 | 0 | OpToUse = ISD::SMUL_LOHI; |
3420 | 43 | } else if (43 HasUMUL_LOHI43 ) { |
3421 | 0 | OpToUse = ISD::UMUL_LOHI; |
3422 | 0 | } |
3423 | 50 | if (OpToUse50 ) { |
3424 | 7 | Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), |
3425 | 7 | Node->getOperand(1))); |
3426 | 7 | break; |
3427 | 7 | } |
3428 | 43 | |
3429 | 43 | SDValue Lo, Hi; |
3430 | 43 | EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); |
3431 | 43 | if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && |
3432 | 43 | TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && |
3433 | 43 | TLI.isOperationLegalOrCustom(ISD::SHL, VT) && |
3434 | 43 | TLI.isOperationLegalOrCustom(ISD::OR, VT) && |
3435 | 43 | TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, |
3436 | 43 | TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { |
3437 | 37 | Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); |
3438 | 37 | Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); |
3439 | 37 | SDValue Shift = |
3440 | 37 | DAG.getConstant(HalfType.getSizeInBits(), dl, |
3441 | 37 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); |
3442 | 37 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); |
3443 | 37 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); |
3444 | 37 | } |
3445 | 43 | break; |
3446 | 43 | } |
3447 | 25 | case ISD::SADDO: |
3448 | 25 | case ISD::SSUBO: { |
3449 | 25 | SDValue LHS = Node->getOperand(0); |
3450 | 25 | SDValue RHS = Node->getOperand(1); |
3451 | 25 | SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? |
3452 | 25 | ISD::ADD12 : ISD::SUB13 , dl, LHS.getValueType(), |
3453 | 25 | LHS, RHS); |
3454 | 25 | Results.push_back(Sum); |
3455 | 25 | EVT ResultType = Node->getValueType(1); |
3456 | 25 | EVT OType = getSetCCResultType(Node->getValueType(0)); |
3457 | 25 | |
3458 | 25 | SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); |
3459 | 25 | |
3460 | 25 | // LHSSign -> LHS >= 0 |
3461 | 25 | // RHSSign -> RHS >= 0 |
3462 | 25 | // SumSign -> Sum >= 0 |
3463 | 25 | // |
3464 | 25 | // Add: |
3465 | 25 | // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) |
3466 | 25 | // Sub: |
3467 | 25 | // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) |
3468 | 25 | SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); |
3469 | 25 | SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); |
3470 | 25 | SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, |
3471 | 25 | Node->getOpcode() == ISD::SADDO ? |
3472 | 25 | ISD::SETEQ12 : ISD::SETNE13 ); |
3473 | 25 | |
3474 | 25 | SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); |
3475 | 25 | SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); |
3476 | 25 | |
3477 | 25 | SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); |
3478 | 25 | Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType)); |
3479 | 25 | break; |
3480 | 25 | } |
3481 | 14 | case ISD::UADDO: |
3482 | 14 | case ISD::USUBO: { |
3483 | 14 | SDValue LHS = Node->getOperand(0); |
3484 | 14 | SDValue RHS = Node->getOperand(1); |
3485 | 14 | SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? |
3486 | 14 | ISD::ADD7 : ISD::SUB7 , dl, LHS.getValueType(), |
3487 | 14 | LHS, RHS); |
3488 | 14 | Results.push_back(Sum); |
3489 | 14 | |
3490 | 14 | EVT ResultType = Node->getValueType(1); |
3491 | 14 | EVT SetCCType = getSetCCResultType(Node->getValueType(0)); |
3492 | 14 | ISD::CondCode CC |
3493 | 14 | = Node->getOpcode() == ISD::UADDO ? ISD::SETULT7 : ISD::SETUGT7 ; |
3494 | 14 | SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); |
3495 | 14 | |
3496 | 14 | Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType)); |
3497 | 14 | break; |
3498 | 14 | } |
3499 | 10 | case ISD::UMULO: |
3500 | 10 | case ISD::SMULO: { |
3501 | 10 | EVT VT = Node->getValueType(0); |
3502 | 10 | EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); |
3503 | 10 | SDValue LHS = Node->getOperand(0); |
3504 | 10 | SDValue RHS = Node->getOperand(1); |
3505 | 10 | SDValue BottomHalf; |
3506 | 10 | SDValue TopHalf; |
3507 | 10 | static const unsigned Ops[2][3] = |
3508 | 10 | { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, |
3509 | 10 | { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; |
3510 | 10 | bool isSigned = Node->getOpcode() == ISD::SMULO; |
3511 | 10 | if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)10 ) { |
3512 | 5 | BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); |
3513 | 5 | TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); |
3514 | 10 | } else if (5 TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)5 ) { |
3515 | 0 | BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, |
3516 | 0 | RHS); |
3517 | 0 | TopHalf = BottomHalf.getValue(1); |
3518 | 5 | } else if (5 TLI.isTypeLegal(WideVT)5 ) { |
3519 | 0 | LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); |
3520 | 0 | RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); |
3521 | 0 | Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); |
3522 | 0 | BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, |
3523 | 0 | DAG.getIntPtrConstant(0, dl)); |
3524 | 0 | TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, |
3525 | 0 | DAG.getIntPtrConstant(1, dl)); |
3526 | 5 | } else { |
3527 | 5 | // We can fall back to a libcall with an illegal type for the MUL if we |
3528 | 5 | // have a libcall big enough. |
3529 | 5 | // Also, we can fall back to a division in some cases, but that's a big |
3530 | 5 | // performance hit in the general case. |
3531 | 5 | RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; |
3532 | 5 | if (WideVT == MVT::i16) |
3533 | 0 | LC = RTLIB::MUL_I16; |
3534 | 5 | else if (5 WideVT == MVT::i325 ) |
3535 | 1 | LC = RTLIB::MUL_I32; |
3536 | 4 | else if (4 WideVT == MVT::i644 ) |
3537 | 4 | LC = RTLIB::MUL_I64; |
3538 | 0 | else if (0 WideVT == MVT::i1280 ) |
3539 | 0 | LC = RTLIB::MUL_I128; |
3540 | 5 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); |
3541 | 5 | |
3542 | 5 | SDValue HiLHS; |
3543 | 5 | SDValue HiRHS; |
3544 | 5 | if (isSigned5 ) { |
3545 | 1 | // The high part is obtained by SRA'ing all but one of the bits of low |
3546 | 1 | // part. |
3547 | 1 | unsigned LoSize = VT.getSizeInBits(); |
3548 | 1 | HiLHS = |
3549 | 1 | DAG.getNode(ISD::SRA, dl, VT, LHS, |
3550 | 1 | DAG.getConstant(LoSize - 1, dl, |
3551 | 1 | TLI.getPointerTy(DAG.getDataLayout()))); |
3552 | 1 | HiRHS = |
3553 | 1 | DAG.getNode(ISD::SRA, dl, VT, RHS, |
3554 | 1 | DAG.getConstant(LoSize - 1, dl, |
3555 | 1 | TLI.getPointerTy(DAG.getDataLayout()))); |
3556 | 5 | } else { |
3557 | 4 | HiLHS = DAG.getConstant(0, dl, VT); |
3558 | 4 | HiRHS = DAG.getConstant(0, dl, VT); |
3559 | 4 | } |
3560 | 5 | |
3561 | 5 | // Here we're passing the 2 arguments explicitly as 4 arguments that are |
3562 | 5 | // pre-lowered to the correct types. This all depends upon WideVT not |
3563 | 5 | // being a legal type for the architecture and thus has to be split to |
3564 | 5 | // two arguments. |
3565 | 5 | SDValue Ret; |
3566 | 5 | if(DAG.getDataLayout().isLittleEndian()5 ) { |
3567 | 5 | // Halves of WideVT are packed into registers in different order |
3568 | 5 | // depending on platform endianness. This is usually handled by |
3569 | 5 | // the C calling convention, but we can't defer to it in |
3570 | 5 | // the legalizer. |
3571 | 5 | SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; |
3572 | 5 | Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); |
3573 | 5 | } else { |
3574 | 0 | SDValue Args[] = { HiLHS, LHS, HiRHS, RHS }; |
3575 | 0 | Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); |
3576 | 0 | } |
3577 | 5 | assert(Ret.getOpcode() == ISD::MERGE_VALUES && |
3578 | 5 | "Ret value is a collection of constituent nodes holding result."); |
3579 | 5 | BottomHalf = Ret.getOperand(0); |
3580 | 5 | TopHalf = Ret.getOperand(1); |
3581 | 5 | } |
3582 | 10 | |
3583 | 10 | if (isSigned10 ) { |
3584 | 5 | Tmp1 = DAG.getConstant( |
3585 | 5 | VT.getSizeInBits() - 1, dl, |
3586 | 5 | TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout())); |
3587 | 5 | Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); |
3588 | 5 | TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1, |
3589 | 5 | ISD::SETNE); |
3590 | 10 | } else { |
3591 | 5 | TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, |
3592 | 5 | DAG.getConstant(0, dl, VT), ISD::SETNE); |
3593 | 5 | } |
3594 | 10 | |
3595 | 10 | // Truncate the result if SetCC returns a larger type than needed. |
3596 | 10 | EVT RType = Node->getValueType(1); |
3597 | 10 | if (RType.getSizeInBits() < TopHalf.getValueSizeInBits()) |
3598 | 1 | TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf); |
3599 | 10 | |
3600 | 10 | assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() && |
3601 | 10 | "Unexpected result type for S/UMULO legalization"); |
3602 | 10 | |
3603 | 10 | Results.push_back(BottomHalf); |
3604 | 10 | Results.push_back(TopHalf); |
3605 | 10 | break; |
3606 | 10 | } |
3607 | 10 | case ISD::BUILD_PAIR: { |
3608 | 10 | EVT PairTy = Node->getValueType(0); |
3609 | 10 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); |
3610 | 10 | Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); |
3611 | 10 | Tmp2 = DAG.getNode( |
3612 | 10 | ISD::SHL, dl, PairTy, Tmp2, |
3613 | 10 | DAG.getConstant(PairTy.getSizeInBits() / 2, dl, |
3614 | 10 | TLI.getShiftAmountTy(PairTy, DAG.getDataLayout()))); |
3615 | 10 | Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); |
3616 | 10 | break; |
3617 | 10 | } |
3618 | 188 | case ISD::SELECT: |
3619 | 188 | Tmp1 = Node->getOperand(0); |
3620 | 188 | Tmp2 = Node->getOperand(1); |
3621 | 188 | Tmp3 = Node->getOperand(2); |
3622 | 188 | if (Tmp1.getOpcode() == ISD::SETCC188 ) { |
3623 | 139 | Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), |
3624 | 139 | Tmp2, Tmp3, |
3625 | 139 | cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); |
3626 | 188 | } else { |
3627 | 49 | Tmp1 = DAG.getSelectCC(dl, Tmp1, |
3628 | 49 | DAG.getConstant(0, dl, Tmp1.getValueType()), |
3629 | 49 | Tmp2, Tmp3, ISD::SETNE); |
3630 | 49 | } |
3631 | 188 | Results.push_back(Tmp1); |
3632 | 188 | break; |
3633 | 4.80k | case ISD::BR_JT: { |
3634 | 4.80k | SDValue Chain = Node->getOperand(0); |
3635 | 4.80k | SDValue Table = Node->getOperand(1); |
3636 | 4.80k | SDValue Index = Node->getOperand(2); |
3637 | 4.80k | |
3638 | 4.80k | const DataLayout &TD = DAG.getDataLayout(); |
3639 | 4.80k | EVT PTy = TLI.getPointerTy(TD); |
3640 | 4.80k | |
3641 | 4.80k | unsigned EntrySize = |
3642 | 4.80k | DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); |
3643 | 4.80k | |
3644 | 4.80k | Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, |
3645 | 4.80k | DAG.getConstant(EntrySize, dl, Index.getValueType())); |
3646 | 4.80k | SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), |
3647 | 4.80k | Index, Table); |
3648 | 4.80k | |
3649 | 4.80k | EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); |
3650 | 4.80k | SDValue LD = DAG.getExtLoad( |
3651 | 4.80k | ISD::SEXTLOAD, dl, PTy, Chain, Addr, |
3652 | 4.80k | MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); |
3653 | 4.80k | Addr = LD; |
3654 | 4.80k | if (TLI.isJumpTableRelative()4.80k ) { |
3655 | 4.66k | // For PIC, the sequence is: |
3656 | 4.66k | // BRIND(load(Jumptable + index) + RelocBase) |
3657 | 4.66k | // RelocBase can be JumpTable, GOT or some sort of global base. |
3658 | 4.66k | Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, |
3659 | 4.66k | TLI.getPICJumpTableRelocBase(Table, DAG)); |
3660 | 4.66k | } |
3661 | 4.80k | Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); |
3662 | 4.80k | Results.push_back(Tmp1); |
3663 | 4.80k | break; |
3664 | 10 | } |
3665 | 32.8k | case ISD::BRCOND: |
3666 | 32.8k | // Expand brcond's setcc into its constituent parts and create a BR_CC |
3667 | 32.8k | // Node. |
3668 | 32.8k | Tmp1 = Node->getOperand(0); |
3669 | 32.8k | Tmp2 = Node->getOperand(1); |
3670 | 32.8k | if (Tmp2.getOpcode() == ISD::SETCC32.8k ) { |
3671 | 294 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, |
3672 | 294 | Tmp1, Tmp2.getOperand(2), |
3673 | 294 | Tmp2.getOperand(0), Tmp2.getOperand(1), |
3674 | 294 | Node->getOperand(2)); |
3675 | 32.8k | } else { |
3676 | 32.5k | // We test only the i1 bit. Skip the AND if UNDEF or another AND. |
3677 | 32.5k | if (Tmp2.isUndef() || |
3678 | 32.5k | (Tmp2.getOpcode() == ISD::AND && |
3679 | 7.40k | isa<ConstantSDNode>(Tmp2.getOperand(1)) && |
3680 | 6.75k | dyn_cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1)) |
3681 | 6.75k | Tmp3 = Tmp2; |
3682 | 32.5k | else |
3683 | 25.7k | Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, |
3684 | 25.7k | DAG.getConstant(1, dl, Tmp2.getValueType())); |
3685 | 32.5k | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, |
3686 | 32.5k | DAG.getCondCode(ISD::SETNE), Tmp3, |
3687 | 32.5k | DAG.getConstant(0, dl, Tmp3.getValueType()), |
3688 | 32.5k | Node->getOperand(2)); |
3689 | 32.5k | } |
3690 | 32.8k | Results.push_back(Tmp1); |
3691 | 32.8k | break; |
3692 | 3.23k | case ISD::SETCC: { |
3693 | 3.23k | Tmp1 = Node->getOperand(0); |
3694 | 3.23k | Tmp2 = Node->getOperand(1); |
3695 | 3.23k | Tmp3 = Node->getOperand(2); |
3696 | 3.23k | bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, |
3697 | 3.23k | Tmp3, NeedInvert, dl); |
3698 | 3.23k | |
3699 | 3.23k | if (Legalized3.23k ) { |
3700 | 313 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the |
3701 | 313 | // condition code, create a new SETCC node. |
3702 | 313 | if (Tmp3.getNode()) |
3703 | 230 | Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), |
3704 | 230 | Tmp1, Tmp2, Tmp3); |
3705 | 313 | |
3706 | 313 | // If we expanded the SETCC by inverting the condition code, then wrap |
3707 | 313 | // the existing SETCC in a NOT to restore the intended condition. |
3708 | 313 | if (NeedInvert) |
3709 | 12 | Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); |
3710 | 313 | |
3711 | 313 | Results.push_back(Tmp1); |
3712 | 313 | break; |
3713 | 313 | } |
3714 | 2.92k | |
3715 | 2.92k | // Otherwise, SETCC for the given comparison type must be completely |
3716 | 2.92k | // illegal; expand it into a SELECT_CC. |
3717 | 2.92k | EVT VT = Node->getValueType(0); |
3718 | 2.92k | int TrueValue; |
3719 | 2.92k | switch (TLI.getBooleanContents(Tmp1->getValueType(0))) { |
3720 | 1.55k | case TargetLowering::ZeroOrOneBooleanContent: |
3721 | 1.55k | case TargetLowering::UndefinedBooleanContent: |
3722 | 1.55k | TrueValue = 1; |
3723 | 1.55k | break; |
3724 | 1.36k | case TargetLowering::ZeroOrNegativeOneBooleanContent: |
3725 | 1.36k | TrueValue = -1; |
3726 | 1.36k | break; |
3727 | 2.92k | } |
3728 | 2.92k | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, |
3729 | 2.92k | DAG.getConstant(TrueValue, dl, VT), |
3730 | 2.92k | DAG.getConstant(0, dl, VT), |
3731 | 2.92k | Tmp3); |
3732 | 2.92k | Results.push_back(Tmp1); |
3733 | 2.92k | break; |
3734 | 2.92k | } |
3735 | 3.51k | case ISD::SELECT_CC: { |
3736 | 3.51k | Tmp1 = Node->getOperand(0); // LHS |
3737 | 3.51k | Tmp2 = Node->getOperand(1); // RHS |
3738 | 3.51k | Tmp3 = Node->getOperand(2); // True |
3739 | 3.51k | Tmp4 = Node->getOperand(3); // False |
3740 | 3.51k | EVT VT = Node->getValueType(0); |
3741 | 3.51k | SDValue CC = Node->getOperand(4); |
3742 | 3.51k | ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); |
3743 | 3.51k | |
3744 | 3.51k | if (TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())3.51k ) { |
3745 | 3.21k | // If the condition code is legal, then we need to expand this |
3746 | 3.21k | // node using SETCC and SELECT. |
3747 | 3.21k | EVT CmpVT = Tmp1.getValueType(); |
3748 | 3.21k | assert(!TLI.isOperationExpand(ISD::SELECT, VT) && |
3749 | 3.21k | "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " |
3750 | 3.21k | "expanded."); |
3751 | 3.21k | EVT CCVT = |
3752 | 3.21k | TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT); |
3753 | 3.21k | SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC); |
3754 | 3.21k | Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); |
3755 | 3.21k | break; |
3756 | 3.21k | } |
3757 | 301 | |
3758 | 301 | // SELECT_CC is legal, so the condition code must not be. |
3759 | 301 | bool Legalized = false; |
3760 | 301 | // Try to legalize by inverting the condition. This is for targets that |
3761 | 301 | // might support an ordered version of a condition, but not the unordered |
3762 | 301 | // version (or vice versa). |
3763 | 301 | ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, |
3764 | 301 | Tmp1.getValueType().isInteger()); |
3765 | 301 | if (TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())301 ) { |
3766 | 256 | // Use the new condition code and swap true and false |
3767 | 256 | Legalized = true; |
3768 | 256 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); |
3769 | 301 | } else { |
3770 | 45 | // If The inverse is not legal, then try to swap the arguments using |
3771 | 45 | // the inverse condition code. |
3772 | 45 | ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); |
3773 | 45 | if (TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())45 ) { |
3774 | 12 | // The swapped inverse condition is legal, so swap true and false, |
3775 | 12 | // lhs and rhs. |
3776 | 12 | Legalized = true; |
3777 | 12 | Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); |
3778 | 12 | } |
3779 | 45 | } |
3780 | 301 | |
3781 | 301 | if (!Legalized301 ) { |
3782 | 33 | Legalized = LegalizeSetCCCondCode( |
3783 | 33 | getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert, |
3784 | 33 | dl); |
3785 | 33 | |
3786 | 33 | assert(Legalized && "Can't legalize SELECT_CC with legal condition!"); |
3787 | 33 | |
3788 | 33 | // If we expanded the SETCC by inverting the condition code, then swap |
3789 | 33 | // the True/False operands to match. |
3790 | 33 | if (NeedInvert) |
3791 | 0 | std::swap(Tmp3, Tmp4); |
3792 | 33 | |
3793 | 33 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the |
3794 | 33 | // condition code, create a new SELECT_CC node. |
3795 | 33 | if (CC.getNode()33 ) { |
3796 | 13 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), |
3797 | 13 | Tmp1, Tmp2, Tmp3, Tmp4, CC); |
3798 | 33 | } else { |
3799 | 20 | Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); |
3800 | 20 | CC = DAG.getCondCode(ISD::SETNE); |
3801 | 20 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, |
3802 | 20 | Tmp2, Tmp3, Tmp4, CC); |
3803 | 20 | } |
3804 | 33 | } |
3805 | 301 | Results.push_back(Tmp1); |
3806 | 301 | break; |
3807 | 301 | } |
3808 | 9 | case ISD::BR_CC: { |
3809 | 9 | Tmp1 = Node->getOperand(0); // Chain |
3810 | 9 | Tmp2 = Node->getOperand(2); // LHS |
3811 | 9 | Tmp3 = Node->getOperand(3); // RHS |
3812 | 9 | Tmp4 = Node->getOperand(1); // CC |
3813 | 9 | |
3814 | 9 | bool Legalized = LegalizeSetCCCondCode(getSetCCResultType( |
3815 | 9 | Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl); |
3816 | 9 | (void)Legalized; |
3817 | 9 | assert(Legalized && "Can't legalize BR_CC with legal condition!"); |
3818 | 9 | |
3819 | 9 | // If we expanded the SETCC by inverting the condition code, then wrap |
3820 | 9 | // the existing SETCC in a NOT to restore the intended condition. |
3821 | 9 | if (NeedInvert) |
3822 | 0 | Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0)); |
3823 | 9 | |
3824 | 9 | // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC |
3825 | 9 | // node. |
3826 | 9 | if (Tmp4.getNode()9 ) { |
3827 | 0 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, |
3828 | 0 | Tmp4, Tmp2, Tmp3, Node->getOperand(4)); |
3829 | 9 | } else { |
3830 | 9 | Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType()); |
3831 | 9 | Tmp4 = DAG.getCondCode(ISD::SETNE); |
3832 | 9 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, |
3833 | 9 | Tmp2, Tmp3, Node->getOperand(4)); |
3834 | 9 | } |
3835 | 9 | Results.push_back(Tmp1); |
3836 | 9 | break; |
3837 | 301 | } |
3838 | 63.0k | case ISD::BUILD_VECTOR: |
3839 | 63.0k | Results.push_back(ExpandBUILD_VECTOR(Node)); |
3840 | 63.0k | break; |
3841 | 10 | case ISD::SRA: |
3842 | 10 | case ISD::SRL: |
3843 | 10 | case ISD::SHL: { |
3844 | 10 | // Scalarize vector SRA/SRL/SHL. |
3845 | 10 | EVT VT = Node->getValueType(0); |
3846 | 10 | assert(VT.isVector() && "Unable to legalize non-vector shift"); |
3847 | 10 | assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"); |
3848 | 10 | unsigned NumElem = VT.getVectorNumElements(); |
3849 | 10 | |
3850 | 10 | SmallVector<SDValue, 8> Scalars; |
3851 | 30 | for (unsigned Idx = 0; Idx < NumElem30 ; Idx++20 ) { |
3852 | 20 | SDValue Ex = DAG.getNode( |
3853 | 20 | ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0), |
3854 | 20 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
3855 | 20 | SDValue Sh = DAG.getNode( |
3856 | 20 | ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1), |
3857 | 20 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); |
3858 | 20 | Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, |
3859 | 20 | VT.getScalarType(), Ex, Sh)); |
3860 | 20 | } |
3861 | 10 | |
3862 | 10 | SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); |
3863 | 10 | ReplaceNode(SDValue(Node, 0), Result); |
3864 | 10 | break; |
3865 | 10 | } |
3866 | 30.2k | case ISD::GLOBAL_OFFSET_TABLE: |
3867 | 30.2k | case ISD::GlobalAddress: |
3868 | 30.2k | case ISD::GlobalTLSAddress: |
3869 | 30.2k | case ISD::ExternalSymbol: |
3870 | 30.2k | case ISD::ConstantPool: |
3871 | 30.2k | case ISD::JumpTable: |
3872 | 30.2k | case ISD::INTRINSIC_W_CHAIN: |
3873 | 30.2k | case ISD::INTRINSIC_WO_CHAIN: |
3874 | 30.2k | case ISD::INTRINSIC_VOID: |
3875 | 30.2k | // FIXME: Custom lowering for these operations shouldn't return null! |
3876 | 30.2k | break; |
3877 | 334k | } |
3878 | 334k | |
3879 | 334k | // Replace the original node with the legalized result. |
3880 | 334k | if (334k Results.empty()334k ) |
3881 | 147k | return false; |
3882 | 186k | |
3883 | 186k | ReplaceNode(Node, Results.data()); |
3884 | 186k | return true; |
3885 | 186k | } |
3886 | | |
3887 | 147k | void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { |
3888 | 147k | SmallVector<SDValue, 8> Results; |
3889 | 147k | SDLoc dl(Node); |
3890 | 147k | SDValue Tmp1, Tmp2, Tmp3, Tmp4; |
3891 | 147k | unsigned Opc = Node->getOpcode(); |
3892 | 147k | switch (Opc) { |
3893 | 13 | case ISD::ATOMIC_FENCE: { |
3894 | 13 | // If the target didn't lower this, lower it to '__sync_synchronize()' call |
3895 | 13 | // FIXME: handle "fence singlethread" more efficiently. |
3896 | 13 | TargetLowering::ArgListTy Args; |
3897 | 13 | |
3898 | 13 | TargetLowering::CallLoweringInfo CLI(DAG); |
3899 | 13 | CLI.setDebugLoc(dl) |
3900 | 13 | .setChain(Node->getOperand(0)) |
3901 | 13 | .setLibCallee( |
3902 | 13 | CallingConv::C, Type::getVoidTy(*DAG.getContext()), |
3903 | 13 | DAG.getExternalSymbol("__sync_synchronize", |
3904 | 13 | TLI.getPointerTy(DAG.getDataLayout())), |
3905 | 13 | std::move(Args)); |
3906 | 13 | |
3907 | 13 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3908 | 13 | |
3909 | 13 | Results.push_back(CallResult.second); |
3910 | 13 | break; |
3911 | 147k | } |
3912 | 147k | // By default, atomic intrinsics are marked Legal and lowered. Targets |
3913 | 147k | // which don't support them directly, however, may want libcalls, in which |
3914 | 147k | // case they mark them Expand, and we get here. |
3915 | 328 | case ISD::ATOMIC_SWAP: |
3916 | 328 | case ISD::ATOMIC_LOAD_ADD: |
3917 | 328 | case ISD::ATOMIC_LOAD_SUB: |
3918 | 328 | case ISD::ATOMIC_LOAD_AND: |
3919 | 328 | case ISD::ATOMIC_LOAD_OR: |
3920 | 328 | case ISD::ATOMIC_LOAD_XOR: |
3921 | 328 | case ISD::ATOMIC_LOAD_NAND: |
3922 | 328 | case ISD::ATOMIC_LOAD_MIN: |
3923 | 328 | case ISD::ATOMIC_LOAD_MAX: |
3924 | 328 | case ISD::ATOMIC_LOAD_UMIN: |
3925 | 328 | case ISD::ATOMIC_LOAD_UMAX: |
3926 | 328 | case ISD::ATOMIC_CMP_SWAP: { |
3927 | 328 | MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); |
3928 | 328 | RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); |
3929 | 328 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"); |
3930 | 328 | |
3931 | 328 | std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false); |
3932 | 328 | Results.push_back(Tmp.first); |
3933 | 328 | Results.push_back(Tmp.second); |
3934 | 328 | break; |
3935 | 328 | } |
3936 | 2 | case ISD::TRAP: { |
3937 | 2 | // If this operation is not supported, lower it to 'abort()' call |
3938 | 2 | TargetLowering::ArgListTy Args; |
3939 | 2 | TargetLowering::CallLoweringInfo CLI(DAG); |
3940 | 2 | CLI.setDebugLoc(dl) |
3941 | 2 | .setChain(Node->getOperand(0)) |
3942 | 2 | .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), |
3943 | 2 | DAG.getExternalSymbol( |
3944 | 2 | "abort", TLI.getPointerTy(DAG.getDataLayout())), |
3945 | 2 | std::move(Args)); |
3946 | 2 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); |
3947 | 2 | |
3948 | 2 | Results.push_back(CallResult.second); |
3949 | 2 | break; |
3950 | 328 | } |
3951 | 45 | case ISD::FMINNUM: |
3952 | 45 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, |
3953 | 45 | RTLIB::FMIN_F80, RTLIB::FMIN_F128, |
3954 | 45 | RTLIB::FMIN_PPCF128)); |
3955 | 45 | break; |
3956 | 101 | case ISD::FMAXNUM: |
3957 | 101 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, |
3958 | 101 | RTLIB::FMAX_F80, RTLIB::FMAX_F128, |
3959 | 101 | RTLIB::FMAX_PPCF128)); |
3960 | 101 | break; |
3961 | 4 | case ISD::FSQRT: |
3962 | 4 | case ISD::STRICT_FSQRT: |
3963 | 4 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, |
3964 | 4 | RTLIB::SQRT_F80, RTLIB::SQRT_F128, |
3965 | 4 | RTLIB::SQRT_PPCF128)); |
3966 | 4 | break; |
3967 | 186 | case ISD::FSIN: |
3968 | 186 | case ISD::STRICT_FSIN: |
3969 | 186 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, |
3970 | 186 | RTLIB::SIN_F80, RTLIB::SIN_F128, |
3971 | 186 | RTLIB::SIN_PPCF128)); |
3972 | 186 | break; |
3973 | 268 | case ISD::FCOS: |
3974 | 268 | case ISD::STRICT_FCOS: |
3975 | 268 | Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, |
3976 | 268 | RTLIB::COS_F80, RTLIB::COS_F128, |
3977 | 268 | RTLIB::COS_PPCF128)); |
3978 | 268 | break; |
3979 | 30 | case ISD::FSINCOS: |
3980 | 30 | // Expand into sincos libcall. |
3981 | 30 | ExpandSinCosLibCall(Node, Results); |
3982 | 30 | break; |
3983 | 56 | case ISD::FLOG: |
3984 | 56 | case ISD::STRICT_FLOG: |
3985 | 56 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, |
3986 | 56 | RTLIB::LOG_F80, RTLIB::LOG_F128, |
3987 | 56 | RTLIB::LOG_PPCF128)); |
3988 | 56 | break; |
3989 | 59 | case ISD::FLOG2: |
3990 | 59 | case ISD::STRICT_FLOG2: |
3991 | 59 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, |
3992 | 59 | RTLIB::LOG2_F80, RTLIB::LOG2_F128, |
3993 | 59 | RTLIB::LOG2_PPCF128)); |
3994 | 59 | break; |
3995 | 60 | case ISD::FLOG10: |
3996 | 60 | case ISD::STRICT_FLOG10: |
3997 | 60 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, |
3998 | 60 | RTLIB::LOG10_F80, RTLIB::LOG10_F128, |
3999 | 60 | RTLIB::LOG10_PPCF128)); |
4000 | 60 | break; |
4001 | 160 | case ISD::FEXP: |
4002 | 160 | case ISD::STRICT_FEXP: |
4003 | 160 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, |
4004 | 160 | RTLIB::EXP_F80, RTLIB::EXP_F128, |
4005 | 160 | RTLIB::EXP_PPCF128)); |
4006 | 160 | break; |
4007 | 83 | case ISD::FEXP2: |
4008 | 83 | case ISD::STRICT_FEXP2: |
4009 | 83 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, |
4010 | 83 | RTLIB::EXP2_F80, RTLIB::EXP2_F128, |
4011 | 83 | RTLIB::EXP2_PPCF128)); |
4012 | 83 | break; |
4013 | 30 | case ISD::FTRUNC: |
4014 | 30 | Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, |
4015 | 30 | RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, |
4016 | 30 | RTLIB::TRUNC_PPCF128)); |
4017 | 30 | break; |
4018 | 38 | case ISD::FFLOOR: |
4019 | 38 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, |
4020 | 38 | RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, |
4021 | 38 | RTLIB::FLOOR_PPCF128)); |
4022 | 38 | break; |
4023 | 20 | case ISD::FCEIL: |
4024 | 20 | Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, |
4025 | 20 | RTLIB::CEIL_F80, RTLIB::CEIL_F128, |
4026 | 20 | RTLIB::CEIL_PPCF128)); |
4027 | 20 | break; |
4028 | 25 | case ISD::FRINT: |
4029 | 25 | case ISD::STRICT_FRINT: |
4030 | 25 | Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, |
4031 | 25 | RTLIB::RINT_F80, RTLIB::RINT_F128, |
4032 | 25 | RTLIB::RINT_PPCF128)); |
4033 | 25 | break; |
4034 | 48 | case ISD::FNEARBYINT: |
4035 | 48 | case ISD::STRICT_FNEARBYINT: |
4036 | 48 | Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, |
4037 | 48 | RTLIB::NEARBYINT_F64, |
4038 | 48 | RTLIB::NEARBYINT_F80, |
4039 | 48 | RTLIB::NEARBYINT_F128, |
4040 | 48 | RTLIB::NEARBYINT_PPCF128)); |
4041 | 48 | break; |
4042 | 17 | case ISD::FROUND: |
4043 | 17 | Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32, |
4044 | 17 | RTLIB::ROUND_F64, |
4045 | 17 | RTLIB::ROUND_F80, |
4046 | 17 | RTLIB::ROUND_F128, |
4047 | 17 | RTLIB::ROUND_PPCF128)); |
4048 | 17 | break; |
4049 | 99 | case ISD::FPOWI: |
4050 | 99 | case ISD::STRICT_FPOWI: |
4051 | 99 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, |
4052 | 99 | RTLIB::POWI_F80, RTLIB::POWI_F128, |
4053 | 99 | RTLIB::POWI_PPCF128)); |
4054 | 99 | break; |
4055 | 177 | case ISD::FPOW: |
4056 | 177 | case ISD::STRICT_FPOW: |
4057 | 177 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, |
4058 | 177 | RTLIB::POW_F80, RTLIB::POW_F128, |
4059 | 177 | RTLIB::POW_PPCF128)); |
4060 | 177 | break; |
4061 | 13 | case ISD::FDIV: |
4062 | 13 | Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, |
4063 | 13 | RTLIB::DIV_F80, RTLIB::DIV_F128, |
4064 | 13 | RTLIB::DIV_PPCF128)); |
4065 | 13 | break; |
4066 | 85 | case ISD::FREM: |
4067 | 85 | Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, |
4068 | 85 | RTLIB::REM_F80, RTLIB::REM_F128, |
4069 | 85 | RTLIB::REM_PPCF128)); |
4070 | 85 | break; |
4071 | 160 | case ISD::FMA: |
4072 | 160 | case ISD::STRICT_FMA: |
4073 | 160 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, |
4074 | 160 | RTLIB::FMA_F80, RTLIB::FMA_F128, |
4075 | 160 | RTLIB::FMA_PPCF128)); |
4076 | 160 | break; |
4077 | 36 | case ISD::FADD: |
4078 | 36 | Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, |
4079 | 36 | RTLIB::ADD_F80, RTLIB::ADD_F128, |
4080 | 36 | RTLIB::ADD_PPCF128)); |
4081 | 36 | break; |
4082 | 81 | case ISD::FMUL: |
4083 | 81 | Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, |
4084 | 81 | RTLIB::MUL_F80, RTLIB::MUL_F128, |
4085 | 81 | RTLIB::MUL_PPCF128)); |
4086 | 81 | break; |
4087 | 157 | case ISD::FP16_TO_FP: |
4088 | 157 | if (Node->getValueType(0) == MVT::f32157 ) { |
4089 | 157 | Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); |
4090 | 157 | } |
4091 | 157 | break; |
4092 | 332 | case ISD::FP_TO_FP16: { |
4093 | 332 | RTLIB::Libcall LC = |
4094 | 332 | RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); |
4095 | 332 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"); |
4096 | 332 | Results.push_back(ExpandLibCall(LC, Node, false)); |
4097 | 332 | break; |
4098 | 160 | } |
4099 | 16 | case ISD::FSUB: |
4100 | 16 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, |
4101 | 16 | RTLIB::SUB_F80, RTLIB::SUB_F128, |
4102 | 16 | RTLIB::SUB_PPCF128)); |
4103 | 16 | break; |
4104 | 19 | case ISD::SREM: |
4105 | 19 | Results.push_back(ExpandIntLibCall(Node, true, |
4106 | 19 | RTLIB::SREM_I8, |
4107 | 19 | RTLIB::SREM_I16, RTLIB::SREM_I32, |
4108 | 19 | RTLIB::SREM_I64, RTLIB::SREM_I128)); |
4109 | 19 | break; |
4110 | 30 | case ISD::UREM: |
4111 | 30 | Results.push_back(ExpandIntLibCall(Node, false, |
4112 | 30 | RTLIB::UREM_I8, |
4113 | 30 | RTLIB::UREM_I16, RTLIB::UREM_I32, |
4114 | 30 | RTLIB::UREM_I64, RTLIB::UREM_I128)); |
4115 | 30 | break; |
4116 | 56 | case ISD::SDIV: |
4117 | 56 | Results.push_back(ExpandIntLibCall(Node, true, |
4118 | 56 | RTLIB::SDIV_I8, |
4119 | 56 | RTLIB::SDIV_I16, RTLIB::SDIV_I32, |
4120 | 56 | RTLIB::SDIV_I64, RTLIB::SDIV_I128)); |
4121 | 56 | break; |
4122 | 52 | case ISD::UDIV: |
4123 | 52 | Results.push_back(ExpandIntLibCall(Node, false, |
4124 | 52 | RTLIB::UDIV_I8, |
4125 | 52 | RTLIB::UDIV_I16, RTLIB::UDIV_I32, |
4126 | 52 | RTLIB::UDIV_I64, RTLIB::UDIV_I128)); |
4127 | 52 | break; |
4128 | 14 | case ISD::SDIVREM: |
4129 | 14 | case ISD::UDIVREM: |
4130 | 14 | // Expand into divrem libcall |
4131 | 14 | ExpandDivRemLibCall(Node, Results); |
4132 | 14 | break; |
4133 | 16 | case ISD::MUL: |
4134 | 16 | Results.push_back(ExpandIntLibCall(Node, false, |
4135 | 16 | RTLIB::MUL_I8, |
4136 | 16 | RTLIB::MUL_I16, RTLIB::MUL_I32, |
4137 | 16 | RTLIB::MUL_I64, RTLIB::MUL_I128)); |
4138 | 16 | break; |
4139 | 147k | } |
4140 | 147k | |
4141 | 147k | // Replace the original node with the legalized result. |
4142 | 147k | if (147k !Results.empty()147k ) |
4143 | 2.91k | ReplaceNode(Node, Results.data()); |
4144 | 147k | } |
4145 | | |
4146 | | // Determine the vector type to use in place of an original scalar element when |
4147 | | // promoting equally sized vectors. |
4148 | | static MVT getPromotedVectorElementType(const TargetLowering &TLI, |
4149 | 2.80k | MVT EltVT, MVT NewEltVT) { |
4150 | 2.80k | unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits(); |
4151 | 2.80k | MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt); |
4152 | 2.80k | assert(TLI.isTypeLegal(MidVT) && "unexpected"); |
4153 | 2.80k | return MidVT; |
4154 | 2.80k | } |
4155 | | |
4156 | 6.94k | void SelectionDAGLegalize::PromoteNode(SDNode *Node) { |
4157 | 6.94k | SmallVector<SDValue, 8> Results; |
4158 | 6.94k | MVT OVT = Node->getSimpleValueType(0); |
4159 | 6.94k | if (Node->getOpcode() == ISD::UINT_TO_FP || |
4160 | 6.88k | Node->getOpcode() == ISD::SINT_TO_FP || |
4161 | 6.85k | Node->getOpcode() == ISD::SETCC || |
4162 | 6.67k | Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || |
4163 | 6.94k | Node->getOpcode() == ISD::INSERT_VECTOR_ELT4.26k ) { |
4164 | 2.67k | OVT = Node->getOperand(0).getSimpleValueType(); |
4165 | 2.67k | } |
4166 | 6.94k | if (Node->getOpcode() == ISD::BR_CC) |
4167 | 1 | OVT = Node->getOperand(2).getSimpleValueType(); |
4168 | 6.94k | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); |
4169 | 6.94k | SDLoc dl(Node); |
4170 | 6.94k | SDValue Tmp1, Tmp2, Tmp3; |
4171 | 6.94k | switch (Node->getOpcode()) { |
4172 | 33 | case ISD::CTTZ: |
4173 | 33 | case ISD::CTTZ_ZERO_UNDEF: |
4174 | 33 | case ISD::CTLZ: |
4175 | 33 | case ISD::CTLZ_ZERO_UNDEF: |
4176 | 33 | case ISD::CTPOP: |
4177 | 33 | // Zero extend the argument. |
4178 | 33 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); |
4179 | 33 | if (Node->getOpcode() == ISD::CTTZ33 ) { |
4180 | 4 | // The count is the same in the promoted type except if the original |
4181 | 4 | // value was zero. This can be handled by setting the bit just off |
4182 | 4 | // the top of the original type. |
4183 | 4 | auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(), |
4184 | 4 | OVT.getSizeInBits()); |
4185 | 4 | Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, |
4186 | 4 | DAG.getConstant(TopBit, dl, NVT)); |
4187 | 4 | } |
4188 | 33 | // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is |
4189 | 33 | // already the correct result. |
4190 | 33 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4191 | 33 | if (Node->getOpcode() == ISD::CTLZ || |
4192 | 33 | Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF28 ) { |
4193 | 11 | // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) |
4194 | 11 | Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, |
4195 | 11 | DAG.getConstant(NVT.getSizeInBits() - |
4196 | 11 | OVT.getSizeInBits(), dl, NVT)); |
4197 | 11 | } |
4198 | 33 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4199 | 33 | break; |
4200 | 0 | case ISD::BITREVERSE: |
4201 | 0 | case ISD::BSWAP: { |
4202 | 0 | unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); |
4203 | 0 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); |
4204 | 0 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4205 | 0 | Tmp1 = DAG.getNode( |
4206 | 0 | ISD::SRL, dl, NVT, Tmp1, |
4207 | 0 | DAG.getConstant(DiffBits, dl, |
4208 | 0 | TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); |
4209 | 0 | Results.push_back(Tmp1); |
4210 | 0 | break; |
4211 | 0 | } |
4212 | 136 | case ISD::FP_TO_UINT: |
4213 | 136 | case ISD::FP_TO_SINT: |
4214 | 136 | Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), |
4215 | 136 | Node->getOpcode() == ISD::FP_TO_SINT, dl); |
4216 | 136 | Results.push_back(Tmp1); |
4217 | 136 | break; |
4218 | 88 | case ISD::UINT_TO_FP: |
4219 | 88 | case ISD::SINT_TO_FP: |
4220 | 88 | Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), |
4221 | 88 | Node->getOpcode() == ISD::SINT_TO_FP, dl); |
4222 | 88 | Results.push_back(Tmp1); |
4223 | 88 | break; |
4224 | 1 | case ISD::VAARG: { |
4225 | 1 | SDValue Chain = Node->getOperand(0); // Get the chain. |
4226 | 1 | SDValue Ptr = Node->getOperand(1); // Get the pointer. |
4227 | 1 | |
4228 | 1 | unsigned TruncOp; |
4229 | 1 | if (OVT.isVector()1 ) { |
4230 | 0 | TruncOp = ISD::BITCAST; |
4231 | 1 | } else { |
4232 | 1 | assert(OVT.isInteger() |
4233 | 1 | && "VAARG promotion is supported only for vectors or integer types"); |
4234 | 1 | TruncOp = ISD::TRUNCATE; |
4235 | 1 | } |
4236 | 1 | |
4237 | 1 | // Perform the larger operation, then convert back |
4238 | 1 | Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), |
4239 | 1 | Node->getConstantOperandVal(3)); |
4240 | 1 | Chain = Tmp1.getValue(1); |
4241 | 1 | |
4242 | 1 | Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); |
4243 | 1 | |
4244 | 1 | // Modified the chain result - switch anything that used the old chain to |
4245 | 1 | // use the new one. |
4246 | 1 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); |
4247 | 1 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); |
4248 | 1 | if (UpdatedNodes1 ) { |
4249 | 0 | UpdatedNodes->insert(Tmp2.getNode()); |
4250 | 0 | UpdatedNodes->insert(Chain.getNode()); |
4251 | 0 | } |
4252 | 1 | ReplacedNode(Node); |
4253 | 1 | break; |
4254 | 88 | } |
4255 | 2.68k | case ISD::MUL: |
4256 | 2.68k | case ISD::SDIV: |
4257 | 2.68k | case ISD::SREM: |
4258 | 2.68k | case ISD::UDIV: |
4259 | 2.68k | case ISD::UREM: |
4260 | 2.68k | case ISD::AND: |
4261 | 2.68k | case ISD::OR: |
4262 | 2.68k | case ISD::XOR: { |
4263 | 2.68k | unsigned ExtOp, TruncOp; |
4264 | 2.68k | if (OVT.isVector()2.68k ) { |
4265 | 2.67k | ExtOp = ISD::BITCAST; |
4266 | 2.67k | TruncOp = ISD::BITCAST; |
4267 | 2.68k | } else { |
4268 | 10 | assert(OVT.isInteger() && "Cannot promote logic operation"); |
4269 | 10 | |
4270 | 10 | switch (Node->getOpcode()) { |
4271 | 1 | default: |
4272 | 1 | ExtOp = ISD::ANY_EXTEND; |
4273 | 1 | break; |
4274 | 3 | case ISD::SDIV: |
4275 | 3 | case ISD::SREM: |
4276 | 3 | ExtOp = ISD::SIGN_EXTEND; |
4277 | 3 | break; |
4278 | 6 | case ISD::UDIV: |
4279 | 6 | case ISD::UREM: |
4280 | 6 | ExtOp = ISD::ZERO_EXTEND; |
4281 | 6 | break; |
4282 | 10 | } |
4283 | 10 | TruncOp = ISD::TRUNCATE; |
4284 | 10 | } |
4285 | 2.68k | // Promote each of the values to the new type. |
4286 | 2.68k | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4287 | 2.68k | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4288 | 2.68k | // Perform the larger operation, then convert back |
4289 | 2.68k | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); |
4290 | 2.68k | Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); |
4291 | 2.68k | break; |
4292 | 2.68k | } |
4293 | 0 | case ISD::UMUL_LOHI: |
4294 | 0 | case ISD::SMUL_LOHI: { |
4295 | 0 | // Promote to a multiply in a wider integer type. |
4296 | 0 | unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND |
4297 | 0 | : ISD::SIGN_EXTEND; |
4298 | 0 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4299 | 0 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4300 | 0 | Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); |
4301 | 0 |
|
4302 | 0 | auto &DL = DAG.getDataLayout(); |
4303 | 0 | unsigned OriginalSize = OVT.getScalarSizeInBits(); |
4304 | 0 | Tmp2 = DAG.getNode( |
4305 | 0 | ISD::SRL, dl, NVT, Tmp1, |
4306 | 0 | DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); |
4307 | 0 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); |
4308 | 0 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); |
4309 | 0 | break; |
4310 | 0 | } |
4311 | 160 | case ISD::SELECT: { |
4312 | 160 | unsigned ExtOp, TruncOp; |
4313 | 160 | if (Node->getValueType(0).isVector() || |
4314 | 160 | Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()160 ) { |
4315 | 152 | ExtOp = ISD::BITCAST; |
4316 | 152 | TruncOp = ISD::BITCAST; |
4317 | 160 | } else if (8 Node->getValueType(0).isInteger()8 ) { |
4318 | 4 | ExtOp = ISD::ANY_EXTEND; |
4319 | 4 | TruncOp = ISD::TRUNCATE; |
4320 | 8 | } else { |
4321 | 4 | ExtOp = ISD::FP_EXTEND; |
4322 | 4 | TruncOp = ISD::FP_ROUND; |
4323 | 4 | } |
4324 | 160 | Tmp1 = Node->getOperand(0); |
4325 | 160 | // Promote each of the values to the new type. |
4326 | 160 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4327 | 160 | Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4328 | 160 | // Perform the larger operation, then round down. |
4329 | 160 | Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); |
4330 | 160 | if (TruncOp != ISD::FP_ROUND) |
4331 | 156 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); |
4332 | 160 | else |
4333 | 4 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, |
4334 | 4 | DAG.getIntPtrConstant(0, dl)); |
4335 | 160 | Results.push_back(Tmp1); |
4336 | 160 | break; |
4337 | 0 | } |
4338 | 402 | case ISD::VECTOR_SHUFFLE: { |
4339 | 402 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); |
4340 | 402 | |
4341 | 402 | // Cast the two input vectors. |
4342 | 402 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); |
4343 | 402 | Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); |
4344 | 402 | |
4345 | 402 | // Convert the shuffle mask to the right # elements. |
4346 | 402 | Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); |
4347 | 402 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); |
4348 | 402 | Results.push_back(Tmp1); |
4349 | 402 | break; |
4350 | 0 | } |
4351 | 177 | case ISD::SETCC: { |
4352 | 177 | unsigned ExtOp = ISD::FP_EXTEND; |
4353 | 177 | if (NVT.isInteger()177 ) { |
4354 | 60 | ISD::CondCode CCCode = |
4355 | 60 | cast<CondCodeSDNode>(Node->getOperand(2))->get(); |
4356 | 60 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND0 : ISD::ZERO_EXTEND60 ; |
4357 | 60 | } |
4358 | 177 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); |
4359 | 177 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); |
4360 | 177 | Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), |
4361 | 177 | Tmp1, Tmp2, Node->getOperand(2))); |
4362 | 177 | break; |
4363 | 0 | } |
4364 | 1 | case ISD::BR_CC: { |
4365 | 1 | unsigned ExtOp = ISD::FP_EXTEND; |
4366 | 1 | if (NVT.isInteger()1 ) { |
4367 | 0 | ISD::CondCode CCCode = |
4368 | 0 | cast<CondCodeSDNode>(Node->getOperand(1))->get(); |
4369 | 0 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND0 : ISD::ZERO_EXTEND0 ; |
4370 | 0 | } |
4371 | 1 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); |
4372 | 1 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); |
4373 | 1 | Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), |
4374 | 1 | Node->getOperand(0), Node->getOperand(1), |
4375 | 1 | Tmp1, Tmp2, Node->getOperand(4))); |
4376 | 1 | break; |
4377 | 0 | } |
4378 | 151 | case ISD::FADD: |
4379 | 151 | case ISD::FSUB: |
4380 | 151 | case ISD::FMUL: |
4381 | 151 | case ISD::FDIV: |
4382 | 151 | case ISD::FREM: |
4383 | 151 | case ISD::FMINNUM: |
4384 | 151 | case ISD::FMAXNUM: |
4385 | 151 | case ISD::FPOW: |
4386 | 151 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4387 | 151 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); |
4388 | 151 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, |
4389 | 151 | Node->getFlags()); |
4390 | 151 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4391 | 151 | Tmp3, DAG.getIntPtrConstant(0, dl))); |
4392 | 151 | break; |
4393 | 25 | case ISD::FMA: |
4394 | 25 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4395 | 25 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); |
4396 | 25 | Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); |
4397 | 25 | Results.push_back( |
4398 | 25 | DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4399 | 25 | DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), |
4400 | 25 | DAG.getIntPtrConstant(0, dl))); |
4401 | 25 | break; |
4402 | 12 | case ISD::FCOPYSIGN: |
4403 | 12 | case ISD::FPOWI: { |
4404 | 12 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4405 | 12 | Tmp2 = Node->getOperand(1); |
4406 | 12 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); |
4407 | 12 | |
4408 | 12 | // fcopysign doesn't change anything but the sign bit, so |
4409 | 12 | // (fp_round (fcopysign (fpext a), b)) |
4410 | 12 | // is as precise as |
4411 | 12 | // (fp_round (fpext a)) |
4412 | 12 | // which is a no-op. Mark it as a TRUNCating FP_ROUND. |
4413 | 12 | const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); |
4414 | 12 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4415 | 12 | Tmp3, DAG.getIntPtrConstant(isTrunc, dl))); |
4416 | 12 | break; |
4417 | 12 | } |
4418 | 163 | case ISD::FFLOOR: |
4419 | 163 | case ISD::FCEIL: |
4420 | 163 | case ISD::FRINT: |
4421 | 163 | case ISD::FNEARBYINT: |
4422 | 163 | case ISD::FROUND: |
4423 | 163 | case ISD::FTRUNC: |
4424 | 163 | case ISD::FNEG: |
4425 | 163 | case ISD::FSQRT: |
4426 | 163 | case ISD::FSIN: |
4427 | 163 | case ISD::FCOS: |
4428 | 163 | case ISD::FLOG: |
4429 | 163 | case ISD::FLOG2: |
4430 | 163 | case ISD::FLOG10: |
4431 | 163 | case ISD::FABS: |
4432 | 163 | case ISD::FEXP: |
4433 | 163 | case ISD::FEXP2: |
4434 | 163 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); |
4435 | 163 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); |
4436 | 163 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, |
4437 | 163 | Tmp2, DAG.getIntPtrConstant(0, dl))); |
4438 | 163 | break; |
4439 | 363 | case ISD::BUILD_VECTOR: { |
4440 | 363 | MVT EltVT = OVT.getVectorElementType(); |
4441 | 363 | MVT NewEltVT = NVT.getVectorElementType(); |
4442 | 363 | |
4443 | 363 | // Handle bitcasts to a different vector type with the same total bit size |
4444 | 363 | // |
4445 | 363 | // e.g. v2i64 = build_vector i64:x, i64:y => v4i32 |
4446 | 363 | // => |
4447 | 363 | // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y)) |
4448 | 363 | |
4449 | 363 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4450 | 363 | "Invalid promote type for build_vector"); |
4451 | 363 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4452 | 363 | |
4453 | 363 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4454 | 363 | |
4455 | 363 | SmallVector<SDValue, 8> NewOps; |
4456 | 1.08k | for (unsigned I = 0, E = Node->getNumOperands(); I != E1.08k ; ++I726 ) { |
4457 | 726 | SDValue Op = Node->getOperand(I); |
4458 | 726 | NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); |
4459 | 726 | } |
4460 | 363 | |
4461 | 363 | SDLoc SL(Node); |
4462 | 363 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps); |
4463 | 363 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); |
4464 | 363 | Results.push_back(CvtVec); |
4465 | 363 | break; |
4466 | 163 | } |
4467 | 2.41k | case ISD::EXTRACT_VECTOR_ELT: { |
4468 | 2.41k | MVT EltVT = OVT.getVectorElementType(); |
4469 | 2.41k | MVT NewEltVT = NVT.getVectorElementType(); |
4470 | 2.41k | |
4471 | 2.41k | // Handle bitcasts to a different vector type with the same total bit size. |
4472 | 2.41k | // |
4473 | 2.41k | // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32 |
4474 | 2.41k | // => |
4475 | 2.41k | // v4i32:castx = bitcast x:v2i64 |
4476 | 2.41k | // |
4477 | 2.41k | // i64 = bitcast |
4478 | 2.41k | // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), |
4479 | 2.41k | // (i32 (extract_vector_elt castx, (2 * y + 1))) |
4480 | 2.41k | // |
4481 | 2.41k | |
4482 | 2.41k | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4483 | 2.41k | "Invalid promote type for extract_vector_elt"); |
4484 | 2.41k | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4485 | 2.41k | |
4486 | 2.41k | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4487 | 2.41k | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
4488 | 2.41k | |
4489 | 2.41k | SDValue Idx = Node->getOperand(1); |
4490 | 2.41k | EVT IdxVT = Idx.getValueType(); |
4491 | 2.41k | SDLoc SL(Node); |
4492 | 2.41k | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT); |
4493 | 2.41k | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); |
4494 | 2.41k | |
4495 | 2.41k | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); |
4496 | 2.41k | |
4497 | 2.41k | SmallVector<SDValue, 8> NewOps; |
4498 | 7.23k | for (unsigned I = 0; I < NewEltsPerOldElt7.23k ; ++I4.82k ) { |
4499 | 4.82k | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); |
4500 | 4.82k | SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); |
4501 | 4.82k | |
4502 | 4.82k | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, |
4503 | 4.82k | CastVec, TmpIdx); |
4504 | 4.82k | NewOps.push_back(Elt); |
4505 | 4.82k | } |
4506 | 2.41k | |
4507 | 2.41k | SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps); |
4508 | 2.41k | Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); |
4509 | 2.41k | break; |
4510 | 163 | } |
4511 | 4 | case ISD::INSERT_VECTOR_ELT: { |
4512 | 4 | MVT EltVT = OVT.getVectorElementType(); |
4513 | 4 | MVT NewEltVT = NVT.getVectorElementType(); |
4514 | 4 | |
4515 | 4 | // Handle bitcasts to a different vector type with the same total bit size |
4516 | 4 | // |
4517 | 4 | // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32 |
4518 | 4 | // => |
4519 | 4 | // v4i32:castx = bitcast x:v2i64 |
4520 | 4 | // v2i32:casty = bitcast y:i64 |
4521 | 4 | // |
4522 | 4 | // v2i64 = bitcast |
4523 | 4 | // (v4i32 insert_vector_elt |
4524 | 4 | // (v4i32 insert_vector_elt v4i32:castx, |
4525 | 4 | // (extract_vector_elt casty, 0), 2 * z), |
4526 | 4 | // (extract_vector_elt casty, 1), (2 * z + 1)) |
4527 | 4 | |
4528 | 4 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && |
4529 | 4 | "Invalid promote type for insert_vector_elt"); |
4530 | 4 | assert(NewEltVT.bitsLT(EltVT) && "not handled"); |
4531 | 4 | |
4532 | 4 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4533 | 4 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); |
4534 | 4 | |
4535 | 4 | SDValue Val = Node->getOperand(1); |
4536 | 4 | SDValue Idx = Node->getOperand(2); |
4537 | 4 | EVT IdxVT = Idx.getValueType(); |
4538 | 4 | SDLoc SL(Node); |
4539 | 4 | |
4540 | 4 | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT); |
4541 | 4 | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); |
4542 | 4 | |
4543 | 4 | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); |
4544 | 4 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); |
4545 | 4 | |
4546 | 4 | SDValue NewVec = CastVec; |
4547 | 12 | for (unsigned I = 0; I < NewEltsPerOldElt12 ; ++I8 ) { |
4548 | 8 | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); |
4549 | 8 | SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); |
4550 | 8 | |
4551 | 8 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, |
4552 | 8 | CastVal, IdxOffset); |
4553 | 8 | |
4554 | 8 | NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, |
4555 | 8 | NewVec, Elt, InEltIdx); |
4556 | 8 | } |
4557 | 4 | |
4558 | 4 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); |
4559 | 4 | break; |
4560 | 163 | } |
4561 | 28 | case ISD::SCALAR_TO_VECTOR: { |
4562 | 28 | MVT EltVT = OVT.getVectorElementType(); |
4563 | 28 | MVT NewEltVT = NVT.getVectorElementType(); |
4564 | 28 | |
4565 | 28 | // Handle bitcasts to different vector type with the same total bit size. |
4566 | 28 | // |
4567 | 28 | // e.g. v2i64 = scalar_to_vector x:i64 |
4568 | 28 | // => |
4569 | 28 | // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef) |
4570 | 28 | // |
4571 | 28 | |
4572 | 28 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); |
4573 | 28 | SDValue Val = Node->getOperand(0); |
4574 | 28 | SDLoc SL(Node); |
4575 | 28 | |
4576 | 28 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); |
4577 | 28 | SDValue Undef = DAG.getUNDEF(MidVT); |
4578 | 28 | |
4579 | 28 | SmallVector<SDValue, 8> NewElts; |
4580 | 28 | NewElts.push_back(CastVal); |
4581 | 56 | for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts56 ; ++I28 ) |
4582 | 28 | NewElts.push_back(Undef); |
4583 | 28 | |
4584 | 28 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); |
4585 | 28 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); |
4586 | 28 | Results.push_back(CvtVec); |
4587 | 28 | break; |
4588 | 6.94k | } |
4589 | 6.94k | } |
4590 | 6.94k | |
4591 | 6.94k | // Replace the original node with the legalized result. |
4592 | 6.94k | if (6.94k !Results.empty()6.94k ) |
4593 | 6.83k | ReplaceNode(Node, Results.data()); |
4594 | 6.94k | } |
4595 | | |
4596 | | /// This is the entry point for the file. |
4597 | 3.42M | void SelectionDAG::Legalize() { |
4598 | 3.42M | AssignTopologicalOrder(); |
4599 | 3.42M | |
4600 | 3.42M | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
4601 | 3.42M | // Use a delete listener to remove nodes which were deleted during |
4602 | 3.42M | // legalization from LegalizeNodes. This is needed to handle the situation |
4603 | 3.42M | // where a new node is allocated by the object pool to the same address of a |
4604 | 3.42M | // previously deleted node. |
4605 | 3.42M | DAGNodeDeletedListener DeleteListener( |
4606 | 3.42M | *this, |
4607 | 244 | [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); }); |
4608 | 3.42M | |
4609 | 3.42M | SelectionDAGLegalize Legalizer(*this, LegalizedNodes); |
4610 | 3.42M | |
4611 | 3.42M | // Visit all the nodes. We start in topological order, so that we see |
4612 | 3.42M | // nodes with their original operands intact. Legalization can produce |
4613 | 3.42M | // new nodes which may themselves need to be legalized. Iterate until all |
4614 | 3.42M | // nodes have been legalized. |
4615 | 9.45M | while (true9.45M ) { |
4616 | 9.45M | bool AnyLegalized = false; |
4617 | 215M | for (auto NI = allnodes_end(); NI != allnodes_begin()215M ;) { |
4618 | 205M | --NI; |
4619 | 205M | |
4620 | 205M | SDNode *N = &*NI; |
4621 | 205M | if (N->use_empty() && 205M N != getRoot().getNode()12.7M ) { |
4622 | 3.30M | ++NI; |
4623 | 3.30M | DeleteNode(N); |
4624 | 3.30M | continue; |
4625 | 3.30M | } |
4626 | 202M | |
4627 | 202M | if (202M LegalizedNodes.insert(N).second202M ) { |
4628 | 75.7M | AnyLegalized = true; |
4629 | 75.7M | Legalizer.LegalizeOp(N); |
4630 | 75.7M | |
4631 | 75.7M | if (N->use_empty() && 75.7M N != getRoot().getNode()7.41M ) { |
4632 | 3.98M | ++NI; |
4633 | 3.98M | DeleteNode(N); |
4634 | 3.98M | } |
4635 | 75.7M | } |
4636 | 205M | } |
4637 | 9.45M | if (!AnyLegalized) |
4638 | 3.42M | break; |
4639 | 9.45M | |
4640 | 9.45M | } |
4641 | 3.42M | |
4642 | 3.42M | // Remove dead nodes now. |
4643 | 3.42M | RemoveDeadNodes(); |
4644 | 3.42M | } |
4645 | | |
4646 | | bool SelectionDAG::LegalizeOp(SDNode *N, |
4647 | 72.8M | SmallSetVector<SDNode *, 16> &UpdatedNodes) { |
4648 | 72.8M | SmallPtrSet<SDNode *, 16> LegalizedNodes; |
4649 | 72.8M | SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); |
4650 | 72.8M | |
4651 | 72.8M | // Directly insert the node in question, and legalize it. This will recurse |
4652 | 72.8M | // as needed through operands. |
4653 | 72.8M | LegalizedNodes.insert(N); |
4654 | 72.8M | Legalizer.LegalizeOp(N); |
4655 | 72.8M | |
4656 | 72.8M | return LegalizedNodes.count(N); |
4657 | 72.8M | } |