/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | // |
10 | | // This implements the SelectionDAG class. |
11 | | // |
12 | | //===----------------------------------------------------------------------===// |
13 | | |
14 | | #include "llvm/CodeGen/SelectionDAG.h" |
15 | | #include "SDNodeDbgValue.h" |
16 | | #include "llvm/ADT/APFloat.h" |
17 | | #include "llvm/ADT/APInt.h" |
18 | | #include "llvm/ADT/APSInt.h" |
19 | | #include "llvm/ADT/ArrayRef.h" |
20 | | #include "llvm/ADT/BitVector.h" |
21 | | #include "llvm/ADT/FoldingSet.h" |
22 | | #include "llvm/ADT/None.h" |
23 | | #include "llvm/ADT/STLExtras.h" |
24 | | #include "llvm/ADT/SmallPtrSet.h" |
25 | | #include "llvm/ADT/SmallVector.h" |
26 | | #include "llvm/ADT/Triple.h" |
27 | | #include "llvm/ADT/Twine.h" |
28 | | #include "llvm/Analysis/ValueTracking.h" |
29 | | #include "llvm/CodeGen/ISDOpcodes.h" |
30 | | #include "llvm/CodeGen/MachineBasicBlock.h" |
31 | | #include "llvm/CodeGen/MachineConstantPool.h" |
32 | | #include "llvm/CodeGen/MachineFrameInfo.h" |
33 | | #include "llvm/CodeGen/MachineFunction.h" |
34 | | #include "llvm/CodeGen/MachineMemOperand.h" |
35 | | #include "llvm/CodeGen/MachineValueType.h" |
36 | | #include "llvm/CodeGen/RuntimeLibcalls.h" |
37 | | #include "llvm/CodeGen/SelectionDAGAddressAnalysis.h" |
38 | | #include "llvm/CodeGen/SelectionDAGNodes.h" |
39 | | #include "llvm/CodeGen/SelectionDAGTargetInfo.h" |
40 | | #include "llvm/CodeGen/ValueTypes.h" |
41 | | #include "llvm/IR/Constant.h" |
42 | | #include "llvm/IR/Constants.h" |
43 | | #include "llvm/IR/DataLayout.h" |
44 | | #include "llvm/IR/DebugInfoMetadata.h" |
45 | | #include "llvm/IR/DebugLoc.h" |
46 | | #include "llvm/IR/DerivedTypes.h" |
47 | | #include "llvm/IR/Function.h" |
48 | | #include "llvm/IR/GlobalValue.h" |
49 | | #include "llvm/IR/Metadata.h" |
50 | | #include "llvm/IR/Type.h" |
51 | | #include "llvm/IR/Value.h" |
52 | | #include "llvm/Support/Casting.h" |
53 | | #include "llvm/Support/CodeGen.h" |
54 | | #include "llvm/Support/Compiler.h" |
55 | | #include "llvm/Support/Debug.h" |
56 | | #include "llvm/Support/ErrorHandling.h" |
57 | | #include "llvm/Support/KnownBits.h" |
58 | | #include "llvm/Support/ManagedStatic.h" |
59 | | #include "llvm/Support/MathExtras.h" |
60 | | #include "llvm/Support/Mutex.h" |
61 | | #include "llvm/Support/raw_ostream.h" |
62 | | #include "llvm/Target/TargetLowering.h" |
63 | | #include "llvm/Target/TargetMachine.h" |
64 | | #include "llvm/Target/TargetOptions.h" |
65 | | #include "llvm/Target/TargetRegisterInfo.h" |
66 | | #include "llvm/Target/TargetSubtargetInfo.h" |
67 | | #include <algorithm> |
68 | | #include <cassert> |
69 | | #include <cstdint> |
70 | | #include <cstdlib> |
71 | | #include <limits> |
72 | | #include <set> |
73 | | #include <string> |
74 | | #include <utility> |
75 | | #include <vector> |
76 | | |
77 | | using namespace llvm; |
78 | | |
79 | | /// makeVTList - Return an instance of the SDVTList struct initialized with the |
80 | | /// specified members. |
81 | 138M | static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { |
82 | 138M | SDVTList Res = {VTs, NumVTs}; |
83 | 138M | return Res; |
84 | 138M | } |
85 | | |
86 | | // Default null implementations of the callbacks. |
87 | 0 | void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} |
88 | 60.3M | void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} |
89 | | |
90 | | #define DEBUG_TYPE "selectiondag" |
91 | | |
92 | 52.9M | static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) { |
93 | 52.9M | DEBUG( |
94 | 52.9M | dbgs() << Msg; |
95 | 52.9M | V.getNode()->dump(G); |
96 | 52.9M | ); |
97 | 52.9M | } |
98 | | |
99 | | //===----------------------------------------------------------------------===// |
100 | | // ConstantFPSDNode Class |
101 | | //===----------------------------------------------------------------------===// |
102 | | |
103 | | /// isExactlyValue - We don't rely on operator== working on double values, as |
104 | | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. |
105 | | /// As such, this method can be used to do an exact bit-for-bit comparison of |
106 | | /// two floating point values. |
107 | 172k | bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { |
108 | 172k | return getValueAPF().bitwiseIsEqual(V); |
109 | 172k | } |
110 | | |
111 | | bool ConstantFPSDNode::isValueValidForType(EVT VT, |
112 | 21.3k | const APFloat& Val) { |
113 | 21.3k | assert(VT.isFloatingPoint() && "Can only convert between FP types"); |
114 | 21.3k | |
115 | 21.3k | // convert modifies in place, so make a copy. |
116 | 21.3k | APFloat Val2 = APFloat(Val); |
117 | 21.3k | bool losesInfo; |
118 | 21.3k | (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), |
119 | 21.3k | APFloat::rmNearestTiesToEven, |
120 | 21.3k | &losesInfo); |
121 | 21.3k | return !losesInfo; |
122 | 21.3k | } |
123 | | |
124 | | //===----------------------------------------------------------------------===// |
125 | | // ISD Namespace |
126 | | //===----------------------------------------------------------------------===// |
127 | | |
128 | 72.5k | bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) { |
129 | 72.5k | auto *BV = dyn_cast<BuildVectorSDNode>(N); |
130 | 72.5k | if (!BV) |
131 | 64.2k | return false; |
132 | 8.26k | |
133 | 8.26k | APInt SplatUndef; |
134 | 8.26k | unsigned SplatBitSize; |
135 | 8.26k | bool HasUndefs; |
136 | 8.26k | unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); |
137 | 8.26k | return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs, |
138 | 8.26k | EltSize) && |
139 | 7.62k | EltSize == SplatBitSize; |
140 | 72.5k | } |
141 | | |
142 | | // FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be |
143 | | // specializations of the more general isConstantSplatVector()? |
144 | | |
145 | 309k | bool ISD::isBuildVectorAllOnes(const SDNode *N) { |
146 | 309k | // Look through a bit convert. |
147 | 379k | while (N->getOpcode() == ISD::BITCAST) |
148 | 70.2k | N = N->getOperand(0).getNode(); |
149 | 309k | |
150 | 309k | if (N->getOpcode() != ISD::BUILD_VECTOR309k ) return false196k ; |
151 | 112k | |
152 | 112k | unsigned i = 0, e = N->getNumOperands(); |
153 | 112k | |
154 | 112k | // Skip over all of the undef values. |
155 | 113k | while (i != e && 113k N->getOperand(i).isUndef()113k ) |
156 | 1.06k | ++i; |
157 | 112k | |
158 | 112k | // Do not accept an all-undef vector. |
159 | 112k | if (i == e112k ) return false25 ; |
160 | 112k | |
161 | 112k | // Do not accept build_vectors that aren't all constants or which have non-~0 |
162 | 112k | // elements. We have to be a bit careful here, as the type of the constant |
163 | 112k | // may not be the same as the type of the vector elements due to type |
164 | 112k | // legalization (the elements are promoted to a legal type for the target and |
165 | 112k | // a vector of a type may be legal when the base element type is not). |
166 | 112k | // We only want to check enough bits to cover the vector elements, because |
167 | 112k | // we care if the resultant vector is all ones, not whether the individual |
168 | 112k | // constants are. |
169 | 112k | SDValue NotZero = N->getOperand(i); |
170 | 112k | unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); |
171 | 112k | if (ConstantSDNode *CN112k = dyn_cast<ConstantSDNode>(NotZero)) { |
172 | 81.5k | if (CN->getAPIntValue().countTrailingOnes() < EltSize) |
173 | 51.4k | return false; |
174 | 31.3k | } else if (ConstantFPSDNode *31.3k CFPN31.3k = dyn_cast<ConstantFPSDNode>(NotZero)) { |
175 | 5.16k | if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) |
176 | 5.06k | return false; |
177 | 31.3k | } else |
178 | 26.1k | return false; |
179 | 30.2k | |
180 | 30.2k | // Okay, we have at least one ~0 value, check to see if the rest match or are |
181 | 30.2k | // undefs. Even with the above element type twiddling, this should be OK, as |
182 | 30.2k | // the same type legalization should have applied to all the elements. |
183 | 133k | for (++i; 30.2k i != e133k ; ++i103k ) |
184 | 108k | if (108k N->getOperand(i) != NotZero && 108k !N->getOperand(i).isUndef()5.24k ) |
185 | 4.74k | return false; |
186 | 25.4k | return true; |
187 | 309k | } |
188 | | |
189 | 894k | bool ISD::isBuildVectorAllZeros(const SDNode *N) { |
190 | 894k | // Look through a bit convert. |
191 | 1.01M | while (N->getOpcode() == ISD::BITCAST) |
192 | 118k | N = N->getOperand(0).getNode(); |
193 | 894k | |
194 | 894k | if (N->getOpcode() != ISD::BUILD_VECTOR894k ) return false678k ; |
195 | 215k | |
196 | 215k | bool IsAllUndef = true; |
197 | 911k | for (const SDValue &Op : N->op_values()) { |
198 | 911k | if (Op.isUndef()) |
199 | 5.71k | continue; |
200 | 905k | IsAllUndef = false; |
201 | 905k | // Do not accept build_vectors that aren't all constants or which have non-0 |
202 | 905k | // elements. We have to be a bit careful here, as the type of the constant |
203 | 905k | // may not be the same as the type of the vector elements due to type |
204 | 905k | // legalization (the elements are promoted to a legal type for the target |
205 | 905k | // and a vector of a type may be legal when the base element type is not). |
206 | 905k | // We only want to check enough bits to cover the vector elements, because |
207 | 905k | // we care if the resultant vector is all zeros, not whether the individual |
208 | 905k | // constants are. |
209 | 905k | unsigned EltSize = N->getValueType(0).getScalarSizeInBits(); |
210 | 905k | if (ConstantSDNode *CN905k = dyn_cast<ConstantSDNode>(Op)) { |
211 | 853k | if (CN->getAPIntValue().countTrailingZeros() < EltSize) |
212 | 104k | return false; |
213 | 52.2k | } else if (ConstantFPSDNode *52.2k CFPN52.2k = dyn_cast<ConstantFPSDNode>(Op)) { |
214 | 21.4k | if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize) |
215 | 5.42k | return false; |
216 | 52.2k | } else |
217 | 30.7k | return false; |
218 | 75.5k | } |
219 | 75.5k | |
220 | 75.5k | // Do not accept an all-undef vector. |
221 | 75.5k | if (75.5k IsAllUndef75.5k ) |
222 | 25 | return false; |
223 | 75.5k | return true; |
224 | 75.5k | } |
225 | | |
226 | 19.1M | bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) { |
227 | 19.1M | if (N->getOpcode() != ISD::BUILD_VECTOR) |
228 | 19.0M | return false; |
229 | 85.6k | |
230 | 85.6k | for (const SDValue &Op : N->op_values()) 85.6k { |
231 | 505k | if (Op.isUndef()) |
232 | 3.01k | continue; |
233 | 502k | if (502k !isa<ConstantSDNode>(Op)502k ) |
234 | 41.7k | return false; |
235 | 43.8k | } |
236 | 43.8k | return true; |
237 | 43.8k | } |
238 | | |
239 | 3.60M | bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) { |
240 | 3.60M | if (N->getOpcode() != ISD::BUILD_VECTOR) |
241 | 3.57M | return false; |
242 | 24.4k | |
243 | 24.4k | for (const SDValue &Op : N->op_values()) 24.4k { |
244 | 66.8k | if (Op.isUndef()) |
245 | 657 | continue; |
246 | 66.1k | if (66.1k !isa<ConstantFPSDNode>(Op)66.1k ) |
247 | 9.68k | return false; |
248 | 14.7k | } |
249 | 14.7k | return true; |
250 | 14.7k | } |
251 | | |
252 | 397k | bool ISD::allOperandsUndef(const SDNode *N) { |
253 | 397k | // Return false if the node has no operands. |
254 | 397k | // This is "logically inconsistent" with the definition of "all" but |
255 | 397k | // is probably the desired behavior. |
256 | 397k | if (N->getNumOperands() == 0) |
257 | 0 | return false; |
258 | 397k | |
259 | 397k | for (const SDValue &Op : N->op_values()) |
260 | 401k | if (401k !Op.isUndef()401k ) |
261 | 397k | return false; |
262 | 75 | |
263 | 75 | return true; |
264 | 75 | } |
265 | | |
266 | 308 | ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) { |
267 | 308 | switch (ExtType) { |
268 | 0 | case ISD::EXTLOAD: |
269 | 0 | return IsFP ? ISD::FP_EXTEND0 : ISD::ANY_EXTEND0 ; |
270 | 113 | case ISD::SEXTLOAD: |
271 | 113 | return ISD::SIGN_EXTEND; |
272 | 195 | case ISD::ZEXTLOAD: |
273 | 195 | return ISD::ZERO_EXTEND; |
274 | 0 | default: |
275 | 0 | break; |
276 | 0 | } |
277 | 0 |
|
278 | 0 | llvm_unreachable0 ("Invalid LoadExtType"); |
279 | 0 | } |
280 | | |
281 | 3.36M | ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { |
282 | 3.36M | // To perform this operation, we just need to swap the L and G bits of the |
283 | 3.36M | // operation. |
284 | 3.36M | unsigned OldL = (Operation >> 2) & 1; |
285 | 3.36M | unsigned OldG = (Operation >> 1) & 1; |
286 | 3.36M | return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits |
287 | 3.36M | (OldL << 1) | // New G bit |
288 | 3.36M | (OldG << 2)); // New L bit. |
289 | 3.36M | } |
290 | | |
291 | 834k | ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { |
292 | 834k | unsigned Operation = Op; |
293 | 834k | if (isInteger) |
294 | 817k | Operation ^= 7; // Flip L, G, E bits, but not U. |
295 | 834k | else |
296 | 16.2k | Operation ^= 15; // Flip all of the condition bits. |
297 | 834k | |
298 | 834k | if (Operation > ISD::SETTRUE2) |
299 | 295 | Operation &= ~8; // Don't let N and U bits get set. |
300 | 834k | |
301 | 834k | return ISD::CondCode(Operation); |
302 | 834k | } |
303 | | |
304 | | /// For an integer comparison, return 1 if the comparison is a signed operation |
305 | | /// and 2 if the result is an unsigned comparison. Return zero if the operation |
306 | | /// does not depend on the sign of the input (setne and seteq). |
307 | 40 | static int isSignedOp(ISD::CondCode Opcode) { |
308 | 40 | switch (Opcode) { |
309 | 0 | default: 0 llvm_unreachable0 ("Illegal integer setcc operation!"); |
310 | 24 | case ISD::SETEQ: |
311 | 24 | case ISD::SETNE: return 0; |
312 | 7 | case ISD::SETLT: |
313 | 7 | case ISD::SETLE: |
314 | 7 | case ISD::SETGT: |
315 | 7 | case ISD::SETGE: return 1; |
316 | 9 | case ISD::SETULT: |
317 | 9 | case ISD::SETULE: |
318 | 9 | case ISD::SETUGT: |
319 | 9 | case ISD::SETUGE: return 2; |
320 | 0 | } |
321 | 0 | } |
322 | | |
323 | | ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, |
324 | 37 | bool IsInteger) { |
325 | 37 | if (IsInteger && 37 (isSignedOp(Op1) | isSignedOp(Op2)) == 319 ) |
326 | 37 | // Cannot fold a signed integer setcc with an unsigned integer setcc. |
327 | 5 | return ISD::SETCC_INVALID; |
328 | 32 | |
329 | 32 | unsigned Op = Op1 | Op2; // Combine all of the condition bits. |
330 | 32 | |
331 | 32 | // If the N and U bits get set, then the resultant comparison DOES suddenly |
332 | 32 | // care about orderedness, and it is true when ordered. |
333 | 32 | if (Op > ISD::SETTRUE2) |
334 | 17 | Op &= ~16; // Clear the U bit if the N bit is set. |
335 | 32 | |
336 | 32 | // Canonicalize illegal integer setcc's. |
337 | 32 | if (IsInteger && 32 Op == ISD::SETUNE14 ) // e.g. SETUGT | SETULT |
338 | 0 | Op = ISD::SETNE; |
339 | 37 | |
340 | 37 | return ISD::CondCode(Op); |
341 | 37 | } |
342 | | |
343 | | ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, |
344 | 27 | bool IsInteger) { |
345 | 27 | if (IsInteger && 27 (isSignedOp(Op1) | isSignedOp(Op2)) == 31 ) |
346 | 27 | // Cannot fold a signed setcc with an unsigned setcc. |
347 | 0 | return ISD::SETCC_INVALID; |
348 | 27 | |
349 | 27 | // Combine all of the condition bits. |
350 | 27 | ISD::CondCode Result = ISD::CondCode(Op1 & Op2); |
351 | 27 | |
352 | 27 | // Canonicalize illegal integer setcc's. |
353 | 27 | if (IsInteger27 ) { |
354 | 1 | switch (Result) { |
355 | 0 | default: break; |
356 | 0 | case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT |
357 | 1 | case ISD::SETOEQ: // SETEQ & SETU[LG]E |
358 | 1 | case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE |
359 | 0 | case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE |
360 | 0 | case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE |
361 | 27 | } |
362 | 27 | } |
363 | 27 | |
364 | 27 | return Result; |
365 | 27 | } |
366 | | |
367 | | //===----------------------------------------------------------------------===// |
368 | | // SDNode Profile Support |
369 | | //===----------------------------------------------------------------------===// |
370 | | |
371 | | /// AddNodeIDOpcode - Add the node opcode to the NodeID data. |
372 | 265M | static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { |
373 | 265M | ID.AddInteger(OpC); |
374 | 265M | } |
375 | | |
376 | | /// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them |
377 | | /// solely with their pointer. |
378 | 265M | static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { |
379 | 265M | ID.AddPointer(VTList.VTs); |
380 | 265M | } |
381 | | |
382 | | /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. |
383 | | static void AddNodeIDOperands(FoldingSetNodeID &ID, |
384 | 157M | ArrayRef<SDValue> Ops) { |
385 | 191M | for (auto& Op : Ops) { |
386 | 191M | ID.AddPointer(Op.getNode()); |
387 | 191M | ID.AddInteger(Op.getResNo()); |
388 | 191M | } |
389 | 157M | } |
390 | | |
391 | | /// AddNodeIDOperands - Various routines for adding operands to the NodeID data. |
392 | | static void AddNodeIDOperands(FoldingSetNodeID &ID, |
393 | 108M | ArrayRef<SDUse> Ops) { |
394 | 144M | for (auto& Op : Ops) { |
395 | 144M | ID.AddPointer(Op.getNode()); |
396 | 144M | ID.AddInteger(Op.getResNo()); |
397 | 144M | } |
398 | 108M | } |
399 | | |
400 | | static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC, |
401 | 157M | SDVTList VTList, ArrayRef<SDValue> OpList) { |
402 | 157M | AddNodeIDOpcode(ID, OpC); |
403 | 157M | AddNodeIDValueTypes(ID, VTList); |
404 | 157M | AddNodeIDOperands(ID, OpList); |
405 | 157M | } |
406 | | |
407 | | /// If this is an SDNode with special info, add this info to the NodeID data. |
408 | 109M | static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { |
409 | 109M | switch (N->getOpcode()) { |
410 | 0 | case ISD::TargetExternalSymbol: |
411 | 0 | case ISD::ExternalSymbol: |
412 | 0 | case ISD::MCSymbol: |
413 | 0 | llvm_unreachable("Should only be used on nodes with operands"); |
414 | 50.7M | default: break; // Normal nodes don't need extra info. |
415 | 27.8M | case ISD::TargetConstant: |
416 | 27.8M | case ISD::Constant: { |
417 | 27.8M | const ConstantSDNode *C = cast<ConstantSDNode>(N); |
418 | 27.8M | ID.AddPointer(C->getConstantIntValue()); |
419 | 27.8M | ID.AddBoolean(C->isOpaque()); |
420 | 27.8M | break; |
421 | 27.8M | } |
422 | 138k | case ISD::TargetConstantFP: |
423 | 138k | case ISD::ConstantFP: |
424 | 138k | ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); |
425 | 138k | break; |
426 | 2.40M | case ISD::TargetGlobalAddress: |
427 | 2.40M | case ISD::GlobalAddress: |
428 | 2.40M | case ISD::TargetGlobalTLSAddress: |
429 | 2.40M | case ISD::GlobalTLSAddress: { |
430 | 2.40M | const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); |
431 | 2.40M | ID.AddPointer(GA->getGlobal()); |
432 | 2.40M | ID.AddInteger(GA->getOffset()); |
433 | 2.40M | ID.AddInteger(GA->getTargetFlags()); |
434 | 2.40M | break; |
435 | 2.40M | } |
436 | 1.19M | case ISD::BasicBlock: |
437 | 1.19M | ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); |
438 | 1.19M | break; |
439 | 15.9M | case ISD::Register: |
440 | 15.9M | ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); |
441 | 15.9M | break; |
442 | 1.13M | case ISD::RegisterMask: |
443 | 1.13M | ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); |
444 | 1.13M | break; |
445 | 1.02k | case ISD::SRCVALUE: |
446 | 1.02k | ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); |
447 | 1.02k | break; |
448 | 969k | case ISD::FrameIndex: |
449 | 969k | case ISD::TargetFrameIndex: |
450 | 969k | ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); |
451 | 969k | break; |
452 | 4.51k | case ISD::JumpTable: |
453 | 4.51k | case ISD::TargetJumpTable: |
454 | 4.51k | ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); |
455 | 4.51k | ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); |
456 | 4.51k | break; |
457 | 122k | case ISD::ConstantPool: |
458 | 122k | case ISD::TargetConstantPool: { |
459 | 122k | const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); |
460 | 122k | ID.AddInteger(CP->getAlignment()); |
461 | 122k | ID.AddInteger(CP->getOffset()); |
462 | 122k | if (CP->isMachineConstantPoolEntry()) |
463 | 103 | CP->getMachineCPVal()->addSelectionDAGCSEId(ID); |
464 | 122k | else |
465 | 122k | ID.AddPointer(CP->getConstVal()); |
466 | 122k | ID.AddInteger(CP->getTargetFlags()); |
467 | 122k | break; |
468 | 122k | } |
469 | 0 | case ISD::TargetIndex: { |
470 | 0 | const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); |
471 | 0 | ID.AddInteger(TI->getIndex()); |
472 | 0 | ID.AddInteger(TI->getOffset()); |
473 | 0 | ID.AddInteger(TI->getTargetFlags()); |
474 | 0 | break; |
475 | 122k | } |
476 | 3.08M | case ISD::LOAD: { |
477 | 3.08M | const LoadSDNode *LD = cast<LoadSDNode>(N); |
478 | 3.08M | ID.AddInteger(LD->getMemoryVT().getRawBits()); |
479 | 3.08M | ID.AddInteger(LD->getRawSubclassData()); |
480 | 3.08M | ID.AddInteger(LD->getPointerInfo().getAddrSpace()); |
481 | 3.08M | break; |
482 | 122k | } |
483 | 5.73M | case ISD::STORE: { |
484 | 5.73M | const StoreSDNode *ST = cast<StoreSDNode>(N); |
485 | 5.73M | ID.AddInteger(ST->getMemoryVT().getRawBits()); |
486 | 5.73M | ID.AddInteger(ST->getRawSubclassData()); |
487 | 5.73M | ID.AddInteger(ST->getPointerInfo().getAddrSpace()); |
488 | 5.73M | break; |
489 | 122k | } |
490 | 23.2k | case ISD::ATOMIC_CMP_SWAP: |
491 | 23.2k | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: |
492 | 23.2k | case ISD::ATOMIC_SWAP: |
493 | 23.2k | case ISD::ATOMIC_LOAD_ADD: |
494 | 23.2k | case ISD::ATOMIC_LOAD_SUB: |
495 | 23.2k | case ISD::ATOMIC_LOAD_AND: |
496 | 23.2k | case ISD::ATOMIC_LOAD_OR: |
497 | 23.2k | case ISD::ATOMIC_LOAD_XOR: |
498 | 23.2k | case ISD::ATOMIC_LOAD_NAND: |
499 | 23.2k | case ISD::ATOMIC_LOAD_MIN: |
500 | 23.2k | case ISD::ATOMIC_LOAD_MAX: |
501 | 23.2k | case ISD::ATOMIC_LOAD_UMIN: |
502 | 23.2k | case ISD::ATOMIC_LOAD_UMAX: |
503 | 23.2k | case ISD::ATOMIC_LOAD: |
504 | 23.2k | case ISD::ATOMIC_STORE: { |
505 | 23.2k | const AtomicSDNode *AT = cast<AtomicSDNode>(N); |
506 | 23.2k | ID.AddInteger(AT->getMemoryVT().getRawBits()); |
507 | 23.2k | ID.AddInteger(AT->getRawSubclassData()); |
508 | 23.2k | ID.AddInteger(AT->getPointerInfo().getAddrSpace()); |
509 | 23.2k | break; |
510 | 23.2k | } |
511 | 147 | case ISD::PREFETCH: { |
512 | 147 | const MemSDNode *PF = cast<MemSDNode>(N); |
513 | 147 | ID.AddInteger(PF->getPointerInfo().getAddrSpace()); |
514 | 147 | break; |
515 | 23.2k | } |
516 | 58.7k | case ISD::VECTOR_SHUFFLE: { |
517 | 58.7k | const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); |
518 | 58.7k | for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); |
519 | 775k | i != e775k ; ++i716k ) |
520 | 716k | ID.AddInteger(SVN->getMaskElt(i)); |
521 | 58.7k | break; |
522 | 23.2k | } |
523 | 75 | case ISD::TargetBlockAddress: |
524 | 75 | case ISD::BlockAddress: { |
525 | 75 | const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); |
526 | 75 | ID.AddPointer(BA->getBlockAddress()); |
527 | 75 | ID.AddInteger(BA->getOffset()); |
528 | 75 | ID.AddInteger(BA->getTargetFlags()); |
529 | 75 | break; |
530 | 109M | } |
531 | 109M | } // end switch (N->getOpcode()) |
532 | 109M | |
533 | 109M | // Target specific memory nodes could also have address spaces to check. |
534 | 109M | if (109M N->isTargetMemoryOpcode()109M ) |
535 | 9.14k | ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); |
536 | 109M | } |
537 | | |
538 | | /// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID |
539 | | /// data. |
540 | 108M | static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { |
541 | 108M | AddNodeIDOpcode(ID, N->getOpcode()); |
542 | 108M | // Add the return value info. |
543 | 108M | AddNodeIDValueTypes(ID, N->getVTList()); |
544 | 108M | // Add the operand info. |
545 | 108M | AddNodeIDOperands(ID, N->ops()); |
546 | 108M | |
547 | 108M | // Handle SDNode leafs with special info. |
548 | 108M | AddNodeIDCustom(ID, N); |
549 | 108M | } |
550 | | |
551 | | //===----------------------------------------------------------------------===// |
552 | | // SelectionDAG Class |
553 | | //===----------------------------------------------------------------------===// |
554 | | |
555 | | /// doNotCSE - Return true if CSE should not be performed for this node. |
556 | 25.9M | static bool doNotCSE(SDNode *N) { |
557 | 25.9M | if (N->getValueType(0) == MVT::Glue) |
558 | 4.81k | return true; // Never CSE anything that produces a flag. |
559 | 25.8M | |
560 | 25.8M | switch (N->getOpcode()) { |
561 | 25.7M | default: break; |
562 | 187k | case ISD::HANDLENODE: |
563 | 187k | case ISD::EH_LABEL: |
564 | 187k | return true; // Never CSE these nodes. |
565 | 25.7M | } |
566 | 25.7M | |
567 | 25.7M | // Check that remaining values produced are not flags. |
568 | 32.6M | for (unsigned i = 1, e = N->getNumValues(); 25.7M i != e32.6M ; ++i6.95M ) |
569 | 15.6M | if (15.6M N->getValueType(i) == MVT::Glue15.6M ) |
570 | 8.66M | return true; // Never CSE anything that produces a flag. |
571 | 25.7M | |
572 | 17.0M | return false; |
573 | 25.9M | } |
574 | | |
575 | | /// RemoveDeadNodes - This method deletes all unreachable nodes in the |
576 | | /// SelectionDAG. |
577 | 14.4M | void SelectionDAG::RemoveDeadNodes() { |
578 | 14.4M | // Create a dummy node (which is not added to allnodes), that adds a reference |
579 | 14.4M | // to the root node, preventing it from being deleted. |
580 | 14.4M | HandleSDNode Dummy(getRoot()); |
581 | 14.4M | |
582 | 14.4M | SmallVector<SDNode*, 128> DeadNodes; |
583 | 14.4M | |
584 | 14.4M | // Add all obviously-dead nodes to the DeadNodes worklist. |
585 | 14.4M | for (SDNode &Node : allnodes()) |
586 | 310M | if (310M Node.use_empty()310M ) |
587 | 1.62M | DeadNodes.push_back(&Node); |
588 | 14.4M | |
589 | 14.4M | RemoveDeadNodes(DeadNodes); |
590 | 14.4M | |
591 | 14.4M | // If the root changed (e.g. it was a dead load, update the root). |
592 | 14.4M | setRoot(Dummy.getValue()); |
593 | 14.4M | } |
594 | | |
595 | | /// RemoveDeadNodes - This method deletes the unreachable nodes in the |
596 | | /// given list, and any nodes that become unreachable as a result. |
597 | 32.8M | void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { |
598 | 32.8M | |
599 | 32.8M | // Process the worklist, deleting the nodes and adding their uses to the |
600 | 32.8M | // worklist. |
601 | 48.5M | while (!DeadNodes.empty()48.5M ) { |
602 | 15.6M | SDNode *N = DeadNodes.pop_back_val(); |
603 | 15.6M | // Skip to next node if we've already managed to delete the node. This could |
604 | 15.6M | // happen if replacing a node causes a node previously added to the node to |
605 | 15.6M | // be deleted. |
606 | 15.6M | if (N->getOpcode() == ISD::DELETED_NODE) |
607 | 0 | continue; |
608 | 15.6M | |
609 | 38.2M | for (DAGUpdateListener *DUL = UpdateListeners; 15.6M DUL38.2M ; DUL = DUL->Next22.5M ) |
610 | 22.5M | DUL->NodeDeleted(N, nullptr); |
611 | 15.6M | |
612 | 15.6M | // Take the node out of the appropriate CSE map. |
613 | 15.6M | RemoveNodeFromCSEMaps(N); |
614 | 15.6M | |
615 | 15.6M | // Next, brutally remove the operand list. This is safe to do, as there are |
616 | 15.6M | // no cycles in the graph. |
617 | 30.6M | for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E30.6M ; ) { |
618 | 14.9M | SDUse &Use = *I++; |
619 | 14.9M | SDNode *Operand = Use.getNode(); |
620 | 14.9M | Use.set(SDValue()); |
621 | 14.9M | |
622 | 14.9M | // Now that we removed this operand, see if there are no uses of it left. |
623 | 14.9M | if (Operand->use_empty()) |
624 | 4.79M | DeadNodes.push_back(Operand); |
625 | 14.9M | } |
626 | 15.6M | |
627 | 15.6M | DeallocateNode(N); |
628 | 15.6M | } |
629 | 32.8M | } |
630 | | |
631 | 1.55M | void SelectionDAG::RemoveDeadNode(SDNode *N){ |
632 | 1.55M | SmallVector<SDNode*, 16> DeadNodes(1, N); |
633 | 1.55M | |
634 | 1.55M | // Create a dummy node that adds a reference to the root node, preventing |
635 | 1.55M | // it from being deleted. (This matters if the root is an operand of the |
636 | 1.55M | // dead node.) |
637 | 1.55M | HandleSDNode Dummy(getRoot()); |
638 | 1.55M | |
639 | 1.55M | RemoveDeadNodes(DeadNodes); |
640 | 1.55M | } |
641 | | |
642 | 30.3M | void SelectionDAG::DeleteNode(SDNode *N) { |
643 | 30.3M | // First take this out of the appropriate CSE map. |
644 | 30.3M | RemoveNodeFromCSEMaps(N); |
645 | 30.3M | |
646 | 30.3M | // Finally, remove uses due to operands of this node, remove from the |
647 | 30.3M | // AllNodes list, and delete the node. |
648 | 30.3M | DeleteNodeNotInCSEMaps(N); |
649 | 30.3M | } |
650 | | |
651 | 30.4M | void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { |
652 | 30.4M | assert(N->getIterator() != AllNodes.begin() && |
653 | 30.4M | "Cannot delete the entry node!"); |
654 | 30.4M | assert(N->use_empty() && "Cannot delete a node that is not dead!"); |
655 | 30.4M | |
656 | 30.4M | // Drop all of the operands and decrement used node's use counts. |
657 | 30.4M | N->DropOperands(); |
658 | 30.4M | |
659 | 30.4M | DeallocateNode(N); |
660 | 30.4M | } |
661 | | |
662 | 115M | void SDDbgInfo::erase(const SDNode *Node) { |
663 | 115M | DbgValMapType::iterator I = DbgValMap.find(Node); |
664 | 115M | if (I == DbgValMap.end()) |
665 | 115M | return; |
666 | 319 | for (auto &Val: I->second) |
667 | 349 | Val->setIsInvalidated(); |
668 | 115M | DbgValMap.erase(I); |
669 | 115M | } |
670 | | |
671 | 115M | void SelectionDAG::DeallocateNode(SDNode *N) { |
672 | 115M | // If we have operands, deallocate them. |
673 | 115M | removeOperands(N); |
674 | 115M | |
675 | 115M | NodeAllocator.Deallocate(AllNodes.remove(N)); |
676 | 115M | |
677 | 115M | // Set the opcode to DELETED_NODE to help catch bugs when node |
678 | 115M | // memory is reallocated. |
679 | 115M | // FIXME: There are places in SDag that have grown a dependency on the opcode |
680 | 115M | // value in the released node. |
681 | 115M | __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType)); |
682 | 115M | N->NodeType = ISD::DELETED_NODE; |
683 | 115M | |
684 | 115M | // If any of the SDDbgValue nodes refer to this SDNode, invalidate |
685 | 115M | // them and forget about that node. |
686 | 115M | DbgInfo->erase(N); |
687 | 115M | } |
688 | | |
689 | | #ifndef NDEBUG |
690 | | /// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. |
691 | | static void VerifySDNode(SDNode *N) { |
692 | | switch (N->getOpcode()) { |
693 | | default: |
694 | | break; |
695 | | case ISD::BUILD_PAIR: { |
696 | | EVT VT = N->getValueType(0); |
697 | | assert(N->getNumValues() == 1 && "Too many results!"); |
698 | | assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && |
699 | | "Wrong return type!"); |
700 | | assert(N->getNumOperands() == 2 && "Wrong number of operands!"); |
701 | | assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && |
702 | | "Mismatched operand types!"); |
703 | | assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && |
704 | | "Wrong operand type!"); |
705 | | assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && |
706 | | "Wrong return type size"); |
707 | | break; |
708 | | } |
709 | | case ISD::BUILD_VECTOR: { |
710 | | assert(N->getNumValues() == 1 && "Too many results!"); |
711 | | assert(N->getValueType(0).isVector() && "Wrong return type!"); |
712 | | assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && |
713 | | "Wrong number of operands!"); |
714 | | EVT EltVT = N->getValueType(0).getVectorElementType(); |
715 | | for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { |
716 | | assert((I->getValueType() == EltVT || |
717 | | (EltVT.isInteger() && I->getValueType().isInteger() && |
718 | | EltVT.bitsLE(I->getValueType()))) && |
719 | | "Wrong operand type!"); |
720 | | assert(I->getValueType() == N->getOperand(0).getValueType() && |
721 | | "Operands must all have the same type"); |
722 | | } |
723 | | break; |
724 | | } |
725 | | } |
726 | | } |
727 | | #endif // NDEBUG |
728 | | |
729 | | /// \brief Insert a newly allocated node into the DAG. |
730 | | /// |
731 | | /// Handles insertion into the all nodes list and CSE map, as well as |
732 | | /// verification and other common operations when a new node is allocated. |
733 | 119M | void SelectionDAG::InsertNode(SDNode *N) { |
734 | 119M | AllNodes.push_back(N); |
735 | | #ifndef NDEBUG |
736 | | N->PersistentId = NextPersistentId++; |
737 | | VerifySDNode(N); |
738 | | #endif |
739 | | } |
740 | | |
741 | | /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that |
742 | | /// correspond to it. This is useful when we're about to delete or repurpose |
743 | | /// the node. We don't want future request for structurally identical nodes |
744 | | /// to return N anymore. |
745 | 92.0M | bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { |
746 | 92.0M | bool Erased = false; |
747 | 92.0M | switch (N->getOpcode()) { |
748 | 172k | case ISD::HANDLENODE: return false; // noop. |
749 | 2.91M | case ISD::CONDCODE: |
750 | 2.91M | assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && |
751 | 2.91M | "Cond code doesn't exist!"); |
752 | 2.91M | Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr; |
753 | 2.91M | CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr; |
754 | 2.91M | break; |
755 | 29.5k | case ISD::ExternalSymbol: |
756 | 29.5k | Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); |
757 | 29.5k | break; |
758 | 253 | case ISD::TargetExternalSymbol: { |
759 | 253 | ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); |
760 | 253 | Erased = TargetExternalSymbols.erase( |
761 | 253 | std::pair<std::string,unsigned char>(ESN->getSymbol(), |
762 | 253 | ESN->getTargetFlags())); |
763 | 253 | break; |
764 | 92.0M | } |
765 | 0 | case ISD::MCSymbol: { |
766 | 0 | auto *MCSN = cast<MCSymbolSDNode>(N); |
767 | 0 | Erased = MCSymbols.erase(MCSN->getMCSymbol()); |
768 | 0 | break; |
769 | 92.0M | } |
770 | 876k | case ISD::VALUETYPE: { |
771 | 876k | EVT VT = cast<VTSDNode>(N)->getVT(); |
772 | 876k | if (VT.isExtended()876k ) { |
773 | 1.64k | Erased = ExtendedValueTypeNodes.erase(VT); |
774 | 876k | } else { |
775 | 874k | Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr; |
776 | 874k | ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr; |
777 | 874k | } |
778 | 876k | break; |
779 | 92.0M | } |
780 | 88.0M | default: |
781 | 88.0M | // Remove it from the CSE Map. |
782 | 88.0M | assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); |
783 | 88.0M | assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); |
784 | 88.0M | Erased = CSEMap.RemoveNode(N); |
785 | 88.0M | break; |
786 | 91.9M | } |
787 | | #ifndef NDEBUG |
788 | | // Verify that the node was actually in one of the CSE maps, unless it has a |
789 | | // flag result (which cannot be CSE'd) or is one of the special cases that are |
790 | | // not subject to CSE. |
791 | | if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && |
792 | | !N->isMachineOpcode() && !doNotCSE(N)) { |
793 | | N->dump(this); |
794 | | dbgs() << "\n"; |
795 | | llvm_unreachable("Node is not in map!"); |
796 | | } |
797 | | #endif |
798 | 0 | return Erased; |
799 | 91.9M | } |
800 | | |
801 | | /// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE |
802 | | /// maps and modified in place. Add it back to the CSE maps, unless an identical |
803 | | /// node already exists, in which case transfer all its users to the existing |
804 | | /// node. This transfer can potentially trigger recursive merging. |
805 | | void |
806 | 25.1M | SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { |
807 | 25.1M | // For node types that aren't CSE'd, just act as if no identical node |
808 | 25.1M | // already exists. |
809 | 25.1M | if (!doNotCSE(N)25.1M ) { |
810 | 16.2M | SDNode *Existing = CSEMap.GetOrInsertNode(N); |
811 | 16.2M | if (Existing != N16.2M ) { |
812 | 64.3k | // If there was already an existing matching node, use ReplaceAllUsesWith |
813 | 64.3k | // to replace the dead one with the existing one. This can cause |
814 | 64.3k | // recursive merging of other unrelated nodes down the line. |
815 | 64.3k | ReplaceAllUsesWith(N, Existing); |
816 | 64.3k | |
817 | 64.3k | // N is now dead. Inform the listeners and delete it. |
818 | 233k | for (DAGUpdateListener *DUL = UpdateListeners; DUL233k ; DUL = DUL->Next168k ) |
819 | 168k | DUL->NodeDeleted(N, Existing); |
820 | 64.3k | DeleteNodeNotInCSEMaps(N); |
821 | 64.3k | return; |
822 | 64.3k | } |
823 | 25.0M | } |
824 | 25.0M | |
825 | 25.0M | // If the node doesn't already exist, we updated it. Inform listeners. |
826 | 86.7M | for (DAGUpdateListener *DUL = UpdateListeners; 25.0M DUL86.7M ; DUL = DUL->Next61.7M ) |
827 | 61.7M | DUL->NodeUpdated(N); |
828 | 25.1M | } |
829 | | |
830 | | /// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
831 | | /// were replaced with those specified. If this node is never memoized, |
832 | | /// return null, otherwise return a pointer to the slot it would take. If a |
833 | | /// node already exists with these operands, the slot will be non-null. |
834 | | SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, |
835 | 21.0k | void *&InsertPos) { |
836 | 21.0k | if (doNotCSE(N)) |
837 | 0 | return nullptr; |
838 | 21.0k | |
839 | 21.0k | SDValue Ops[] = { Op }; |
840 | 21.0k | FoldingSetNodeID ID; |
841 | 21.0k | AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
842 | 21.0k | AddNodeIDCustom(ID, N); |
843 | 21.0k | SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
844 | 21.0k | if (Node) |
845 | 16 | Node->intersectFlagsWith(N->getFlags()); |
846 | 21.0k | return Node; |
847 | 21.0k | } |
848 | | |
849 | | /// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
850 | | /// were replaced with those specified. If this node is never memoized, |
851 | | /// return null, otherwise return a pointer to the slot it would take. If a |
852 | | /// node already exists with these operands, the slot will be non-null. |
853 | | SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, |
854 | | SDValue Op1, SDValue Op2, |
855 | 32.1k | void *&InsertPos) { |
856 | 32.1k | if (doNotCSE(N)) |
857 | 0 | return nullptr; |
858 | 32.1k | |
859 | 32.1k | SDValue Ops[] = { Op1, Op2 }; |
860 | 32.1k | FoldingSetNodeID ID; |
861 | 32.1k | AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
862 | 32.1k | AddNodeIDCustom(ID, N); |
863 | 32.1k | SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
864 | 32.1k | if (Node) |
865 | 853 | Node->intersectFlagsWith(N->getFlags()); |
866 | 32.1k | return Node; |
867 | 32.1k | } |
868 | | |
869 | | /// FindModifiedNodeSlot - Find a slot for the specified node if its operands |
870 | | /// were replaced with those specified. If this node is never memoized, |
871 | | /// return null, otherwise return a pointer to the slot it would take. If a |
872 | | /// node already exists with these operands, the slot will be non-null. |
873 | | SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops, |
874 | 720k | void *&InsertPos) { |
875 | 720k | if (doNotCSE(N)) |
876 | 10.5k | return nullptr; |
877 | 709k | |
878 | 709k | FoldingSetNodeID ID; |
879 | 709k | AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops); |
880 | 709k | AddNodeIDCustom(ID, N); |
881 | 709k | SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos); |
882 | 709k | if (Node) |
883 | 596 | Node->intersectFlagsWith(N->getFlags()); |
884 | 720k | return Node; |
885 | 720k | } |
886 | | |
887 | 356k | unsigned SelectionDAG::getEVTAlignment(EVT VT) const { |
888 | 356k | Type *Ty = VT == MVT::iPTR ? |
889 | 0 | PointerType::get(Type::getInt8Ty(*getContext()), 0) : |
890 | 356k | VT.getTypeForEVT(*getContext()); |
891 | 356k | |
892 | 356k | return getDataLayout().getABITypeAlignment(Ty); |
893 | 356k | } |
894 | | |
895 | | // EntryNode could meaningfully have debug info if we can find it... |
896 | | SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) |
897 | | : TM(tm), OptLevel(OL), |
898 | | EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), |
899 | 35.1k | Root(getEntryNode()) { |
900 | 35.1k | InsertNode(&EntryNode); |
901 | 35.1k | DbgInfo = new SDDbgInfo(); |
902 | 35.1k | } |
903 | | |
904 | | void SelectionDAG::init(MachineFunction &NewMF, |
905 | | OptimizationRemarkEmitter &NewORE, |
906 | 437k | Pass *PassPtr) { |
907 | 437k | MF = &NewMF; |
908 | 437k | SDAGISelPass = PassPtr; |
909 | 437k | ORE = &NewORE; |
910 | 437k | TLI = getSubtarget().getTargetLowering(); |
911 | 437k | TSI = getSubtarget().getSelectionDAGInfo(); |
912 | 437k | Context = &MF->getFunction()->getContext(); |
913 | 437k | } |
914 | | |
915 | 35.0k | SelectionDAG::~SelectionDAG() { |
916 | 35.0k | assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); |
917 | 35.0k | allnodes_clear(); |
918 | 35.0k | OperandRecycler.clear(OperandAllocator); |
919 | 35.0k | delete DbgInfo; |
920 | 35.0k | } |
921 | | |
922 | 3.46M | void SelectionDAG::allnodes_clear() { |
923 | 3.46M | assert(&*AllNodes.begin() == &EntryNode); |
924 | 3.46M | AllNodes.remove(AllNodes.begin()); |
925 | 73.0M | while (!AllNodes.empty()) |
926 | 69.5M | DeallocateNode(&AllNodes.front()); |
927 | | #ifndef NDEBUG |
928 | | NextPersistentId = 0; |
929 | | #endif |
930 | | } |
931 | | |
932 | | SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, |
933 | 31.1M | void *&InsertPos) { |
934 | 31.1M | SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); |
935 | 31.1M | if (N31.1M ) { |
936 | 10.6M | switch (N->getOpcode()) { |
937 | 10.6M | default: break; |
938 | 0 | case ISD::Constant: |
939 | 0 | case ISD::ConstantFP: |
940 | 0 | llvm_unreachable("Querying for Constant and ConstantFP nodes requires " |
941 | 31.1M | "debug location. Use another overload."); |
942 | 31.1M | } |
943 | 31.1M | } |
944 | 31.1M | return N; |
945 | 31.1M | } |
946 | | |
947 | | SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, |
948 | 125M | const SDLoc &DL, void *&InsertPos) { |
949 | 125M | SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos); |
950 | 125M | if (N125M ) { |
951 | 27.5M | switch (N->getOpcode()) { |
952 | 6.49M | case ISD::Constant: |
953 | 6.49M | case ISD::ConstantFP: |
954 | 6.49M | // Erase debug location from the node if the node is used at several |
955 | 6.49M | // different places. Do not propagate one location to all uses as it |
956 | 6.49M | // will cause a worse single stepping debugging experience. |
957 | 6.49M | if (N->getDebugLoc() != DL.getDebugLoc()) |
958 | 84.0k | N->setDebugLoc(DebugLoc()); |
959 | 6.49M | break; |
960 | 21.0M | default: |
961 | 21.0M | // When the node's point of use is located earlier in the instruction |
962 | 21.0M | // sequence than its prior point of use, update its debug info to the |
963 | 21.0M | // earlier location. |
964 | 21.0M | if (DL.getIROrder() && 21.0M DL.getIROrder() < N->getIROrder()13.5M ) |
965 | 258k | N->setDebugLoc(DL.getDebugLoc()); |
966 | 6.49M | break; |
967 | 125M | } |
968 | 125M | } |
969 | 125M | return N; |
970 | 125M | } |
971 | | |
972 | 3.42M | void SelectionDAG::clear() { |
973 | 3.42M | allnodes_clear(); |
974 | 3.42M | OperandRecycler.clear(OperandAllocator); |
975 | 3.42M | OperandAllocator.Reset(); |
976 | 3.42M | CSEMap.clear(); |
977 | 3.42M | |
978 | 3.42M | ExtendedValueTypeNodes.clear(); |
979 | 3.42M | ExternalSymbols.clear(); |
980 | 3.42M | TargetExternalSymbols.clear(); |
981 | 3.42M | MCSymbols.clear(); |
982 | 3.42M | std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), |
983 | 3.42M | static_cast<CondCodeSDNode*>(nullptr)); |
984 | 3.42M | std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), |
985 | 3.42M | static_cast<SDNode*>(nullptr)); |
986 | 3.42M | |
987 | 3.42M | EntryNode.UseList = nullptr; |
988 | 3.42M | InsertNode(&EntryNode); |
989 | 3.42M | Root = getEntryNode(); |
990 | 3.42M | DbgInfo->clear(); |
991 | 3.42M | } |
992 | | |
993 | 6 | SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) { |
994 | 6 | return VT.bitsGT(Op.getValueType()) |
995 | 0 | ? getNode(ISD::FP_EXTEND, DL, VT, Op) |
996 | 6 | : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL)); |
997 | 6 | } |
998 | | |
999 | 74.7k | SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1000 | 74.7k | return VT.bitsGT(Op.getValueType()) ? |
1001 | 29.8k | getNode(ISD::ANY_EXTEND, DL, VT, Op) : |
1002 | 44.8k | getNode(ISD::TRUNCATE, DL, VT, Op); |
1003 | 74.7k | } |
1004 | | |
1005 | 840k | SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1006 | 840k | return VT.bitsGT(Op.getValueType()) ? |
1007 | 34.4k | getNode(ISD::SIGN_EXTEND, DL, VT, Op) : |
1008 | 806k | getNode(ISD::TRUNCATE, DL, VT, Op); |
1009 | 840k | } |
1010 | | |
1011 | 321k | SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { |
1012 | 321k | return VT.bitsGT(Op.getValueType()) ? |
1013 | 14.1k | getNode(ISD::ZERO_EXTEND, DL, VT, Op) : |
1014 | 307k | getNode(ISD::TRUNCATE, DL, VT, Op); |
1015 | 321k | } |
1016 | | |
1017 | | SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, |
1018 | 9.09k | EVT OpVT) { |
1019 | 9.09k | if (VT.bitsLE(Op.getValueType())) |
1020 | 8.85k | return getNode(ISD::TRUNCATE, SL, VT, Op); |
1021 | 233 | |
1022 | 233 | TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT); |
1023 | 233 | return getNode(TLI->getExtendForContent(BType), SL, VT, Op); |
1024 | 233 | } |
1025 | | |
1026 | 893k | SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { |
1027 | 893k | assert(!VT.isVector() && |
1028 | 893k | "getZeroExtendInReg should use the vector element type instead of " |
1029 | 893k | "the vector type!"); |
1030 | 893k | if (Op.getValueType() == VT893k ) return Op23 ; |
1031 | 893k | unsigned BitWidth = Op.getScalarValueSizeInBits(); |
1032 | 893k | APInt Imm = APInt::getLowBitsSet(BitWidth, |
1033 | 893k | VT.getSizeInBits()); |
1034 | 893k | return getNode(ISD::AND, DL, Op.getValueType(), Op, |
1035 | 893k | getConstant(Imm, DL, Op.getValueType())); |
1036 | 893k | } |
1037 | | |
1038 | | SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL, |
1039 | 228 | EVT VT) { |
1040 | 228 | assert(VT.isVector() && "This DAG node is restricted to vector types."); |
1041 | 228 | assert(VT.getSizeInBits() == Op.getValueSizeInBits() && |
1042 | 228 | "The sizes of the input and result must match in order to perform the " |
1043 | 228 | "extend in-register."); |
1044 | 228 | assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() && |
1045 | 228 | "The destination vector type must have fewer lanes than the input."); |
1046 | 228 | return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op); |
1047 | 228 | } |
1048 | | |
1049 | | SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, const SDLoc &DL, |
1050 | 1.87k | EVT VT) { |
1051 | 1.87k | assert(VT.isVector() && "This DAG node is restricted to vector types."); |
1052 | 1.87k | assert(VT.getSizeInBits() == Op.getValueSizeInBits() && |
1053 | 1.87k | "The sizes of the input and result must match in order to perform the " |
1054 | 1.87k | "extend in-register."); |
1055 | 1.87k | assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() && |
1056 | 1.87k | "The destination vector type must have fewer lanes than the input."); |
1057 | 1.87k | return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op); |
1058 | 1.87k | } |
1059 | | |
1060 | | SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL, |
1061 | 1.54k | EVT VT) { |
1062 | 1.54k | assert(VT.isVector() && "This DAG node is restricted to vector types."); |
1063 | 1.54k | assert(VT.getSizeInBits() == Op.getValueSizeInBits() && |
1064 | 1.54k | "The sizes of the input and result must match in order to perform the " |
1065 | 1.54k | "extend in-register."); |
1066 | 1.54k | assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() && |
1067 | 1.54k | "The destination vector type must have fewer lanes than the input."); |
1068 | 1.54k | return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op); |
1069 | 1.54k | } |
1070 | | |
1071 | | /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). |
1072 | 9.71k | SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { |
1073 | 9.71k | EVT EltVT = VT.getScalarType(); |
1074 | 9.71k | SDValue NegOne = |
1075 | 9.71k | getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT); |
1076 | 9.71k | return getNode(ISD::XOR, DL, VT, Val, NegOne); |
1077 | 9.71k | } |
1078 | | |
1079 | 12 | SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) { |
1080 | 12 | EVT EltVT = VT.getScalarType(); |
1081 | 12 | SDValue TrueValue; |
1082 | 12 | switch (TLI->getBooleanContents(VT)) { |
1083 | 0 | case TargetLowering::ZeroOrOneBooleanContent: |
1084 | 0 | case TargetLowering::UndefinedBooleanContent: |
1085 | 0 | TrueValue = getConstant(1, DL, VT); |
1086 | 0 | break; |
1087 | 12 | case TargetLowering::ZeroOrNegativeOneBooleanContent: |
1088 | 12 | TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, |
1089 | 12 | VT); |
1090 | 12 | break; |
1091 | 12 | } |
1092 | 12 | return getNode(ISD::XOR, DL, VT, Val, TrueValue); |
1093 | 12 | } |
1094 | | |
1095 | | SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT, |
1096 | 28.8M | bool isT, bool isO) { |
1097 | 28.8M | EVT EltVT = VT.getScalarType(); |
1098 | 28.8M | assert((EltVT.getSizeInBits() >= 64 || |
1099 | 28.8M | (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && |
1100 | 28.8M | "getConstant with a uint64_t value that doesn't fit in the type!"); |
1101 | 28.8M | return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO); |
1102 | 28.8M | } |
1103 | | |
1104 | | SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT, |
1105 | 33.8M | bool isT, bool isO) { |
1106 | 33.8M | return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO); |
1107 | 33.8M | } |
1108 | | |
1109 | | SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL, |
1110 | 39.7M | EVT VT, bool isT, bool isO) { |
1111 | 39.7M | assert(VT.isInteger() && "Cannot create FP integer constant!"); |
1112 | 39.7M | |
1113 | 39.7M | EVT EltVT = VT.getScalarType(); |
1114 | 39.7M | const ConstantInt *Elt = &Val; |
1115 | 39.7M | |
1116 | 39.7M | // In some cases the vector type is legal but the element type is illegal and |
1117 | 39.7M | // needs to be promoted, for example v8i8 on ARM. In this case, promote the |
1118 | 39.7M | // inserted value (the type does not need to match the vector element type). |
1119 | 39.7M | // Any extra bits introduced will be truncated away. |
1120 | 39.7M | if (VT.isVector() && 39.7M TLI->getTypeAction(*getContext(), EltVT) == |
1121 | 39.7M | TargetLowering::TypePromoteInteger) { |
1122 | 7.10k | EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); |
1123 | 7.10k | APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits()); |
1124 | 7.10k | Elt = ConstantInt::get(*getContext(), NewVal); |
1125 | 7.10k | } |
1126 | 39.7M | // In other cases the element type is illegal and needs to be expanded, for |
1127 | 39.7M | // example v2i64 on MIPS32. In this case, find the nearest legal type, split |
1128 | 39.7M | // the value into n parts and use a vector type with n-times the elements. |
1129 | 39.7M | // Then bitcast to the type requested. |
1130 | 39.7M | // Legalizing constants too early makes the DAGCombiner's job harder so we |
1131 | 39.7M | // only legalize if the DAG tells us we must produce legal types. |
1132 | 39.7M | else if (39.7M NewNodesMustHaveLegalTypes && 39.7M VT.isVector()17.6M && |
1133 | 39.9k | TLI->getTypeAction(*getContext(), EltVT) == |
1134 | 39.7M | TargetLowering::TypeExpandInteger) { |
1135 | 183 | const APInt &NewVal = Elt->getValue(); |
1136 | 183 | EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); |
1137 | 183 | unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits(); |
1138 | 183 | unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits; |
1139 | 183 | EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts); |
1140 | 183 | |
1141 | 183 | // Check the temporary vector is the correct size. If this fails then |
1142 | 183 | // getTypeToTransformTo() probably returned a type whose size (in bits) |
1143 | 183 | // isn't a power-of-2 factor of the requested type size. |
1144 | 183 | assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits()); |
1145 | 183 | |
1146 | 183 | SmallVector<SDValue, 2> EltParts; |
1147 | 549 | for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements()549 ; ++i366 ) { |
1148 | 366 | EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits) |
1149 | 366 | .zextOrTrunc(ViaEltSizeInBits), DL, |
1150 | 366 | ViaEltVT, isT, isO)); |
1151 | 366 | } |
1152 | 183 | |
1153 | 183 | // EltParts is currently in little endian order. If we actually want |
1154 | 183 | // big-endian order then reverse it now. |
1155 | 183 | if (getDataLayout().isBigEndian()) |
1156 | 56 | std::reverse(EltParts.begin(), EltParts.end()); |
1157 | 183 | |
1158 | 183 | // The elements must be reversed when the element order is different |
1159 | 183 | // to the endianness of the elements (because the BITCAST is itself a |
1160 | 183 | // vector shuffle in this situation). However, we do not need any code to |
1161 | 183 | // perform this reversal because getConstant() is producing a vector |
1162 | 183 | // splat. |
1163 | 183 | // This situation occurs in MIPS MSA. |
1164 | 183 | |
1165 | 183 | SmallVector<SDValue, 8> Ops; |
1166 | 733 | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e733 ; ++i550 ) |
1167 | 550 | Ops.insert(Ops.end(), EltParts.begin(), EltParts.end()); |
1168 | 39.7M | |
1169 | 39.7M | SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops)); |
1170 | 39.7M | NewSDValueDbgMsg(V, "Creating constant: ", this); |
1171 | 39.7M | return V; |
1172 | 39.7M | } |
1173 | 39.7M | |
1174 | 39.7M | assert(Elt->getBitWidth() == EltVT.getSizeInBits() && |
1175 | 39.7M | "APInt size does not match type size!"); |
1176 | 39.7M | unsigned Opc = isT ? ISD::TargetConstant21.3M : ISD::Constant18.3M ; |
1177 | 39.7M | FoldingSetNodeID ID; |
1178 | 39.7M | AddNodeIDNode(ID, Opc, getVTList(EltVT), None); |
1179 | 39.7M | ID.AddPointer(Elt); |
1180 | 39.7M | ID.AddBoolean(isO); |
1181 | 39.7M | void *IP = nullptr; |
1182 | 39.7M | SDNode *N = nullptr; |
1183 | 39.7M | if ((N = FindNodeOrInsertPos(ID, DL, IP))) |
1184 | 17.8M | if (17.8M !VT.isVector()17.8M ) |
1185 | 17.7M | return SDValue(N, 0); |
1186 | 22.0M | |
1187 | 22.0M | if (22.0M !N22.0M ) { |
1188 | 21.9M | N = newSDNode<ConstantSDNode>(isT, isO, Elt, DL.getDebugLoc(), EltVT); |
1189 | 21.9M | CSEMap.InsertNode(N, IP); |
1190 | 21.9M | InsertNode(N); |
1191 | 21.9M | } |
1192 | 22.0M | |
1193 | 22.0M | SDValue Result(N, 0); |
1194 | 22.0M | if (VT.isVector()) |
1195 | 61.1k | Result = getSplatBuildVector(VT, DL, Result); |
1196 | 39.7M | |
1197 | 39.7M | NewSDValueDbgMsg(Result, "Creating constant: ", this); |
1198 | 39.7M | return Result; |
1199 | 39.7M | } |
1200 | | |
1201 | | SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL, |
1202 | 6.62M | bool isTarget) { |
1203 | 6.62M | return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget); |
1204 | 6.62M | } |
1205 | | |
1206 | | SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT, |
1207 | 11.1k | bool isTarget) { |
1208 | 11.1k | return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget); |
1209 | 11.1k | } |
1210 | | |
1211 | | SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL, |
1212 | 162k | EVT VT, bool isTarget) { |
1213 | 162k | assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); |
1214 | 162k | |
1215 | 162k | EVT EltVT = VT.getScalarType(); |
1216 | 162k | |
1217 | 162k | // Do the map lookup using the actual bit pattern for the floating point |
1218 | 162k | // value, so that we don't have problems with 0.0 comparing equal to -0.0, and |
1219 | 162k | // we don't have issues with SNANs. |
1220 | 162k | unsigned Opc = isTarget ? ISD::TargetConstantFP34.4k : ISD::ConstantFP128k ; |
1221 | 162k | FoldingSetNodeID ID; |
1222 | 162k | AddNodeIDNode(ID, Opc, getVTList(EltVT), None); |
1223 | 162k | ID.AddPointer(&V); |
1224 | 162k | void *IP = nullptr; |
1225 | 162k | SDNode *N = nullptr; |
1226 | 162k | if ((N = FindNodeOrInsertPos(ID, DL, IP))) |
1227 | 4.17k | if (4.17k !VT.isVector()4.17k ) |
1228 | 3.00k | return SDValue(N, 0); |
1229 | 159k | |
1230 | 159k | if (159k !N159k ) { |
1231 | 158k | N = newSDNode<ConstantFPSDNode>(isTarget, &V, DL.getDebugLoc(), EltVT); |
1232 | 158k | CSEMap.InsertNode(N, IP); |
1233 | 158k | InsertNode(N); |
1234 | 158k | } |
1235 | 159k | |
1236 | 159k | SDValue Result(N, 0); |
1237 | 159k | if (VT.isVector()) |
1238 | 2.07k | Result = getSplatBuildVector(VT, DL, Result); |
1239 | 162k | NewSDValueDbgMsg(Result, "Creating fp constant: ", this); |
1240 | 162k | return Result; |
1241 | 162k | } |
1242 | | |
1243 | | SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT, |
1244 | 5.66k | bool isTarget) { |
1245 | 5.66k | EVT EltVT = VT.getScalarType(); |
1246 | 5.66k | if (EltVT == MVT::f32) |
1247 | 3.61k | return getConstantFP(APFloat((float)Val), DL, VT, isTarget); |
1248 | 2.05k | else if (2.05k EltVT == MVT::f642.05k ) |
1249 | 1.74k | return getConstantFP(APFloat(Val), DL, VT, isTarget); |
1250 | 306 | else if (306 EltVT == MVT::f80 || 306 EltVT == MVT::f128168 || EltVT == MVT::ppcf128161 || |
1251 | 306 | EltVT == MVT::f16159 ) { |
1252 | 306 | bool Ignored; |
1253 | 306 | APFloat APF = APFloat(Val); |
1254 | 306 | APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, |
1255 | 306 | &Ignored); |
1256 | 306 | return getConstantFP(APF, DL, VT, isTarget); |
1257 | 306 | } else |
1258 | 306 | llvm_unreachable("Unsupported type in getConstantFP"); |
1259 | 5.66k | } |
1260 | | |
1261 | | SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, |
1262 | | EVT VT, int64_t Offset, bool isTargetGA, |
1263 | 6.30M | unsigned char TargetFlags) { |
1264 | 6.30M | assert((TargetFlags == 0 || isTargetGA) && |
1265 | 6.30M | "Cannot set target flags on target-independent globals"); |
1266 | 6.30M | |
1267 | 6.30M | // Truncate (with sign-extension) the offset value to the pointer size. |
1268 | 6.30M | unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); |
1269 | 6.30M | if (BitWidth < 64) |
1270 | 132k | Offset = SignExtend64(Offset, BitWidth); |
1271 | 6.30M | |
1272 | 6.30M | unsigned Opc; |
1273 | 6.30M | if (GV->isThreadLocal()) |
1274 | 1.66k | Opc = isTargetGA ? 1.66k ISD::TargetGlobalTLSAddress900 : ISD::GlobalTLSAddress763 ; |
1275 | 6.30M | else |
1276 | 6.29M | Opc = isTargetGA ? 6.29M ISD::TargetGlobalAddress3.68M : ISD::GlobalAddress2.61M ; |
1277 | 6.30M | |
1278 | 6.30M | FoldingSetNodeID ID; |
1279 | 6.30M | AddNodeIDNode(ID, Opc, getVTList(VT), None); |
1280 | 6.30M | ID.AddPointer(GV); |
1281 | 6.30M | ID.AddInteger(Offset); |
1282 | 6.30M | ID.AddInteger(TargetFlags); |
1283 | 6.30M | void *IP = nullptr; |
1284 | 6.30M | if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
1285 | 293k | return SDValue(E, 0); |
1286 | 6.00M | |
1287 | 6.00M | auto *N = newSDNode<GlobalAddressSDNode>( |
1288 | 6.00M | Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags); |
1289 | 6.00M | CSEMap.InsertNode(N, IP); |
1290 | 6.00M | InsertNode(N); |
1291 | 6.00M | return SDValue(N, 0); |
1292 | 6.00M | } |
1293 | | |
1294 | 1.43M | SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { |
1295 | 1.43M | unsigned Opc = isTarget ? ISD::TargetFrameIndex927k : ISD::FrameIndex505k ; |
1296 | 1.43M | FoldingSetNodeID ID; |
1297 | 1.43M | AddNodeIDNode(ID, Opc, getVTList(VT), None); |
1298 | 1.43M | ID.AddInteger(FI); |
1299 | 1.43M | void *IP = nullptr; |
1300 | 1.43M | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1301 | 431k | return SDValue(E, 0); |
1302 | 1.00M | |
1303 | 1.00M | auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget); |
1304 | 1.00M | CSEMap.InsertNode(N, IP); |
1305 | 1.00M | InsertNode(N); |
1306 | 1.00M | return SDValue(N, 0); |
1307 | 1.00M | } |
1308 | | |
1309 | | SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, |
1310 | 15.1k | unsigned char TargetFlags) { |
1311 | 15.1k | assert((TargetFlags == 0 || isTarget) && |
1312 | 15.1k | "Cannot set target flags on target-independent jump tables"); |
1313 | 15.1k | unsigned Opc = isTarget ? ISD::TargetJumpTable9.96k : ISD::JumpTable5.16k ; |
1314 | 15.1k | FoldingSetNodeID ID; |
1315 | 15.1k | AddNodeIDNode(ID, Opc, getVTList(VT), None); |
1316 | 15.1k | ID.AddInteger(JTI); |
1317 | 15.1k | ID.AddInteger(TargetFlags); |
1318 | 15.1k | void *IP = nullptr; |
1319 | 15.1k | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1320 | 238 | return SDValue(E, 0); |
1321 | 14.8k | |
1322 | 14.8k | auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags); |
1323 | 14.8k | CSEMap.InsertNode(N, IP); |
1324 | 14.8k | InsertNode(N); |
1325 | 14.8k | return SDValue(N, 0); |
1326 | 14.8k | } |
1327 | | |
1328 | | SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, |
1329 | | unsigned Alignment, int Offset, |
1330 | | bool isTarget, |
1331 | 287k | unsigned char TargetFlags) { |
1332 | 287k | assert((TargetFlags == 0 || isTarget) && |
1333 | 287k | "Cannot set target flags on target-independent globals"); |
1334 | 287k | if (Alignment == 0) |
1335 | 98.2k | Alignment = MF->getFunction()->optForSize() |
1336 | 1.97k | ? getDataLayout().getABITypeAlignment(C->getType()) |
1337 | 96.2k | : getDataLayout().getPrefTypeAlignment(C->getType()); |
1338 | 287k | unsigned Opc = isTarget ? ISD::TargetConstantPool191k : ISD::ConstantPool96.6k ; |
1339 | 287k | FoldingSetNodeID ID; |
1340 | 287k | AddNodeIDNode(ID, Opc, getVTList(VT), None); |
1341 | 287k | ID.AddInteger(Alignment); |
1342 | 287k | ID.AddInteger(Offset); |
1343 | 287k | ID.AddPointer(C); |
1344 | 287k | ID.AddInteger(TargetFlags); |
1345 | 287k | void *IP = nullptr; |
1346 | 287k | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1347 | 2.97k | return SDValue(E, 0); |
1348 | 284k | |
1349 | 284k | auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment, |
1350 | 284k | TargetFlags); |
1351 | 284k | CSEMap.InsertNode(N, IP); |
1352 | 284k | InsertNode(N); |
1353 | 284k | return SDValue(N, 0); |
1354 | 284k | } |
1355 | | |
1356 | | SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, |
1357 | | unsigned Alignment, int Offset, |
1358 | | bool isTarget, |
1359 | 356 | unsigned char TargetFlags) { |
1360 | 356 | assert((TargetFlags == 0 || isTarget) && |
1361 | 356 | "Cannot set target flags on target-independent globals"); |
1362 | 356 | if (Alignment == 0) |
1363 | 0 | Alignment = getDataLayout().getPrefTypeAlignment(C->getType()); |
1364 | 356 | unsigned Opc = isTarget ? ISD::TargetConstantPool339 : ISD::ConstantPool17 ; |
1365 | 356 | FoldingSetNodeID ID; |
1366 | 356 | AddNodeIDNode(ID, Opc, getVTList(VT), None); |
1367 | 356 | ID.AddInteger(Alignment); |
1368 | 356 | ID.AddInteger(Offset); |
1369 | 356 | C->addSelectionDAGCSEId(ID); |
1370 | 356 | ID.AddInteger(TargetFlags); |
1371 | 356 | void *IP = nullptr; |
1372 | 356 | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1373 | 0 | return SDValue(E, 0); |
1374 | 356 | |
1375 | 356 | auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment, |
1376 | 356 | TargetFlags); |
1377 | 356 | CSEMap.InsertNode(N, IP); |
1378 | 356 | InsertNode(N); |
1379 | 356 | return SDValue(N, 0); |
1380 | 356 | } |
1381 | | |
1382 | | SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, |
1383 | 0 | unsigned char TargetFlags) { |
1384 | 0 | FoldingSetNodeID ID; |
1385 | 0 | AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None); |
1386 | 0 | ID.AddInteger(Index); |
1387 | 0 | ID.AddInteger(Offset); |
1388 | 0 | ID.AddInteger(TargetFlags); |
1389 | 0 | void *IP = nullptr; |
1390 | 0 | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1391 | 0 | return SDValue(E, 0); |
1392 | 0 |
|
1393 | 0 | auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags); |
1394 | 0 | CSEMap.InsertNode(N, IP); |
1395 | 0 | InsertNode(N); |
1396 | 0 | return SDValue(N, 0); |
1397 | 0 | } |
1398 | | |
1399 | 3.91M | SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { |
1400 | 3.91M | FoldingSetNodeID ID; |
1401 | 3.91M | AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None); |
1402 | 3.91M | ID.AddPointer(MBB); |
1403 | 3.91M | void *IP = nullptr; |
1404 | 3.91M | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1405 | 5 | return SDValue(E, 0); |
1406 | 3.91M | |
1407 | 3.91M | auto *N = newSDNode<BasicBlockSDNode>(MBB); |
1408 | 3.91M | CSEMap.InsertNode(N, IP); |
1409 | 3.91M | InsertNode(N); |
1410 | 3.91M | return SDValue(N, 0); |
1411 | 3.91M | } |
1412 | | |
1413 | 1.36M | SDValue SelectionDAG::getValueType(EVT VT) { |
1414 | 1.36M | if (VT.isSimple() && 1.36M (unsigned)VT.getSimpleVT().SimpleTy >= |
1415 | 1.35M | ValueTypeNodes.size()) |
1416 | 24.4k | ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); |
1417 | 1.36M | |
1418 | 1.36M | SDNode *&N = VT.isExtended() ? |
1419 | 1.36M | ExtendedValueTypeNodes[VT]2.05k : ValueTypeNodes[VT.getSimpleVT().SimpleTy]1.35M ; |
1420 | 1.36M | |
1421 | 1.36M | if (N1.36M ) return SDValue(N, 0)483k ; |
1422 | 876k | N = newSDNode<VTSDNode>(VT); |
1423 | 876k | InsertNode(N); |
1424 | 876k | return SDValue(N, 0); |
1425 | 876k | } |
1426 | | |
1427 | 33.3k | SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { |
1428 | 33.3k | SDNode *&N = ExternalSymbols[Sym]; |
1429 | 33.3k | if (N33.3k ) return SDValue(N, 0)3.78k ; |
1430 | 29.5k | N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT); |
1431 | 29.5k | InsertNode(N); |
1432 | 29.5k | return SDValue(N, 0); |
1433 | 29.5k | } |
1434 | | |
1435 | 65 | SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) { |
1436 | 65 | SDNode *&N = MCSymbols[Sym]; |
1437 | 65 | if (N) |
1438 | 16 | return SDValue(N, 0); |
1439 | 49 | N = newSDNode<MCSymbolSDNode>(Sym, VT); |
1440 | 49 | InsertNode(N); |
1441 | 49 | return SDValue(N, 0); |
1442 | 49 | } |
1443 | | |
1444 | | SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, |
1445 | 47.3k | unsigned char TargetFlags) { |
1446 | 47.3k | SDNode *&N = |
1447 | 47.3k | TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, |
1448 | 47.3k | TargetFlags)]; |
1449 | 47.3k | if (N47.3k ) return SDValue(N, 0)9.23k ; |
1450 | 38.0k | N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT); |
1451 | 38.0k | InsertNode(N); |
1452 | 38.0k | return SDValue(N, 0); |
1453 | 38.0k | } |
1454 | | |
1455 | 3.18M | SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { |
1456 | 3.18M | if ((unsigned)Cond >= CondCodeNodes.size()) |
1457 | 37.8k | CondCodeNodes.resize(Cond+1); |
1458 | 3.18M | |
1459 | 3.18M | if (!CondCodeNodes[Cond]3.18M ) { |
1460 | 2.91M | auto *N = newSDNode<CondCodeSDNode>(Cond); |
1461 | 2.91M | CondCodeNodes[Cond] = N; |
1462 | 2.91M | InsertNode(N); |
1463 | 2.91M | } |
1464 | 3.18M | |
1465 | 3.18M | return SDValue(CondCodeNodes[Cond], 0); |
1466 | 3.18M | } |
1467 | | |
1468 | | /// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that |
1469 | | /// point at N1 to point at N2 and indices that point at N2 to point at N1. |
1470 | 915 | static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) { |
1471 | 915 | std::swap(N1, N2); |
1472 | 915 | ShuffleVectorSDNode::commuteMask(M); |
1473 | 915 | } |
1474 | | |
1475 | | SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, |
1476 | 106k | SDValue N2, ArrayRef<int> Mask) { |
1477 | 106k | assert(VT.getVectorNumElements() == Mask.size() && |
1478 | 106k | "Must have the same number of vector elements as mask elements!"); |
1479 | 106k | assert(VT == N1.getValueType() && VT == N2.getValueType() && |
1480 | 106k | "Invalid VECTOR_SHUFFLE"); |
1481 | 106k | |
1482 | 106k | // Canonicalize shuffle undef, undef -> undef |
1483 | 106k | if (N1.isUndef() && 106k N2.isUndef()352 ) |
1484 | 140 | return getUNDEF(VT); |
1485 | 106k | |
1486 | 106k | // Validate that all indices in Mask are within the range of the elements |
1487 | 106k | // input to the shuffle. |
1488 | 106k | int NElts = Mask.size(); |
1489 | 106k | assert(llvm::all_of(Mask, [&](int M) { return M < (NElts * 2); }) && |
1490 | 106k | "Index out of range"); |
1491 | 106k | |
1492 | 106k | // Copy the mask so we can do any needed cleanup. |
1493 | 106k | SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end()); |
1494 | 106k | |
1495 | 106k | // Canonicalize shuffle v, v -> v, undef |
1496 | 106k | if (N1 == N2106k ) { |
1497 | 3.52k | N2 = getUNDEF(VT); |
1498 | 32.6k | for (int i = 0; i != NElts32.6k ; ++i29.1k ) |
1499 | 29.1k | if (29.1k MaskVec[i] >= NElts29.1k ) MaskVec[i] -= NElts1.54k ; |
1500 | 3.52k | } |
1501 | 106k | |
1502 | 106k | // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. |
1503 | 106k | if (N1.isUndef()) |
1504 | 212 | commuteShuffle(N1, N2, MaskVec); |
1505 | 106k | |
1506 | 106k | // If shuffling a splat, try to blend the splat instead. We do this here so |
1507 | 106k | // that even when this arises during lowering we don't have to re-handle it. |
1508 | 10.0k | auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) { |
1509 | 10.0k | BitVector UndefElements; |
1510 | 10.0k | SDValue Splat = BV->getSplatValue(&UndefElements); |
1511 | 10.0k | if (!Splat) |
1512 | 862 | return; |
1513 | 9.20k | |
1514 | 122k | for (int i = 0; 9.20k i < NElts122k ; ++i113k ) { |
1515 | 113k | if (MaskVec[i] < Offset || 113k MaskVec[i] >= (Offset + NElts)88.6k ) |
1516 | 36.6k | continue; |
1517 | 77.0k | |
1518 | 77.0k | // If this input comes from undef, mark it as such. |
1519 | 77.0k | if (77.0k UndefElements[MaskVec[i] - Offset]77.0k ) { |
1520 | 25 | MaskVec[i] = -1; |
1521 | 25 | continue; |
1522 | 25 | } |
1523 | 76.9k | |
1524 | 76.9k | // If we can blend a non-undef lane, use that instead. |
1525 | 76.9k | if (76.9k !UndefElements[i]76.9k ) |
1526 | 75.5k | MaskVec[i] = i + Offset; |
1527 | 113k | } |
1528 | 10.0k | }; |
1529 | 106k | if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1)) |
1530 | 4.59k | BlendSplat(N1BV, 0); |
1531 | 106k | if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2)) |
1532 | 5.47k | BlendSplat(N2BV, NElts); |
1533 | 106k | |
1534 | 106k | // Canonicalize all index into lhs, -> shuffle lhs, undef |
1535 | 106k | // Canonicalize all index into rhs, -> shuffle rhs, undef |
1536 | 106k | bool AllLHS = true, AllRHS = true; |
1537 | 106k | bool N2Undef = N2.isUndef(); |
1538 | 960k | for (int i = 0; i != NElts960k ; ++i854k ) { |
1539 | 854k | if (MaskVec[i] >= NElts854k ) { |
1540 | 141k | if (N2Undef) |
1541 | 2.82k | MaskVec[i] = -1; |
1542 | 141k | else |
1543 | 139k | AllLHS = false; |
1544 | 854k | } else if (712k MaskVec[i] >= 0712k ) { |
1545 | 526k | AllRHS = false; |
1546 | 526k | } |
1547 | 854k | } |
1548 | 106k | if (AllLHS && 106k AllRHS70.8k ) |
1549 | 45 | return getUNDEF(VT); |
1550 | 106k | if (106k AllLHS && 106k !N2Undef70.7k ) |
1551 | 2.66k | N2 = getUNDEF(VT); |
1552 | 106k | if (AllRHS106k ) { |
1553 | 703 | N1 = getUNDEF(VT); |
1554 | 703 | commuteShuffle(N1, N2, MaskVec); |
1555 | 703 | } |
1556 | 106k | // Reset our undef status after accounting for the mask. |
1557 | 106k | N2Undef = N2.isUndef(); |
1558 | 106k | // Re-check whether both sides ended up undef. |
1559 | 106k | if (N1.isUndef() && 106k N2Undef0 ) |
1560 | 0 | return getUNDEF(VT); |
1561 | 106k | |
1562 | 106k | // If Identity shuffle return that node. |
1563 | 106k | bool Identity = true, AllSame = true; |
1564 | 960k | for (int i = 0; i != NElts960k ; ++i854k ) { |
1565 | 854k | if (MaskVec[i] >= 0 && 854k MaskVec[i] != i665k ) Identity = false364k ; |
1566 | 854k | if (MaskVec[i] != MaskVec[0]854k ) AllSame = false645k ; |
1567 | 854k | } |
1568 | 106k | if (Identity && 106k NElts31.8k ) |
1569 | 31.8k | return N1; |
1570 | 74.2k | |
1571 | 74.2k | // Shuffling a constant splat doesn't change the result. |
1572 | 74.2k | if (74.2k N2Undef74.2k ) { |
1573 | 39.6k | SDValue V = N1; |
1574 | 39.6k | |
1575 | 39.6k | // Look through any bitcasts. We check that these don't change the number |
1576 | 39.6k | // (and size) of elements and just changes their types. |
1577 | 50.2k | while (V.getOpcode() == ISD::BITCAST) |
1578 | 10.5k | V = V->getOperand(0); |
1579 | 39.6k | |
1580 | 39.6k | // A splat should always show up as a build vector node. |
1581 | 39.6k | if (auto *BV39.6k = dyn_cast<BuildVectorSDNode>(V)) { |
1582 | 835 | BitVector UndefElements; |
1583 | 835 | SDValue Splat = BV->getSplatValue(&UndefElements); |
1584 | 835 | // If this is a splat of an undef, shuffling it is also undef. |
1585 | 835 | if (Splat && 835 Splat.isUndef()113 ) |
1586 | 0 | return getUNDEF(VT); |
1587 | 835 | |
1588 | 835 | bool SameNumElts = |
1589 | 835 | V.getValueType().getVectorNumElements() == VT.getVectorNumElements(); |
1590 | 835 | |
1591 | 835 | // We only have a splat which can skip shuffles if there is a splatted |
1592 | 835 | // value and no undef lanes rearranged by the shuffle. |
1593 | 835 | if (Splat && 835 UndefElements.none()113 ) { |
1594 | 25 | // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the |
1595 | 25 | // number of elements match or the value splatted is a zero constant. |
1596 | 25 | if (SameNumElts) |
1597 | 4 | return N1; |
1598 | 21 | if (auto *21 C21 = dyn_cast<ConstantSDNode>(Splat)) |
1599 | 20 | if (20 C->isNullValue()20 ) |
1600 | 13 | return N1; |
1601 | 818 | } |
1602 | 818 | |
1603 | 818 | // If the shuffle itself creates a splat, build the vector directly. |
1604 | 818 | if (818 AllSame && 818 SameNumElts289 ) { |
1605 | 249 | EVT BuildVT = BV->getValueType(0); |
1606 | 249 | const SDValue &Splatted = BV->getOperand(MaskVec[0]); |
1607 | 249 | SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted); |
1608 | 249 | |
1609 | 249 | // We may have jumped through bitcasts, so the type of the |
1610 | 249 | // BUILD_VECTOR may not match the type of the shuffle. |
1611 | 249 | if (BuildVT != VT) |
1612 | 0 | NewBV = getNode(ISD::BITCAST, dl, VT, NewBV); |
1613 | 249 | return NewBV; |
1614 | 249 | } |
1615 | 73.9k | } |
1616 | 39.6k | } |
1617 | 73.9k | |
1618 | 73.9k | FoldingSetNodeID ID; |
1619 | 73.9k | SDValue Ops[2] = { N1, N2 }; |
1620 | 73.9k | AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops); |
1621 | 791k | for (int i = 0; i != NElts791k ; ++i717k ) |
1622 | 717k | ID.AddInteger(MaskVec[i]); |
1623 | 73.9k | |
1624 | 73.9k | void* IP = nullptr; |
1625 | 73.9k | if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
1626 | 3.16k | return SDValue(E, 0); |
1627 | 70.7k | |
1628 | 70.7k | // Allocate the mask array for the node out of the BumpPtrAllocator, since |
1629 | 70.7k | // SDNode doesn't have access to it. This memory will be "leaked" when |
1630 | 70.7k | // the node is deallocated, but recovered when the NodeAllocator is released. |
1631 | 70.7k | int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); |
1632 | 70.7k | std::copy(MaskVec.begin(), MaskVec.end(), MaskAlloc); |
1633 | 70.7k | |
1634 | 70.7k | auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(), |
1635 | 70.7k | dl.getDebugLoc(), MaskAlloc); |
1636 | 70.7k | createOperands(N, Ops); |
1637 | 70.7k | |
1638 | 70.7k | CSEMap.InsertNode(N, IP); |
1639 | 70.7k | InsertNode(N); |
1640 | 70.7k | return SDValue(N, 0); |
1641 | 70.7k | } |
1642 | | |
1643 | 4.32k | SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) { |
1644 | 4.32k | MVT VT = SV.getSimpleValueType(0); |
1645 | 4.32k | SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end()); |
1646 | 4.32k | ShuffleVectorSDNode::commuteMask(MaskVec); |
1647 | 4.32k | |
1648 | 4.32k | SDValue Op0 = SV.getOperand(0); |
1649 | 4.32k | SDValue Op1 = SV.getOperand(1); |
1650 | 4.32k | return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec); |
1651 | 4.32k | } |
1652 | | |
1653 | 23.6M | SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { |
1654 | 23.6M | FoldingSetNodeID ID; |
1655 | 23.6M | AddNodeIDNode(ID, ISD::Register, getVTList(VT), None); |
1656 | 23.6M | ID.AddInteger(RegNo); |
1657 | 23.6M | void *IP = nullptr; |
1658 | 23.6M | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1659 | 9.48M | return SDValue(E, 0); |
1660 | 14.1M | |
1661 | 14.1M | auto *N = newSDNode<RegisterSDNode>(RegNo, VT); |
1662 | 14.1M | CSEMap.InsertNode(N, IP); |
1663 | 14.1M | InsertNode(N); |
1664 | 14.1M | return SDValue(N, 0); |
1665 | 14.1M | } |
1666 | | |
1667 | 1.79M | SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { |
1668 | 1.79M | FoldingSetNodeID ID; |
1669 | 1.79M | AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None); |
1670 | 1.79M | ID.AddPointer(RegMask); |
1671 | 1.79M | void *IP = nullptr; |
1672 | 1.79M | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1673 | 697k | return SDValue(E, 0); |
1674 | 1.09M | |
1675 | 1.09M | auto *N = newSDNode<RegisterMaskSDNode>(RegMask); |
1676 | 1.09M | CSEMap.InsertNode(N, IP); |
1677 | 1.09M | InsertNode(N); |
1678 | 1.09M | return SDValue(N, 0); |
1679 | 1.09M | } |
1680 | | |
1681 | | SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root, |
1682 | 37.0k | MCSymbol *Label) { |
1683 | 37.0k | return getLabelNode(ISD::EH_LABEL, dl, Root, Label); |
1684 | 37.0k | } |
1685 | | |
1686 | | SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl, |
1687 | 37.0k | SDValue Root, MCSymbol *Label) { |
1688 | 37.0k | FoldingSetNodeID ID; |
1689 | 37.0k | SDValue Ops[] = { Root }; |
1690 | 37.0k | AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops); |
1691 | 37.0k | ID.AddPointer(Label); |
1692 | 37.0k | void *IP = nullptr; |
1693 | 37.0k | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1694 | 0 | return SDValue(E, 0); |
1695 | 37.0k | |
1696 | 37.0k | auto *N = newSDNode<LabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label); |
1697 | 37.0k | createOperands(N, Ops); |
1698 | 37.0k | |
1699 | 37.0k | CSEMap.InsertNode(N, IP); |
1700 | 37.0k | InsertNode(N); |
1701 | 37.0k | return SDValue(N, 0); |
1702 | 37.0k | } |
1703 | | |
1704 | | SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, |
1705 | | int64_t Offset, |
1706 | | bool isTarget, |
1707 | 246 | unsigned char TargetFlags) { |
1708 | 246 | unsigned Opc = isTarget ? ISD::TargetBlockAddress136 : ISD::BlockAddress110 ; |
1709 | 246 | |
1710 | 246 | FoldingSetNodeID ID; |
1711 | 246 | AddNodeIDNode(ID, Opc, getVTList(VT), None); |
1712 | 246 | ID.AddPointer(BA); |
1713 | 246 | ID.AddInteger(Offset); |
1714 | 246 | ID.AddInteger(TargetFlags); |
1715 | 246 | void *IP = nullptr; |
1716 | 246 | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1717 | 2 | return SDValue(E, 0); |
1718 | 244 | |
1719 | 244 | auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags); |
1720 | 244 | CSEMap.InsertNode(N, IP); |
1721 | 244 | InsertNode(N); |
1722 | 244 | return SDValue(N, 0); |
1723 | 244 | } |
1724 | | |
1725 | 1.21k | SDValue SelectionDAG::getSrcValue(const Value *V) { |
1726 | 1.21k | assert((!V || V->getType()->isPointerTy()) && |
1727 | 1.21k | "SrcValue is not a pointer?"); |
1728 | 1.21k | |
1729 | 1.21k | FoldingSetNodeID ID; |
1730 | 1.21k | AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None); |
1731 | 1.21k | ID.AddPointer(V); |
1732 | 1.21k | |
1733 | 1.21k | void *IP = nullptr; |
1734 | 1.21k | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1735 | 365 | return SDValue(E, 0); |
1736 | 849 | |
1737 | 849 | auto *N = newSDNode<SrcValueSDNode>(V); |
1738 | 849 | CSEMap.InsertNode(N, IP); |
1739 | 849 | InsertNode(N); |
1740 | 849 | return SDValue(N, 0); |
1741 | 849 | } |
1742 | | |
1743 | 11.8k | SDValue SelectionDAG::getMDNode(const MDNode *MD) { |
1744 | 11.8k | FoldingSetNodeID ID; |
1745 | 11.8k | AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None); |
1746 | 11.8k | ID.AddPointer(MD); |
1747 | 11.8k | |
1748 | 11.8k | void *IP = nullptr; |
1749 | 11.8k | if (SDNode *E = FindNodeOrInsertPos(ID, IP)) |
1750 | 0 | return SDValue(E, 0); |
1751 | 11.8k | |
1752 | 11.8k | auto *N = newSDNode<MDNodeSDNode>(MD); |
1753 | 11.8k | CSEMap.InsertNode(N, IP); |
1754 | 11.8k | InsertNode(N); |
1755 | 11.8k | return SDValue(N, 0); |
1756 | 11.8k | } |
1757 | | |
1758 | 176k | SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) { |
1759 | 176k | if (VT == V.getValueType()) |
1760 | 60.4k | return V; |
1761 | 116k | |
1762 | 116k | return getNode(ISD::BITCAST, SDLoc(V), VT, V); |
1763 | 116k | } |
1764 | | |
1765 | | SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, |
1766 | 199 | unsigned SrcAS, unsigned DestAS) { |
1767 | 199 | SDValue Ops[] = {Ptr}; |
1768 | 199 | FoldingSetNodeID ID; |
1769 | 199 | AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops); |
1770 | 199 | ID.AddInteger(SrcAS); |
1771 | 199 | ID.AddInteger(DestAS); |
1772 | 199 | |
1773 | 199 | void *IP = nullptr; |
1774 | 199 | if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
1775 | 0 | return SDValue(E, 0); |
1776 | 199 | |
1777 | 199 | auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(), |
1778 | 199 | VT, SrcAS, DestAS); |
1779 | 199 | createOperands(N, Ops); |
1780 | 199 | |
1781 | 199 | CSEMap.InsertNode(N, IP); |
1782 | 199 | InsertNode(N); |
1783 | 199 | return SDValue(N, 0); |
1784 | 199 | } |
1785 | | |
1786 | | /// getShiftAmountOperand - Return the specified value casted to |
1787 | | /// the target's desired shift amount type. |
1788 | 942k | SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { |
1789 | 942k | EVT OpTy = Op.getValueType(); |
1790 | 942k | EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout()); |
1791 | 942k | if (OpTy == ShTy || 942k OpTy.isVector()15.1k ) return Op927k ; |
1792 | 15.1k | |
1793 | 15.1k | return getZExtOrTrunc(Op, SDLoc(Op), ShTy); |
1794 | 15.1k | } |
1795 | | |
1796 | 27 | SDValue SelectionDAG::expandVAArg(SDNode *Node) { |
1797 | 27 | SDLoc dl(Node); |
1798 | 27 | const TargetLowering &TLI = getTargetLoweringInfo(); |
1799 | 27 | const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue(); |
1800 | 27 | EVT VT = Node->getValueType(0); |
1801 | 27 | SDValue Tmp1 = Node->getOperand(0); |
1802 | 27 | SDValue Tmp2 = Node->getOperand(1); |
1803 | 27 | unsigned Align = Node->getConstantOperandVal(3); |
1804 | 27 | |
1805 | 27 | SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1, |
1806 | 27 | Tmp2, MachinePointerInfo(V)); |
1807 | 27 | SDValue VAList = VAListLoad; |
1808 | 27 | |
1809 | 27 | if (Align > TLI.getMinStackArgumentAlignment()27 ) { |
1810 | 11 | assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2"); |
1811 | 11 | |
1812 | 11 | VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, |
1813 | 11 | getConstant(Align - 1, dl, VAList.getValueType())); |
1814 | 11 | |
1815 | 11 | VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList, |
1816 | 11 | getConstant(-(int64_t)Align, dl, VAList.getValueType())); |
1817 | 11 | } |
1818 | 27 | |
1819 | 27 | // Increment the pointer, VAList, to the next vaarg |
1820 | 27 | Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList, |
1821 | 27 | getConstant(getDataLayout().getTypeAllocSize( |
1822 | 27 | VT.getTypeForEVT(*getContext())), |
1823 | 27 | dl, VAList.getValueType())); |
1824 | 27 | // Store the incremented VAList to the legalized pointer |
1825 | 27 | Tmp1 = |
1826 | 27 | getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V)); |
1827 | 27 | // Load the actual argument out of the pointer VAList |
1828 | 27 | return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo()); |
1829 | 27 | } |
1830 | | |
1831 | 35 | SDValue SelectionDAG::expandVACopy(SDNode *Node) { |
1832 | 35 | SDLoc dl(Node); |
1833 | 35 | const TargetLowering &TLI = getTargetLoweringInfo(); |
1834 | 35 | // This defaults to loading a pointer from the input and storing it to the |
1835 | 35 | // output, returning the chain. |
1836 | 35 | const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue(); |
1837 | 35 | const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue(); |
1838 | 35 | SDValue Tmp1 = |
1839 | 35 | getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0), |
1840 | 35 | Node->getOperand(2), MachinePointerInfo(VS)); |
1841 | 35 | return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1), |
1842 | 35 | MachinePointerInfo(VD)); |
1843 | 35 | } |
1844 | | |
1845 | 2.62k | SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { |
1846 | 2.62k | MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); |
1847 | 2.62k | unsigned ByteSize = VT.getStoreSize(); |
1848 | 2.62k | Type *Ty = VT.getTypeForEVT(*getContext()); |
1849 | 2.62k | unsigned StackAlign = |
1850 | 2.62k | std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign); |
1851 | 2.62k | |
1852 | 2.62k | int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false); |
1853 | 2.62k | return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); |
1854 | 2.62k | } |
1855 | | |
1856 | 3.92k | SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { |
1857 | 3.92k | unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize()); |
1858 | 3.92k | Type *Ty1 = VT1.getTypeForEVT(*getContext()); |
1859 | 3.92k | Type *Ty2 = VT2.getTypeForEVT(*getContext()); |
1860 | 3.92k | const DataLayout &DL = getDataLayout(); |
1861 | 3.92k | unsigned Align = |
1862 | 3.92k | std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2)); |
1863 | 3.92k | |
1864 | 3.92k | MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); |
1865 | 3.92k | int FrameIdx = MFI.CreateStackObject(Bytes, Align, false); |
1866 | 3.92k | return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout())); |
1867 | 3.92k | } |
1868 | | |
1869 | | SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, |
1870 | 3.50M | ISD::CondCode Cond, const SDLoc &dl) { |
1871 | 3.50M | // These setcc operations always fold. |
1872 | 3.50M | switch (Cond) { |
1873 | 3.44M | default: break; |
1874 | 37 | case ISD::SETFALSE: |
1875 | 37 | case ISD::SETFALSE2: return getConstant(0, dl, VT); |
1876 | 42 | case ISD::SETTRUE: |
1877 | 42 | case ISD::SETTRUE2: { |
1878 | 42 | TargetLowering::BooleanContent Cnt = |
1879 | 42 | TLI->getBooleanContents(N1->getValueType(0)); |
1880 | 42 | return getConstant( |
1881 | 42 | Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL21 : 121 , dl, |
1882 | 42 | VT); |
1883 | 42 | } |
1884 | 42 | |
1885 | 54.5k | case ISD::SETOEQ: |
1886 | 54.5k | case ISD::SETOGT: |
1887 | 54.5k | case ISD::SETOGE: |
1888 | 54.5k | case ISD::SETOLT: |
1889 | 54.5k | case ISD::SETOLE: |
1890 | 54.5k | case ISD::SETONE: |
1891 | 54.5k | case ISD::SETO: |
1892 | 54.5k | case ISD::SETUO: |
1893 | 54.5k | case ISD::SETUEQ: |
1894 | 54.5k | case ISD::SETUNE: |
1895 | 54.5k | assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); |
1896 | 54.5k | break; |
1897 | 3.50M | } |
1898 | 3.50M | |
1899 | 3.50M | if (ConstantSDNode *3.50M N2C3.50M = dyn_cast<ConstantSDNode>(N2)) { |
1900 | 2.71M | const APInt &C2 = N2C->getAPIntValue(); |
1901 | 2.71M | if (ConstantSDNode *N1C2.71M = dyn_cast<ConstantSDNode>(N1)) { |
1902 | 1.81k | const APInt &C1 = N1C->getAPIntValue(); |
1903 | 1.81k | |
1904 | 1.81k | switch (Cond) { |
1905 | 0 | default: 0 llvm_unreachable0 ("Unknown integer setcc!"); |
1906 | 218 | case ISD::SETEQ: return getConstant(C1 == C2, dl, VT); |
1907 | 120 | case ISD::SETNE: return getConstant(C1 != C2, dl, VT); |
1908 | 198 | case ISD::SETULT: return getConstant(C1.ult(C2), dl, VT); |
1909 | 222 | case ISD::SETUGT: return getConstant(C1.ugt(C2), dl, VT); |
1910 | 194 | case ISD::SETULE: return getConstant(C1.ule(C2), dl, VT); |
1911 | 176 | case ISD::SETUGE: return getConstant(C1.uge(C2), dl, VT); |
1912 | 196 | case ISD::SETLT: return getConstant(C1.slt(C2), dl, VT); |
1913 | 194 | case ISD::SETGT: return getConstant(C1.sgt(C2), dl, VT); |
1914 | 143 | case ISD::SETLE: return getConstant(C1.sle(C2), dl, VT); |
1915 | 153 | case ISD::SETGE: return getConstant(C1.sge(C2), dl, VT); |
1916 | 3.49M | } |
1917 | 3.49M | } |
1918 | 2.71M | } |
1919 | 3.49M | if (ConstantFPSDNode *3.49M N1C3.49M = dyn_cast<ConstantFPSDNode>(N1)) { |
1920 | 449 | if (ConstantFPSDNode *N2C449 = dyn_cast<ConstantFPSDNode>(N2)) { |
1921 | 230 | APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); |
1922 | 230 | switch (Cond) { |
1923 | 0 | default: break; |
1924 | 4 | case ISD::SETEQ: if (4 R==APFloat::cmpUnordered4 ) |
1925 | 0 | return getUNDEF(VT); |
1926 | 4 | LLVM_FALLTHROUGH4 ; |
1927 | 91 | case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, dl, VT); |
1928 | 4 | case ISD::SETNE: if (4 R==APFloat::cmpUnordered4 ) |
1929 | 0 | return getUNDEF(VT); |
1930 | 4 | LLVM_FALLTHROUGH4 ; |
1931 | 4 | case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || |
1932 | 0 | R==APFloat::cmpLessThan, dl, VT); |
1933 | 0 | case ISD::SETLT: if (0 R==APFloat::cmpUnordered0 ) |
1934 | 0 | return getUNDEF(VT); |
1935 | 0 | LLVM_FALLTHROUGH0 ; |
1936 | 22 | case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, dl, VT); |
1937 | 0 | case ISD::SETGT: if (0 R==APFloat::cmpUnordered0 ) |
1938 | 0 | return getUNDEF(VT); |
1939 | 0 | LLVM_FALLTHROUGH0 ; |
1940 | 2 | case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, dl, VT); |
1941 | 0 | case ISD::SETLE: if (0 R==APFloat::cmpUnordered0 ) |
1942 | 0 | return getUNDEF(VT); |
1943 | 0 | LLVM_FALLTHROUGH0 ; |
1944 | 4 | case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || |
1945 | 4 | R==APFloat::cmpEqual, dl, VT); |
1946 | 0 | case ISD::SETGE: if (0 R==APFloat::cmpUnordered0 ) |
1947 | 0 | return getUNDEF(VT); |
1948 | 0 | LLVM_FALLTHROUGH0 ; |
1949 | 5 | case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || |
1950 | 5 | R==APFloat::cmpEqual, dl, VT); |
1951 | 8 | case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, dl, VT); |
1952 | 31 | case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, dl, VT); |
1953 | 0 | case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || |
1954 | 0 | R==APFloat::cmpEqual, dl, VT); |
1955 | 18 | case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, dl, VT); |
1956 | 6 | case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || |
1957 | 6 | R==APFloat::cmpLessThan, dl, VT); |
1958 | 36 | case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || |
1959 | 36 | R==APFloat::cmpUnordered, dl, VT); |
1960 | 1 | case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, dl, VT); |
1961 | 2 | case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, dl, VT); |
1962 | 449 | } |
1963 | 219 | } else { |
1964 | 219 | // Ensure that the constant occurs on the RHS. |
1965 | 219 | ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); |
1966 | 219 | MVT CompVT = N1.getValueType().getSimpleVT(); |
1967 | 219 | if (!TLI->isCondCodeLegal(SwappedCond, CompVT)) |
1968 | 103 | return SDValue(); |
1969 | 116 | |
1970 | 116 | return getSetCC(dl, VT, N2, N1, SwappedCond); |
1971 | 116 | } |
1972 | 449 | } |
1973 | 3.49M | |
1974 | 3.49M | // Could not fold it. |
1975 | 3.49M | return SDValue(); |
1976 | 3.49M | } |
1977 | | |
1978 | | /// See if the specified operand can be simplified with the knowledge that only |
1979 | | /// the bits specified by Mask are used. |
1980 | 1.86M | SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) { |
1981 | 1.86M | switch (V.getOpcode()) { |
1982 | 1.35M | default: |
1983 | 1.35M | break; |
1984 | 344k | case ISD::Constant: { |
1985 | 344k | const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode()); |
1986 | 344k | assert(CV && "Const value should be ConstSDNode."); |
1987 | 344k | const APInt &CVal = CV->getAPIntValue(); |
1988 | 344k | APInt NewVal = CVal & Mask; |
1989 | 344k | if (NewVal != CVal) |
1990 | 786 | return getConstant(NewVal, SDLoc(V), V.getValueType()); |
1991 | 343k | break; |
1992 | 343k | } |
1993 | 23.1k | case ISD::OR: |
1994 | 23.1k | case ISD::XOR: |
1995 | 23.1k | // If the LHS or RHS don't contribute bits to the or, drop them. |
1996 | 23.1k | if (MaskedValueIsZero(V.getOperand(0), Mask)) |
1997 | 712 | return V.getOperand(1); |
1998 | 22.4k | if (22.4k MaskedValueIsZero(V.getOperand(1), Mask)22.4k ) |
1999 | 942 | return V.getOperand(0); |
2000 | 21.4k | break; |
2001 | 122k | case ISD::SRL: |
2002 | 122k | // Only look at single-use SRLs. |
2003 | 122k | if (!V.getNode()->hasOneUse()) |
2004 | 13.1k | break; |
2005 | 109k | if (ConstantSDNode *109k RHSC109k = dyn_cast<ConstantSDNode>(V.getOperand(1))) { |
2006 | 108k | // See if we can recursively simplify the LHS. |
2007 | 108k | unsigned Amt = RHSC->getZExtValue(); |
2008 | 108k | |
2009 | 108k | // Watch out for shift count overflow though. |
2010 | 108k | if (Amt >= Mask.getBitWidth()) |
2011 | 1 | break; |
2012 | 108k | APInt NewMask = Mask << Amt; |
2013 | 108k | if (SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask)) |
2014 | 1.56k | return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS, |
2015 | 1.56k | V.getOperand(1)); |
2016 | 107k | } |
2017 | 107k | break; |
2018 | 12.8k | case ISD::AND: { |
2019 | 12.8k | // X & -1 -> X (ignoring bits which aren't demanded). |
2020 | 12.8k | ConstantSDNode *AndVal = isConstOrConstSplat(V.getOperand(1)); |
2021 | 12.8k | if (AndVal && 12.8k Mask.isSubsetOf(AndVal->getAPIntValue())12.0k ) |
2022 | 1.79k | return V.getOperand(0); |
2023 | 11.0k | break; |
2024 | 11.0k | } |
2025 | 12.8k | case ISD::ANY_EXTEND: { |
2026 | 12.8k | SDValue Src = V.getOperand(0); |
2027 | 12.8k | unsigned SrcBitWidth = Src.getScalarValueSizeInBits(); |
2028 | 12.8k | // Being conservative here - only peek through if we only demand bits in the |
2029 | 12.8k | // non-extended source (even though the extended bits are technically undef). |
2030 | 12.8k | if (Mask.getActiveBits() > SrcBitWidth) |
2031 | 30 | break; |
2032 | 12.8k | APInt SrcMask = Mask.trunc(SrcBitWidth); |
2033 | 12.8k | if (SDValue DemandedSrc = GetDemandedBits(Src, SrcMask)) |
2034 | 151 | return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc); |
2035 | 12.6k | break; |
2036 | 12.6k | } |
2037 | 1.86M | } |
2038 | 1.86M | return SDValue(); |
2039 | 1.86M | } |
2040 | | |
2041 | | /// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We |
2042 | | /// use this predicate to simplify operations downstream. |
2043 | 507k | bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { |
2044 | 507k | unsigned BitWidth = Op.getScalarValueSizeInBits(); |
2045 | 507k | return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth); |
2046 | 507k | } |
2047 | | |
2048 | | /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use |
2049 | | /// this predicate to simplify operations downstream. Mask is known to be zero |
2050 | | /// for bits that V cannot have. |
2051 | | bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, |
2052 | 3.71M | unsigned Depth) const { |
2053 | 3.71M | KnownBits Known; |
2054 | 3.71M | computeKnownBits(Op, Known, Depth); |
2055 | 3.71M | return Mask.isSubsetOf(Known.Zero); |
2056 | 3.71M | } |
2057 | | |
2058 | | /// If a SHL/SRA/SRL node has a constant or splat constant shift amount that |
2059 | | /// is less than the element bit-width of the shift node, return it. |
2060 | 4.95M | static const APInt *getValidShiftAmountConstant(SDValue V) { |
2061 | 4.95M | if (ConstantSDNode *SA4.95M = isConstOrConstSplat(V.getOperand(1))) { |
2062 | 4.76M | // Shifting more than the bitwidth is not valid. |
2063 | 4.76M | const APInt &ShAmt = SA->getAPIntValue(); |
2064 | 4.76M | if (ShAmt.ult(V.getScalarValueSizeInBits())) |
2065 | 4.76M | return &ShAmt; |
2066 | 193k | } |
2067 | 193k | return nullptr; |
2068 | 193k | } |
2069 | | |
2070 | | /// Determine which bits of Op are known to be either zero or one and return |
2071 | | /// them in Known. For vectors, the known bits are those that are shared by |
2072 | | /// every vector element. |
2073 | | void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known, |
2074 | 54.7M | unsigned Depth) const { |
2075 | 54.7M | EVT VT = Op.getValueType(); |
2076 | 54.7M | APInt DemandedElts = VT.isVector() |
2077 | 751k | ? APInt::getAllOnesValue(VT.getVectorNumElements()) |
2078 | 54.0M | : APInt(1, 1); |
2079 | 54.7M | computeKnownBits(Op, Known, DemandedElts, Depth); |
2080 | 54.7M | } |
2081 | | |
2082 | | /// Determine which bits of Op are known to be either zero or one and return |
2083 | | /// them in Known. The DemandedElts argument allows us to only collect the known |
2084 | | /// bits that are shared by the requested vector elements. |
2085 | | void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known, |
2086 | | const APInt &DemandedElts, |
2087 | 107M | unsigned Depth) const { |
2088 | 107M | unsigned BitWidth = Op.getScalarValueSizeInBits(); |
2089 | 107M | |
2090 | 107M | Known = KnownBits(BitWidth); // Don't know anything. |
2091 | 107M | if (Depth == 6) |
2092 | 2.73M | return; // Limit search depth. |
2093 | 104M | |
2094 | 104M | KnownBits Known2; |
2095 | 104M | unsigned NumElts = DemandedElts.getBitWidth(); |
2096 | 104M | |
2097 | 104M | if (!DemandedElts) |
2098 | 0 | return; // No demanded elts, better to assume we don't know anything. |
2099 | 104M | |
2100 | 104M | unsigned Opcode = Op.getOpcode(); |
2101 | 104M | switch (Opcode) { |
2102 | 23.0M | case ISD::Constant: |
2103 | 23.0M | // We know all of the bits for a constant! |
2104 | 23.0M | Known.One = cast<ConstantSDNode>(Op)->getAPIntValue(); |
2105 | 23.0M | Known.Zero = ~Known.One; |
2106 | 23.0M | break; |
2107 | 377k | case ISD::BUILD_VECTOR: |
2108 | 377k | // Collect the known bits that are shared by every demanded vector element. |
2109 | 377k | assert(NumElts == Op.getValueType().getVectorNumElements() && |
2110 | 377k | "Unexpected vector size"); |
2111 | 377k | Known.Zero.setAllBits(); Known.One.setAllBits(); |
2112 | 2.22M | for (unsigned i = 0, e = Op.getNumOperands(); i != e2.22M ; ++i1.84M ) { |
2113 | 2.00M | if (!DemandedElts[i]) |
2114 | 1.15M | continue; |
2115 | 853k | |
2116 | 853k | SDValue SrcOp = Op.getOperand(i); |
2117 | 853k | computeKnownBits(SrcOp, Known2, Depth + 1); |
2118 | 853k | |
2119 | 853k | // BUILD_VECTOR can implicitly truncate sources, we must handle this. |
2120 | 853k | if (SrcOp.getValueSizeInBits() != BitWidth853k ) { |
2121 | 51.0k | assert(SrcOp.getValueSizeInBits() > BitWidth && |
2122 | 51.0k | "Expected BUILD_VECTOR implicit truncation"); |
2123 | 51.0k | Known2 = Known2.trunc(BitWidth); |
2124 | 51.0k | } |
2125 | 853k | |
2126 | 853k | // Known bits are the values that are shared by every demanded element. |
2127 | 853k | Known.One &= Known2.One; |
2128 | 853k | Known.Zero &= Known2.Zero; |
2129 | 853k | |
2130 | 853k | // If we don't know any bits, early out. |
2131 | 853k | if (!Known.One && 853k !Known.Zero311k ) |
2132 | 164k | break; |
2133 | 2.00M | } |
2134 | 377k | break; |
2135 | 32.9k | case ISD::VECTOR_SHUFFLE: { |
2136 | 32.9k | // Collect the known bits that are shared by every vector element referenced |
2137 | 32.9k | // by the shuffle. |
2138 | 32.9k | APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0); |
2139 | 32.9k | Known.Zero.setAllBits(); Known.One.setAllBits(); |
2140 | 32.9k | const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); |
2141 | 32.9k | assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); |
2142 | 405k | for (unsigned i = 0; i != NumElts405k ; ++i372k ) { |
2143 | 377k | if (!DemandedElts[i]) |
2144 | 205k | continue; |
2145 | 172k | |
2146 | 172k | int M = SVN->getMaskElt(i); |
2147 | 172k | if (M < 0172k ) { |
2148 | 4.95k | // For UNDEF elements, we don't know anything about the common state of |
2149 | 4.95k | // the shuffle result. |
2150 | 4.95k | Known.resetAll(); |
2151 | 4.95k | DemandedLHS.clearAllBits(); |
2152 | 4.95k | DemandedRHS.clearAllBits(); |
2153 | 4.95k | break; |
2154 | 4.95k | } |
2155 | 167k | |
2156 | 167k | if (167k (unsigned)M < NumElts167k ) |
2157 | 125k | DemandedLHS.setBit((unsigned)M % NumElts); |
2158 | 167k | else |
2159 | 41.3k | DemandedRHS.setBit((unsigned)M % NumElts); |
2160 | 377k | } |
2161 | 32.9k | // Known bits are the values that are shared by every demanded element. |
2162 | 32.9k | if (!!DemandedLHS32.9k ) { |
2163 | 24.9k | SDValue LHS = Op.getOperand(0); |
2164 | 24.9k | computeKnownBits(LHS, Known2, DemandedLHS, Depth + 1); |
2165 | 24.9k | Known.One &= Known2.One; |
2166 | 24.9k | Known.Zero &= Known2.Zero; |
2167 | 24.9k | } |
2168 | 32.9k | // If we don't know any bits, early out. |
2169 | 32.9k | if (!Known.One && 32.9k !Known.Zero29.8k ) |
2170 | 28.7k | break; |
2171 | 4.18k | if (4.18k !!DemandedRHS4.18k ) { |
2172 | 3.18k | SDValue RHS = Op.getOperand(1); |
2173 | 3.18k | computeKnownBits(RHS, Known2, DemandedRHS, Depth + 1); |
2174 | 3.18k | Known.One &= Known2.One; |
2175 | 3.18k | Known.Zero &= Known2.Zero; |
2176 | 3.18k | } |
2177 | 4.18k | break; |
2178 | 4.18k | } |
2179 | 21.9k | case ISD::CONCAT_VECTORS: { |
2180 | 21.9k | // Split DemandedElts and test each of the demanded subvectors. |
2181 | 21.9k | Known.Zero.setAllBits(); Known.One.setAllBits(); |
2182 | 21.9k | EVT SubVectorVT = Op.getOperand(0).getValueType(); |
2183 | 21.9k | unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); |
2184 | 21.9k | unsigned NumSubVectors = Op.getNumOperands(); |
2185 | 29.8k | for (unsigned i = 0; i != NumSubVectors29.8k ; ++i7.91k ) { |
2186 | 27.9k | APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts); |
2187 | 27.9k | DemandedSub = DemandedSub.trunc(NumSubVectorElts); |
2188 | 27.9k | if (!!DemandedSub27.9k ) { |
2189 | 24.5k | SDValue Sub = Op.getOperand(i); |
2190 | 24.5k | computeKnownBits(Sub, Known2, DemandedSub, Depth + 1); |
2191 | 24.5k | Known.One &= Known2.One; |
2192 | 24.5k | Known.Zero &= Known2.Zero; |
2193 | 24.5k | } |
2194 | 27.9k | // If we don't know any bits, early out. |
2195 | 27.9k | if (!Known.One && 27.9k !Known.Zero24.6k ) |
2196 | 20.0k | break; |
2197 | 27.9k | } |
2198 | 21.9k | break; |
2199 | 4.18k | } |
2200 | 142k | case ISD::EXTRACT_SUBVECTOR: { |
2201 | 142k | // If we know the element index, just demand that subvector elements, |
2202 | 142k | // otherwise demand them all. |
2203 | 142k | SDValue Src = Op.getOperand(0); |
2204 | 142k | ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1)); |
2205 | 142k | unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
2206 | 142k | if (SubIdx && 142k SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)142k ) { |
2207 | 142k | // Offset the demanded elts by the subvector index. |
2208 | 142k | uint64_t Idx = SubIdx->getZExtValue(); |
2209 | 142k | APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx); |
2210 | 142k | computeKnownBits(Src, Known, DemandedSrc, Depth + 1); |
2211 | 142k | } else { |
2212 | 0 | computeKnownBits(Src, Known, Depth + 1); |
2213 | 0 | } |
2214 | 142k | break; |
2215 | 4.18k | } |
2216 | 485k | case ISD::BITCAST: { |
2217 | 485k | SDValue N0 = Op.getOperand(0); |
2218 | 485k | unsigned SubBitWidth = N0.getScalarValueSizeInBits(); |
2219 | 485k | |
2220 | 485k | // Ignore bitcasts from floating point. |
2221 | 485k | if (!N0.getValueType().isInteger()) |
2222 | 61.8k | break; |
2223 | 423k | |
2224 | 423k | // Fast handling of 'identity' bitcasts. |
2225 | 423k | if (423k BitWidth == SubBitWidth423k ) { |
2226 | 143 | computeKnownBits(N0, Known, DemandedElts, Depth + 1); |
2227 | 143 | break; |
2228 | 143 | } |
2229 | 423k | |
2230 | 423k | // Support big-endian targets when it becomes useful. |
2231 | 423k | bool IsLE = getDataLayout().isLittleEndian(); |
2232 | 423k | if (!IsLE) |
2233 | 3.98k | break; |
2234 | 419k | |
2235 | 419k | // Bitcast 'small element' vector to 'large element' scalar/vector. |
2236 | 419k | if (419k (BitWidth % SubBitWidth) == 0419k ) { |
2237 | 233k | assert(N0.getValueType().isVector() && "Expected bitcast from vector"); |
2238 | 233k | |
2239 | 233k | // Collect known bits for the (larger) output by collecting the known |
2240 | 233k | // bits from each set of sub elements and shift these into place. |
2241 | 233k | // We need to separately call computeKnownBits for each set of |
2242 | 233k | // sub elements as the knownbits for each is likely to be different. |
2243 | 233k | unsigned SubScale = BitWidth / SubBitWidth; |
2244 | 233k | APInt SubDemandedElts(NumElts * SubScale, 0); |
2245 | 705k | for (unsigned i = 0; i != NumElts705k ; ++i472k ) |
2246 | 472k | if (472k DemandedElts[i]472k ) |
2247 | 434k | SubDemandedElts.setBit(i * SubScale); |
2248 | 233k | |
2249 | 1.09M | for (unsigned i = 0; i != SubScale1.09M ; ++i862k ) { |
2250 | 862k | computeKnownBits(N0, Known2, SubDemandedElts.shl(i), |
2251 | 862k | Depth + 1); |
2252 | 862k | Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * i); |
2253 | 862k | Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * i); |
2254 | 862k | } |
2255 | 233k | } |
2256 | 419k | |
2257 | 419k | // Bitcast 'large element' scalar/vector to 'small element' vector. |
2258 | 419k | if ((SubBitWidth % BitWidth) == 0419k ) { |
2259 | 185k | assert(Op.getValueType().isVector() && "Expected bitcast to vector"); |
2260 | 185k | |
2261 | 185k | // Collect known bits for the (smaller) output by collecting the known |
2262 | 185k | // bits from the overlapping larger input elements and extracting the |
2263 | 185k | // sub sections we actually care about. |
2264 | 185k | unsigned SubScale = SubBitWidth / BitWidth; |
2265 | 185k | APInt SubDemandedElts(NumElts / SubScale, 0); |
2266 | 5.36M | for (unsigned i = 0; i != NumElts5.36M ; ++i5.18M ) |
2267 | 5.18M | if (5.18M DemandedElts[i]5.18M ) |
2268 | 584k | SubDemandedElts.setBit(i / SubScale); |
2269 | 185k | |
2270 | 185k | computeKnownBits(N0, Known2, SubDemandedElts, Depth + 1); |
2271 | 185k | |
2272 | 185k | Known.Zero.setAllBits(); Known.One.setAllBits(); |
2273 | 2.34M | for (unsigned i = 0; i != NumElts2.34M ; ++i2.15M ) |
2274 | 2.33M | if (2.33M DemandedElts[i]2.33M ) { |
2275 | 227k | unsigned Offset = (i % SubScale) * BitWidth; |
2276 | 227k | Known.One &= Known2.One.lshr(Offset).trunc(BitWidth); |
2277 | 227k | Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth); |
2278 | 227k | // If we don't know any bits, early out. |
2279 | 227k | if (!Known.One && 227k !Known.Zero221k ) |
2280 | 179k | break; |
2281 | 2.33M | } |
2282 | 185k | } |
2283 | 419k | break; |
2284 | 419k | } |
2285 | 2.22M | case ISD::AND: |
2286 | 2.22M | // If either the LHS or the RHS are Zero, the result is zero. |
2287 | 2.22M | computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1); |
2288 | 2.22M | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2289 | 2.22M | |
2290 | 2.22M | // Output known-1 bits are only known if set in both the LHS & RHS. |
2291 | 2.22M | Known.One &= Known2.One; |
2292 | 2.22M | // Output known-0 are known to be clear if zero in either the LHS | RHS. |
2293 | 2.22M | Known.Zero |= Known2.Zero; |
2294 | 2.22M | break; |
2295 | 611k | case ISD::OR: |
2296 | 611k | computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1); |
2297 | 611k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2298 | 611k | |
2299 | 611k | // Output known-0 bits are only known if clear in both the LHS & RHS. |
2300 | 611k | Known.Zero &= Known2.Zero; |
2301 | 611k | // Output known-1 are known to be set if set in either the LHS | RHS. |
2302 | 611k | Known.One |= Known2.One; |
2303 | 611k | break; |
2304 | 518k | case ISD::XOR: { |
2305 | 518k | computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1); |
2306 | 518k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2307 | 518k | |
2308 | 518k | // Output known-0 bits are known if clear or set in both the LHS & RHS. |
2309 | 518k | APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One); |
2310 | 518k | // Output known-1 are known to be set if set in only one of the LHS, RHS. |
2311 | 518k | Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero); |
2312 | 518k | Known.Zero = KnownZeroOut; |
2313 | 518k | break; |
2314 | 419k | } |
2315 | 1.99M | case ISD::MUL: { |
2316 | 1.99M | computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1); |
2317 | 1.99M | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2318 | 1.99M | |
2319 | 1.99M | // If low bits are zero in either operand, output low known-0 bits. |
2320 | 1.99M | // Also compute a conservative estimate for high known-0 bits. |
2321 | 1.99M | // More trickiness is possible, but this is sufficient for the |
2322 | 1.99M | // interesting case of alignment computation. |
2323 | 1.99M | unsigned TrailZ = Known.countMinTrailingZeros() + |
2324 | 1.99M | Known2.countMinTrailingZeros(); |
2325 | 1.99M | unsigned LeadZ = std::max(Known.countMinLeadingZeros() + |
2326 | 1.99M | Known2.countMinLeadingZeros(), |
2327 | 1.99M | BitWidth) - BitWidth; |
2328 | 1.99M | |
2329 | 1.99M | Known.resetAll(); |
2330 | 1.99M | Known.Zero.setLowBits(std::min(TrailZ, BitWidth)); |
2331 | 1.99M | Known.Zero.setHighBits(std::min(LeadZ, BitWidth)); |
2332 | 1.99M | break; |
2333 | 419k | } |
2334 | 59.2k | case ISD::UDIV: { |
2335 | 59.2k | // For the purposes of computing leading zeros we can conservatively |
2336 | 59.2k | // treat a udiv as a logical right shift by the power of 2 known to |
2337 | 59.2k | // be less than the denominator. |
2338 | 59.2k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2339 | 59.2k | unsigned LeadZ = Known2.countMinLeadingZeros(); |
2340 | 59.2k | |
2341 | 59.2k | computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); |
2342 | 59.2k | unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros(); |
2343 | 59.2k | if (RHSMaxLeadingZeros != BitWidth) |
2344 | 38.2k | LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1); |
2345 | 59.2k | |
2346 | 59.2k | Known.Zero.setHighBits(LeadZ); |
2347 | 59.2k | break; |
2348 | 419k | } |
2349 | 308k | case ISD::SELECT: |
2350 | 308k | computeKnownBits(Op.getOperand(2), Known, Depth+1); |
2351 | 308k | // If we don't know any bits, early out. |
2352 | 308k | if (!Known.One && 308k !Known.Zero173k ) |
2353 | 152k | break; |
2354 | 156k | computeKnownBits(Op.getOperand(1), Known2, Depth+1); |
2355 | 156k | |
2356 | 156k | // Only known if known in both the LHS and RHS. |
2357 | 156k | Known.One &= Known2.One; |
2358 | 156k | Known.Zero &= Known2.Zero; |
2359 | 156k | break; |
2360 | 221k | case ISD::SELECT_CC: |
2361 | 221k | computeKnownBits(Op.getOperand(3), Known, Depth+1); |
2362 | 221k | // If we don't know any bits, early out. |
2363 | 221k | if (!Known.One && 221k !Known.Zero121k ) |
2364 | 108k | break; |
2365 | 113k | computeKnownBits(Op.getOperand(2), Known2, Depth+1); |
2366 | 113k | |
2367 | 113k | // Only known if known in both the LHS and RHS. |
2368 | 113k | Known.One &= Known2.One; |
2369 | 113k | Known.Zero &= Known2.Zero; |
2370 | 113k | break; |
2371 | 2.39k | case ISD::SMULO: |
2372 | 2.39k | case ISD::UMULO: |
2373 | 2.39k | if (Op.getResNo() != 1) |
2374 | 60 | break; |
2375 | 2.33k | // The boolean result conforms to getBooleanContents. |
2376 | 2.33k | // If we know the result of a setcc has the top bits zero, use this info. |
2377 | 2.33k | // We know that we have an integer-based boolean since these operations |
2378 | 2.33k | // are only available for integer. |
2379 | 2.33k | if (2.33k TLI->getBooleanContents(Op.getValueType().isVector(), false) == |
2380 | 2.33k | TargetLowering::ZeroOrOneBooleanContent && |
2381 | 2.33k | BitWidth > 1) |
2382 | 2.31k | Known.Zero.setBitsFrom(1); |
2383 | 2.33k | break; |
2384 | 779k | case ISD::SETCC: |
2385 | 779k | // If we know the result of a setcc has the top bits zero, use this info. |
2386 | 779k | if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == |
2387 | 779k | TargetLowering::ZeroOrOneBooleanContent && |
2388 | 682k | BitWidth > 1) |
2389 | 643k | Known.Zero.setBitsFrom(1); |
2390 | 779k | break; |
2391 | 3.23M | case ISD::SHL: |
2392 | 3.23M | if (const APInt *ShAmt3.23M = getValidShiftAmountConstant(Op)) { |
2393 | 3.11M | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2394 | 3.11M | Known.Zero <<= *ShAmt; |
2395 | 3.11M | Known.One <<= *ShAmt; |
2396 | 3.11M | // Low bits are known zero. |
2397 | 3.11M | Known.Zero.setLowBits(ShAmt->getZExtValue()); |
2398 | 3.11M | } |
2399 | 3.23M | break; |
2400 | 1.25M | case ISD::SRL: |
2401 | 1.25M | if (const APInt *ShAmt1.25M = getValidShiftAmountConstant(Op)) { |
2402 | 1.20M | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2403 | 1.20M | Known.Zero.lshrInPlace(*ShAmt); |
2404 | 1.20M | Known.One.lshrInPlace(*ShAmt); |
2405 | 1.20M | // High bits are known zero. |
2406 | 1.20M | Known.Zero.setHighBits(ShAmt->getZExtValue()); |
2407 | 1.20M | } |
2408 | 1.25M | break; |
2409 | 465k | case ISD::SRA: |
2410 | 465k | if (const APInt *ShAmt465k = getValidShiftAmountConstant(Op)) { |
2411 | 439k | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2412 | 439k | Known.Zero.lshrInPlace(*ShAmt); |
2413 | 439k | Known.One.lshrInPlace(*ShAmt); |
2414 | 439k | // If we know the value of the sign bit, then we know it is copied across |
2415 | 439k | // the high bits by the shift amount. |
2416 | 439k | APInt SignMask = APInt::getSignMask(BitWidth); |
2417 | 439k | SignMask.lshrInPlace(*ShAmt); // Adjust to where it is now in the mask. |
2418 | 439k | if (Known.Zero.intersects(SignMask)439k ) { |
2419 | 10 | Known.Zero.setHighBits(ShAmt->getZExtValue());// New bits are known zero. |
2420 | 439k | } else if (439k Known.One.intersects(SignMask)439k ) { |
2421 | 1 | Known.One.setHighBits(ShAmt->getZExtValue()); // New bits are known one. |
2422 | 1 | } |
2423 | 439k | } |
2424 | 465k | break; |
2425 | 164k | case ISD::SIGN_EXTEND_INREG: { |
2426 | 164k | EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); |
2427 | 164k | unsigned EBits = EVT.getScalarSizeInBits(); |
2428 | 164k | |
2429 | 164k | // Sign extension. Compute the demanded bits in the result that are not |
2430 | 164k | // present in the input. |
2431 | 164k | APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); |
2432 | 164k | |
2433 | 164k | APInt InSignMask = APInt::getSignMask(EBits); |
2434 | 164k | APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); |
2435 | 164k | |
2436 | 164k | // If the sign extended bits are demanded, we know that the sign |
2437 | 164k | // bit is demanded. |
2438 | 164k | InSignMask = InSignMask.zext(BitWidth); |
2439 | 164k | if (NewBits.getBoolValue()) |
2440 | 164k | InputDemandedBits |= InSignMask; |
2441 | 164k | |
2442 | 164k | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2443 | 164k | Known.One &= InputDemandedBits; |
2444 | 164k | Known.Zero &= InputDemandedBits; |
2445 | 164k | |
2446 | 164k | // If the sign bit of the input is known set or clear, then we know the |
2447 | 164k | // top bits of the result. |
2448 | 164k | if (Known.Zero.intersects(InSignMask)164k ) { // Input sign bit known clear |
2449 | 8 | Known.Zero |= NewBits; |
2450 | 8 | Known.One &= ~NewBits; |
2451 | 164k | } else if (164k Known.One.intersects(InSignMask)164k ) { // Input sign bit known set |
2452 | 2 | Known.One |= NewBits; |
2453 | 2 | Known.Zero &= ~NewBits; |
2454 | 164k | } else { // Input sign bit unknown |
2455 | 164k | Known.Zero &= ~NewBits; |
2456 | 164k | Known.One &= ~NewBits; |
2457 | 164k | } |
2458 | 164k | break; |
2459 | 2.33k | } |
2460 | 4.34k | case ISD::CTTZ: |
2461 | 4.34k | case ISD::CTTZ_ZERO_UNDEF: { |
2462 | 4.34k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2463 | 4.34k | // If we have a known 1, its position is our upper bound. |
2464 | 4.34k | unsigned PossibleTZ = Known2.countMaxTrailingZeros(); |
2465 | 4.34k | unsigned LowBits = Log2_32(PossibleTZ) + 1; |
2466 | 4.34k | Known.Zero.setBitsFrom(LowBits); |
2467 | 4.34k | break; |
2468 | 4.34k | } |
2469 | 56.9k | case ISD::CTLZ: |
2470 | 56.9k | case ISD::CTLZ_ZERO_UNDEF: { |
2471 | 56.9k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2472 | 56.9k | // If we have a known 1, its position is our upper bound. |
2473 | 56.9k | unsigned PossibleLZ = Known2.countMaxLeadingZeros(); |
2474 | 56.9k | unsigned LowBits = Log2_32(PossibleLZ) + 1; |
2475 | 56.9k | Known.Zero.setBitsFrom(LowBits); |
2476 | 56.9k | break; |
2477 | 56.9k | } |
2478 | 1.04k | case ISD::CTPOP: { |
2479 | 1.04k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2480 | 1.04k | // If we know some of the bits are zero, they can't be one. |
2481 | 1.04k | unsigned PossibleOnes = Known2.countMaxPopulation(); |
2482 | 1.04k | Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1); |
2483 | 1.04k | break; |
2484 | 56.9k | } |
2485 | 9.47M | case ISD::LOAD: { |
2486 | 9.47M | LoadSDNode *LD = cast<LoadSDNode>(Op); |
2487 | 9.47M | // If this is a ZEXTLoad and we are looking at the loaded value. |
2488 | 9.47M | if (ISD::isZEXTLoad(Op.getNode()) && 9.47M Op.getResNo() == 01.15M ) { |
2489 | 1.14M | EVT VT = LD->getMemoryVT(); |
2490 | 1.14M | unsigned MemBits = VT.getScalarSizeInBits(); |
2491 | 1.14M | Known.Zero.setBitsFrom(MemBits); |
2492 | 9.47M | } else if (const MDNode *8.33M Ranges8.33M = LD->getRanges()) { |
2493 | 65.6k | if (LD->getExtensionType() == ISD::NON_EXTLOAD) |
2494 | 1.25k | computeKnownBitsFromRangeMetadata(*Ranges, Known); |
2495 | 8.33M | } |
2496 | 9.47M | break; |
2497 | 56.9k | } |
2498 | 5.93k | case ISD::ZERO_EXTEND_VECTOR_INREG: { |
2499 | 5.93k | EVT InVT = Op.getOperand(0).getValueType(); |
2500 | 5.93k | unsigned InBits = InVT.getScalarSizeInBits(); |
2501 | 5.93k | Known = Known.trunc(InBits); |
2502 | 5.93k | computeKnownBits(Op.getOperand(0), Known, |
2503 | 5.93k | DemandedElts.zext(InVT.getVectorNumElements()), |
2504 | 5.93k | Depth + 1); |
2505 | 5.93k | Known = Known.zext(BitWidth); |
2506 | 5.93k | Known.Zero.setBitsFrom(InBits); |
2507 | 5.93k | break; |
2508 | 56.9k | } |
2509 | 816k | case ISD::ZERO_EXTEND: { |
2510 | 816k | EVT InVT = Op.getOperand(0).getValueType(); |
2511 | 816k | unsigned InBits = InVT.getScalarSizeInBits(); |
2512 | 816k | Known = Known.trunc(InBits); |
2513 | 816k | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2514 | 816k | Known = Known.zext(BitWidth); |
2515 | 816k | Known.Zero.setBitsFrom(InBits); |
2516 | 816k | break; |
2517 | 56.9k | } |
2518 | 56.9k | // TODO ISD::SIGN_EXTEND_VECTOR_INREG |
2519 | 1.15M | case ISD::SIGN_EXTEND: { |
2520 | 1.15M | EVT InVT = Op.getOperand(0).getValueType(); |
2521 | 1.15M | unsigned InBits = InVT.getScalarSizeInBits(); |
2522 | 1.15M | |
2523 | 1.15M | Known = Known.trunc(InBits); |
2524 | 1.15M | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2525 | 1.15M | |
2526 | 1.15M | // If the sign bit is known to be zero or one, then sext will extend |
2527 | 1.15M | // it to the top bits, else it will just zext. |
2528 | 1.15M | Known = Known.sext(BitWidth); |
2529 | 1.15M | break; |
2530 | 56.9k | } |
2531 | 216k | case ISD::ANY_EXTEND: { |
2532 | 216k | EVT InVT = Op.getOperand(0).getValueType(); |
2533 | 216k | unsigned InBits = InVT.getScalarSizeInBits(); |
2534 | 216k | Known = Known.trunc(InBits); |
2535 | 216k | computeKnownBits(Op.getOperand(0), Known, Depth+1); |
2536 | 216k | Known = Known.zext(BitWidth); |
2537 | 216k | break; |
2538 | 56.9k | } |
2539 | 1.63M | case ISD::TRUNCATE: { |
2540 | 1.63M | EVT InVT = Op.getOperand(0).getValueType(); |
2541 | 1.63M | unsigned InBits = InVT.getScalarSizeInBits(); |
2542 | 1.63M | Known = Known.zext(InBits); |
2543 | 1.63M | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2544 | 1.63M | Known = Known.trunc(BitWidth); |
2545 | 1.63M | break; |
2546 | 56.9k | } |
2547 | 970k | case ISD::AssertZext: { |
2548 | 970k | EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); |
2549 | 970k | APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); |
2550 | 970k | computeKnownBits(Op.getOperand(0), Known, Depth+1); |
2551 | 970k | Known.Zero |= (~InMask); |
2552 | 970k | Known.One &= (~Known.Zero); |
2553 | 970k | break; |
2554 | 56.9k | } |
2555 | 26 | case ISD::FGETSIGN: |
2556 | 26 | // All bits are zero except the low bit. |
2557 | 26 | Known.Zero.setBitsFrom(1); |
2558 | 26 | break; |
2559 | 6.02k | case ISD::USUBO: |
2560 | 6.02k | case ISD::SSUBO: |
2561 | 6.02k | if (Op.getResNo() == 16.02k ) { |
2562 | 4.39k | // If we know the result of a setcc has the top bits zero, use this info. |
2563 | 4.39k | if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == |
2564 | 4.39k | TargetLowering::ZeroOrOneBooleanContent && |
2565 | 154 | BitWidth > 1) |
2566 | 123 | Known.Zero.setBitsFrom(1); |
2567 | 4.39k | break; |
2568 | 4.39k | } |
2569 | 1.62k | LLVM_FALLTHROUGH1.62k ; |
2570 | 854k | case ISD::SUB: |
2571 | 854k | case ISD::SUBC: { |
2572 | 854k | if (ConstantSDNode *CLHS854k = isConstOrConstSplat(Op.getOperand(0))) { |
2573 | 205k | // We know that the top bits of C-X are clear if X contains less bits |
2574 | 205k | // than C (i.e. no wrap-around can happen). For example, 20-X is |
2575 | 205k | // positive if we can prove that X is >= 0 and < 16. |
2576 | 205k | if (CLHS->getAPIntValue().isNonNegative()205k ) { |
2577 | 195k | unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); |
2578 | 195k | // NLZ can't be BitWidth with no sign bit |
2579 | 195k | APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); |
2580 | 195k | computeKnownBits(Op.getOperand(1), Known2, DemandedElts, |
2581 | 195k | Depth + 1); |
2582 | 195k | |
2583 | 195k | // If all of the MaskV bits are known to be zero, then we know the |
2584 | 195k | // output top bits are zero, because we now know that the output is |
2585 | 195k | // from [0-C]. |
2586 | 195k | if ((Known2.Zero & MaskV) == MaskV195k ) { |
2587 | 2.48k | unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); |
2588 | 2.48k | // Top bits known zero. |
2589 | 2.48k | Known.Zero.setHighBits(NLZ2); |
2590 | 2.48k | } |
2591 | 195k | } |
2592 | 205k | } |
2593 | 854k | |
2594 | 854k | // If low bits are know to be zero in both operands, then we know they are |
2595 | 854k | // going to be 0 in the result. Both addition and complement operations |
2596 | 854k | // preserve the low zero bits. |
2597 | 854k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2598 | 854k | unsigned KnownZeroLow = Known2.countMinTrailingZeros(); |
2599 | 854k | if (KnownZeroLow == 0) |
2600 | 671k | break; |
2601 | 182k | |
2602 | 182k | computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); |
2603 | 182k | KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros()); |
2604 | 182k | Known.Zero.setLowBits(KnownZeroLow); |
2605 | 182k | break; |
2606 | 182k | } |
2607 | 13.3k | case ISD::UADDO: |
2608 | 13.3k | case ISD::SADDO: |
2609 | 13.3k | case ISD::ADDCARRY: |
2610 | 13.3k | if (Op.getResNo() == 113.3k ) { |
2611 | 1.28k | // If we know the result of a setcc has the top bits zero, use this info. |
2612 | 1.28k | if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == |
2613 | 1.28k | TargetLowering::ZeroOrOneBooleanContent && |
2614 | 1.14k | BitWidth > 1) |
2615 | 1.10k | Known.Zero.setBitsFrom(1); |
2616 | 1.28k | break; |
2617 | 1.28k | } |
2618 | 12.0k | LLVM_FALLTHROUGH12.0k ; |
2619 | 14.0M | case ISD::ADD: |
2620 | 14.0M | case ISD::ADDC: |
2621 | 14.0M | case ISD::ADDE: { |
2622 | 14.0M | // Output known-0 bits are known if clear or set in both the low clear bits |
2623 | 14.0M | // common to both LHS & RHS. For example, 8+(X<<3) is known to have the |
2624 | 14.0M | // low 3 bits clear. |
2625 | 14.0M | // Output known-0 bits are also known if the top bits of each input are |
2626 | 14.0M | // known to be clear. For example, if one input has the top 10 bits clear |
2627 | 14.0M | // and the other has the top 8 bits clear, we know the top 7 bits of the |
2628 | 14.0M | // output must be clear. |
2629 | 14.0M | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2630 | 14.0M | unsigned KnownZeroHigh = Known2.countMinLeadingZeros(); |
2631 | 14.0M | unsigned KnownZeroLow = Known2.countMinTrailingZeros(); |
2632 | 14.0M | |
2633 | 14.0M | computeKnownBits(Op.getOperand(1), Known2, DemandedElts, |
2634 | 14.0M | Depth + 1); |
2635 | 14.0M | KnownZeroHigh = std::min(KnownZeroHigh, Known2.countMinLeadingZeros()); |
2636 | 14.0M | KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros()); |
2637 | 14.0M | |
2638 | 14.0M | if (Opcode == ISD::ADDE || 14.0M Opcode == ISD::ADDCARRY14.0M ) { |
2639 | 5.17k | // With ADDE and ADDCARRY, a carry bit may be added in, so we can only |
2640 | 5.17k | // use this information if we know (at least) that the low two bits are |
2641 | 5.17k | // clear. We then return to the caller that the low bit is unknown but |
2642 | 5.17k | // that other bits are known zero. |
2643 | 5.17k | if (KnownZeroLow >= 2) |
2644 | 641 | Known.Zero.setBits(1, KnownZeroLow); |
2645 | 5.17k | break; |
2646 | 5.17k | } |
2647 | 14.0M | |
2648 | 14.0M | Known.Zero.setLowBits(KnownZeroLow); |
2649 | 14.0M | if (KnownZeroHigh > 1) |
2650 | 249k | Known.Zero.setHighBits(KnownZeroHigh - 1); |
2651 | 14.0M | break; |
2652 | 14.0M | } |
2653 | 6.20k | case ISD::SREM: |
2654 | 6.20k | if (ConstantSDNode *Rem6.20k = isConstOrConstSplat(Op.getOperand(1))) { |
2655 | 3.11k | const APInt &RA = Rem->getAPIntValue().abs(); |
2656 | 3.11k | if (RA.isPowerOf2()3.11k ) { |
2657 | 604 | APInt LowBits = RA - 1; |
2658 | 604 | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2659 | 604 | |
2660 | 604 | // The low bits of the first operand are unchanged by the srem. |
2661 | 604 | Known.Zero = Known2.Zero & LowBits; |
2662 | 604 | Known.One = Known2.One & LowBits; |
2663 | 604 | |
2664 | 604 | // If the first operand is non-negative or has all low bits zero, then |
2665 | 604 | // the upper bits are all zero. |
2666 | 604 | if (Known2.Zero[BitWidth-1] || 604 ((Known2.Zero & LowBits) == LowBits)604 ) |
2667 | 2 | Known.Zero |= ~LowBits; |
2668 | 604 | |
2669 | 604 | // If the first operand is negative and not all low bits are zero, then |
2670 | 604 | // the upper bits are all one. |
2671 | 604 | if (Known2.One[BitWidth-1] && 604 ((Known2.One & LowBits) != 0)0 ) |
2672 | 0 | Known.One |= ~LowBits; |
2673 | 604 | assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?"); |
2674 | 604 | } |
2675 | 3.11k | } |
2676 | 6.20k | break; |
2677 | 10.3k | case ISD::UREM: { |
2678 | 10.3k | if (ConstantSDNode *Rem10.3k = isConstOrConstSplat(Op.getOperand(1))) { |
2679 | 7.31k | const APInt &RA = Rem->getAPIntValue(); |
2680 | 7.31k | if (RA.isPowerOf2()7.31k ) { |
2681 | 8 | APInt LowBits = (RA - 1); |
2682 | 8 | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2683 | 8 | |
2684 | 8 | // The upper bits are all zero, the lower ones are unchanged. |
2685 | 8 | Known.Zero = Known2.Zero | ~LowBits; |
2686 | 8 | Known.One = Known2.One & LowBits; |
2687 | 8 | break; |
2688 | 8 | } |
2689 | 10.2k | } |
2690 | 10.2k | |
2691 | 10.2k | // Since the result is less than or equal to either operand, any leading |
2692 | 10.2k | // zero bits in either operand must also exist in the result. |
2693 | 10.2k | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2694 | 10.2k | computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); |
2695 | 10.2k | |
2696 | 10.2k | uint32_t Leaders = |
2697 | 10.2k | std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros()); |
2698 | 10.2k | Known.resetAll(); |
2699 | 10.2k | Known.Zero.setHighBits(Leaders); |
2700 | 10.2k | break; |
2701 | 10.2k | } |
2702 | 50 | case ISD::EXTRACT_ELEMENT: { |
2703 | 50 | computeKnownBits(Op.getOperand(0), Known, Depth+1); |
2704 | 50 | const unsigned Index = Op.getConstantOperandVal(1); |
2705 | 50 | const unsigned BitWidth = Op.getValueSizeInBits(); |
2706 | 50 | |
2707 | 50 | // Remove low part of known bits mask |
2708 | 50 | Known.Zero = Known.Zero.getHiBits(Known.Zero.getBitWidth() - Index * BitWidth); |
2709 | 50 | Known.One = Known.One.getHiBits(Known.One.getBitWidth() - Index * BitWidth); |
2710 | 50 | |
2711 | 50 | // Remove high part of known bit mask |
2712 | 50 | Known = Known.trunc(BitWidth); |
2713 | 50 | break; |
2714 | 10.2k | } |
2715 | 393k | case ISD::EXTRACT_VECTOR_ELT: { |
2716 | 393k | SDValue InVec = Op.getOperand(0); |
2717 | 393k | SDValue EltNo = Op.getOperand(1); |
2718 | 393k | EVT VecVT = InVec.getValueType(); |
2719 | 393k | const unsigned BitWidth = Op.getValueSizeInBits(); |
2720 | 393k | const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); |
2721 | 393k | const unsigned NumSrcElts = VecVT.getVectorNumElements(); |
2722 | 393k | // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know |
2723 | 393k | // anything about the extended bits. |
2724 | 393k | if (BitWidth > EltBitWidth) |
2725 | 63.7k | Known = Known.trunc(EltBitWidth); |
2726 | 393k | ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo); |
2727 | 393k | if (ConstEltNo && 393k ConstEltNo->getAPIntValue().ult(NumSrcElts)392k ) { |
2728 | 392k | // If we know the element index, just demand that vector element. |
2729 | 392k | unsigned Idx = ConstEltNo->getZExtValue(); |
2730 | 392k | APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx); |
2731 | 392k | computeKnownBits(InVec, Known, DemandedElt, Depth + 1); |
2732 | 393k | } else { |
2733 | 535 | // Unknown element index, so ignore DemandedElts and demand them all. |
2734 | 535 | computeKnownBits(InVec, Known, Depth + 1); |
2735 | 535 | } |
2736 | 393k | if (BitWidth > EltBitWidth) |
2737 | 63.7k | Known = Known.zext(BitWidth); |
2738 | 393k | break; |
2739 | 10.2k | } |
2740 | 8.18k | case ISD::INSERT_VECTOR_ELT: { |
2741 | 8.18k | SDValue InVec = Op.getOperand(0); |
2742 | 8.18k | SDValue InVal = Op.getOperand(1); |
2743 | 8.18k | SDValue EltNo = Op.getOperand(2); |
2744 | 8.18k | |
2745 | 8.18k | ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo); |
2746 | 8.18k | if (CEltNo && 8.18k CEltNo->getAPIntValue().ult(NumElts)8.17k ) { |
2747 | 8.17k | // If we know the element index, split the demand between the |
2748 | 8.17k | // source vector and the inserted element. |
2749 | 8.17k | Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth); |
2750 | 8.17k | unsigned EltIdx = CEltNo->getZExtValue(); |
2751 | 8.17k | |
2752 | 8.17k | // If we demand the inserted element then add its common known bits. |
2753 | 8.17k | if (DemandedElts[EltIdx]8.17k ) { |
2754 | 7.10k | computeKnownBits(InVal, Known2, Depth + 1); |
2755 | 7.10k | Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth()); |
2756 | 7.10k | Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth()); |
2757 | 7.10k | } |
2758 | 8.17k | |
2759 | 8.17k | // If we demand the source vector then add its common known bits, ensuring |
2760 | 8.17k | // that we don't demand the inserted element. |
2761 | 8.17k | APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx)); |
2762 | 8.17k | if (!!VectorElts8.17k ) { |
2763 | 4.61k | computeKnownBits(InVec, Known2, VectorElts, Depth + 1); |
2764 | 4.61k | Known.One &= Known2.One; |
2765 | 4.61k | Known.Zero &= Known2.Zero; |
2766 | 4.61k | } |
2767 | 8.18k | } else { |
2768 | 4 | // Unknown element index, so ignore DemandedElts and demand them all. |
2769 | 4 | computeKnownBits(InVec, Known, Depth + 1); |
2770 | 4 | computeKnownBits(InVal, Known2, Depth + 1); |
2771 | 4 | Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth()); |
2772 | 4 | Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth()); |
2773 | 4 | } |
2774 | 8.18k | break; |
2775 | 10.2k | } |
2776 | 499 | case ISD::BITREVERSE: { |
2777 | 499 | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2778 | 499 | Known.Zero = Known2.Zero.reverseBits(); |
2779 | 499 | Known.One = Known2.One.reverseBits(); |
2780 | 499 | break; |
2781 | 10.2k | } |
2782 | 6.35k | case ISD::BSWAP: { |
2783 | 6.35k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2784 | 6.35k | Known.Zero = Known2.Zero.byteSwap(); |
2785 | 6.35k | Known.One = Known2.One.byteSwap(); |
2786 | 6.35k | break; |
2787 | 10.2k | } |
2788 | 2.11k | case ISD::ABS: { |
2789 | 2.11k | computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1); |
2790 | 2.11k | |
2791 | 2.11k | // If the source's MSB is zero then we know the rest of the bits already. |
2792 | 2.11k | if (Known2.isNonNegative()2.11k ) { |
2793 | 0 | Known.Zero = Known2.Zero; |
2794 | 0 | Known.One = Known2.One; |
2795 | 0 | break; |
2796 | 0 | } |
2797 | 2.11k | |
2798 | 2.11k | // We only know that the absolute values's MSB will be zero iff there is |
2799 | 2.11k | // a set bit that isn't the sign bit (otherwise it could be INT_MIN). |
2800 | 2.11k | Known2.One.clearSignBit(); |
2801 | 2.11k | if (Known2.One.getBoolValue()2.11k ) { |
2802 | 2 | Known.Zero = APInt::getSignMask(BitWidth); |
2803 | 2 | break; |
2804 | 2 | } |
2805 | 2.11k | break; |
2806 | 2.11k | } |
2807 | 626 | case ISD::UMIN: { |
2808 | 626 | computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1); |
2809 | 626 | computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); |
2810 | 626 | |
2811 | 626 | // UMIN - we know that the result will have the maximum of the |
2812 | 626 | // known zero leading bits of the inputs. |
2813 | 626 | unsigned LeadZero = Known.countMinLeadingZeros(); |
2814 | 626 | LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros()); |
2815 | 626 | |
2816 | 626 | Known.Zero &= Known2.Zero; |
2817 | 626 | Known.One &= Known2.One; |
2818 | 626 | Known.Zero.setHighBits(LeadZero); |
2819 | 626 | break; |
2820 | 2.11k | } |
2821 | 303 | case ISD::UMAX: { |
2822 | 303 | computeKnownBits(Op.getOperand(0), Known, DemandedElts, |
2823 | 303 | Depth + 1); |
2824 | 303 | computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); |
2825 | 303 | |
2826 | 303 | // UMAX - we know that the result will have the maximum of the |
2827 | 303 | // known one leading bits of the inputs. |
2828 | 303 | unsigned LeadOne = Known.countMinLeadingOnes(); |
2829 | 303 | LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes()); |
2830 | 303 | |
2831 | 303 | Known.Zero &= Known2.Zero; |
2832 | 303 | Known.One &= Known2.One; |
2833 | 303 | Known.One.setHighBits(LeadOne); |
2834 | 303 | break; |
2835 | 2.11k | } |
2836 | 6.09k | case ISD::SMIN: |
2837 | 6.09k | case ISD::SMAX: { |
2838 | 6.09k | computeKnownBits(Op.getOperand(0), Known, DemandedElts, |
2839 | 6.09k | Depth + 1); |
2840 | 6.09k | // If we don't know any bits, early out. |
2841 | 6.09k | if (!Known.One && 6.09k !Known.Zero6.09k ) |
2842 | 6.09k | break; |
2843 | 0 | computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1); |
2844 | 0 | Known.Zero &= Known2.Zero; |
2845 | 0 | Known.One &= Known2.One; |
2846 | 0 | break; |
2847 | 0 | } |
2848 | 4.09M | case ISD::FrameIndex: |
2849 | 4.09M | case ISD::TargetFrameIndex: |
2850 | 4.09M | if (unsigned Align4.09M = InferPtrAlignment(Op)) { |
2851 | 4.09M | // The low bits are known zero if the pointer is aligned. |
2852 | 4.09M | Known.Zero.setLowBits(Log2_32(Align)); |
2853 | 4.09M | break; |
2854 | 4.09M | } |
2855 | 0 | break; |
2856 | 0 |
|
2857 | 34.7M | default: |
2858 | 34.7M | if (Opcode < ISD::BUILTIN_OP_END) |
2859 | 32.5M | break; |
2860 | 2.19M | LLVM_FALLTHROUGH2.19M ; |
2861 | 2.55M | case ISD::INTRINSIC_WO_CHAIN: |
2862 | 2.55M | case ISD::INTRINSIC_W_CHAIN: |
2863 | 2.55M | case ISD::INTRINSIC_VOID: |
2864 | 2.55M | // Allow the target to implement this method for its nodes. |
2865 | 2.55M | TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth); |
2866 | 2.55M | break; |
2867 | 104M | } |
2868 | 104M | |
2869 | 104M | assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?"); |
2870 | 104M | } |
2871 | | |
2872 | | SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0, |
2873 | 4.43k | SDValue N1) const { |
2874 | 4.43k | // X + 0 never overflow |
2875 | 4.43k | if (isNullConstant(N1)) |
2876 | 0 | return OFK_Never; |
2877 | 4.43k | |
2878 | 4.43k | KnownBits N1Known; |
2879 | 4.43k | computeKnownBits(N1, N1Known); |
2880 | 4.43k | if (N1Known.Zero.getBoolValue()4.43k ) { |
2881 | 1.56k | KnownBits N0Known; |
2882 | 1.56k | computeKnownBits(N0, N0Known); |
2883 | 1.56k | |
2884 | 1.56k | bool overflow; |
2885 | 1.56k | (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow); |
2886 | 1.56k | if (!overflow) |
2887 | 124 | return OFK_Never; |
2888 | 4.31k | } |
2889 | 4.31k | |
2890 | 4.31k | // mulhi + 1 never overflow |
2891 | 4.31k | if (4.31k N0.getOpcode() == ISD::UMUL_LOHI && 4.31k N0.getResNo() == 1593 && |
2892 | 95 | (~N1Known.Zero & 0x01) == ~N1Known.Zero) |
2893 | 42 | return OFK_Never; |
2894 | 4.27k | |
2895 | 4.27k | if (4.27k N1.getOpcode() == ISD::UMUL_LOHI && 4.27k N1.getResNo() == 1346 ) { |
2896 | 113 | KnownBits N0Known; |
2897 | 113 | computeKnownBits(N0, N0Known); |
2898 | 113 | |
2899 | 113 | if ((~N0Known.Zero & 0x01) == ~N0Known.Zero) |
2900 | 0 | return OFK_Never; |
2901 | 4.27k | } |
2902 | 4.27k | |
2903 | 4.27k | return OFK_Sometime; |
2904 | 4.27k | } |
2905 | | |
2906 | 250k | bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const { |
2907 | 250k | EVT OpVT = Val.getValueType(); |
2908 | 250k | unsigned BitWidth = OpVT.getScalarSizeInBits(); |
2909 | 250k | |
2910 | 250k | // Is the constant a known power of 2? |
2911 | 250k | if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val)) |
2912 | 242k | return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); |
2913 | 8.33k | |
2914 | 8.33k | // A left-shift of a constant one will have exactly one bit set because |
2915 | 8.33k | // shifting the bit off the end is undefined. |
2916 | 8.33k | if (8.33k Val.getOpcode() == ISD::SHL8.33k ) { |
2917 | 179 | auto *C = isConstOrConstSplat(Val.getOperand(0)); |
2918 | 179 | if (C && 179 C->getAPIntValue() == 144 ) |
2919 | 41 | return true; |
2920 | 8.29k | } |
2921 | 8.29k | |
2922 | 8.29k | // Similarly, a logical right-shift of a constant sign-bit will have exactly |
2923 | 8.29k | // one bit set. |
2924 | 8.29k | if (8.29k Val.getOpcode() == ISD::SRL8.29k ) { |
2925 | 23 | auto *C = isConstOrConstSplat(Val.getOperand(0)); |
2926 | 23 | if (C && 23 C->getAPIntValue().isSignMask()5 ) |
2927 | 5 | return true; |
2928 | 8.29k | } |
2929 | 8.29k | |
2930 | 8.29k | // Are all operands of a build vector constant powers of two? |
2931 | 8.29k | if (8.29k Val.getOpcode() == ISD::BUILD_VECTOR8.29k ) |
2932 | 2.79k | if (2.79k llvm::all_of(Val->ops(), [BitWidth](SDValue E) 2.79k { |
2933 | 5.74k | if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E)) |
2934 | 5.72k | return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2(); |
2935 | 17 | return false; |
2936 | 17 | })) |
2937 | 315 | return true; |
2938 | 7.97k | |
2939 | 7.97k | // More could be done here, though the above checks are enough |
2940 | 7.97k | // to handle some common cases. |
2941 | 7.97k | |
2942 | 7.97k | // Fall back to computeKnownBits to catch other known cases. |
2943 | 7.97k | KnownBits Known; |
2944 | 7.97k | computeKnownBits(Val, Known); |
2945 | 87 | return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1); |
2946 | 250k | } |
2947 | | |
2948 | 4.33M | unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const { |
2949 | 4.33M | EVT VT = Op.getValueType(); |
2950 | 4.33M | APInt DemandedElts = VT.isVector() |
2951 | 43.4k | ? APInt::getAllOnesValue(VT.getVectorNumElements()) |
2952 | 4.28M | : APInt(1, 1); |
2953 | 4.33M | return ComputeNumSignBits(Op, DemandedElts, Depth); |
2954 | 4.33M | } |
2955 | | |
2956 | | unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, |
2957 | 4.37M | unsigned Depth) const { |
2958 | 4.37M | EVT VT = Op.getValueType(); |
2959 | 4.37M | assert(VT.isInteger() && "Invalid VT!"); |
2960 | 4.37M | unsigned VTBits = VT.getScalarSizeInBits(); |
2961 | 4.37M | unsigned NumElts = DemandedElts.getBitWidth(); |
2962 | 4.37M | unsigned Tmp, Tmp2; |
2963 | 4.37M | unsigned FirstAnswer = 1; |
2964 | 4.37M | |
2965 | 4.37M | if (Depth == 6) |
2966 | 2.21k | return 1; // Limit search depth. |
2967 | 4.36M | |
2968 | 4.36M | if (4.36M !DemandedElts4.36M ) |
2969 | 0 | return 1; // No demanded elts, better to assume we don't know anything. |
2970 | 4.36M | |
2971 | 4.36M | switch (Op.getOpcode()) { |
2972 | 2.30M | default: break; |
2973 | 55.1k | case ISD::AssertSext: |
2974 | 55.1k | Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); |
2975 | 55.1k | return VTBits-Tmp+1; |
2976 | 73.2k | case ISD::AssertZext: |
2977 | 73.2k | Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); |
2978 | 73.2k | return VTBits-Tmp; |
2979 | 4.36M | |
2980 | 821k | case ISD::Constant: { |
2981 | 821k | const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); |
2982 | 821k | return Val.getNumSignBits(); |
2983 | 4.36M | } |
2984 | 4.36M | |
2985 | 11.7k | case ISD::BUILD_VECTOR: |
2986 | 11.7k | Tmp = VTBits; |
2987 | 31.8k | for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && 31.8k (Tmp > 1)29.7k ; ++i20.1k ) { |
2988 | 20.1k | if (!DemandedElts[i]) |
2989 | 3.54k | continue; |
2990 | 16.5k | |
2991 | 16.5k | SDValue SrcOp = Op.getOperand(i); |
2992 | 16.5k | Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1); |
2993 | 16.5k | |
2994 | 16.5k | // BUILD_VECTOR can implicitly truncate sources, we must handle this. |
2995 | 16.5k | if (SrcOp.getValueSizeInBits() != VTBits16.5k ) { |
2996 | 242 | assert(SrcOp.getValueSizeInBits() > VTBits && |
2997 | 242 | "Expected BUILD_VECTOR implicit truncation"); |
2998 | 242 | unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits; |
2999 | 242 | Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits116 : 1126 ); |
3000 | 242 | } |
3001 | 20.1k | Tmp = std::min(Tmp, Tmp2); |
3002 | 20.1k | } |
3003 | 11.7k | return Tmp; |
3004 | 4.36M | |
3005 | 355 | case ISD::VECTOR_SHUFFLE: { |
3006 | 355 | // Collect the minimum number of sign bits that are shared by every vector |
3007 | 355 | // element referenced by the shuffle. |
3008 | 355 | APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0); |
3009 | 355 | const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op); |
3010 | 355 | assert(NumElts == SVN->getMask().size() && "Unexpected vector size"); |
3011 | 2.58k | for (unsigned i = 0; i != NumElts2.58k ; ++i2.23k ) { |
3012 | 2.27k | int M = SVN->getMaskElt(i); |
3013 | 2.27k | if (!DemandedElts[i]) |
3014 | 383 | continue; |
3015 | 1.89k | // For UNDEF elements, we don't know anything about the common state of |
3016 | 1.89k | // the shuffle result. |
3017 | 1.89k | if (1.89k M < 01.89k ) |
3018 | 42 | return 1; |
3019 | 1.85k | if (1.85k (unsigned)M < NumElts1.85k ) |
3020 | 1.28k | DemandedLHS.setBit((unsigned)M % NumElts); |
3021 | 1.85k | else |
3022 | 564 | DemandedRHS.setBit((unsigned)M % NumElts); |
3023 | 2.27k | } |
3024 | 313 | Tmp = std::numeric_limits<unsigned>::max(); |
3025 | 313 | if (!!DemandedLHS) |
3026 | 308 | Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1); |
3027 | 313 | if (!!DemandedRHS313 ) { |
3028 | 151 | Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1); |
3029 | 151 | Tmp = std::min(Tmp, Tmp2); |
3030 | 151 | } |
3031 | 313 | // If we don't know anything, early out and try computeKnownBits fall-back. |
3032 | 313 | if (Tmp == 1) |
3033 | 194 | break; |
3034 | 313 | assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
3035 | 119 | return Tmp; |
3036 | 119 | } |
3037 | 119 | |
3038 | 16.0k | case ISD::BITCAST: { |
3039 | 16.0k | SDValue N0 = Op.getOperand(0); |
3040 | 16.0k | unsigned SrcBits = N0.getScalarValueSizeInBits(); |
3041 | 16.0k | |
3042 | 16.0k | // Ignore bitcasts from floating point. |
3043 | 16.0k | if (!N0.getValueType().isInteger()) |
3044 | 2.30k | break; |
3045 | 13.7k | |
3046 | 13.7k | // Fast handling of 'identity' bitcasts. |
3047 | 13.7k | if (13.7k VTBits == SrcBits13.7k ) |
3048 | 1 | return ComputeNumSignBits(N0, DemandedElts, Depth + 1); |
3049 | 13.7k | |
3050 | 13.7k | // Bitcast 'large element' scalar/vector to 'small element' vector. |
3051 | 13.7k | // TODO: Handle cases other than 'sign splat' when we have a use case. |
3052 | 13.7k | // Requires handling of DemandedElts and Endianness. |
3053 | 13.7k | if (13.7k (SrcBits % VTBits) == 013.7k ) { |
3054 | 6.25k | assert(Op.getValueType().isVector() && "Expected bitcast to vector"); |
3055 | 6.25k | Tmp = ComputeNumSignBits(N0, Depth + 1); |
3056 | 6.25k | if (Tmp == SrcBits) |
3057 | 2.03k | return VTBits; |
3058 | 11.7k | } |
3059 | 11.7k | break; |
3060 | 11.7k | } |
3061 | 11.7k | |
3062 | 51.9k | case ISD::SIGN_EXTEND: |
3063 | 51.9k | case ISD::SIGN_EXTEND_VECTOR_INREG: |
3064 | 51.9k | Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits(); |
3065 | 51.9k | return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; |
3066 | 51.9k | |
3067 | 14.3k | case ISD::SIGN_EXTEND_INREG: |
3068 | 14.3k | // Max of the input and what this extends. |
3069 | 14.3k | Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits(); |
3070 | 14.3k | Tmp = VTBits-Tmp+1; |
3071 | 14.3k | |
3072 | 14.3k | Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
3073 | 14.3k | return std::max(Tmp, Tmp2); |
3074 | 51.9k | |
3075 | 13.0k | case ISD::SRA: |
3076 | 13.0k | Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); |
3077 | 13.0k | // SRA X, C -> adds C sign bits. |
3078 | 13.0k | if (ConstantSDNode *C13.0k = isConstOrConstSplat(Op.getOperand(1))) { |
3079 | 10.9k | APInt ShiftVal = C->getAPIntValue(); |
3080 | 10.9k | ShiftVal += Tmp; |
3081 | 10.9k | Tmp = ShiftVal.uge(VTBits) ? VTBits2.52k : ShiftVal.getZExtValue()8.47k ; |
3082 | 10.9k | } |
3083 | 13.0k | return Tmp; |
3084 | 87.0k | case ISD::SHL: |
3085 | 87.0k | if (ConstantSDNode *C87.0k = isConstOrConstSplat(Op.getOperand(1))) { |
3086 | 76.1k | // shl destroys sign bits. |
3087 | 76.1k | Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
3088 | 76.1k | if (C->getAPIntValue().uge(VTBits) || // Bad shift. |
3089 | 76.1k | C->getAPIntValue().uge(Tmp)76.1k ) break38.2k ; // Shifted all sign bits out. |
3090 | 37.9k | return Tmp - C->getZExtValue(); |
3091 | 37.9k | } |
3092 | 10.9k | break; |
3093 | 115k | case ISD::AND: |
3094 | 115k | case ISD::OR: |
3095 | 115k | case ISD::XOR: // NOT is handled here. |
3096 | 115k | // Logical binary ops preserve the number of sign bits at the worst. |
3097 | 115k | Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
3098 | 115k | if (Tmp != 1115k ) { |
3099 | 46.7k | Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); |
3100 | 46.7k | FirstAnswer = std::min(Tmp, Tmp2); |
3101 | 46.7k | // We computed what we know about the sign bits as our first |
3102 | 46.7k | // answer. Now proceed to the generic code that uses |
3103 | 46.7k | // computeKnownBits, and pick whichever answer is better. |
3104 | 46.7k | } |
3105 | 115k | break; |
3106 | 115k | |
3107 | 2.57k | case ISD::SELECT: |
3108 | 2.57k | Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); |
3109 | 2.57k | if (Tmp == 12.57k ) return 1917 ; // Early out. |
3110 | 1.65k | Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); |
3111 | 1.65k | return std::min(Tmp, Tmp2); |
3112 | 957 | case ISD::SELECT_CC: |
3113 | 957 | Tmp = ComputeNumSignBits(Op.getOperand(2), Depth+1); |
3114 | 957 | if (Tmp == 1957 ) return 1274 ; // Early out. |
3115 | 683 | Tmp2 = ComputeNumSignBits(Op.getOperand(3), Depth+1); |
3116 | 683 | return std::min(Tmp, Tmp2); |
3117 | 110 | case ISD::SMIN: |
3118 | 110 | case ISD::SMAX: |
3119 | 110 | case ISD::UMIN: |
3120 | 110 | case ISD::UMAX: |
3121 | 110 | Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
3122 | 110 | if (Tmp == 1) |
3123 | 10 | return 1; // Early out. |
3124 | 100 | Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1); |
3125 | 100 | return std::min(Tmp, Tmp2); |
3126 | 126 | case ISD::SADDO: |
3127 | 126 | case ISD::UADDO: |
3128 | 126 | case ISD::SSUBO: |
3129 | 126 | case ISD::USUBO: |
3130 | 126 | case ISD::SMULO: |
3131 | 126 | case ISD::UMULO: |
3132 | 126 | if (Op.getResNo() != 1) |
3133 | 52 | break; |
3134 | 74 | // The boolean result conforms to getBooleanContents. Fall through. |
3135 | 74 | // If setcc returns 0/-1, all bits are sign bits. |
3136 | 74 | // We know that we have an integer-based boolean since these operations |
3137 | 74 | // are only available for integer. |
3138 | 74 | if (74 TLI->getBooleanContents(Op.getValueType().isVector(), false) == |
3139 | 74 | TargetLowering::ZeroOrNegativeOneBooleanContent) |
3140 | 1 | return VTBits; |
3141 | 73 | break; |
3142 | 4.83k | case ISD::SETCC: |
3143 | 4.83k | // If setcc returns 0/-1, all bits are sign bits. |
3144 | 4.83k | if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) == |
3145 | 4.83k | TargetLowering::ZeroOrNegativeOneBooleanContent) |
3146 | 3.97k | return VTBits; |
3147 | 853 | break; |
3148 | 262 | case ISD::ROTL: |
3149 | 262 | case ISD::ROTR: |
3150 | 262 | if (ConstantSDNode *C262 = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { |
3151 | 248 | unsigned RotAmt = C->getAPIntValue().urem(VTBits); |
3152 | 248 | |
3153 | 248 | // Handle rotate right by N like a rotate left by 32-N. |
3154 | 248 | if (Op.getOpcode() == ISD::ROTR) |
3155 | 230 | RotAmt = (VTBits - RotAmt) % VTBits; |
3156 | 248 | |
3157 | 248 | // If we aren't rotating out all of the known-in sign bits, return the |
3158 | 248 | // number that are left. This handles rotl(sext(x), 1) for example. |
3159 | 248 | Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
3160 | 248 | if (Tmp > (RotAmt + 1)248 ) return (Tmp - RotAmt)0 ; |
3161 | 262 | } |
3162 | 262 | break; |
3163 | 608k | case ISD::ADD: |
3164 | 608k | case ISD::ADDC: |
3165 | 608k | // Add can have at most one carry bit. Thus we know that the output |
3166 | 608k | // is, at worst, one more bit than the inputs. |
3167 | 608k | Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
3168 | 608k | if (Tmp == 1608k ) return 1567k ; // Early out. |
3169 | 41.2k | |
3170 | 41.2k | // Special case decrementing a value (ADD X, -1): |
3171 | 41.2k | if (ConstantSDNode *41.2k CRHS41.2k = dyn_cast<ConstantSDNode>(Op.getOperand(1))) |
3172 | 29.2k | if (29.2k CRHS->isAllOnesValue()29.2k ) { |
3173 | 2.24k | KnownBits Known; |
3174 | 2.24k | computeKnownBits(Op.getOperand(0), Known, Depth+1); |
3175 | 2.24k | |
3176 | 2.24k | // If the input is known to be 0 or 1, the output is 0/-1, which is all |
3177 | 2.24k | // sign bits set. |
3178 | 2.24k | if ((Known.Zero | 1).isAllOnesValue()) |
3179 | 26 | return VTBits; |
3180 | 2.22k | |
3181 | 2.22k | // If we are subtracting one from a positive number, there is no carry |
3182 | 2.22k | // out of the result. |
3183 | 2.22k | if (2.22k Known.isNonNegative()2.22k ) |
3184 | 1.26k | return Tmp; |
3185 | 39.9k | } |
3186 | 39.9k | |
3187 | 39.9k | Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); |
3188 | 39.9k | if (Tmp2 == 139.9k ) return 13.49k ; |
3189 | 36.4k | return std::min(Tmp, Tmp2)-1; |
3190 | 36.4k | |
3191 | 62.0k | case ISD::SUB: |
3192 | 62.0k | Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); |
3193 | 62.0k | if (Tmp2 == 162.0k ) return 147.7k ; |
3194 | 14.3k | |
3195 | 14.3k | // Handle NEG. |
3196 | 14.3k | if (ConstantSDNode *14.3k CLHS14.3k = isConstOrConstSplat(Op.getOperand(0))) |
3197 | 1.98k | if (1.98k CLHS->isNullValue()1.98k ) { |
3198 | 970 | KnownBits Known; |
3199 | 970 | computeKnownBits(Op.getOperand(1), Known, Depth+1); |
3200 | 970 | // If the input is known to be 0 or 1, the output is 0/-1, which is all |
3201 | 970 | // sign bits set. |
3202 | 970 | if ((Known.Zero | 1).isAllOnesValue()) |
3203 | 64 | return VTBits; |
3204 | 906 | |
3205 | 906 | // If the input is known to be positive (the sign bit is known clear), |
3206 | 906 | // the output of the NEG has the same number of sign bits as the input. |
3207 | 906 | if (906 Known.isNonNegative()906 ) |
3208 | 390 | return Tmp2; |
3209 | 13.9k | |
3210 | 13.9k | // Otherwise, we treat this like a SUB. |
3211 | 13.9k | } |
3212 | 13.9k | |
3213 | 13.9k | // Sub can have at most one carry bit. Thus we know that the output |
3214 | 13.9k | // is, at worst, one more bit than the inputs. |
3215 | 13.9k | Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
3216 | 13.9k | if (Tmp == 113.9k ) return 16.91k ; // Early out. |
3217 | 7.00k | return std::min(Tmp, Tmp2)-1; |
3218 | 95.6k | case ISD::TRUNCATE: { |
3219 | 95.6k | // Check if the sign bits of source go down as far as the truncated value. |
3220 | 95.6k | unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits(); |
3221 | 95.6k | unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1); |
3222 | 95.6k | if (NumSrcSignBits > (NumSrcBits - VTBits)) |
3223 | 36.0k | return NumSrcSignBits - (NumSrcBits - VTBits); |
3224 | 59.6k | break; |
3225 | 59.6k | } |
3226 | 24 | case ISD::EXTRACT_ELEMENT: { |
3227 | 24 | const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1); |
3228 | 24 | const int BitWidth = Op.getValueSizeInBits(); |
3229 | 24 | const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth; |
3230 | 24 | |
3231 | 24 | // Get reverse index (starting from 1), Op1 value indexes elements from |
3232 | 24 | // little end. Sign starts at big end. |
3233 | 24 | const int rIndex = Items - 1 - Op.getConstantOperandVal(1); |
3234 | 24 | |
3235 | 24 | // If the sign portion ends in our element the subtraction gives correct |
3236 | 24 | // result. Otherwise it gives either negative or > bitwidth result |
3237 | 24 | return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0); |
3238 | 59.6k | } |
3239 | 224 | case ISD::INSERT_VECTOR_ELT: { |
3240 | 224 | SDValue InVec = Op.getOperand(0); |
3241 | 224 | SDValue InVal = Op.getOperand(1); |
3242 | 224 | SDValue EltNo = Op.getOperand(2); |
3243 | 224 | unsigned NumElts = InVec.getValueType().getVectorNumElements(); |
3244 | 224 | |
3245 | 224 | ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo); |
3246 | 224 | if (CEltNo && 224 CEltNo->getAPIntValue().ult(NumElts)220 ) { |
3247 | 220 | // If we know the element index, split the demand between the |
3248 | 220 | // source vector and the inserted element. |
3249 | 220 | unsigned EltIdx = CEltNo->getZExtValue(); |
3250 | 220 | |
3251 | 220 | // If we demand the inserted element then get its sign bits. |
3252 | 220 | Tmp = std::numeric_limits<unsigned>::max(); |
3253 | 220 | if (DemandedElts[EltIdx]220 ) { |
3254 | 218 | // TODO - handle implicit truncation of inserted elements. |
3255 | 218 | if (InVal.getScalarValueSizeInBits() != VTBits) |
3256 | 0 | break; |
3257 | 218 | Tmp = ComputeNumSignBits(InVal, Depth + 1); |
3258 | 218 | } |
3259 | 220 | |
3260 | 220 | // If we demand the source vector then get its sign bits, and determine |
3261 | 220 | // the minimum. |
3262 | 220 | APInt VectorElts = DemandedElts; |
3263 | 220 | VectorElts.clearBit(EltIdx); |
3264 | 220 | if (!!VectorElts220 ) { |
3265 | 177 | Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1); |
3266 | 177 | Tmp = std::min(Tmp, Tmp2); |
3267 | 177 | } |
3268 | 224 | } else { |
3269 | 4 | // Unknown element index, so ignore DemandedElts and demand them all. |
3270 | 4 | Tmp = ComputeNumSignBits(InVec, Depth + 1); |
3271 | 4 | Tmp2 = ComputeNumSignBits(InVal, Depth + 1); |
3272 | 4 | Tmp = std::min(Tmp, Tmp2); |
3273 | 4 | } |
3274 | 224 | assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
3275 | 224 | return Tmp; |
3276 | 224 | } |
3277 | 29.8k | case ISD::EXTRACT_VECTOR_ELT: { |
3278 | 29.8k | SDValue InVec = Op.getOperand(0); |
3279 | 29.8k | SDValue EltNo = Op.getOperand(1); |
3280 | 29.8k | EVT VecVT = InVec.getValueType(); |
3281 | 29.8k | const unsigned BitWidth = Op.getValueSizeInBits(); |
3282 | 29.8k | const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits(); |
3283 | 29.8k | const unsigned NumSrcElts = VecVT.getVectorNumElements(); |
3284 | 29.8k | |
3285 | 29.8k | // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know |
3286 | 29.8k | // anything about sign bits. But if the sizes match we can derive knowledge |
3287 | 29.8k | // about sign bits from the vector operand. |
3288 | 29.8k | if (BitWidth != EltBitWidth) |
3289 | 8.85k | break; |
3290 | 21.0k | |
3291 | 21.0k | // If we know the element index, just demand that vector element, else for |
3292 | 21.0k | // an unknown element index, ignore DemandedElts and demand them all. |
3293 | 21.0k | APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts); |
3294 | 21.0k | ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo); |
3295 | 21.0k | if (ConstEltNo && 21.0k ConstEltNo->getAPIntValue().ult(NumSrcElts)21.0k ) |
3296 | 21.0k | DemandedSrcElts = |
3297 | 21.0k | APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue()); |
3298 | 21.0k | |
3299 | 21.0k | return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1); |
3300 | 21.0k | } |
3301 | 1.66k | case ISD::EXTRACT_SUBVECTOR: { |
3302 | 1.66k | // If we know the element index, just demand that subvector elements, |
3303 | 1.66k | // otherwise demand them all. |
3304 | 1.66k | SDValue Src = Op.getOperand(0); |
3305 | 1.66k | ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1)); |
3306 | 1.66k | unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); |
3307 | 1.66k | if (SubIdx && 1.66k SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)1.66k ) { |
3308 | 1.66k | // Offset the demanded elts by the subvector index. |
3309 | 1.66k | uint64_t Idx = SubIdx->getZExtValue(); |
3310 | 1.66k | APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx); |
3311 | 1.66k | return ComputeNumSignBits(Src, DemandedSrc, Depth + 1); |
3312 | 1.66k | } |
3313 | 0 | return ComputeNumSignBits(Src, Depth + 1); |
3314 | 0 | } |
3315 | 959 | case ISD::CONCAT_VECTORS: |
3316 | 959 | // Determine the minimum number of sign bits across all demanded |
3317 | 959 | // elts of the input vectors. Early out if the result is already 1. |
3318 | 959 | Tmp = std::numeric_limits<unsigned>::max(); |
3319 | 959 | EVT SubVectorVT = Op.getOperand(0).getValueType(); |
3320 | 959 | unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements(); |
3321 | 959 | unsigned NumSubVectors = Op.getNumOperands(); |
3322 | 2.15k | for (unsigned i = 0; (i < NumSubVectors) && 2.15k (Tmp > 1)1.91k ; ++i1.19k ) { |
3323 | 1.19k | APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts); |
3324 | 1.19k | DemandedSub = DemandedSub.trunc(NumSubVectorElts); |
3325 | 1.19k | if (!DemandedSub) |
3326 | 139 | continue; |
3327 | 1.05k | Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1); |
3328 | 1.05k | Tmp = std::min(Tmp, Tmp2); |
3329 | 1.05k | } |
3330 | 355 | assert(Tmp <= VTBits && "Failed to determine minimum sign bits"); |
3331 | 355 | return Tmp; |
3332 | 2.54M | } |
3333 | 2.54M | |
3334 | 2.54M | // If we are looking at the loaded value of the SDNode. |
3335 | 2.54M | if (2.54M Op.getResNo() == 02.54M ) { |
3336 | 2.48M | // Handle LOADX separately here. EXTLOAD case will fallthrough. |
3337 | 2.48M | if (LoadSDNode *LD2.48M = dyn_cast<LoadSDNode>(Op)) { |
3338 | 856k | unsigned ExtType = LD->getExtensionType(); |
3339 | 856k | switch (ExtType) { |
3340 | 723k | default: break; |
3341 | 54.7k | case ISD::SEXTLOAD: // '17' bits known |
3342 | 54.7k | Tmp = LD->getMemoryVT().getScalarSizeInBits(); |
3343 | 54.7k | return VTBits-Tmp+1; |
3344 | 77.7k | case ISD::ZEXTLOAD: // '16' bits known |
3345 | 77.7k | Tmp = LD->getMemoryVT().getScalarSizeInBits(); |
3346 | 77.7k | return VTBits-Tmp; |
3347 | 2.41M | } |
3348 | 2.41M | } |
3349 | 2.48M | } |
3350 | 2.41M | |
3351 | 2.41M | // Allow the target to implement this method for its nodes. |
3352 | 2.41M | if (2.41M Op.getOpcode() >= ISD::BUILTIN_OP_END || |
3353 | 2.27M | Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || |
3354 | 2.27M | Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || |
3355 | 2.41M | Op.getOpcode() == ISD::INTRINSIC_VOID2.24M ) { |
3356 | 175k | unsigned NumBits = |
3357 | 175k | TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth); |
3358 | 175k | if (NumBits > 1) |
3359 | 3.88k | FirstAnswer = std::max(FirstAnswer, NumBits); |
3360 | 175k | } |
3361 | 2.41M | |
3362 | 2.41M | // Finally, if we can prove that the top bits of the result are 0's or 1's, |
3363 | 2.41M | // use this information. |
3364 | 2.41M | KnownBits Known; |
3365 | 2.41M | computeKnownBits(Op, Known, DemandedElts, Depth); |
3366 | 2.41M | |
3367 | 2.41M | APInt Mask; |
3368 | 2.41M | if (Known.isNonNegative()2.41M ) { // sign bit is 0 |
3369 | 205k | Mask = Known.Zero; |
3370 | 2.41M | } else if (2.21M Known.isNegative()2.21M ) { // sign bit is 1; |
3371 | 587 | Mask = Known.One; |
3372 | 2.21M | } else { |
3373 | 2.21M | // Nothing known. |
3374 | 2.21M | return FirstAnswer; |
3375 | 2.21M | } |
3376 | 205k | |
3377 | 205k | // Okay, we know that the sign bit in Mask is set. Use CLZ to determine |
3378 | 205k | // the number of identical bits in the top of the input value. |
3379 | 205k | Mask = ~Mask; |
3380 | 205k | Mask <<= Mask.getBitWidth()-VTBits; |
3381 | 205k | // Return # leading zeros. We use 'min' here in case Val was zero before |
3382 | 205k | // shifting. We don't want to return '64' as for an i32 "0". |
3383 | 205k | return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); |
3384 | 205k | } |
3385 | | |
3386 | 17.6M | bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { |
3387 | 17.6M | if ((Op.getOpcode() != ISD::ADD && 17.6M Op.getOpcode() != ISD::OR3.67M ) || |
3388 | 14.1M | !isa<ConstantSDNode>(Op.getOperand(1))) |
3389 | 4.24M | return false; |
3390 | 13.3M | |
3391 | 13.3M | if (13.3M Op.getOpcode() == ISD::OR && |
3392 | 190k | !MaskedValueIsZero(Op.getOperand(0), |
3393 | 190k | cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) |
3394 | 176 | return false; |
3395 | 13.3M | |
3396 | 13.3M | return true; |
3397 | 13.3M | } |
3398 | | |
3399 | 593 | bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { |
3400 | 593 | // If we're told that NaNs won't happen, assume they won't. |
3401 | 593 | if (getTarget().Options.NoNaNsFPMath) |
3402 | 194 | return true; |
3403 | 399 | |
3404 | 399 | if (399 Op->getFlags().hasNoNaNs()399 ) |
3405 | 26 | return true; |
3406 | 373 | |
3407 | 373 | // If the value is a constant, we can obviously see if it is a NaN or not. |
3408 | 373 | if (const ConstantFPSDNode *373 C373 = dyn_cast<ConstantFPSDNode>(Op)) |
3409 | 43 | return !C->getValueAPF().isNaN(); |
3410 | 330 | |
3411 | 330 | // TODO: Recognize more cases here. |
3412 | 330 | |
3413 | 330 | return false; |
3414 | 330 | } |
3415 | | |
3416 | 172 | bool SelectionDAG::isKnownNeverZero(SDValue Op) const { |
3417 | 172 | // If the value is a constant, we can obviously see if it is a zero or not. |
3418 | 172 | if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) |
3419 | 26 | return !C->isZero(); |
3420 | 146 | |
3421 | 146 | // TODO: Recognize more cases here. |
3422 | 146 | switch (Op.getOpcode()) { |
3423 | 146 | default: break; |
3424 | 0 | case ISD::OR: |
3425 | 0 | if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) |
3426 | 0 | return !C->isNullValue(); |
3427 | 0 | break; |
3428 | 146 | } |
3429 | 146 | |
3430 | 146 | return false; |
3431 | 146 | } |
3432 | | |
3433 | 14.4k | bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { |
3434 | 14.4k | // Check the obvious case. |
3435 | 14.4k | if (A == B14.4k ) return true3.86k ; |
3436 | 10.5k | |
3437 | 10.5k | // For for negative and positive zero. |
3438 | 10.5k | if (const ConstantFPSDNode *10.5k CA10.5k = dyn_cast<ConstantFPSDNode>(A)) |
3439 | 1.25k | if (const ConstantFPSDNode *1.25k CB1.25k = dyn_cast<ConstantFPSDNode>(B)) |
3440 | 114 | if (114 CA->isZero() && 114 CB->isZero()4 ) return true4 ; |
3441 | 10.5k | |
3442 | 10.5k | // Otherwise they may not be equal. |
3443 | 10.5k | return false; |
3444 | 10.5k | } |
3445 | | |
3446 | 8.64M | bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const { |
3447 | 8.64M | assert(A.getValueType() == B.getValueType() && |
3448 | 8.64M | "Values must have the same type"); |
3449 | 8.64M | KnownBits AKnown, BKnown; |
3450 | 8.64M | computeKnownBits(A, AKnown); |
3451 | 8.64M | computeKnownBits(B, BKnown); |
3452 | 8.64M | return (AKnown.Zero | BKnown.Zero).isAllOnesValue(); |
3453 | 8.64M | } |
3454 | | |
3455 | | static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT, |
3456 | | ArrayRef<SDValue> Ops, |
3457 | 50.1k | SelectionDAG &DAG) { |
3458 | 50.1k | assert(!Ops.empty() && "Can't concatenate an empty list of vectors!"); |
3459 | 50.1k | assert(llvm::all_of(Ops, |
3460 | 50.1k | [Ops](SDValue Op) { |
3461 | 50.1k | return Ops[0].getValueType() == Op.getValueType(); |
3462 | 50.1k | }) && |
3463 | 50.1k | "Concatenation of vectors with inconsistent value types!"); |
3464 | 50.1k | assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) == |
3465 | 50.1k | VT.getVectorNumElements() && |
3466 | 50.1k | "Incorrect element count in vector concatenation!"); |
3467 | 50.1k | |
3468 | 50.1k | if (Ops.size() == 1) |
3469 | 0 | return Ops[0]; |
3470 | 50.1k | |
3471 | 50.1k | // Concat of UNDEFs is UNDEF. |
3472 | 50.6k | if (50.1k llvm::all_of(Ops, [](SDValue Op) 50.1k { return Op.isUndef(); }50.6k )) |
3473 | 150 | return DAG.getUNDEF(VT); |
3474 | 49.9k | |
3475 | 49.9k | // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be |
3476 | 49.9k | // simplified to one big BUILD_VECTOR. |
3477 | 49.9k | // FIXME: Add support for SCALAR_TO_VECTOR as well. |
3478 | 49.9k | EVT SVT = VT.getScalarType(); |
3479 | 49.9k | SmallVector<SDValue, 16> Elts; |
3480 | 52.0k | for (SDValue Op : Ops) { |
3481 | 52.0k | EVT OpVT = Op.getValueType(); |
3482 | 52.0k | if (Op.isUndef()) |
3483 | 529 | Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT)); |
3484 | 51.5k | else if (51.5k Op.getOpcode() == ISD::BUILD_VECTOR51.5k ) |
3485 | 2.10k | Elts.append(Op->op_begin(), Op->op_end()); |
3486 | 51.5k | else |
3487 | 49.3k | return SDValue(); |
3488 | 567 | } |
3489 | 567 | |
3490 | 567 | // BUILD_VECTOR requires all inputs to be of the same type, find the |
3491 | 567 | // maximum type and extend them all. |
3492 | 567 | for (SDValue Op : Elts) |
3493 | 10.4k | SVT = (SVT.bitsLT(Op.getValueType()) ? 10.4k Op.getValueType()145 : SVT10.2k ); |
3494 | 567 | |
3495 | 567 | if (SVT.bitsGT(VT.getScalarType())) |
3496 | 145 | for (SDValue &Op : Elts) |
3497 | 4.44k | Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT) |
3498 | 0 | ? DAG.getZExtOrTrunc(Op, DL, SVT) |
3499 | 4.44k | : DAG.getSExtOrTrunc(Op, DL, SVT); |
3500 | 50.1k | |
3501 | 50.1k | SDValue V = DAG.getBuildVector(VT, DL, Elts); |
3502 | 50.1k | NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG); |
3503 | 50.1k | return V; |
3504 | 50.1k | } |
3505 | | |
3506 | | /// Gets or creates the specified node. |
3507 | 7.11M | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) { |
3508 | 7.11M | FoldingSetNodeID ID; |
3509 | 7.11M | AddNodeIDNode(ID, Opcode, getVTList(VT), None); |
3510 | 7.11M | void *IP = nullptr; |
3511 | 7.11M | if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
3512 | 5.47M | return SDValue(E, 0); |
3513 | 1.64M | |
3514 | 1.64M | auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), |
3515 | 1.64M | getVTList(VT)); |
3516 | 1.64M | CSEMap.InsertNode(N, IP); |
3517 | 1.64M | |
3518 | 1.64M | InsertNode(N); |
3519 | 1.64M | SDValue V = SDValue(N, 0); |
3520 | 1.64M | NewSDValueDbgMsg(V, "Creating new node: ", this); |
3521 | 1.64M | return V; |
3522 | 1.64M | } |
3523 | | |
3524 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
3525 | 20.1M | SDValue Operand, const SDNodeFlags Flags) { |
3526 | 20.1M | // Constant fold unary operations with an integer constant operand. Even |
3527 | 20.1M | // opaque constant will be folded, because the folding of unary operations |
3528 | 20.1M | // doesn't create new constants with different values. Nevertheless, the |
3529 | 20.1M | // opaque flag is preserved during folding to prevent future folding with |
3530 | 20.1M | // other constants. |
3531 | 20.1M | if (ConstantSDNode *C20.1M = dyn_cast<ConstantSDNode>(Operand)) { |
3532 | 920k | const APInt &Val = C->getAPIntValue(); |
3533 | 920k | switch (Opcode) { |
3534 | 48.3k | default: break; |
3535 | 432k | case ISD::SIGN_EXTEND: |
3536 | 432k | return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT, |
3537 | 432k | C->isTargetOpcode(), C->isOpaque()); |
3538 | 400k | case ISD::ANY_EXTEND: |
3539 | 400k | case ISD::ZERO_EXTEND: |
3540 | 400k | case ISD::TRUNCATE: |
3541 | 400k | return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT, |
3542 | 400k | C->isTargetOpcode(), C->isOpaque()); |
3543 | 411 | case ISD::UINT_TO_FP: |
3544 | 411 | case ISD::SINT_TO_FP: { |
3545 | 411 | APFloat apf(EVTToAPFloatSemantics(VT), |
3546 | 411 | APInt::getNullValue(VT.getSizeInBits())); |
3547 | 411 | (void)apf.convertFromAPInt(Val, |
3548 | 411 | Opcode==ISD::SINT_TO_FP, |
3549 | 411 | APFloat::rmNearestTiesToEven); |
3550 | 411 | return getConstantFP(apf, DL, VT); |
3551 | 411 | } |
3552 | 10.2k | case ISD::BITCAST: |
3553 | 10.2k | if (VT == MVT::f16 && 10.2k C->getValueType(0) == MVT::i169 ) |
3554 | 9 | return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT); |
3555 | 10.2k | if (10.2k VT == MVT::f32 && 10.2k C->getValueType(0) == MVT::i32319 ) |
3556 | 319 | return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT); |
3557 | 9.96k | if (9.96k VT == MVT::f64 && 9.96k C->getValueType(0) == MVT::i64107 ) |
3558 | 107 | return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT); |
3559 | 9.85k | if (9.85k VT == MVT::f128 && 9.85k C->getValueType(0) == MVT::i1280 ) |
3560 | 0 | return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT); |
3561 | 9.85k | break; |
3562 | 190 | case ISD::ABS: |
3563 | 190 | return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(), |
3564 | 190 | C->isOpaque()); |
3565 | 403 | case ISD::BITREVERSE: |
3566 | 403 | return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(), |
3567 | 403 | C->isOpaque()); |
3568 | 170 | case ISD::BSWAP: |
3569 | 170 | return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(), |
3570 | 170 | C->isOpaque()); |
3571 | 390 | case ISD::CTPOP: |
3572 | 390 | return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(), |
3573 | 390 | C->isOpaque()); |
3574 | 25.5k | case ISD::CTLZ: |
3575 | 25.5k | case ISD::CTLZ_ZERO_UNDEF: |
3576 | 25.5k | return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(), |
3577 | 25.5k | C->isOpaque()); |
3578 | 1.32k | case ISD::CTTZ: |
3579 | 1.32k | case ISD::CTTZ_ZERO_UNDEF: |
3580 | 1.32k | return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(), |
3581 | 1.32k | C->isOpaque()); |
3582 | 281 | case ISD::FP16_TO_FP: { |
3583 | 281 | bool Ignored; |
3584 | 281 | APFloat FPV(APFloat::IEEEhalf(), |
3585 | 281 | (Val.getBitWidth() == 16) ? Val281 : Val.trunc(16)0 ); |
3586 | 281 | |
3587 | 281 | // This can return overflow, underflow, or inexact; we don't care. |
3588 | 281 | // FIXME need to be more flexible about rounding mode. |
3589 | 281 | (void)FPV.convert(EVTToAPFloatSemantics(VT), |
3590 | 281 | APFloat::rmNearestTiesToEven, &Ignored); |
3591 | 281 | return getConstantFP(FPV, DL, VT); |
3592 | 19.2M | } |
3593 | 920k | } |
3594 | 920k | } |
3595 | 19.2M | |
3596 | 19.2M | // Constant fold unary operations with a floating point constant operand. |
3597 | 19.2M | if (ConstantFPSDNode *19.2M C19.2M = dyn_cast<ConstantFPSDNode>(Operand)) { |
3598 | 6.30k | APFloat V = C->getValueAPF(); // make copy |
3599 | 6.30k | switch (Opcode) { |
3600 | 317 | case ISD::FNEG: |
3601 | 317 | V.changeSign(); |
3602 | 317 | return getConstantFP(V, DL, VT); |
3603 | 120 | case ISD::FABS: |
3604 | 120 | V.clearSign(); |
3605 | 120 | return getConstantFP(V, DL, VT); |
3606 | 16 | case ISD::FCEIL: { |
3607 | 16 | APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); |
3608 | 16 | if (fs == APFloat::opOK || 16 fs == APFloat::opInexact0 ) |
3609 | 16 | return getConstantFP(V, DL, VT); |
3610 | 0 | break; |
3611 | 0 | } |
3612 | 13 | case ISD::FTRUNC: { |
3613 | 13 | APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); |
3614 | 13 | if (fs == APFloat::opOK || 13 fs == APFloat::opInexact0 ) |
3615 | 13 | return getConstantFP(V, DL, VT); |
3616 | 0 | break; |
3617 | 0 | } |
3618 | 13 | case ISD::FFLOOR: { |
3619 | 13 | APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); |
3620 | 13 | if (fs == APFloat::opOK || 13 fs == APFloat::opInexact0 ) |
3621 | 13 | return getConstantFP(V, DL, VT); |
3622 | 0 | break; |
3623 | 0 | } |
3624 | 104 | case ISD::FP_EXTEND: { |
3625 | 104 | bool ignored; |
3626 | 104 | // This can return overflow, underflow, or inexact; we don't care. |
3627 | 104 | // FIXME need to be more flexible about rounding mode. |
3628 | 104 | (void)V.convert(EVTToAPFloatSemantics(VT), |
3629 | 104 | APFloat::rmNearestTiesToEven, &ignored); |
3630 | 104 | return getConstantFP(V, DL, VT); |
3631 | 0 | } |
3632 | 625 | case ISD::FP_TO_SINT: |
3633 | 625 | case ISD::FP_TO_UINT: { |
3634 | 625 | bool ignored; |
3635 | 625 | APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT); |
3636 | 625 | // FIXME need to be more flexible about rounding mode. |
3637 | 625 | APFloat::opStatus s = |
3638 | 625 | V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored); |
3639 | 625 | if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual |
3640 | 28 | break; |
3641 | 597 | return getConstant(IntVal, DL, VT); |
3642 | 597 | } |
3643 | 3.73k | case ISD::BITCAST: |
3644 | 3.73k | if (VT == MVT::i16 && 3.73k C->getValueType(0) == MVT::f16193 ) |
3645 | 193 | return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT); |
3646 | 3.54k | else if (3.54k VT == MVT::i32 && 3.54k C->getValueType(0) == MVT::f322.55k ) |
3647 | 2.55k | return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT); |
3648 | 990 | else if (990 VT == MVT::i64 && 990 C->getValueType(0) == MVT::f64798 ) |
3649 | 798 | return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT); |
3650 | 192 | break; |
3651 | 18 | case ISD::FP_TO_FP16: { |
3652 | 18 | bool Ignored; |
3653 | 18 | // This can return overflow, underflow, or inexact; we don't care. |
3654 | 18 | // FIXME need to be more flexible about rounding mode. |
3655 | 18 | (void)V.convert(APFloat::IEEEhalf(), |
3656 | 18 | APFloat::rmNearestTiesToEven, &Ignored); |
3657 | 18 | return getConstant(V.bitcastToAPInt(), DL, VT); |
3658 | 19.2M | } |
3659 | 6.30k | } |
3660 | 6.30k | } |
3661 | 19.2M | |
3662 | 19.2M | // Constant fold unary operations with a vector integer or float operand. |
3663 | 19.2M | if (BuildVectorSDNode *19.2M BV19.2M = dyn_cast<BuildVectorSDNode>(Operand)) { |
3664 | 191k | if (BV->isConstant()191k ) { |
3665 | 33.2k | switch (Opcode) { |
3666 | 31.3k | default: |
3667 | 31.3k | // FIXME: Entirely reasonable to perform folding of other unary |
3668 | 31.3k | // operations here as the need arises. |
3669 | 31.3k | break; |
3670 | 1.85k | case ISD::FNEG: |
3671 | 1.85k | case ISD::FABS: |
3672 | 1.85k | case ISD::FCEIL: |
3673 | 1.85k | case ISD::FTRUNC: |
3674 | 1.85k | case ISD::FFLOOR: |
3675 | 1.85k | case ISD::FP_EXTEND: |
3676 | 1.85k | case ISD::FP_TO_SINT: |
3677 | 1.85k | case ISD::FP_TO_UINT: |
3678 | 1.85k | case ISD::TRUNCATE: |
3679 | 1.85k | case ISD::UINT_TO_FP: |
3680 | 1.85k | case ISD::SINT_TO_FP: |
3681 | 1.85k | case ISD::ABS: |
3682 | 1.85k | case ISD::BITREVERSE: |
3683 | 1.85k | case ISD::BSWAP: |
3684 | 1.85k | case ISD::CTLZ: |
3685 | 1.85k | case ISD::CTLZ_ZERO_UNDEF: |
3686 | 1.85k | case ISD::CTTZ: |
3687 | 1.85k | case ISD::CTTZ_ZERO_UNDEF: |
3688 | 1.85k | case ISD::CTPOP: { |
3689 | 1.85k | SDValue Ops = { Operand }; |
3690 | 1.85k | if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) |
3691 | 1.79k | return Fold; |
3692 | 19.2M | } |
3693 | 33.2k | } |
3694 | 33.2k | } |
3695 | 191k | } |
3696 | 19.2M | |
3697 | 19.2M | unsigned OpOpcode = Operand.getNode()->getOpcode(); |
3698 | 19.2M | switch (Opcode) { |
3699 | 12.7M | case ISD::TokenFactor: |
3700 | 12.7M | case ISD::MERGE_VALUES: |
3701 | 12.7M | case ISD::CONCAT_VECTORS: |
3702 | 12.7M | return Operand; // Factor, merge or concat of one node? No need. |
3703 | 0 | case ISD::FP_ROUND: 0 llvm_unreachable0 ("Invalid method to make FP_ROUND node"); |
3704 | 33.4k | case ISD::FP_EXTEND: |
3705 | 33.4k | assert(VT.isFloatingPoint() && |
3706 | 33.4k | Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); |
3707 | 33.4k | if (Operand.getValueType() == VT33.4k ) return Operand2.29k ; // noop conversion. |
3708 | 33.4k | assert((!VT.isVector() || |
3709 | 31.1k | VT.getVectorNumElements() == |
3710 | 31.1k | Operand.getValueType().getVectorNumElements()) && |
3711 | 31.1k | "Vector element count mismatch!"); |
3712 | 31.1k | assert(Operand.getValueType().bitsLT(VT) && |
3713 | 31.1k | "Invalid fpext node, dst < src!"); |
3714 | 31.1k | if (Operand.isUndef()) |
3715 | 53 | return getUNDEF(VT); |
3716 | 31.1k | break; |
3717 | 340k | case ISD::SIGN_EXTEND: |
3718 | 340k | assert(VT.isInteger() && Operand.getValueType().isInteger() && |
3719 | 340k | "Invalid SIGN_EXTEND!"); |
3720 | 340k | if (Operand.getValueType() == VT340k ) return Operand3.49k ; // noop extension |
3721 | 340k | assert((!VT.isVector() || |
3722 | 336k | VT.getVectorNumElements() == |
3723 | 336k | Operand.getValueType().getVectorNumElements()) && |
3724 | 336k | "Vector element count mismatch!"); |
3725 | 336k | assert(Operand.getValueType().bitsLT(VT) && |
3726 | 336k | "Invalid sext node, dst < src!"); |
3727 | 336k | if (OpOpcode == ISD::SIGN_EXTEND || 336k OpOpcode == ISD::ZERO_EXTEND336k ) |
3728 | 394 | return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); |
3729 | 336k | else if (336k OpOpcode == ISD::UNDEF336k ) |
3730 | 336k | // sext(undef) = 0, because the top bits will all be the same. |
3731 | 123 | return getConstant(0, DL, VT); |
3732 | 336k | break; |
3733 | 684k | case ISD::ZERO_EXTEND: |
3734 | 684k | assert(VT.isInteger() && Operand.getValueType().isInteger() && |
3735 | 684k | "Invalid ZERO_EXTEND!"); |
3736 | 684k | if (Operand.getValueType() == VT684k ) return Operand15.3k ; // noop extension |
3737 | 684k | assert((!VT.isVector() || |
3738 | 669k | VT.getVectorNumElements() == |
3739 | 669k | Operand.getValueType().getVectorNumElements()) && |
3740 | 669k | "Vector element count mismatch!"); |
3741 | 669k | assert(Operand.getValueType().bitsLT(VT) && |
3742 | 669k | "Invalid zext node, dst < src!"); |
3743 | 669k | if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) |
3744 | 4.05k | return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0)); |
3745 | 665k | else if (665k OpOpcode == ISD::UNDEF665k ) |
3746 | 665k | // zext(undef) = 0, because the top bits will be zero. |
3747 | 882 | return getConstant(0, DL, VT); |
3748 | 664k | break; |
3749 | 734k | case ISD::ANY_EXTEND: |
3750 | 734k | assert(VT.isInteger() && Operand.getValueType().isInteger() && |
3751 | 734k | "Invalid ANY_EXTEND!"); |
3752 | 734k | if (Operand.getValueType() == VT734k ) return Operand452k ; // noop extension |
3753 | 734k | assert((!VT.isVector() || |
3754 | 282k | VT.getVectorNumElements() == |
3755 | 282k | Operand.getValueType().getVectorNumElements()) && |
3756 | 282k | "Vector element count mismatch!"); |
3757 | 282k | assert(Operand.getValueType().bitsLT(VT) && |
3758 | 282k | "Invalid anyext node, dst < src!"); |
3759 | 282k | |
3760 | 282k | if (OpOpcode == ISD::ZERO_EXTEND || 282k OpOpcode == ISD::SIGN_EXTEND280k || |
3761 | 279k | OpOpcode == ISD::ANY_EXTEND) |
3762 | 282k | // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) |
3763 | 24.2k | return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); |
3764 | 257k | else if (257k OpOpcode == ISD::UNDEF257k ) |
3765 | 967 | return getUNDEF(VT); |
3766 | 257k | |
3767 | 257k | // (ext (trunx x)) -> x |
3768 | 257k | if (257k OpOpcode == ISD::TRUNCATE257k ) { |
3769 | 21.5k | SDValue OpOp = Operand.getOperand(0); |
3770 | 21.5k | if (OpOp.getValueType() == VT) |
3771 | 16.3k | return OpOp; |
3772 | 240k | } |
3773 | 240k | break; |
3774 | 2.26M | case ISD::TRUNCATE: |
3775 | 2.26M | assert(VT.isInteger() && Operand.getValueType().isInteger() && |
3776 | 2.26M | "Invalid TRUNCATE!"); |
3777 | 2.26M | if (Operand.getValueType() == VT2.26M ) return Operand1.22M ; // noop truncate |
3778 | 2.26M | assert((!VT.isVector() || |
3779 | 1.04M | VT.getVectorNumElements() == |
3780 | 1.04M | Operand.getValueType().getVectorNumElements()) && |
3781 | 1.04M | "Vector element count mismatch!"); |
3782 | 1.04M | assert(Operand.getValueType().bitsGT(VT) && |
3783 | 1.04M | "Invalid truncate node, src < dst!"); |
3784 | 1.04M | if (OpOpcode == ISD::TRUNCATE) |
3785 | 37.0k | return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); |
3786 | 1.00M | if (1.00M OpOpcode == ISD::ZERO_EXTEND || 1.00M OpOpcode == ISD::SIGN_EXTEND1.00M || |
3787 | 1.00M | OpOpcode == ISD::ANY_EXTEND997k ) { |
3788 | 19.9k | // If the source is smaller than the dest, we still need an extend. |
3789 | 19.9k | if (Operand.getOperand(0).getValueType().getScalarType() |
3790 | 19.9k | .bitsLT(VT.getScalarType())) |
3791 | 4.70k | return getNode(OpOpcode, DL, VT, Operand.getOperand(0)); |
3792 | 15.2k | if (15.2k Operand.getOperand(0).getValueType().bitsGT(VT)15.2k ) |
3793 | 785 | return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0)); |
3794 | 14.4k | return Operand.getOperand(0); |
3795 | 14.4k | } |
3796 | 984k | if (984k OpOpcode == ISD::UNDEF984k ) |
3797 | 12.0k | return getUNDEF(VT); |
3798 | 972k | break; |
3799 | 569 | case ISD::ABS: |
3800 | 569 | assert(VT.isInteger() && VT == Operand.getValueType() && |
3801 | 569 | "Invalid ABS!"); |
3802 | 569 | if (OpOpcode == ISD::UNDEF) |
3803 | 1 | return getUNDEF(VT); |
3804 | 568 | break; |
3805 | 1.52k | case ISD::BSWAP: |
3806 | 1.52k | assert(VT.isInteger() && VT == Operand.getValueType() && |
3807 | 1.52k | "Invalid BSWAP!"); |
3808 | 1.52k | assert((VT.getScalarSizeInBits() % 16 == 0) && |
3809 | 1.52k | "BSWAP types must be a multiple of 16 bits!"); |
3810 | 1.52k | if (OpOpcode == ISD::UNDEF) |
3811 | 4 | return getUNDEF(VT); |
3812 | 1.52k | break; |
3813 | 714 | case ISD::BITREVERSE: |
3814 | 714 | assert(VT.isInteger() && VT == Operand.getValueType() && |
3815 | 714 | "Invalid BITREVERSE!"); |
3816 | 714 | if (OpOpcode == ISD::UNDEF) |
3817 | 4 | return getUNDEF(VT); |
3818 | 710 | break; |
3819 | 757k | case ISD::BITCAST: |
3820 | 757k | // Basic sanity checking. |
3821 | 757k | assert(VT.getSizeInBits() == Operand.getValueSizeInBits() && |
3822 | 757k | "Cannot BITCAST between types of different sizes!"); |
3823 | 757k | if (VT == Operand.getValueType()757k ) return Operand200k ; // noop conversion. |
3824 | 556k | if (556k OpOpcode == ISD::BITCAST556k ) // bitconv(bitconv(x)) -> bitconv(x) |
3825 | 32.1k | return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); |
3826 | 524k | if (524k OpOpcode == ISD::UNDEF524k ) |
3827 | 5.41k | return getUNDEF(VT); |
3828 | 518k | break; |
3829 | 40.8k | case ISD::SCALAR_TO_VECTOR: |
3830 | 40.8k | assert(VT.isVector() && !Operand.getValueType().isVector() && |
3831 | 40.8k | (VT.getVectorElementType() == Operand.getValueType() || |
3832 | 40.8k | (VT.getVectorElementType().isInteger() && |
3833 | 40.8k | Operand.getValueType().isInteger() && |
3834 | 40.8k | VT.getVectorElementType().bitsLE(Operand.getValueType()))) && |
3835 | 40.8k | "Illegal SCALAR_TO_VECTOR node!"); |
3836 | 40.8k | if (OpOpcode == ISD::UNDEF) |
3837 | 1 | return getUNDEF(VT); |
3838 | 40.8k | // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. |
3839 | 40.8k | if (40.8k OpOpcode == ISD::EXTRACT_VECTOR_ELT && |
3840 | 16.5k | isa<ConstantSDNode>(Operand.getOperand(1)) && |
3841 | 16.3k | Operand.getConstantOperandVal(1) == 0 && |
3842 | 5.87k | Operand.getOperand(0).getValueType() == VT) |
3843 | 644 | return Operand.getOperand(0); |
3844 | 40.2k | break; |
3845 | 8.39k | case ISD::FNEG: |
3846 | 8.39k | // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 |
3847 | 8.39k | if (getTarget().Options.UnsafeFPMath && 8.39k OpOpcode == ISD::FSUB646 ) |
3848 | 8.39k | // FIXME: FNEG has no fast-math-flags to propagate; use the FSUB's flags? |
3849 | 59 | return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1), |
3850 | 59 | Operand.getOperand(0), Operand.getNode()->getFlags()); |
3851 | 8.33k | if (8.33k OpOpcode == ISD::FNEG8.33k ) // --X -> X |
3852 | 0 | return Operand.getOperand(0); |
3853 | 8.33k | break; |
3854 | 4.17k | case ISD::FABS: |
3855 | 4.17k | if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) |
3856 | 0 | return getNode(ISD::FABS, DL, VT, Operand.getOperand(0)); |
3857 | 4.17k | break; |
3858 | 4.51M | } |
3859 | 4.51M | |
3860 | 4.51M | SDNode *N; |
3861 | 4.51M | SDVTList VTs = getVTList(VT); |
3862 | 4.51M | SDValue Ops[] = {Operand}; |
3863 | 4.51M | if (VT != MVT::Glue4.51M ) { // Don't CSE flag producing nodes |
3864 | 4.51M | FoldingSetNodeID ID; |
3865 | 4.51M | AddNodeIDNode(ID, Opcode, VTs, Ops); |
3866 | 4.51M | void *IP = nullptr; |
3867 | 4.51M | if (SDNode *E4.51M = FindNodeOrInsertPos(ID, DL, IP)) { |
3868 | 282k | E->intersectFlagsWith(Flags); |
3869 | 282k | return SDValue(E, 0); |
3870 | 282k | } |
3871 | 4.23M | |
3872 | 4.23M | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
3873 | 4.23M | N->setFlags(Flags); |
3874 | 4.23M | createOperands(N, Ops); |
3875 | 4.23M | CSEMap.InsertNode(N, IP); |
3876 | 4.51M | } else { |
3877 | 1.26k | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
3878 | 1.26k | createOperands(N, Ops); |
3879 | 1.26k | } |
3880 | 4.51M | |
3881 | 4.23M | InsertNode(N); |
3882 | 4.23M | SDValue V = SDValue(N, 0); |
3883 | 4.23M | NewSDValueDbgMsg(V, "Creating new node: ", this); |
3884 | 4.23M | return V; |
3885 | 20.1M | } |
3886 | | |
3887 | | static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1, |
3888 | 1.07M | const APInt &C2) { |
3889 | 1.07M | switch (Opcode) { |
3890 | 592k | case ISD::ADD: return std::make_pair(C1 + C2, true); |
3891 | 25.2k | case ISD::SUB: return std::make_pair(C1 - C2, true); |
3892 | 4.97k | case ISD::MUL: return std::make_pair(C1 * C2, true); |
3893 | 266k | case ISD::AND: return std::make_pair(C1 & C2, true); |
3894 | 1.56k | case ISD::OR: return std::make_pair(C1 | C2, true); |
3895 | 2.50k | case ISD::XOR: return std::make_pair(C1 ^ C2, true); |
3896 | 26.3k | case ISD::SHL: return std::make_pair(C1 << C2, true); |
3897 | 13.6k | case ISD::SRL: return std::make_pair(C1.lshr(C2), true); |
3898 | 927 | case ISD::SRA: return std::make_pair(C1.ashr(C2), true); |
3899 | 16 | case ISD::ROTL: return std::make_pair(C1.rotl(C2), true); |
3900 | 0 | case ISD::ROTR: return std::make_pair(C1.rotr(C2), true); |
3901 | 840 | case ISD::SMIN: return std::make_pair(C1.sle(C2) ? 840 C1446 : C2394 , true); |
3902 | 844 | case ISD::SMAX: return std::make_pair(C1.sge(C2) ? 844 C1496 : C2348 , true); |
3903 | 824 | case ISD::UMIN: return std::make_pair(C1.ule(C2) ? 824 C1443 : C2381 , true); |
3904 | 825 | case ISD::UMAX: return std::make_pair(C1.uge(C2) ? 825 C1481 : C2344 , true); |
3905 | 28 | case ISD::UDIV: |
3906 | 28 | if (!C2.getBoolValue()) |
3907 | 0 | break; |
3908 | 28 | return std::make_pair(C1.udiv(C2), true); |
3909 | 32 | case ISD::UREM: |
3910 | 32 | if (!C2.getBoolValue()) |
3911 | 0 | break; |
3912 | 32 | return std::make_pair(C1.urem(C2), true); |
3913 | 99 | case ISD::SDIV: |
3914 | 99 | if (!C2.getBoolValue()) |
3915 | 0 | break; |
3916 | 99 | return std::make_pair(C1.sdiv(C2), true); |
3917 | 57 | case ISD::SREM: |
3918 | 57 | if (!C2.getBoolValue()) |
3919 | 0 | break; |
3920 | 57 | return std::make_pair(C1.srem(C2), true); |
3921 | 136k | } |
3922 | 136k | return std::make_pair(APInt(1, 0), false); |
3923 | 136k | } |
3924 | | |
3925 | | SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, |
3926 | | EVT VT, const ConstantSDNode *Cst1, |
3927 | 1.07M | const ConstantSDNode *Cst2) { |
3928 | 1.07M | if (Cst1->isOpaque() || 1.07M Cst2->isOpaque()1.07M ) |
3929 | 1.40k | return SDValue(); |
3930 | 1.07M | |
3931 | 1.07M | std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(), |
3932 | 1.07M | Cst2->getAPIntValue()); |
3933 | 1.07M | if (!Folded.second) |
3934 | 136k | return SDValue(); |
3935 | 937k | return getConstant(Folded.first, DL, VT); |
3936 | 937k | } |
3937 | | |
3938 | | SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, |
3939 | | const GlobalAddressSDNode *GA, |
3940 | 298k | const SDNode *N2) { |
3941 | 298k | if (GA->getOpcode() != ISD::GlobalAddress) |
3942 | 11 | return SDValue(); |
3943 | 298k | if (298k !TLI->isOffsetFoldingLegal(GA)298k ) |
3944 | 295k | return SDValue(); |
3945 | 2.64k | const ConstantSDNode *Cst2 = dyn_cast<ConstantSDNode>(N2); |
3946 | 2.64k | if (!Cst2) |
3947 | 1.38k | return SDValue(); |
3948 | 1.26k | int64_t Offset = Cst2->getSExtValue(); |
3949 | 1.26k | switch (Opcode) { |
3950 | 1.25k | case ISD::ADD: break; |
3951 | 1 | case ISD::SUB: Offset = -uint64_t(Offset); break; |
3952 | 5 | default: return SDValue(); |
3953 | 1.25k | } |
3954 | 1.25k | return getGlobalAddress(GA->getGlobal(), SDLoc(Cst2), VT, |
3955 | 1.25k | GA->getOffset() + uint64_t(Offset)); |
3956 | 1.25k | } |
3957 | | |
3958 | 20.2M | bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) { |
3959 | 20.2M | switch (Opcode) { |
3960 | 87.4k | case ISD::SDIV: |
3961 | 87.4k | case ISD::UDIV: |
3962 | 87.4k | case ISD::SREM: |
3963 | 87.4k | case ISD::UREM: { |
3964 | 87.4k | // If a divisor is zero/undef or any element of a divisor vector is |
3965 | 87.4k | // zero/undef, the whole op is undef. |
3966 | 87.4k | assert(Ops.size() == 2 && "Div/rem should have 2 operands"); |
3967 | 87.4k | SDValue Divisor = Ops[1]; |
3968 | 87.4k | if (Divisor.isUndef() || 87.4k isNullConstant(Divisor)87.4k ) |
3969 | 37 | return true; |
3970 | 87.4k | |
3971 | 87.4k | return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) && |
3972 | 1.34k | llvm::any_of(Divisor->op_values(), |
3973 | 13.0k | [](SDValue V) { return V.isUndef() || |
3974 | 13.0k | isNullConstant(V); }); |
3975 | 87.4k | // TODO: Handle signed overflow. |
3976 | 87.4k | } |
3977 | 87.4k | // TODO: Handle oversized shifts. |
3978 | 20.1M | default: |
3979 | 20.1M | return false; |
3980 | 0 | } |
3981 | 0 | } |
3982 | | |
3983 | | SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, |
3984 | | EVT VT, SDNode *Cst1, |
3985 | 17.4M | SDNode *Cst2) { |
3986 | 17.4M | // If the opcode is a target-specific ISD node, there's nothing we can |
3987 | 17.4M | // do here and the operand rules may not line up with the below, so |
3988 | 17.4M | // bail early. |
3989 | 17.4M | if (Opcode >= ISD::BUILTIN_OP_END) |
3990 | 1.10M | return SDValue(); |
3991 | 16.3M | |
3992 | 16.3M | if (16.3M isUndef(Opcode, {SDValue(Cst1, 0), SDValue(Cst2, 0)})16.3M ) |
3993 | 76 | return getUNDEF(VT); |
3994 | 16.3M | |
3995 | 16.3M | // Handle the case of two scalars. |
3996 | 16.3M | if (const ConstantSDNode *16.3M Scalar116.3M = dyn_cast<ConstantSDNode>(Cst1)) { |
3997 | 1.15M | if (const ConstantSDNode *Scalar21.15M = dyn_cast<ConstantSDNode>(Cst2)) { |
3998 | 1.06M | SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2); |
3999 | 1.06M | assert((!Folded || !VT.isVector()) && |
4000 | 1.06M | "Can't fold vectors ops with scalar operands"); |
4001 | 1.06M | return Folded; |
4002 | 1.06M | } |
4003 | 15.2M | } |
4004 | 15.2M | |
4005 | 15.2M | // fold (add Sym, c) -> Sym+c |
4006 | 15.2M | if (GlobalAddressSDNode *15.2M GA15.2M = dyn_cast<GlobalAddressSDNode>(Cst1)) |
4007 | 297k | return FoldSymbolOffset(Opcode, VT, GA, Cst2); |
4008 | 14.9M | if (14.9M TLI->isCommutativeBinOp(Opcode)14.9M ) |
4009 | 7.28M | if (GlobalAddressSDNode *7.28M GA7.28M = dyn_cast<GlobalAddressSDNode>(Cst2)) |
4010 | 1.11k | return FoldSymbolOffset(Opcode, VT, GA, Cst1); |
4011 | 14.9M | |
4012 | 14.9M | // For vectors extract each constant element into Inputs so we can constant |
4013 | 14.9M | // fold them individually. |
4014 | 14.9M | BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); |
4015 | 14.9M | BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); |
4016 | 14.9M | if (!BV1 || 14.9M !BV254.1k ) |
4017 | 14.9M | return SDValue(); |
4018 | 19.6k | |
4019 | 14.9M | assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); |
4020 | 19.6k | |
4021 | 19.6k | EVT SVT = VT.getScalarType(); |
4022 | 19.6k | EVT LegalSVT = SVT; |
4023 | 19.6k | if (NewNodesMustHaveLegalTypes && 19.6k LegalSVT.isInteger()8.61k ) { |
4024 | 8.61k | LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); |
4025 | 8.61k | if (LegalSVT.bitsLT(SVT)) |
4026 | 0 | return SDValue(); |
4027 | 19.6k | } |
4028 | 19.6k | SmallVector<SDValue, 4> Outputs; |
4029 | 31.8k | for (unsigned I = 0, E = BV1->getNumOperands(); I != E31.8k ; ++I12.2k ) { |
4030 | 29.6k | SDValue V1 = BV1->getOperand(I); |
4031 | 29.6k | SDValue V2 = BV2->getOperand(I); |
4032 | 29.6k | |
4033 | 29.6k | if (SVT.isInteger()29.6k ) { |
4034 | 28.5k | if (V1->getValueType(0).bitsGT(SVT)) |
4035 | 1.20k | V1 = getNode(ISD::TRUNCATE, DL, SVT, V1); |
4036 | 28.5k | if (V2->getValueType(0).bitsGT(SVT)) |
4037 | 1.60k | V2 = getNode(ISD::TRUNCATE, DL, SVT, V2); |
4038 | 28.5k | } |
4039 | 29.6k | |
4040 | 29.6k | if (V1->getValueType(0) != SVT || 29.6k V2->getValueType(0) != SVT29.6k ) |
4041 | 0 | return SDValue(); |
4042 | 29.6k | |
4043 | 29.6k | // Fold one vector element. |
4044 | 29.6k | SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2); |
4045 | 29.6k | if (LegalSVT != SVT) |
4046 | 939 | ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult); |
4047 | 29.6k | |
4048 | 29.6k | // Scalar folding only succeeded if the result is a constant or UNDEF. |
4049 | 29.6k | if (!ScalarResult.isUndef() && 29.6k ScalarResult.getOpcode() != ISD::Constant29.6k && |
4050 | 17.9k | ScalarResult.getOpcode() != ISD::ConstantFP) |
4051 | 17.3k | return SDValue(); |
4052 | 12.2k | Outputs.push_back(ScalarResult); |
4053 | 12.2k | } |
4054 | 19.6k | |
4055 | 2.21k | assert(VT.getVectorNumElements() == Outputs.size() && |
4056 | 2.21k | "Vector size mismatch!"); |
4057 | 2.21k | |
4058 | 2.21k | // We may have a vector type but a scalar result. Create a splat. |
4059 | 2.21k | Outputs.resize(VT.getVectorNumElements(), Outputs.back()); |
4060 | 2.21k | |
4061 | 2.21k | // Build a big vector out of the scalar elements we generated. |
4062 | 2.21k | return getBuildVector(VT, SDLoc(), Outputs); |
4063 | 17.4M | } |
4064 | | |
4065 | | // TODO: Merge with FoldConstantArithmetic |
4066 | | SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode, |
4067 | | const SDLoc &DL, EVT VT, |
4068 | | ArrayRef<SDValue> Ops, |
4069 | 3.83M | const SDNodeFlags Flags) { |
4070 | 3.83M | // If the opcode is a target-specific ISD node, there's nothing we can |
4071 | 3.83M | // do here and the operand rules may not line up with the below, so |
4072 | 3.83M | // bail early. |
4073 | 3.83M | if (Opcode >= ISD::BUILTIN_OP_END) |
4074 | 0 | return SDValue(); |
4075 | 3.83M | |
4076 | 3.83M | if (3.83M isUndef(Opcode, Ops)3.83M ) |
4077 | 1 | return getUNDEF(VT); |
4078 | 3.83M | |
4079 | 3.83M | // We can only fold vectors - maybe merge with FoldConstantArithmetic someday? |
4080 | 3.83M | if (3.83M !VT.isVector()3.83M ) |
4081 | 3.46M | return SDValue(); |
4082 | 375k | |
4083 | 375k | unsigned NumElts = VT.getVectorNumElements(); |
4084 | 375k | |
4085 | 3.67k | auto IsScalarOrSameVectorSize = [&](const SDValue &Op) { |
4086 | 3.67k | return !Op.getValueType().isVector() || |
4087 | 3.28k | Op.getValueType().getVectorNumElements() == NumElts; |
4088 | 3.67k | }; |
4089 | 375k | |
4090 | 395k | auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) { |
4091 | 395k | BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op); |
4092 | 395k | return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) || |
4093 | 395k | (BV && 395k BV->isConstant()43.6k ); |
4094 | 395k | }; |
4095 | 375k | |
4096 | 375k | // All operands must be vector types with the same number of elements as |
4097 | 375k | // the result type and must be either UNDEF or a build vector of constant |
4098 | 375k | // or UNDEF scalars. |
4099 | 375k | if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) || |
4100 | 2.56k | !llvm::all_of(Ops, IsScalarOrSameVectorSize)) |
4101 | 372k | return SDValue(); |
4102 | 2.56k | |
4103 | 2.56k | // If we are comparing vectors, then the result needs to be a i1 boolean |
4104 | 2.56k | // that is then sign-extended back to the legal result type. |
4105 | 2.56k | EVT SVT = (Opcode == ISD::SETCC ? 2.56k MVT::i1397 : VT.getScalarType()2.17k ); |
4106 | 2.56k | |
4107 | 2.56k | // Find legal integer scalar type for constant promotion and |
4108 | 2.56k | // ensure that its scalar size is at least as large as source. |
4109 | 2.56k | EVT LegalSVT = VT.getScalarType(); |
4110 | 2.56k | if (NewNodesMustHaveLegalTypes && 2.56k LegalSVT.isInteger()571 ) { |
4111 | 543 | LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT); |
4112 | 543 | if (LegalSVT.bitsLT(VT.getScalarType())) |
4113 | 0 | return SDValue(); |
4114 | 2.56k | } |
4115 | 2.56k | |
4116 | 2.56k | // Constant fold each scalar lane separately. |
4117 | 2.56k | SmallVector<SDValue, 4> ScalarResults; |
4118 | 22.3k | for (unsigned i = 0; i != NumElts22.3k ; i++19.8k ) { |
4119 | 19.9k | SmallVector<SDValue, 4> ScalarOps; |
4120 | 25.8k | for (SDValue Op : Ops) { |
4121 | 25.8k | EVT InSVT = Op.getValueType().getScalarType(); |
4122 | 25.8k | BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op); |
4123 | 25.8k | if (!InBV25.8k ) { |
4124 | 1.62k | // We've checked that this is UNDEF or a constant of some kind. |
4125 | 1.62k | if (Op.isUndef()) |
4126 | 120 | ScalarOps.push_back(getUNDEF(InSVT)); |
4127 | 1.62k | else |
4128 | 1.50k | ScalarOps.push_back(Op); |
4129 | 1.62k | continue; |
4130 | 1.62k | } |
4131 | 24.2k | |
4132 | 24.2k | SDValue ScalarOp = InBV->getOperand(i); |
4133 | 24.2k | EVT ScalarVT = ScalarOp.getValueType(); |
4134 | 24.2k | |
4135 | 24.2k | // Build vector (integer) scalar operands may need implicit |
4136 | 24.2k | // truncation - do this before constant folding. |
4137 | 24.2k | if (ScalarVT.isInteger() && 24.2k ScalarVT.bitsGT(InSVT)23.1k ) |
4138 | 424 | ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp); |
4139 | 25.8k | |
4140 | 25.8k | ScalarOps.push_back(ScalarOp); |
4141 | 25.8k | } |
4142 | 19.9k | |
4143 | 19.9k | // Constant fold the scalar operands. |
4144 | 19.9k | SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags); |
4145 | 19.9k | |
4146 | 19.9k | // Legalize the (integer) scalar constant if necessary. |
4147 | 19.9k | if (LegalSVT != SVT) |
4148 | 6.05k | ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult); |
4149 | 19.9k | |
4150 | 19.9k | // Scalar folding only succeeded if the result is a constant or UNDEF. |
4151 | 19.9k | if (!ScalarResult.isUndef() && 19.9k ScalarResult.getOpcode() != ISD::Constant19.7k && |
4152 | 593 | ScalarResult.getOpcode() != ISD::ConstantFP) |
4153 | 153 | return SDValue(); |
4154 | 19.8k | ScalarResults.push_back(ScalarResult); |
4155 | 19.8k | } |
4156 | 2.56k | |
4157 | 2.41k | SDValue V = getBuildVector(VT, DL, ScalarResults); |
4158 | 2.41k | NewSDValueDbgMsg(V, "New node fold constant vector: ", this); |
4159 | 2.41k | return V; |
4160 | 3.83M | } |
4161 | | |
4162 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
4163 | 21.6M | SDValue N1, SDValue N2, const SDNodeFlags Flags) { |
4164 | 21.6M | ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1); |
4165 | 21.6M | ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2); |
4166 | 21.6M | ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); |
4167 | 21.6M | ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); |
4168 | 21.6M | |
4169 | 21.6M | // Canonicalize constant to RHS if commutative. |
4170 | 21.6M | if (TLI->isCommutativeBinOp(Opcode)21.6M ) { |
4171 | 11.5M | if (N1C && 11.5M !N2C419k ) { |
4172 | 16.8k | std::swap(N1C, N2C); |
4173 | 16.8k | std::swap(N1, N2); |
4174 | 11.5M | } else if (11.5M N1CFP && 11.5M !N2CFP1.37k ) { |
4175 | 747 | std::swap(N1CFP, N2CFP); |
4176 | 747 | std::swap(N1, N2); |
4177 | 747 | } |
4178 | 11.5M | } |
4179 | 21.6M | |
4180 | 21.6M | switch (Opcode) { |
4181 | 3.67M | default: break; |
4182 | 2.95M | case ISD::TokenFactor: |
4183 | 2.95M | assert(VT == MVT::Other && N1.getValueType() == MVT::Other && |
4184 | 2.95M | N2.getValueType() == MVT::Other && "Invalid token factor!"); |
4185 | 2.95M | // Fold trivial token factors. |
4186 | 2.95M | if (N1.getOpcode() == ISD::EntryToken2.95M ) return N210.8k ; |
4187 | 2.94M | if (2.94M N2.getOpcode() == ISD::EntryToken2.94M ) return N11.08k ; |
4188 | 2.94M | if (2.94M N1 == N22.94M ) return N1107k ; |
4189 | 2.83M | break; |
4190 | 43.3k | case ISD::CONCAT_VECTORS: { |
4191 | 43.3k | // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. |
4192 | 43.3k | SDValue Ops[] = {N1, N2}; |
4193 | 43.3k | if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) |
4194 | 482 | return V; |
4195 | 42.8k | break; |
4196 | 42.8k | } |
4197 | 1.24M | case ISD::AND: |
4198 | 1.24M | assert(VT.isInteger() && "This operator does not apply to FP types!"); |
4199 | 1.24M | assert(N1.getValueType() == N2.getValueType() && |
4200 | 1.24M | N1.getValueType() == VT && "Binary operator types must match!"); |
4201 | 1.24M | // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's |
4202 | 1.24M | // worth handling here. |
4203 | 1.24M | if (N2C && 1.24M N2C->isNullValue()1.08M ) |
4204 | 3.79k | return N2; |
4205 | 1.24M | if (1.24M N2C && 1.24M N2C->isAllOnesValue()1.08M ) // X & -1 -> X |
4206 | 3.17k | return N1; |
4207 | 1.24M | break; |
4208 | 10.1M | case ISD::OR: |
4209 | 10.1M | case ISD::XOR: |
4210 | 10.1M | case ISD::ADD: |
4211 | 10.1M | case ISD::SUB: |
4212 | 10.1M | assert(VT.isInteger() && "This operator does not apply to FP types!"); |
4213 | 10.1M | assert(N1.getValueType() == N2.getValueType() && |
4214 | 10.1M | N1.getValueType() == VT && "Binary operator types must match!"); |
4215 | 10.1M | // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so |
4216 | 10.1M | // it's worth handling here. |
4217 | 10.1M | if (N2C && 10.1M N2C->isNullValue()8.92M ) |
4218 | 3.58M | return N1; |
4219 | 6.53M | break; |
4220 | 318k | case ISD::UDIV: |
4221 | 318k | case ISD::UREM: |
4222 | 318k | case ISD::MULHU: |
4223 | 318k | case ISD::MULHS: |
4224 | 318k | case ISD::MUL: |
4225 | 318k | case ISD::SDIV: |
4226 | 318k | case ISD::SREM: |
4227 | 318k | case ISD::SMIN: |
4228 | 318k | case ISD::SMAX: |
4229 | 318k | case ISD::UMIN: |
4230 | 318k | case ISD::UMAX: |
4231 | 318k | assert(VT.isInteger() && "This operator does not apply to FP types!"); |
4232 | 318k | assert(N1.getValueType() == N2.getValueType() && |
4233 | 318k | N1.getValueType() == VT && "Binary operator types must match!"); |
4234 | 318k | break; |
4235 | 192k | case ISD::FADD: |
4236 | 192k | case ISD::FSUB: |
4237 | 192k | case ISD::FMUL: |
4238 | 192k | case ISD::FDIV: |
4239 | 192k | case ISD::FREM: |
4240 | 192k | if (getTarget().Options.UnsafeFPMath192k ) { |
4241 | 5.51k | if (Opcode == ISD::FADD5.51k ) { |
4242 | 1.57k | // x+0 --> x |
4243 | 1.57k | if (N2CFP && 1.57k N2CFP->getValueAPF().isZero()263 ) |
4244 | 16 | return N1; |
4245 | 3.93k | } else if (3.93k Opcode == ISD::FSUB3.93k ) { |
4246 | 674 | // x-0 --> x |
4247 | 674 | if (N2CFP && 674 N2CFP->getValueAPF().isZero()10 ) |
4248 | 0 | return N1; |
4249 | 3.26k | } else if (3.26k Opcode == ISD::FMUL3.26k ) { |
4250 | 2.77k | // x*0 --> 0 |
4251 | 2.77k | if (N2CFP && 2.77k N2CFP->isZero()498 ) |
4252 | 6 | return N2; |
4253 | 2.76k | // x*1 --> x |
4254 | 2.76k | if (2.76k N2CFP && 2.76k N2CFP->isExactlyValue(1.0)492 ) |
4255 | 61 | return N1; |
4256 | 192k | } |
4257 | 5.51k | } |
4258 | 192k | assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); |
4259 | 192k | assert(N1.getValueType() == N2.getValueType() && |
4260 | 192k | N1.getValueType() == VT && "Binary operator types must match!"); |
4261 | 192k | break; |
4262 | 2.13k | case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. |
4263 | 2.13k | assert(N1.getValueType() == VT && |
4264 | 2.13k | N1.getValueType().isFloatingPoint() && |
4265 | 2.13k | N2.getValueType().isFloatingPoint() && |
4266 | 2.13k | "Invalid FCOPYSIGN!"); |
4267 | 2.13k | break; |
4268 | 786k | case ISD::SHL: |
4269 | 786k | case ISD::SRA: |
4270 | 786k | case ISD::SRL: |
4271 | 786k | case ISD::ROTL: |
4272 | 786k | case ISD::ROTR: |
4273 | 786k | assert(VT == N1.getValueType() && |
4274 | 786k | "Shift operators return type must be the same as their first arg"); |
4275 | 786k | assert(VT.isInteger() && N2.getValueType().isInteger() && |
4276 | 786k | "Shifts only work on integers"); |
4277 | 786k | assert((!VT.isVector() || VT == N2.getValueType()) && |
4278 | 786k | "Vector shift amounts must be in the same as their first arg"); |
4279 | 786k | // Verify that the shift amount VT is bit enough to hold valid shift |
4280 | 786k | // amounts. This catches things like trying to shift an i1024 value by an |
4281 | 786k | // i8, which is easy to fall into in generic code that uses |
4282 | 786k | // TLI.getShiftAmount(). |
4283 | 786k | assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) && |
4284 | 786k | "Invalid use of small shift amount with oversized value!"); |
4285 | 786k | |
4286 | 786k | // Always fold shifts of i1 values so the code generator doesn't need to |
4287 | 786k | // handle them. Since we know the size of the shift has to be less than the |
4288 | 786k | // size of the value, the shift/rotate count is guaranteed to be zero. |
4289 | 786k | if (VT == MVT::i1) |
4290 | 45 | return N1; |
4291 | 786k | if (786k N2C && 786k N2C->isNullValue()711k ) |
4292 | 16.1k | return N1; |
4293 | 770k | break; |
4294 | 8 | case ISD::FP_ROUND_INREG: { |
4295 | 8 | EVT EVT = cast<VTSDNode>(N2)->getVT(); |
4296 | 8 | assert(VT == N1.getValueType() && "Not an inreg round!"); |
4297 | 8 | assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && |
4298 | 8 | "Cannot FP_ROUND_INREG integer types"); |
4299 | 8 | assert(EVT.isVector() == VT.isVector() && |
4300 | 8 | "FP_ROUND_INREG type should be vector iff the operand " |
4301 | 8 | "type is vector!"); |
4302 | 8 | assert((!EVT.isVector() || |
4303 | 8 | EVT.getVectorNumElements() == VT.getVectorNumElements()) && |
4304 | 8 | "Vector element counts must match in FP_ROUND_INREG"); |
4305 | 8 | assert(EVT.bitsLE(VT) && "Not rounding down!"); |
4306 | 8 | (void)EVT; |
4307 | 8 | if (cast<VTSDNode>(N2)->getVT() == VT8 ) return N10 ; // Not actually rounding. |
4308 | 8 | break; |
4309 | 8 | } |
4310 | 13.1k | case ISD::FP_ROUND: |
4311 | 13.1k | assert(VT.isFloatingPoint() && |
4312 | 13.1k | N1.getValueType().isFloatingPoint() && |
4313 | 13.1k | VT.bitsLE(N1.getValueType()) && |
4314 | 13.1k | N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) && |
4315 | 13.1k | "Invalid FP_ROUND!"); |
4316 | 13.1k | if (N1.getValueType() == VT13.1k ) return N177 ; // noop conversion. |
4317 | 13.0k | break; |
4318 | 1.29M | case ISD::AssertSext: |
4319 | 1.29M | case ISD::AssertZext: { |
4320 | 1.29M | EVT EVT = cast<VTSDNode>(N2)->getVT(); |
4321 | 1.29M | assert(VT == N1.getValueType() && "Not an inreg extend!"); |
4322 | 1.29M | assert(VT.isInteger() && EVT.isInteger() && |
4323 | 1.29M | "Cannot *_EXTEND_INREG FP types"); |
4324 | 1.29M | assert(!EVT.isVector() && |
4325 | 1.29M | "AssertSExt/AssertZExt type should be the vector element type " |
4326 | 1.29M | "rather than the vector type!"); |
4327 | 1.29M | assert(EVT.bitsLE(VT) && "Not extending!"); |
4328 | 1.29M | if (VT == EVT1.29M ) return N1703k ; // noop assertion. |
4329 | 590k | break; |
4330 | 590k | } |
4331 | 65.1k | case ISD::SIGN_EXTEND_INREG: { |
4332 | 65.1k | EVT EVT = cast<VTSDNode>(N2)->getVT(); |
4333 | 65.1k | assert(VT == N1.getValueType() && "Not an inreg extend!"); |
4334 | 65.1k | assert(VT.isInteger() && EVT.isInteger() && |
4335 | 65.1k | "Cannot *_EXTEND_INREG FP types"); |
4336 | 65.1k | assert(EVT.isVector() == VT.isVector() && |
4337 | 65.1k | "SIGN_EXTEND_INREG type should be vector iff the operand " |
4338 | 65.1k | "type is vector!"); |
4339 | 65.1k | assert((!EVT.isVector() || |
4340 | 65.1k | EVT.getVectorNumElements() == VT.getVectorNumElements()) && |
4341 | 65.1k | "Vector element counts must match in SIGN_EXTEND_INREG"); |
4342 | 65.1k | assert(EVT.bitsLE(VT) && "Not extending!"); |
4343 | 65.1k | if (EVT == VT65.1k ) return N190 ; // Not actually extending |
4344 | 65.0k | |
4345 | 65.0k | auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) 65.0k { |
4346 | 11.7k | unsigned FromBits = EVT.getScalarSizeInBits(); |
4347 | 11.7k | Val <<= Val.getBitWidth() - FromBits; |
4348 | 11.7k | Val.ashrInPlace(Val.getBitWidth() - FromBits); |
4349 | 11.7k | return getConstant(Val, DL, ConstantVT); |
4350 | 11.7k | }; |
4351 | 65.0k | |
4352 | 65.0k | if (N1C65.0k ) { |
4353 | 10.2k | const APInt &Val = N1C->getAPIntValue(); |
4354 | 10.2k | return SignExtendInReg(Val, VT); |
4355 | 10.2k | } |
4356 | 54.7k | if (54.7k ISD::isBuildVectorOfConstantSDNodes(N1.getNode())54.7k ) { |
4357 | 347 | SmallVector<SDValue, 8> Ops; |
4358 | 347 | llvm::EVT OpVT = N1.getOperand(0).getValueType(); |
4359 | 1.92k | for (int i = 0, e = VT.getVectorNumElements(); i != e1.92k ; ++i1.58k ) { |
4360 | 1.58k | SDValue Op = N1.getOperand(i); |
4361 | 1.58k | if (Op.isUndef()1.58k ) { |
4362 | 42 | Ops.push_back(getUNDEF(OpVT)); |
4363 | 42 | continue; |
4364 | 42 | } |
4365 | 1.54k | ConstantSDNode *C = cast<ConstantSDNode>(Op); |
4366 | 1.54k | APInt Val = C->getAPIntValue(); |
4367 | 1.54k | Ops.push_back(SignExtendInReg(Val, OpVT)); |
4368 | 1.54k | } |
4369 | 347 | return getBuildVector(VT, DL, Ops); |
4370 | 347 | } |
4371 | 54.4k | break; |
4372 | 54.4k | } |
4373 | 516k | case ISD::EXTRACT_VECTOR_ELT: |
4374 | 516k | // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. |
4375 | 516k | if (N1.isUndef()) |
4376 | 14.3k | return getUNDEF(VT); |
4377 | 501k | |
4378 | 501k | // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF |
4379 | 501k | if (501k N2C && 501k N2C->getZExtValue() >= N1.getValueType().getVectorNumElements()500k ) |
4380 | 84 | return getUNDEF(VT); |
4381 | 501k | |
4382 | 501k | // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is |
4383 | 501k | // expanding copies of large vectors from registers. |
4384 | 501k | if (501k N2C && |
4385 | 500k | N1.getOpcode() == ISD::CONCAT_VECTORS && |
4386 | 501k | N1.getNumOperands() > 03.79k ) { |
4387 | 3.79k | unsigned Factor = |
4388 | 3.79k | N1.getOperand(0).getValueType().getVectorNumElements(); |
4389 | 3.79k | return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, |
4390 | 3.79k | N1.getOperand(N2C->getZExtValue() / Factor), |
4391 | 3.79k | getConstant(N2C->getZExtValue() % Factor, DL, |
4392 | 3.79k | N2.getValueType())); |
4393 | 3.79k | } |
4394 | 497k | |
4395 | 497k | // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is |
4396 | 497k | // expanding large vector constants. |
4397 | 497k | if (497k N2C && 497k N1.getOpcode() == ISD::BUILD_VECTOR496k ) { |
4398 | 125k | SDValue Elt = N1.getOperand(N2C->getZExtValue()); |
4399 | 125k | |
4400 | 125k | if (VT != Elt.getValueType()) |
4401 | 125k | // If the vector element type is not legal, the BUILD_VECTOR operands |
4402 | 125k | // are promoted and implicitly truncated, and the result implicitly |
4403 | 125k | // extended. Make that explicit here. |
4404 | 6.36k | Elt = getAnyExtOrTrunc(Elt, DL, VT); |
4405 | 125k | |
4406 | 125k | return Elt; |
4407 | 125k | } |
4408 | 372k | |
4409 | 372k | // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector |
4410 | 372k | // operations are lowered to scalars. |
4411 | 372k | if (372k N1.getOpcode() == ISD::INSERT_VECTOR_ELT372k ) { |
4412 | 4.13k | // If the indices are the same, return the inserted element else |
4413 | 4.13k | // if the indices are known different, extract the element from |
4414 | 4.13k | // the original vector. |
4415 | 4.13k | SDValue N1Op2 = N1.getOperand(2); |
4416 | 4.13k | ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2); |
4417 | 4.13k | |
4418 | 4.13k | if (N1Op2C && 4.13k N2C3.02k ) { |
4419 | 3.01k | if (N1Op2C->getZExtValue() == N2C->getZExtValue()3.01k ) { |
4420 | 504 | if (VT == N1.getOperand(1).getValueType()) |
4421 | 487 | return N1.getOperand(1); |
4422 | 504 | else |
4423 | 17 | return getSExtOrTrunc(N1.getOperand(1), DL, VT); |
4424 | 2.51k | } |
4425 | 2.51k | |
4426 | 2.51k | return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); |
4427 | 2.51k | } |
4428 | 4.13k | } |
4429 | 369k | |
4430 | 369k | // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed |
4431 | 369k | // when vector types are scalarized and v1iX is legal. |
4432 | 369k | // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx) |
4433 | 369k | if (369k N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && |
4434 | 369k | N1.getValueType().getVectorNumElements() == 1106k ) { |
4435 | 26 | return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), |
4436 | 26 | N1.getOperand(1)); |
4437 | 26 | } |
4438 | 369k | break; |
4439 | 61.3k | case ISD::EXTRACT_ELEMENT: |
4440 | 61.3k | assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); |
4441 | 61.3k | assert(!N1.getValueType().isVector() && !VT.isVector() && |
4442 | 61.3k | (N1.getValueType().isInteger() == VT.isInteger()) && |
4443 | 61.3k | N1.getValueType() != VT && |
4444 | 61.3k | "Wrong types for EXTRACT_ELEMENT!"); |
4445 | 61.3k | |
4446 | 61.3k | // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding |
4447 | 61.3k | // 64-bit integers into 32-bit parts. Instead of building the extract of |
4448 | 61.3k | // the BUILD_PAIR, only to have legalize rip it apart, just do it now. |
4449 | 61.3k | if (N1.getOpcode() == ISD::BUILD_PAIR) |
4450 | 12.3k | return N1.getOperand(N2C->getZExtValue()); |
4451 | 48.9k | |
4452 | 48.9k | // EXTRACT_ELEMENT of a constant int is also very common. |
4453 | 48.9k | if (48.9k N1C48.9k ) { |
4454 | 11.0k | unsigned ElementSize = VT.getSizeInBits(); |
4455 | 11.0k | unsigned Shift = ElementSize * N2C->getZExtValue(); |
4456 | 11.0k | APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift); |
4457 | 11.0k | return getConstant(ShiftedVal.trunc(ElementSize), DL, VT); |
4458 | 11.0k | } |
4459 | 37.9k | break; |
4460 | 328k | case ISD::EXTRACT_SUBVECTOR: |
4461 | 328k | if (VT.isSimple() && 328k N1.getValueType().isSimple()327k ) { |
4462 | 291k | assert(VT.isVector() && N1.getValueType().isVector() && |
4463 | 291k | "Extract subvector VTs must be a vectors!"); |
4464 | 291k | assert(VT.getVectorElementType() == |
4465 | 291k | N1.getValueType().getVectorElementType() && |
4466 | 291k | "Extract subvector VTs must have the same element type!"); |
4467 | 291k | assert(VT.getSimpleVT() <= N1.getSimpleValueType() && |
4468 | 291k | "Extract subvector must be from larger vector to smaller vector!"); |
4469 | 291k | |
4470 | 291k | if (N2C291k ) { |
4471 | 291k | assert((VT.getVectorNumElements() + N2C->getZExtValue() |
4472 | 291k | <= N1.getValueType().getVectorNumElements()) |
4473 | 291k | && "Extract subvector overflow!"); |
4474 | 291k | } |
4475 | 291k | |
4476 | 291k | // Trivial extraction. |
4477 | 291k | if (VT.getSimpleVT() == N1.getSimpleValueType()) |
4478 | 37.6k | return N1; |
4479 | 253k | |
4480 | 253k | // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF. |
4481 | 253k | if (253k N1.isUndef()253k ) |
4482 | 1.07k | return getUNDEF(VT); |
4483 | 252k | |
4484 | 252k | // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of |
4485 | 252k | // the concat have the same type as the extract. |
4486 | 252k | if (252k N2C && 252k N1.getOpcode() == ISD::CONCAT_VECTORS252k && |
4487 | 11.2k | N1.getNumOperands() > 0 && |
4488 | 252k | VT == N1.getOperand(0).getValueType()11.2k ) { |
4489 | 9.45k | unsigned Factor = VT.getVectorNumElements(); |
4490 | 9.45k | return N1.getOperand(N2C->getZExtValue() / Factor); |
4491 | 9.45k | } |
4492 | 242k | |
4493 | 242k | // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created |
4494 | 242k | // during shuffle legalization. |
4495 | 242k | if (242k N1.getOpcode() == ISD::INSERT_SUBVECTOR && 242k N2 == N1.getOperand(2)240 && |
4496 | 186 | VT == N1.getOperand(1).getValueType()) |
4497 | 186 | return N1.getOperand(1); |
4498 | 280k | } |
4499 | 280k | break; |
4500 | 16.9M | } |
4501 | 16.9M | |
4502 | 16.9M | // Perform trivial constant folding. |
4503 | 16.9M | if (SDValue 16.9M SV16.9M = |
4504 | 16.9M | FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode())) |
4505 | 434k | return SV; |
4506 | 16.5M | |
4507 | 16.5M | // Constant fold FP operations. |
4508 | 16.5M | bool HasFPExceptions = TLI->hasFloatingPointExceptions(); |
4509 | 16.5M | if (N1CFP16.5M ) { |
4510 | 30.5k | if (N2CFP30.5k ) { |
4511 | 10.7k | APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); |
4512 | 10.7k | APFloat::opStatus s; |
4513 | 10.7k | switch (Opcode) { |
4514 | 305 | case ISD::FADD: |
4515 | 305 | s = V1.add(V2, APFloat::rmNearestTiesToEven); |
4516 | 305 | if (!HasFPExceptions || 305 s != APFloat::opInvalidOp305 ) |
4517 | 305 | return getConstantFP(V1, DL, VT); |
4518 | 0 | break; |
4519 | 27 | case ISD::FSUB: |
4520 | 27 | s = V1.subtract(V2, APFloat::rmNearestTiesToEven); |
4521 | 27 | if (!HasFPExceptions || 27 s!=APFloat::opInvalidOp27 ) |
4522 | 27 | return getConstantFP(V1, DL, VT); |
4523 | 0 | break; |
4524 | 263 | case ISD::FMUL: |
4525 | 263 | s = V1.multiply(V2, APFloat::rmNearestTiesToEven); |
4526 | 263 | if (!HasFPExceptions || 263 s!=APFloat::opInvalidOp252 ) |
4527 | 263 | return getConstantFP(V1, DL, VT); |
4528 | 0 | break; |
4529 | 39 | case ISD::FDIV: |
4530 | 39 | s = V1.divide(V2, APFloat::rmNearestTiesToEven); |
4531 | 39 | if (!HasFPExceptions || 39 (s!=APFloat::opInvalidOp && |
4532 | 39 | s!=APFloat::opDivByZero23 )) { |
4533 | 18 | return getConstantFP(V1, DL, VT); |
4534 | 18 | } |
4535 | 21 | break; |
4536 | 25 | case ISD::FREM : |
4537 | 25 | s = V1.mod(V2); |
4538 | 25 | if (!HasFPExceptions || 25 (s!=APFloat::opInvalidOp && |
4539 | 25 | s!=APFloat::opDivByZero11 )) { |
4540 | 11 | return getConstantFP(V1, DL, VT); |
4541 | 11 | } |
4542 | 14 | break; |
4543 | 4 | case ISD::FCOPYSIGN: |
4544 | 4 | V1.copySign(V2); |
4545 | 4 | return getConstantFP(V1, DL, VT); |
4546 | 10.0k | default: break; |
4547 | 29.9k | } |
4548 | 29.9k | } |
4549 | 29.9k | |
4550 | 29.9k | if (29.9k Opcode == ISD::FP_ROUND29.9k ) { |
4551 | 4 | APFloat V = N1CFP->getValueAPF(); // make copy |
4552 | 4 | bool ignored; |
4553 | 4 | // This can return overflow, underflow, or inexact; we don't care. |
4554 | 4 | // FIXME need to be more flexible about rounding mode. |
4555 | 4 | (void)V.convert(EVTToAPFloatSemantics(VT), |
4556 | 4 | APFloat::rmNearestTiesToEven, &ignored); |
4557 | 4 | return getConstantFP(V, DL, VT); |
4558 | 4 | } |
4559 | 16.5M | } |
4560 | 16.5M | |
4561 | 16.5M | // Canonicalize an UNDEF to the RHS, even over a constant. |
4562 | 16.5M | if (16.5M N1.isUndef()16.5M ) { |
4563 | 5.10k | if (TLI->isCommutativeBinOp(Opcode)5.10k ) { |
4564 | 2.85k | std::swap(N1, N2); |
4565 | 5.10k | } else { |
4566 | 2.25k | switch (Opcode) { |
4567 | 129 | case ISD::FP_ROUND_INREG: |
4568 | 129 | case ISD::SIGN_EXTEND_INREG: |
4569 | 129 | case ISD::SUB: |
4570 | 129 | case ISD::FSUB: |
4571 | 129 | case ISD::FDIV: |
4572 | 129 | case ISD::FREM: |
4573 | 129 | case ISD::SRA: |
4574 | 129 | return N1; // fold op(undef, arg2) -> undef |
4575 | 353 | case ISD::UDIV: |
4576 | 353 | case ISD::SDIV: |
4577 | 353 | case ISD::UREM: |
4578 | 353 | case ISD::SREM: |
4579 | 353 | case ISD::SRL: |
4580 | 353 | case ISD::SHL: |
4581 | 353 | if (!VT.isVector()) |
4582 | 270 | return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0 |
4583 | 83 | // For vectors, we can't easily build an all zero vector, just return |
4584 | 83 | // the LHS. |
4585 | 83 | return N2; |
4586 | 2.25k | } |
4587 | 2.25k | } |
4588 | 5.10k | } |
4589 | 16.5M | |
4590 | 16.5M | // Fold a bunch of operators when the RHS is undef. |
4591 | 16.5M | if (16.5M N2.isUndef()16.5M ) { |
4592 | 20.9k | switch (Opcode) { |
4593 | 480 | case ISD::XOR: |
4594 | 480 | if (N1.isUndef()) |
4595 | 480 | // Handle undef ^ undef -> 0 special case. This is a common |
4596 | 480 | // idiom (misuse). |
4597 | 6 | return getConstant(0, DL, VT); |
4598 | 474 | LLVM_FALLTHROUGH474 ; |
4599 | 1.95k | case ISD::ADD: |
4600 | 1.95k | case ISD::ADDC: |
4601 | 1.95k | case ISD::ADDE: |
4602 | 1.95k | case ISD::SUB: |
4603 | 1.95k | case ISD::UDIV: |
4604 | 1.95k | case ISD::SDIV: |
4605 | 1.95k | case ISD::UREM: |
4606 | 1.95k | case ISD::SREM: |
4607 | 1.95k | return N2; // fold op(arg1, undef) -> undef |
4608 | 1.31k | case ISD::FADD: |
4609 | 1.31k | case ISD::FSUB: |
4610 | 1.31k | case ISD::FMUL: |
4611 | 1.31k | case ISD::FDIV: |
4612 | 1.31k | case ISD::FREM: |
4613 | 1.31k | if (getTarget().Options.UnsafeFPMath) |
4614 | 5 | return N2; |
4615 | 1.31k | break; |
4616 | 387 | case ISD::MUL: |
4617 | 387 | case ISD::AND: |
4618 | 387 | case ISD::SRL: |
4619 | 387 | case ISD::SHL: |
4620 | 387 | if (!VT.isVector()) |
4621 | 258 | return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0 |
4622 | 129 | // For vectors, we can't easily build an all zero vector, just return |
4623 | 129 | // the LHS. |
4624 | 129 | return N1; |
4625 | 81 | case ISD::OR: |
4626 | 81 | if (!VT.isVector()) |
4627 | 41 | return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL, VT); |
4628 | 40 | // For vectors, we can't easily build an all one vector, just return |
4629 | 40 | // the LHS. |
4630 | 40 | return N1; |
4631 | 6 | case ISD::SRA: |
4632 | 6 | return N1; |
4633 | 16.5M | } |
4634 | 16.5M | } |
4635 | 16.5M | |
4636 | 16.5M | // Memoize this node if possible. |
4637 | 16.5M | SDNode *N; |
4638 | 16.5M | SDVTList VTs = getVTList(VT); |
4639 | 16.5M | SDValue Ops[] = {N1, N2}; |
4640 | 16.5M | if (VT != MVT::Glue16.5M ) { |
4641 | 16.4M | FoldingSetNodeID ID; |
4642 | 16.4M | AddNodeIDNode(ID, Opcode, VTs, Ops); |
4643 | 16.4M | void *IP = nullptr; |
4644 | 16.4M | if (SDNode *E16.4M = FindNodeOrInsertPos(ID, DL, IP)) { |
4645 | 1.29M | E->intersectFlagsWith(Flags); |
4646 | 1.29M | return SDValue(E, 0); |
4647 | 1.29M | } |
4648 | 15.1M | |
4649 | 15.1M | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
4650 | 15.1M | N->setFlags(Flags); |
4651 | 15.1M | createOperands(N, Ops); |
4652 | 15.1M | CSEMap.InsertNode(N, IP); |
4653 | 16.5M | } else { |
4654 | 27.3k | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
4655 | 27.3k | createOperands(N, Ops); |
4656 | 27.3k | } |
4657 | 16.5M | |
4658 | 15.2M | InsertNode(N); |
4659 | 15.2M | SDValue V = SDValue(N, 0); |
4660 | 15.2M | NewSDValueDbgMsg(V, "Creating new node: ", this); |
4661 | 15.2M | return V; |
4662 | 21.6M | } |
4663 | | |
4664 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
4665 | 9.97M | SDValue N1, SDValue N2, SDValue N3) { |
4666 | 9.97M | // Perform various simplifications. |
4667 | 9.97M | switch (Opcode) { |
4668 | 3.55k | case ISD::FMA: { |
4669 | 3.55k | ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); |
4670 | 3.55k | ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); |
4671 | 3.55k | ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); |
4672 | 3.55k | if (N1CFP && 3.55k N2CFP55 && N3CFP15 ) { |
4673 | 9 | APFloat V1 = N1CFP->getValueAPF(); |
4674 | 9 | const APFloat &V2 = N2CFP->getValueAPF(); |
4675 | 9 | const APFloat &V3 = N3CFP->getValueAPF(); |
4676 | 9 | APFloat::opStatus s = |
4677 | 9 | V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); |
4678 | 9 | if (!TLI->hasFloatingPointExceptions() || 9 s != APFloat::opInvalidOp9 ) |
4679 | 9 | return getConstantFP(V1, DL, VT); |
4680 | 3.54k | } |
4681 | 3.54k | break; |
4682 | 3.54k | } |
4683 | 22 | case ISD::CONCAT_VECTORS: { |
4684 | 22 | // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. |
4685 | 22 | SDValue Ops[] = {N1, N2, N3}; |
4686 | 22 | if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) |
4687 | 2 | return V; |
4688 | 20 | break; |
4689 | 20 | } |
4690 | 3.50M | case ISD::SETCC: { |
4691 | 3.50M | // Use FoldSetCC to simplify SETCC's. |
4692 | 3.50M | if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL)) |
4693 | 2.23k | return V; |
4694 | 3.49M | // Vector constant folding. |
4695 | 3.49M | SDValue Ops[] = {N1, N2, N3}; |
4696 | 3.49M | if (SDValue V3.49M = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) { |
4697 | 340 | NewSDValueDbgMsg(V, "New node vector constant folding: ", this); |
4698 | 340 | return V; |
4699 | 340 | } |
4700 | 3.49M | break; |
4701 | 3.49M | } |
4702 | 164k | case ISD::SELECT: |
4703 | 164k | if (ConstantSDNode *N1C164k = dyn_cast<ConstantSDNode>(N1)) { |
4704 | 400 | if (N1C->getZExtValue()) |
4705 | 138 | return N2; // select true, X, Y -> X |
4706 | 262 | return N3; // select false, X, Y -> Y |
4707 | 262 | } |
4708 | 163k | |
4709 | 163k | if (163k N2 == N3163k ) return N2135 ; // select C, X, X -> X |
4710 | 163k | break; |
4711 | 0 | case ISD::VECTOR_SHUFFLE: |
4712 | 0 | llvm_unreachable("should use getVectorShuffle constructor!"); |
4713 | 78.4k | case ISD::INSERT_VECTOR_ELT: { |
4714 | 78.4k | ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3); |
4715 | 78.4k | // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF |
4716 | 78.4k | if (N3C && 78.4k N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()78.1k ) |
4717 | 12 | return getUNDEF(VT); |
4718 | 78.4k | break; |
4719 | 78.4k | } |
4720 | 66.3k | case ISD::INSERT_SUBVECTOR: { |
4721 | 66.3k | SDValue Index = N3; |
4722 | 66.3k | if (VT.isSimple() && 66.3k N1.getValueType().isSimple()66.3k |
4723 | 66.3k | && N2.getValueType().isSimple()66.3k ) { |
4724 | 66.3k | assert(VT.isVector() && N1.getValueType().isVector() && |
4725 | 66.3k | N2.getValueType().isVector() && |
4726 | 66.3k | "Insert subvector VTs must be a vectors"); |
4727 | 66.3k | assert(VT == N1.getValueType() && |
4728 | 66.3k | "Dest and insert subvector source types must match!"); |
4729 | 66.3k | assert(N2.getSimpleValueType() <= N1.getSimpleValueType() && |
4730 | 66.3k | "Insert subvector must be from smaller vector to larger vector!"); |
4731 | 66.3k | if (isa<ConstantSDNode>(Index)66.3k ) { |
4732 | 66.3k | assert((N2.getValueType().getVectorNumElements() + |
4733 | 66.3k | cast<ConstantSDNode>(Index)->getZExtValue() |
4734 | 66.3k | <= VT.getVectorNumElements()) |
4735 | 66.3k | && "Insert subvector overflow!"); |
4736 | 66.3k | } |
4737 | 66.3k | |
4738 | 66.3k | // Trivial insertion. |
4739 | 66.3k | if (VT.getSimpleVT() == N2.getSimpleValueType()) |
4740 | 101 | return N2; |
4741 | 66.2k | } |
4742 | 66.2k | break; |
4743 | 66.2k | } |
4744 | 0 | case ISD::BITCAST: |
4745 | 0 | // Fold bit_convert nodes from a type to themselves. |
4746 | 0 | if (N1.getValueType() == VT) |
4747 | 0 | return N1; |
4748 | 0 | break; |
4749 | 9.97M | } |
4750 | 9.97M | |
4751 | 9.97M | // Memoize node if it doesn't produce a flag. |
4752 | 9.97M | SDNode *N; |
4753 | 9.97M | SDVTList VTs = getVTList(VT); |
4754 | 9.97M | SDValue Ops[] = {N1, N2, N3}; |
4755 | 9.97M | if (VT != MVT::Glue9.97M ) { |
4756 | 9.97M | FoldingSetNodeID ID; |
4757 | 9.97M | AddNodeIDNode(ID, Opcode, VTs, Ops); |
4758 | 9.97M | void *IP = nullptr; |
4759 | 9.97M | if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
4760 | 330k | return SDValue(E, 0); |
4761 | 9.64M | |
4762 | 9.64M | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
4763 | 9.64M | createOperands(N, Ops); |
4764 | 9.64M | CSEMap.InsertNode(N, IP); |
4765 | 9.97M | } else { |
4766 | 2.80k | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
4767 | 2.80k | createOperands(N, Ops); |
4768 | 2.80k | } |
4769 | 9.97M | |
4770 | 9.64M | InsertNode(N); |
4771 | 9.64M | SDValue V = SDValue(N, 0); |
4772 | 9.64M | NewSDValueDbgMsg(V, "Creating new node: ", this); |
4773 | 9.64M | return V; |
4774 | 9.97M | } |
4775 | | |
4776 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
4777 | 1.03M | SDValue N1, SDValue N2, SDValue N3, SDValue N4) { |
4778 | 1.03M | SDValue Ops[] = { N1, N2, N3, N4 }; |
4779 | 1.03M | return getNode(Opcode, DL, VT, Ops); |
4780 | 1.03M | } |
4781 | | |
4782 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
4783 | | SDValue N1, SDValue N2, SDValue N3, SDValue N4, |
4784 | 2.08M | SDValue N5) { |
4785 | 2.08M | SDValue Ops[] = { N1, N2, N3, N4, N5 }; |
4786 | 2.08M | return getNode(Opcode, DL, VT, Ops); |
4787 | 2.08M | } |
4788 | | |
4789 | | /// getStackArgumentTokenFactor - Compute a TokenFactor to force all |
4790 | | /// the incoming stack arguments to be loaded from the stack. |
4791 | 154 | SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { |
4792 | 154 | SmallVector<SDValue, 8> ArgChains; |
4793 | 154 | |
4794 | 154 | // Include the original chain at the beginning of the list. When this is |
4795 | 154 | // used by target LowerCall hooks, this helps legalize find the |
4796 | 154 | // CALLSEQ_BEGIN node. |
4797 | 154 | ArgChains.push_back(Chain); |
4798 | 154 | |
4799 | 154 | // Add a chain value for each stack argument. |
4800 | 154 | for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), |
4801 | 738 | UE = getEntryNode().getNode()->use_end(); U != UE738 ; ++U584 ) |
4802 | 584 | if (LoadSDNode *584 L584 = dyn_cast<LoadSDNode>(*U)) |
4803 | 153 | if (FrameIndexSDNode *153 FI153 = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) |
4804 | 102 | if (102 FI->getIndex() < 0102 ) |
4805 | 102 | ArgChains.push_back(SDValue(L, 1)); |
4806 | 154 | |
4807 | 154 | // Build a tokenfactor for all the chains. |
4808 | 154 | return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); |
4809 | 154 | } |
4810 | | |
4811 | | /// getMemsetValue - Vectorized representation of the memset value |
4812 | | /// operand. |
4813 | | static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, |
4814 | 16.5k | const SDLoc &dl) { |
4815 | 16.5k | assert(!Value.isUndef()); |
4816 | 16.5k | |
4817 | 16.5k | unsigned NumBits = VT.getScalarSizeInBits(); |
4818 | 16.5k | if (ConstantSDNode *C16.5k = dyn_cast<ConstantSDNode>(Value)) { |
4819 | 16.4k | assert(C->getAPIntValue().getBitWidth() == 8); |
4820 | 16.4k | APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); |
4821 | 16.4k | if (VT.isInteger()) |
4822 | 16.4k | return DAG.getConstant(Val, dl, VT); |
4823 | 31 | return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl, |
4824 | 31 | VT); |
4825 | 31 | } |
4826 | 110 | |
4827 | 16.5k | assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?"); |
4828 | 110 | EVT IntVT = VT.getScalarType(); |
4829 | 110 | if (!IntVT.isInteger()) |
4830 | 1 | IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits()); |
4831 | 110 | |
4832 | 110 | Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value); |
4833 | 110 | if (NumBits > 8110 ) { |
4834 | 94 | // Use a multiplication with 0x010101... to extend the input to the |
4835 | 94 | // required length. |
4836 | 94 | APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); |
4837 | 94 | Value = DAG.getNode(ISD::MUL, dl, IntVT, Value, |
4838 | 94 | DAG.getConstant(Magic, dl, IntVT)); |
4839 | 94 | } |
4840 | 110 | |
4841 | 110 | if (VT != Value.getValueType() && 110 !VT.isInteger()18 ) |
4842 | 1 | Value = DAG.getBitcast(VT.getScalarType(), Value); |
4843 | 110 | if (VT != Value.getValueType()) |
4844 | 18 | Value = DAG.getSplatBuildVector(VT, dl, Value); |
4845 | 16.5k | |
4846 | 16.5k | return Value; |
4847 | 16.5k | } |
4848 | | |
4849 | | /// getMemsetStringVal - Similar to getMemsetValue. Except this is only |
4850 | | /// used when a memcpy is turned into a memset when the source is a constant |
4851 | | /// string ptr. |
4852 | | static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, |
4853 | | const TargetLowering &TLI, |
4854 | 753 | const ConstantDataArraySlice &Slice) { |
4855 | 753 | // Handle vector with all elements zero. |
4856 | 753 | if (Slice.Array == nullptr753 ) { |
4857 | 70 | if (VT.isInteger()) |
4858 | 64 | return DAG.getConstant(0, dl, VT); |
4859 | 6 | else if (6 VT == MVT::f32 || 6 VT == MVT::f646 || VT == MVT::f1283 ) |
4860 | 6 | return DAG.getConstantFP(0.0, dl, VT); |
4861 | 0 | else if (0 VT.isVector()0 ) { |
4862 | 0 | unsigned NumElts = VT.getVectorNumElements(); |
4863 | 0 | MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i320 : MVT::i640 ; |
4864 | 0 | return DAG.getNode(ISD::BITCAST, dl, VT, |
4865 | 0 | DAG.getConstant(0, dl, |
4866 | 0 | EVT::getVectorVT(*DAG.getContext(), |
4867 | 0 | EltVT, NumElts))); |
4868 | 0 | } else |
4869 | 0 | llvm_unreachable("Expected type!"); |
4870 | 70 | } |
4871 | 753 | |
4872 | 683 | assert(!VT.isVector() && "Can't handle vector type here!"); |
4873 | 683 | unsigned NumVTBits = VT.getSizeInBits(); |
4874 | 683 | unsigned NumVTBytes = NumVTBits / 8; |
4875 | 683 | unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length)); |
4876 | 683 | |
4877 | 683 | APInt Val(NumVTBits, 0); |
4878 | 683 | if (DAG.getDataLayout().isLittleEndian()683 ) { |
4879 | 3.15k | for (unsigned i = 0; i != NumBytes3.15k ; ++i2.47k ) |
4880 | 2.47k | Val |= (uint64_t)(unsigned char)Slice[i] << i*8; |
4881 | 0 | } else { |
4882 | 0 | for (unsigned i = 0; i != NumBytes0 ; ++i0 ) |
4883 | 0 | Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8; |
4884 | 0 | } |
4885 | 683 | |
4886 | 683 | // If the "cost" of materializing the integer immediate is less than the cost |
4887 | 683 | // of a load, then it is cost effective to turn the load into the immediate. |
4888 | 683 | Type *Ty = VT.getTypeForEVT(*DAG.getContext()); |
4889 | 683 | if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty)) |
4890 | 562 | return DAG.getConstant(Val, dl, VT); |
4891 | 121 | return SDValue(nullptr, 0); |
4892 | 121 | } |
4893 | | |
4894 | | SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset, |
4895 | 85.6k | const SDLoc &DL) { |
4896 | 85.6k | EVT VT = Base.getValueType(); |
4897 | 85.6k | return getNode(ISD::ADD, DL, VT, Base, getConstant(Offset, DL, VT)); |
4898 | 85.6k | } |
4899 | | |
4900 | | /// Returns true if memcpy source is constant data. |
4901 | 13.5k | static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { |
4902 | 13.5k | uint64_t SrcDelta = 0; |
4903 | 13.5k | GlobalAddressSDNode *G = nullptr; |
4904 | 13.5k | if (Src.getOpcode() == ISD::GlobalAddress) |
4905 | 976 | G = cast<GlobalAddressSDNode>(Src); |
4906 | 12.6k | else if (12.6k Src.getOpcode() == ISD::ADD && |
4907 | 1.69k | Src.getOperand(0).getOpcode() == ISD::GlobalAddress && |
4908 | 12.6k | Src.getOperand(1).getOpcode() == ISD::Constant201 ) { |
4909 | 187 | G = cast<GlobalAddressSDNode>(Src.getOperand(0)); |
4910 | 187 | SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); |
4911 | 187 | } |
4912 | 13.5k | if (!G) |
4913 | 12.4k | return false; |
4914 | 1.16k | |
4915 | 1.16k | return getConstantDataArrayInfo(G->getGlobal(), Slice, 8, |
4916 | 1.16k | SrcDelta + G->getOffset()); |
4917 | 1.16k | } |
4918 | | |
4919 | | /// Determines the optimal series of memory ops to replace the memset / memcpy. |
4920 | | /// Return true if the number of memory ops is below the threshold (Limit). |
4921 | | /// It returns the types of the sequence of memory ops to perform |
4922 | | /// memset / memcpy by reference. |
4923 | | static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, |
4924 | | unsigned Limit, uint64_t Size, |
4925 | | unsigned DstAlign, unsigned SrcAlign, |
4926 | | bool IsMemset, |
4927 | | bool ZeroMemset, |
4928 | | bool MemcpyStrSrc, |
4929 | | bool AllowOverlap, |
4930 | | unsigned DstAS, unsigned SrcAS, |
4931 | | SelectionDAG &DAG, |
4932 | 35.3k | const TargetLowering &TLI) { |
4933 | 35.3k | assert((SrcAlign == 0 || SrcAlign >= DstAlign) && |
4934 | 35.3k | "Expecting memcpy / memset source to meet alignment requirement!"); |
4935 | 35.3k | // If 'SrcAlign' is zero, that means the memory operation does not need to |
4936 | 35.3k | // load the value, i.e. memset or memcpy from constant string. Otherwise, |
4937 | 35.3k | // it's the inferred alignment of the source. 'DstAlign', on the other hand, |
4938 | 35.3k | // is the specified alignment of the memory operation. If it is zero, that |
4939 | 35.3k | // means it's possible to change the alignment of the destination. |
4940 | 35.3k | // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does |
4941 | 35.3k | // not need to be loaded. |
4942 | 35.3k | EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, |
4943 | 35.3k | IsMemset, ZeroMemset, MemcpyStrSrc, |
4944 | 35.3k | DAG.getMachineFunction()); |
4945 | 35.3k | |
4946 | 35.3k | if (VT == MVT::Other35.3k ) { |
4947 | 637 | // Use the largest integer type whose alignment constraints are satisfied. |
4948 | 637 | // We only need to check DstAlign here as SrcAlign is always greater or |
4949 | 637 | // equal to DstAlign (or zero). |
4950 | 637 | VT = MVT::i64; |
4951 | 1.02k | while (DstAlign && 1.02k DstAlign < VT.getSizeInBits() / 81.00k && |
4952 | 803 | !TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign)) |
4953 | 392 | VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); |
4954 | 637 | assert(VT.isInteger()); |
4955 | 637 | |
4956 | 637 | // Find the largest legal integer type. |
4957 | 637 | MVT LVT = MVT::i64; |
4958 | 1.03k | while (!TLI.isTypeLegal(LVT)) |
4959 | 398 | LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); |
4960 | 637 | assert(LVT.isInteger()); |
4961 | 637 | |
4962 | 637 | // If the type we've chosen is larger than the largest legal integer type |
4963 | 637 | // then use that instead. |
4964 | 637 | if (VT.bitsGT(LVT)) |
4965 | 63 | VT = LVT; |
4966 | 637 | } |
4967 | 35.3k | |
4968 | 35.3k | unsigned NumMemOps = 0; |
4969 | 176k | while (Size != 0176k ) { |
4970 | 153k | unsigned VTSize = VT.getSizeInBits() / 8; |
4971 | 165k | while (VTSize > Size165k ) { |
4972 | 11.8k | // For now, only use non-vector load / store's for the left-over pieces. |
4973 | 11.8k | EVT NewVT = VT; |
4974 | 11.8k | unsigned NewVTSize; |
4975 | 11.8k | |
4976 | 11.8k | bool Found = false; |
4977 | 11.8k | if (VT.isVector() || 11.8k VT.isFloatingPoint()11.7k ) { |
4978 | 6.33k | NewVT = (VT.getSizeInBits() > 64) ? MVT::i646.29k : MVT::i3244 ; |
4979 | 6.33k | if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && |
4980 | 6.26k | TLI.isSafeMemOpType(NewVT.getSimpleVT())) |
4981 | 6.26k | Found = true; |
4982 | 71 | else if (71 NewVT == MVT::i64 && |
4983 | 71 | TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && |
4984 | 71 | TLI.isSafeMemOpType(MVT::f64)13 ) { |
4985 | 12 | // i64 is usually not legal on 32-bit targets, but f64 may be. |
4986 | 12 | NewVT = MVT::f64; |
4987 | 12 | Found = true; |
4988 | 12 | } |
4989 | 6.33k | } |
4990 | 11.8k | |
4991 | 11.8k | if (!Found11.8k ) { |
4992 | 5.56k | do { |
4993 | 5.56k | NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); |
4994 | 5.56k | if (NewVT == MVT::i8) |
4995 | 399 | break; |
4996 | 5.16k | } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); |
4997 | 5.56k | } |
4998 | 11.4k | NewVTSize = NewVT.getSizeInBits() / 8; |
4999 | 11.4k | |
5000 | 11.4k | // If the new VT cannot cover all of the remaining bits, then consider |
5001 | 11.4k | // issuing a (or a pair of) unaligned and overlapping load / store. |
5002 | 11.4k | // FIXME: Only does this for 64-bit or more since we don't have proper |
5003 | 11.4k | // cost model for unaligned load / store. |
5004 | 11.4k | bool Fast; |
5005 | 11.4k | if (NumMemOps && 11.4k AllowOverlap11.4k && |
5006 | 11.4k | VTSize >= 811.3k && NewVTSize < Size10.4k && |
5007 | 11.4k | TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, &Fast)225 && Fast221 ) |
5008 | 207 | VTSize = Size; |
5009 | 11.2k | else { |
5010 | 11.2k | VT = NewVT; |
5011 | 11.2k | VTSize = NewVTSize; |
5012 | 11.2k | } |
5013 | 11.8k | } |
5014 | 153k | |
5015 | 153k | if (153k ++NumMemOps > Limit153k ) |
5016 | 11.9k | return false; |
5017 | 141k | |
5018 | 141k | MemOps.push_back(VT); |
5019 | 141k | Size -= VTSize; |
5020 | 141k | } |
5021 | 35.3k | |
5022 | 22.9k | return true; |
5023 | 35.3k | } |
5024 | | |
5025 | 35.3k | static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { |
5026 | 35.3k | // On Darwin, -Os means optimize for size without hurting performance, so |
5027 | 35.3k | // only really optimize for size when -Oz (MinSize) is used. |
5028 | 35.3k | if (MF.getTarget().getTargetTriple().isOSDarwin()) |
5029 | 34.3k | return MF.getFunction()->optForMinSize(); |
5030 | 973 | return MF.getFunction()->optForSize(); |
5031 | 973 | } |
5032 | | |
5033 | | static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, |
5034 | | SDValue Chain, SDValue Dst, SDValue Src, |
5035 | | uint64_t Size, unsigned Align, |
5036 | | bool isVol, bool AlwaysInline, |
5037 | | MachinePointerInfo DstPtrInfo, |
5038 | 13.5k | MachinePointerInfo SrcPtrInfo) { |
5039 | 13.5k | // Turn a memcpy of undef to nop. |
5040 | 13.5k | if (Src.isUndef()) |
5041 | 3 | return Chain; |
5042 | 13.5k | |
5043 | 13.5k | // Expand memcpy to a series of load and store ops if the size operand falls |
5044 | 13.5k | // below a certain threshold. |
5045 | 13.5k | // TODO: In the AlwaysInline case, if the size is big then generate a loop |
5046 | 13.5k | // rather than maybe a humongous number of loads and stores. |
5047 | 13.5k | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
5048 | 13.5k | const DataLayout &DL = DAG.getDataLayout(); |
5049 | 13.5k | LLVMContext &C = *DAG.getContext(); |
5050 | 13.5k | std::vector<EVT> MemOps; |
5051 | 13.5k | bool DstAlignCanChange = false; |
5052 | 13.5k | MachineFunction &MF = DAG.getMachineFunction(); |
5053 | 13.5k | MachineFrameInfo &MFI = MF.getFrameInfo(); |
5054 | 13.5k | bool OptSize = shouldLowerMemFuncForSize(MF); |
5055 | 13.5k | FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); |
5056 | 13.5k | if (FI && 13.5k !MFI.isFixedObjectIndex(FI->getIndex())1.26k ) |
5057 | 1.26k | DstAlignCanChange = true; |
5058 | 13.5k | unsigned SrcAlign = DAG.InferPtrAlignment(Src); |
5059 | 13.5k | if (Align > SrcAlign) |
5060 | 8.86k | SrcAlign = Align; |
5061 | 13.5k | ConstantDataArraySlice Slice; |
5062 | 13.5k | bool CopyFromConstant = isMemSrcFromConstant(Src, Slice); |
5063 | 462 | bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; |
5064 | 13.5k | unsigned Limit = AlwaysInline ? ~0U3 : TLI.getMaxStoresPerMemcpy(OptSize)13.5k ; |
5065 | 13.5k | |
5066 | 13.5k | if (!FindOptimalMemOpLowering(MemOps, Limit, Size, |
5067 | 13.5k | (DstAlignCanChange ? 01.26k : Align12.3k ), |
5068 | 13.5k | (isZeroConstant ? 024 : SrcAlign13.5k ), |
5069 | 13.5k | false, false, CopyFromConstant, true, |
5070 | 13.5k | DstPtrInfo.getAddrSpace(), |
5071 | 13.5k | SrcPtrInfo.getAddrSpace(), |
5072 | 13.5k | DAG, TLI)) |
5073 | 6.84k | return SDValue(); |
5074 | 6.74k | |
5075 | 6.74k | if (6.74k DstAlignCanChange6.74k ) { |
5076 | 860 | Type *Ty = MemOps[0].getTypeForEVT(C); |
5077 | 860 | unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty); |
5078 | 860 | |
5079 | 860 | // Don't promote to an alignment that would require dynamic stack |
5080 | 860 | // realignment. |
5081 | 860 | const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); |
5082 | 860 | if (!TRI->needsStackRealignment(MF)) |
5083 | 837 | while (837 NewAlign > Align && |
5084 | 734 | DL.exceedsNaturalStackAlignment(NewAlign)) |
5085 | 9 | NewAlign /= 2; |
5086 | 860 | |
5087 | 860 | if (NewAlign > Align860 ) { |
5088 | 730 | // Give the stack frame object a larger alignment if needed. |
5089 | 730 | if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign) |
5090 | 560 | MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
5091 | 730 | Align = NewAlign; |
5092 | 730 | } |
5093 | 860 | } |
5094 | 6.74k | |
5095 | 6.74k | MachineMemOperand::Flags MMOFlags = |
5096 | 6.74k | isVol ? MachineMemOperand::MOVolatile1 : MachineMemOperand::MONone6.74k ; |
5097 | 6.74k | SmallVector<SDValue, 8> OutChains; |
5098 | 6.74k | unsigned NumMemOps = MemOps.size(); |
5099 | 6.74k | uint64_t SrcOff = 0, DstOff = 0; |
5100 | 17.3k | for (unsigned i = 0; i != NumMemOps17.3k ; ++i10.5k ) { |
5101 | 10.5k | EVT VT = MemOps[i]; |
5102 | 10.5k | unsigned VTSize = VT.getSizeInBits() / 8; |
5103 | 10.5k | SDValue Value, Store; |
5104 | 10.5k | |
5105 | 10.5k | if (VTSize > Size10.5k ) { |
5106 | 174 | // Issuing an unaligned load / store pair that overlaps with the previous |
5107 | 174 | // pair. Adjust the offset accordingly. |
5108 | 174 | assert(i == NumMemOps-1 && i != 0); |
5109 | 174 | SrcOff -= VTSize - Size; |
5110 | 174 | DstOff -= VTSize - Size; |
5111 | 174 | } |
5112 | 10.5k | |
5113 | 10.5k | if (CopyFromConstant && |
5114 | 10.5k | (isZeroConstant || 918 (VT.isInteger() && 890 !VT.isVector()740 ))) { |
5115 | 753 | // It's unlikely a store of a vector immediate can be done in a single |
5116 | 753 | // instruction. It would require a load from a constantpool first. |
5117 | 753 | // We only handle zero vectors here. |
5118 | 753 | // FIXME: Handle other cases where store of vector immediate is done in |
5119 | 753 | // a single instruction. |
5120 | 753 | ConstantDataArraySlice SubSlice; |
5121 | 753 | if (SrcOff < Slice.Length753 ) { |
5122 | 709 | SubSlice = Slice; |
5123 | 709 | SubSlice.move(SrcOff); |
5124 | 753 | } else { |
5125 | 44 | // This is an out-of-bounds access and hence UB. Pretend we read zero. |
5126 | 44 | SubSlice.Array = nullptr; |
5127 | 44 | SubSlice.Offset = 0; |
5128 | 44 | SubSlice.Length = VTSize; |
5129 | 44 | } |
5130 | 753 | Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice); |
5131 | 753 | if (Value.getNode()) |
5132 | 632 | Store = DAG.getStore(Chain, dl, Value, |
5133 | 632 | DAG.getMemBasePlusOffset(Dst, DstOff, dl), |
5134 | 632 | DstPtrInfo.getWithOffset(DstOff), Align, |
5135 | 632 | MMOFlags); |
5136 | 753 | } |
5137 | 10.5k | |
5138 | 10.5k | if (!Store.getNode()10.5k ) { |
5139 | 9.94k | // The type might not be legal for the target. This should only happen |
5140 | 9.94k | // if the type is smaller than a legal type, as on PPC, so the right |
5141 | 9.94k | // thing to do is generate a LoadExt/StoreTrunc pair. These simplify |
5142 | 9.94k | // to Load/Store if NVT==VT. |
5143 | 9.94k | // FIXME does the case above also need this? |
5144 | 9.94k | EVT NVT = TLI.getTypeToTransformTo(C, VT); |
5145 | 9.94k | assert(NVT.bitsGE(VT)); |
5146 | 9.94k | |
5147 | 9.94k | bool isDereferenceable = |
5148 | 9.94k | SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); |
5149 | 9.94k | MachineMemOperand::Flags SrcMMOFlags = MMOFlags; |
5150 | 9.94k | if (isDereferenceable) |
5151 | 5.85k | SrcMMOFlags |= MachineMemOperand::MODereferenceable; |
5152 | 9.94k | |
5153 | 9.94k | Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, |
5154 | 9.94k | DAG.getMemBasePlusOffset(Src, SrcOff, dl), |
5155 | 9.94k | SrcPtrInfo.getWithOffset(SrcOff), VT, |
5156 | 9.94k | MinAlign(SrcAlign, SrcOff), SrcMMOFlags); |
5157 | 9.94k | OutChains.push_back(Value.getValue(1)); |
5158 | 9.94k | Store = DAG.getTruncStore( |
5159 | 9.94k | Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl), |
5160 | 9.94k | DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags); |
5161 | 9.94k | } |
5162 | 10.5k | OutChains.push_back(Store); |
5163 | 10.5k | SrcOff += VTSize; |
5164 | 10.5k | DstOff += VTSize; |
5165 | 10.5k | Size -= VTSize; |
5166 | 10.5k | } |
5167 | 13.5k | |
5168 | 13.5k | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
5169 | 13.5k | } |
5170 | | |
5171 | | static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, |
5172 | | SDValue Chain, SDValue Dst, SDValue Src, |
5173 | | uint64_t Size, unsigned Align, |
5174 | | bool isVol, bool AlwaysInline, |
5175 | | MachinePointerInfo DstPtrInfo, |
5176 | 147 | MachinePointerInfo SrcPtrInfo) { |
5177 | 147 | // Turn a memmove of undef to nop. |
5178 | 147 | if (Src.isUndef()) |
5179 | 0 | return Chain; |
5180 | 147 | |
5181 | 147 | // Expand memmove to a series of load and store ops if the size operand falls |
5182 | 147 | // below a certain threshold. |
5183 | 147 | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
5184 | 147 | const DataLayout &DL = DAG.getDataLayout(); |
5185 | 147 | LLVMContext &C = *DAG.getContext(); |
5186 | 147 | std::vector<EVT> MemOps; |
5187 | 147 | bool DstAlignCanChange = false; |
5188 | 147 | MachineFunction &MF = DAG.getMachineFunction(); |
5189 | 147 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
5190 | 147 | bool OptSize = shouldLowerMemFuncForSize(MF); |
5191 | 147 | FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); |
5192 | 147 | if (FI && 147 !MFI.isFixedObjectIndex(FI->getIndex())1 ) |
5193 | 1 | DstAlignCanChange = true; |
5194 | 147 | unsigned SrcAlign = DAG.InferPtrAlignment(Src); |
5195 | 147 | if (Align > SrcAlign) |
5196 | 134 | SrcAlign = Align; |
5197 | 147 | unsigned Limit = AlwaysInline ? ~0U0 : TLI.getMaxStoresPerMemmove(OptSize)147 ; |
5198 | 147 | |
5199 | 147 | if (!FindOptimalMemOpLowering(MemOps, Limit, Size, |
5200 | 147 | (DstAlignCanChange ? 01 : Align146 ), SrcAlign, |
5201 | 147 | false, false, false, false, |
5202 | 147 | DstPtrInfo.getAddrSpace(), |
5203 | 147 | SrcPtrInfo.getAddrSpace(), |
5204 | 147 | DAG, TLI)) |
5205 | 57 | return SDValue(); |
5206 | 90 | |
5207 | 90 | if (90 DstAlignCanChange90 ) { |
5208 | 1 | Type *Ty = MemOps[0].getTypeForEVT(C); |
5209 | 1 | unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty); |
5210 | 1 | if (NewAlign > Align1 ) { |
5211 | 1 | // Give the stack frame object a larger alignment if needed. |
5212 | 1 | if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign) |
5213 | 0 | MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
5214 | 1 | Align = NewAlign; |
5215 | 1 | } |
5216 | 1 | } |
5217 | 90 | |
5218 | 90 | MachineMemOperand::Flags MMOFlags = |
5219 | 90 | isVol ? MachineMemOperand::MOVolatile0 : MachineMemOperand::MONone90 ; |
5220 | 90 | uint64_t SrcOff = 0, DstOff = 0; |
5221 | 90 | SmallVector<SDValue, 8> LoadValues; |
5222 | 90 | SmallVector<SDValue, 8> LoadChains; |
5223 | 90 | SmallVector<SDValue, 8> OutChains; |
5224 | 90 | unsigned NumMemOps = MemOps.size(); |
5225 | 226 | for (unsigned i = 0; i < NumMemOps226 ; i++136 ) { |
5226 | 136 | EVT VT = MemOps[i]; |
5227 | 136 | unsigned VTSize = VT.getSizeInBits() / 8; |
5228 | 136 | SDValue Value; |
5229 | 136 | |
5230 | 136 | bool isDereferenceable = |
5231 | 136 | SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL); |
5232 | 136 | MachineMemOperand::Flags SrcMMOFlags = MMOFlags; |
5233 | 136 | if (isDereferenceable) |
5234 | 31 | SrcMMOFlags |= MachineMemOperand::MODereferenceable; |
5235 | 136 | |
5236 | 136 | Value = |
5237 | 136 | DAG.getLoad(VT, dl, Chain, DAG.getMemBasePlusOffset(Src, SrcOff, dl), |
5238 | 136 | SrcPtrInfo.getWithOffset(SrcOff), SrcAlign, SrcMMOFlags); |
5239 | 136 | LoadValues.push_back(Value); |
5240 | 136 | LoadChains.push_back(Value.getValue(1)); |
5241 | 136 | SrcOff += VTSize; |
5242 | 136 | } |
5243 | 90 | Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); |
5244 | 90 | OutChains.clear(); |
5245 | 226 | for (unsigned i = 0; i < NumMemOps226 ; i++136 ) { |
5246 | 136 | EVT VT = MemOps[i]; |
5247 | 136 | unsigned VTSize = VT.getSizeInBits() / 8; |
5248 | 136 | SDValue Store; |
5249 | 136 | |
5250 | 136 | Store = DAG.getStore(Chain, dl, LoadValues[i], |
5251 | 136 | DAG.getMemBasePlusOffset(Dst, DstOff, dl), |
5252 | 136 | DstPtrInfo.getWithOffset(DstOff), Align, MMOFlags); |
5253 | 136 | OutChains.push_back(Store); |
5254 | 136 | DstOff += VTSize; |
5255 | 136 | } |
5256 | 147 | |
5257 | 147 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
5258 | 147 | } |
5259 | | |
5260 | | /// \brief Lower the call to 'memset' intrinsic function into a series of store |
5261 | | /// operations. |
5262 | | /// |
5263 | | /// \param DAG Selection DAG where lowered code is placed. |
5264 | | /// \param dl Link to corresponding IR location. |
5265 | | /// \param Chain Control flow dependency. |
5266 | | /// \param Dst Pointer to destination memory location. |
5267 | | /// \param Src Value of byte to write into the memory. |
5268 | | /// \param Size Number of bytes to write. |
5269 | | /// \param Align Alignment of the destination in bytes. |
5270 | | /// \param isVol True if destination is volatile. |
5271 | | /// \param DstPtrInfo IR information on the memory pointer. |
5272 | | /// \returns New head in the control flow, if lowering was successful, empty |
5273 | | /// SDValue otherwise. |
5274 | | /// |
5275 | | /// The function tries to replace 'llvm.memset' intrinsic with several store |
5276 | | /// operations and value calculation code. This is usually profitable for small |
5277 | | /// memory size. |
5278 | | static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, |
5279 | | SDValue Chain, SDValue Dst, SDValue Src, |
5280 | | uint64_t Size, unsigned Align, bool isVol, |
5281 | 21.5k | MachinePointerInfo DstPtrInfo) { |
5282 | 21.5k | // Turn a memset of undef to nop. |
5283 | 21.5k | if (Src.isUndef()) |
5284 | 1 | return Chain; |
5285 | 21.5k | |
5286 | 21.5k | // Expand memset to a series of load/store ops if the size operand |
5287 | 21.5k | // falls below a certain threshold. |
5288 | 21.5k | const TargetLowering &TLI = DAG.getTargetLoweringInfo(); |
5289 | 21.5k | std::vector<EVT> MemOps; |
5290 | 21.5k | bool DstAlignCanChange = false; |
5291 | 21.5k | MachineFunction &MF = DAG.getMachineFunction(); |
5292 | 21.5k | MachineFrameInfo &MFI = MF.getFrameInfo(); |
5293 | 21.5k | bool OptSize = shouldLowerMemFuncForSize(MF); |
5294 | 21.5k | FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); |
5295 | 21.5k | if (FI && 21.5k !MFI.isFixedObjectIndex(FI->getIndex())3.62k ) |
5296 | 3.62k | DstAlignCanChange = true; |
5297 | 21.5k | bool IsZeroVal = |
5298 | 21.4k | isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); |
5299 | 21.5k | if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), |
5300 | 21.5k | Size, (DstAlignCanChange ? 03.62k : Align17.9k ), 0, |
5301 | 21.5k | true, IsZeroVal, false, true, |
5302 | 21.5k | DstPtrInfo.getAddrSpace(), ~0u, |
5303 | 21.5k | DAG, TLI)) |
5304 | 5.03k | return SDValue(); |
5305 | 16.5k | |
5306 | 16.5k | if (16.5k DstAlignCanChange16.5k ) { |
5307 | 3.36k | Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); |
5308 | 3.36k | unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty); |
5309 | 3.36k | if (NewAlign > Align3.36k ) { |
5310 | 2.90k | // Give the stack frame object a larger alignment if needed. |
5311 | 2.90k | if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign) |
5312 | 2.87k | MFI.setObjectAlignment(FI->getIndex(), NewAlign); |
5313 | 2.90k | Align = NewAlign; |
5314 | 2.90k | } |
5315 | 3.36k | } |
5316 | 16.5k | |
5317 | 16.5k | SmallVector<SDValue, 8> OutChains; |
5318 | 16.5k | uint64_t DstOff = 0; |
5319 | 16.5k | unsigned NumMemOps = MemOps.size(); |
5320 | 16.5k | |
5321 | 16.5k | // Find the largest store and generate the bit pattern for it. |
5322 | 16.5k | EVT LargestVT = MemOps[0]; |
5323 | 63.3k | for (unsigned i = 1; i < NumMemOps63.3k ; i++46.8k ) |
5324 | 46.8k | if (46.8k MemOps[i].bitsGT(LargestVT)46.8k ) |
5325 | 0 | LargestVT = MemOps[i]; |
5326 | 16.5k | SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); |
5327 | 16.5k | |
5328 | 79.9k | for (unsigned i = 0; i < NumMemOps79.9k ; i++63.3k ) { |
5329 | 63.3k | EVT VT = MemOps[i]; |
5330 | 63.3k | unsigned VTSize = VT.getSizeInBits() / 8; |
5331 | 63.3k | if (VTSize > Size63.3k ) { |
5332 | 33 | // Issuing an unaligned load / store pair that overlaps with the previous |
5333 | 33 | // pair. Adjust the offset accordingly. |
5334 | 33 | assert(i == NumMemOps-1 && i != 0); |
5335 | 33 | DstOff -= VTSize - Size; |
5336 | 33 | } |
5337 | 63.3k | |
5338 | 63.3k | // If this store is smaller than the largest store see whether we can get |
5339 | 63.3k | // the smaller value for free with a truncate. |
5340 | 63.3k | SDValue Value = MemSetValue; |
5341 | 63.3k | if (VT.bitsLT(LargestVT)63.3k ) { |
5342 | 3.85k | if (!LargestVT.isVector() && 3.85k !VT.isVector()3.84k && |
5343 | 3.84k | TLI.isTruncateFree(LargestVT, VT)) |
5344 | 3.83k | Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); |
5345 | 3.85k | else |
5346 | 18 | Value = getMemsetValue(Src, VT, DAG, dl); |
5347 | 3.85k | } |
5348 | 63.3k | assert(Value.getValueType() == VT && "Value with wrong type."); |
5349 | 63.3k | SDValue Store = DAG.getStore( |
5350 | 63.3k | Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl), |
5351 | 63.3k | DstPtrInfo.getWithOffset(DstOff), Align, |
5352 | 63.3k | isVol ? MachineMemOperand::MOVolatile0 : MachineMemOperand::MONone63.3k ); |
5353 | 63.3k | OutChains.push_back(Store); |
5354 | 63.3k | DstOff += VT.getSizeInBits() / 8; |
5355 | 63.3k | Size -= VTSize; |
5356 | 63.3k | } |
5357 | 21.5k | |
5358 | 21.5k | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); |
5359 | 21.5k | } |
5360 | | |
5361 | | static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, |
5362 | 18.8k | unsigned AS) { |
5363 | 18.8k | // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all |
5364 | 18.8k | // pointer operands can be losslessly bitcasted to pointers of address space 0 |
5365 | 18.8k | if (AS != 0 && 18.8k !TLI->isNoopAddrSpaceCast(AS, 0)0 ) { |
5366 | 0 | report_fatal_error("cannot lower memory intrinsic in address space " + |
5367 | 0 | Twine(AS)); |
5368 | 0 | } |
5369 | 18.8k | } |
5370 | | |
5371 | | SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, |
5372 | | SDValue Src, SDValue Size, unsigned Align, |
5373 | | bool isVol, bool AlwaysInline, bool isTailCall, |
5374 | | MachinePointerInfo DstPtrInfo, |
5375 | 15.4k | MachinePointerInfo SrcPtrInfo) { |
5376 | 15.4k | assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); |
5377 | 15.4k | |
5378 | 15.4k | // Check to see if we should lower the memcpy to loads and stores first. |
5379 | 15.4k | // For cases within the target-specified limits, this is the best choice. |
5380 | 15.4k | ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); |
5381 | 15.4k | if (ConstantSize15.4k ) { |
5382 | 13.6k | // Memcpy with size zero? Just return the original chain. |
5383 | 13.6k | if (ConstantSize->isNullValue()) |
5384 | 5 | return Chain; |
5385 | 13.5k | |
5386 | 13.5k | SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, |
5387 | 13.5k | ConstantSize->getZExtValue(),Align, |
5388 | 13.5k | isVol, false, DstPtrInfo, SrcPtrInfo); |
5389 | 13.5k | if (Result.getNode()) |
5390 | 6.74k | return Result; |
5391 | 8.65k | } |
5392 | 8.65k | |
5393 | 8.65k | // Then check to see if we should lower the memcpy with target-specific |
5394 | 8.65k | // code. If the target chooses to do this, this is the next best. |
5395 | 8.65k | if (8.65k TSI8.65k ) { |
5396 | 8.65k | SDValue Result = TSI->EmitTargetCodeForMemcpy( |
5397 | 8.65k | *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline, |
5398 | 8.65k | DstPtrInfo, SrcPtrInfo); |
5399 | 8.65k | if (Result.getNode()) |
5400 | 182 | return Result; |
5401 | 8.47k | } |
5402 | 8.47k | |
5403 | 8.47k | // If we really need inline code and the target declined to provide it, |
5404 | 8.47k | // use a (potentially long) sequence of loads and stores. |
5405 | 8.47k | if (8.47k AlwaysInline8.47k ) { |
5406 | 3 | assert(ConstantSize && "AlwaysInline requires a constant size!"); |
5407 | 3 | return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, |
5408 | 3 | ConstantSize->getZExtValue(), Align, isVol, |
5409 | 3 | true, DstPtrInfo, SrcPtrInfo); |
5410 | 3 | } |
5411 | 8.46k | |
5412 | 8.46k | checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
5413 | 8.46k | checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); |
5414 | 8.46k | |
5415 | 8.46k | // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc |
5416 | 8.46k | // memcpy is not guaranteed to be safe. libc memcpys aren't required to |
5417 | 8.46k | // respect volatile, so they may do things like read or write memory |
5418 | 8.46k | // beyond the given memory regions. But fixing this isn't easy, and most |
5419 | 8.46k | // people don't care. |
5420 | 8.46k | |
5421 | 8.46k | // Emit a library call. |
5422 | 8.46k | TargetLowering::ArgListTy Args; |
5423 | 8.46k | TargetLowering::ArgListEntry Entry; |
5424 | 8.46k | Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
5425 | 8.46k | Entry.Node = Dst; Args.push_back(Entry); |
5426 | 8.46k | Entry.Node = Src; Args.push_back(Entry); |
5427 | 8.46k | Entry.Node = Size; Args.push_back(Entry); |
5428 | 8.46k | // FIXME: pass in SDLoc |
5429 | 8.46k | TargetLowering::CallLoweringInfo CLI(*this); |
5430 | 8.46k | CLI.setDebugLoc(dl) |
5431 | 8.46k | .setChain(Chain) |
5432 | 8.46k | .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY), |
5433 | 8.46k | Dst.getValueType().getTypeForEVT(*getContext()), |
5434 | 8.46k | getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), |
5435 | 8.46k | TLI->getPointerTy(getDataLayout())), |
5436 | 8.46k | std::move(Args)) |
5437 | 8.46k | .setDiscardResult() |
5438 | 8.46k | .setTailCall(isTailCall); |
5439 | 8.46k | |
5440 | 8.46k | std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); |
5441 | 8.46k | return CallResult.second; |
5442 | 8.46k | } |
5443 | | |
5444 | | SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, |
5445 | | SDValue Src, SDValue Size, unsigned Align, |
5446 | | bool isVol, bool isTailCall, |
5447 | | MachinePointerInfo DstPtrInfo, |
5448 | 442 | MachinePointerInfo SrcPtrInfo) { |
5449 | 442 | assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); |
5450 | 442 | |
5451 | 442 | // Check to see if we should lower the memmove to loads and stores first. |
5452 | 442 | // For cases within the target-specified limits, this is the best choice. |
5453 | 442 | ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); |
5454 | 442 | if (ConstantSize442 ) { |
5455 | 147 | // Memmove with size zero? Just return the original chain. |
5456 | 147 | if (ConstantSize->isNullValue()) |
5457 | 0 | return Chain; |
5458 | 147 | |
5459 | 147 | SDValue Result = |
5460 | 147 | getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, |
5461 | 147 | ConstantSize->getZExtValue(), Align, isVol, |
5462 | 147 | false, DstPtrInfo, SrcPtrInfo); |
5463 | 147 | if (Result.getNode()) |
5464 | 90 | return Result; |
5465 | 352 | } |
5466 | 352 | |
5467 | 352 | // Then check to see if we should lower the memmove with target-specific |
5468 | 352 | // code. If the target chooses to do this, this is the next best. |
5469 | 352 | if (352 TSI352 ) { |
5470 | 352 | SDValue Result = TSI->EmitTargetCodeForMemmove( |
5471 | 352 | *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo); |
5472 | 352 | if (Result.getNode()) |
5473 | 48 | return Result; |
5474 | 304 | } |
5475 | 304 | |
5476 | 304 | checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
5477 | 304 | checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace()); |
5478 | 304 | |
5479 | 304 | // FIXME: If the memmove is volatile, lowering it to plain libc memmove may |
5480 | 304 | // not be safe. See memcpy above for more details. |
5481 | 304 | |
5482 | 304 | // Emit a library call. |
5483 | 304 | TargetLowering::ArgListTy Args; |
5484 | 304 | TargetLowering::ArgListEntry Entry; |
5485 | 304 | Entry.Ty = getDataLayout().getIntPtrType(*getContext()); |
5486 | 304 | Entry.Node = Dst; Args.push_back(Entry); |
5487 | 304 | Entry.Node = Src; Args.push_back(Entry); |
5488 | 304 | Entry.Node = Size; Args.push_back(Entry); |
5489 | 304 | // FIXME: pass in SDLoc |
5490 | 304 | TargetLowering::CallLoweringInfo CLI(*this); |
5491 | 304 | CLI.setDebugLoc(dl) |
5492 | 304 | .setChain(Chain) |
5493 | 304 | .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE), |
5494 | 304 | Dst.getValueType().getTypeForEVT(*getContext()), |
5495 | 304 | getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), |
5496 | 304 | TLI->getPointerTy(getDataLayout())), |
5497 | 304 | std::move(Args)) |
5498 | 304 | .setDiscardResult() |
5499 | 304 | .setTailCall(isTailCall); |
5500 | 304 | |
5501 | 304 | std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); |
5502 | 304 | return CallResult.second; |
5503 | 304 | } |
5504 | | |
5505 | | SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, |
5506 | | SDValue Src, SDValue Size, unsigned Align, |
5507 | | bool isVol, bool isTailCall, |
5508 | 29.5k | MachinePointerInfo DstPtrInfo) { |
5509 | 29.5k | assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); |
5510 | 29.5k | |
5511 | 29.5k | // Check to see if we should lower the memset to stores first. |
5512 | 29.5k | // For cases within the target-specified limits, this is the best choice. |
5513 | 29.5k | ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); |
5514 | 29.5k | if (ConstantSize29.5k ) { |
5515 | 21.5k | // Memset with size zero? Just return the original chain. |
5516 | 21.5k | if (ConstantSize->isNullValue()) |
5517 | 8 | return Chain; |
5518 | 21.5k | |
5519 | 21.5k | SDValue Result = |
5520 | 21.5k | getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), |
5521 | 21.5k | Align, isVol, DstPtrInfo); |
5522 | 21.5k | |
5523 | 21.5k | if (Result.getNode()) |
5524 | 16.5k | return Result; |
5525 | 12.9k | } |
5526 | 12.9k | |
5527 | 12.9k | // Then check to see if we should lower the memset with target-specific |
5528 | 12.9k | // code. If the target chooses to do this, this is the next best. |
5529 | 12.9k | if (12.9k TSI12.9k ) { |
5530 | 12.9k | SDValue Result = TSI->EmitTargetCodeForMemset( |
5531 | 12.9k | *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo); |
5532 | 12.9k | if (Result.getNode()) |
5533 | 11.6k | return Result; |
5534 | 1.34k | } |
5535 | 1.34k | |
5536 | 1.34k | checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); |
5537 | 1.34k | |
5538 | 1.34k | // Emit a library call. |
5539 | 1.34k | Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext()); |
5540 | 1.34k | TargetLowering::ArgListTy Args; |
5541 | 1.34k | TargetLowering::ArgListEntry Entry; |
5542 | 1.34k | Entry.Node = Dst; Entry.Ty = IntPtrTy; |
5543 | 1.34k | Args.push_back(Entry); |
5544 | 1.34k | Entry.Node = Src; |
5545 | 1.34k | Entry.Ty = Src.getValueType().getTypeForEVT(*getContext()); |
5546 | 1.34k | Args.push_back(Entry); |
5547 | 1.34k | Entry.Node = Size; |
5548 | 1.34k | Entry.Ty = IntPtrTy; |
5549 | 1.34k | Args.push_back(Entry); |
5550 | 1.34k | |
5551 | 1.34k | // FIXME: pass in SDLoc |
5552 | 1.34k | TargetLowering::CallLoweringInfo CLI(*this); |
5553 | 1.34k | CLI.setDebugLoc(dl) |
5554 | 1.34k | .setChain(Chain) |
5555 | 1.34k | .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET), |
5556 | 1.34k | Dst.getValueType().getTypeForEVT(*getContext()), |
5557 | 1.34k | getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), |
5558 | 1.34k | TLI->getPointerTy(getDataLayout())), |
5559 | 1.34k | std::move(Args)) |
5560 | 1.34k | .setDiscardResult() |
5561 | 1.34k | .setTailCall(isTailCall); |
5562 | 1.34k | |
5563 | 1.34k | std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); |
5564 | 1.34k | return CallResult.second; |
5565 | 1.34k | } |
5566 | | |
5567 | | SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
5568 | | SDVTList VTList, ArrayRef<SDValue> Ops, |
5569 | 26.5k | MachineMemOperand *MMO) { |
5570 | 26.5k | FoldingSetNodeID ID; |
5571 | 26.5k | ID.AddInteger(MemVT.getRawBits()); |
5572 | 26.5k | AddNodeIDNode(ID, Opcode, VTList, Ops); |
5573 | 26.5k | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
5574 | 26.5k | void* IP = nullptr; |
5575 | 26.5k | if (SDNode *E26.5k = FindNodeOrInsertPos(ID, dl, IP)) { |
5576 | 0 | cast<AtomicSDNode>(E)->refineAlignment(MMO); |
5577 | 0 | return SDValue(E, 0); |
5578 | 0 | } |
5579 | 26.5k | |
5580 | 26.5k | auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
5581 | 26.5k | VTList, MemVT, MMO); |
5582 | 26.5k | createOperands(N, Ops); |
5583 | 26.5k | |
5584 | 26.5k | CSEMap.InsertNode(N, IP); |
5585 | 26.5k | InsertNode(N); |
5586 | 26.5k | return SDValue(N, 0); |
5587 | 26.5k | } |
5588 | | |
5589 | | SDValue SelectionDAG::getAtomicCmpSwap( |
5590 | | unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, |
5591 | | SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, |
5592 | | unsigned Alignment, AtomicOrdering SuccessOrdering, |
5593 | 2.47k | AtomicOrdering FailureOrdering, SyncScope::ID SSID) { |
5594 | 2.47k | assert(Opcode == ISD::ATOMIC_CMP_SWAP || |
5595 | 2.47k | Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
5596 | 2.47k | assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); |
5597 | 2.47k | |
5598 | 2.47k | if (Alignment == 0) // Ensure that codegen never sees alignment 0 |
5599 | 2.47k | Alignment = getEVTAlignment(MemVT); |
5600 | 2.47k | |
5601 | 2.47k | MachineFunction &MF = getMachineFunction(); |
5602 | 2.47k | |
5603 | 2.47k | // FIXME: Volatile isn't really correct; we should keep track of atomic |
5604 | 2.47k | // orderings in the memoperand. |
5605 | 2.47k | auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad | |
5606 | 2.47k | MachineMemOperand::MOStore; |
5607 | 2.47k | MachineMemOperand *MMO = |
5608 | 2.47k | MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, |
5609 | 2.47k | AAMDNodes(), nullptr, SSID, SuccessOrdering, |
5610 | 2.47k | FailureOrdering); |
5611 | 2.47k | |
5612 | 2.47k | return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO); |
5613 | 2.47k | } |
5614 | | |
5615 | | SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, |
5616 | | EVT MemVT, SDVTList VTs, SDValue Chain, |
5617 | | SDValue Ptr, SDValue Cmp, SDValue Swp, |
5618 | 6.31k | MachineMemOperand *MMO) { |
5619 | 6.31k | assert(Opcode == ISD::ATOMIC_CMP_SWAP || |
5620 | 6.31k | Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); |
5621 | 6.31k | assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); |
5622 | 6.31k | |
5623 | 6.31k | SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; |
5624 | 6.31k | return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
5625 | 6.31k | } |
5626 | | |
5627 | | SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
5628 | | SDValue Chain, SDValue Ptr, SDValue Val, |
5629 | | const Value *PtrVal, unsigned Alignment, |
5630 | | AtomicOrdering Ordering, |
5631 | 17.2k | SyncScope::ID SSID) { |
5632 | 17.2k | if (Alignment == 0) // Ensure that codegen never sees alignment 0 |
5633 | 5.10k | Alignment = getEVTAlignment(MemVT); |
5634 | 17.2k | |
5635 | 17.2k | MachineFunction &MF = getMachineFunction(); |
5636 | 17.2k | // An atomic store does not load. An atomic load does not store. |
5637 | 17.2k | // (An atomicrmw obviously both loads and stores.) |
5638 | 17.2k | // For now, atomics are considered to be volatile always, and they are |
5639 | 17.2k | // chained as such. |
5640 | 17.2k | // FIXME: Volatile isn't really correct; we should keep track of atomic |
5641 | 17.2k | // orderings in the memoperand. |
5642 | 17.2k | auto Flags = MachineMemOperand::MOVolatile; |
5643 | 17.2k | if (Opcode != ISD::ATOMIC_STORE) |
5644 | 5.10k | Flags |= MachineMemOperand::MOLoad; |
5645 | 17.2k | if (Opcode != ISD::ATOMIC_LOAD) |
5646 | 17.2k | Flags |= MachineMemOperand::MOStore; |
5647 | 17.2k | |
5648 | 17.2k | MachineMemOperand *MMO = |
5649 | 17.2k | MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, |
5650 | 17.2k | MemVT.getStoreSize(), Alignment, AAMDNodes(), |
5651 | 17.2k | nullptr, SSID, Ordering); |
5652 | 17.2k | |
5653 | 17.2k | return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO); |
5654 | 17.2k | } |
5655 | | |
5656 | | SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
5657 | | SDValue Chain, SDValue Ptr, SDValue Val, |
5658 | 18.8k | MachineMemOperand *MMO) { |
5659 | 18.8k | assert((Opcode == ISD::ATOMIC_LOAD_ADD || |
5660 | 18.8k | Opcode == ISD::ATOMIC_LOAD_SUB || |
5661 | 18.8k | Opcode == ISD::ATOMIC_LOAD_AND || |
5662 | 18.8k | Opcode == ISD::ATOMIC_LOAD_OR || |
5663 | 18.8k | Opcode == ISD::ATOMIC_LOAD_XOR || |
5664 | 18.8k | Opcode == ISD::ATOMIC_LOAD_NAND || |
5665 | 18.8k | Opcode == ISD::ATOMIC_LOAD_MIN || |
5666 | 18.8k | Opcode == ISD::ATOMIC_LOAD_MAX || |
5667 | 18.8k | Opcode == ISD::ATOMIC_LOAD_UMIN || |
5668 | 18.8k | Opcode == ISD::ATOMIC_LOAD_UMAX || |
5669 | 18.8k | Opcode == ISD::ATOMIC_SWAP || |
5670 | 18.8k | Opcode == ISD::ATOMIC_STORE) && |
5671 | 18.8k | "Invalid Atomic Op"); |
5672 | 18.8k | |
5673 | 18.8k | EVT VT = Val.getValueType(); |
5674 | 18.8k | |
5675 | 12.3k | SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : |
5676 | 6.49k | getVTList(VT, MVT::Other); |
5677 | 18.8k | SDValue Ops[] = {Chain, Ptr, Val}; |
5678 | 18.8k | return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
5679 | 18.8k | } |
5680 | | |
5681 | | SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, |
5682 | | EVT VT, SDValue Chain, SDValue Ptr, |
5683 | 1.35k | MachineMemOperand *MMO) { |
5684 | 1.35k | assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); |
5685 | 1.35k | |
5686 | 1.35k | SDVTList VTs = getVTList(VT, MVT::Other); |
5687 | 1.35k | SDValue Ops[] = {Chain, Ptr}; |
5688 | 1.35k | return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); |
5689 | 1.35k | } |
5690 | | |
5691 | | /// getMergeValues - Create a MERGE_VALUES node from the given operands. |
5692 | 959k | SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) { |
5693 | 959k | if (Ops.size() == 1) |
5694 | 908k | return Ops[0]; |
5695 | 51.6k | |
5696 | 51.6k | SmallVector<EVT, 4> VTs; |
5697 | 51.6k | VTs.reserve(Ops.size()); |
5698 | 157k | for (unsigned i = 0; i < Ops.size()157k ; ++i105k ) |
5699 | 105k | VTs.push_back(Ops[i].getValueType()); |
5700 | 959k | return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops); |
5701 | 959k | } |
5702 | | |
5703 | | SDValue SelectionDAG::getMemIntrinsicNode( |
5704 | | unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops, |
5705 | | EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol, |
5706 | 126k | bool ReadMem, bool WriteMem, unsigned Size) { |
5707 | 126k | if (Align == 0) // Ensure that codegen never sees alignment 0 |
5708 | 29.5k | Align = getEVTAlignment(MemVT); |
5709 | 126k | |
5710 | 126k | MachineFunction &MF = getMachineFunction(); |
5711 | 126k | auto Flags = MachineMemOperand::MONone; |
5712 | 126k | if (WriteMem) |
5713 | 74.5k | Flags |= MachineMemOperand::MOStore; |
5714 | 126k | if (ReadMem) |
5715 | 52.1k | Flags |= MachineMemOperand::MOLoad; |
5716 | 126k | if (Vol) |
5717 | 94.3k | Flags |= MachineMemOperand::MOVolatile; |
5718 | 126k | if (!Size) |
5719 | 126k | Size = MemVT.getStoreSize(); |
5720 | 126k | MachineMemOperand *MMO = |
5721 | 126k | MF.getMachineMemOperand(PtrInfo, Flags, Size, Align); |
5722 | 126k | |
5723 | 126k | return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); |
5724 | 126k | } |
5725 | | |
5726 | | SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, |
5727 | | SDVTList VTList, |
5728 | | ArrayRef<SDValue> Ops, EVT MemVT, |
5729 | 134k | MachineMemOperand *MMO) { |
5730 | 134k | assert((Opcode == ISD::INTRINSIC_VOID || |
5731 | 134k | Opcode == ISD::INTRINSIC_W_CHAIN || |
5732 | 134k | Opcode == ISD::PREFETCH || |
5733 | 134k | Opcode == ISD::LIFETIME_START || |
5734 | 134k | Opcode == ISD::LIFETIME_END || |
5735 | 134k | ((int)Opcode <= std::numeric_limits<int>::max() && |
5736 | 134k | (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && |
5737 | 134k | "Opcode is not a memory-accessing opcode!"); |
5738 | 134k | |
5739 | 134k | // Memoize the node unless it returns a flag. |
5740 | 134k | MemIntrinsicSDNode *N; |
5741 | 134k | if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue134k ) { |
5742 | 132k | FoldingSetNodeID ID; |
5743 | 132k | AddNodeIDNode(ID, Opcode, VTList, Ops); |
5744 | 132k | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
5745 | 132k | void *IP = nullptr; |
5746 | 132k | if (SDNode *E132k = FindNodeOrInsertPos(ID, dl, IP)) { |
5747 | 5 | cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); |
5748 | 5 | return SDValue(E, 0); |
5749 | 5 | } |
5750 | 132k | |
5751 | 132k | N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
5752 | 132k | VTList, MemVT, MMO); |
5753 | 132k | createOperands(N, Ops); |
5754 | 132k | |
5755 | 132k | CSEMap.InsertNode(N, IP); |
5756 | 134k | } else { |
5757 | 2.37k | N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), |
5758 | 2.37k | VTList, MemVT, MMO); |
5759 | 2.37k | createOperands(N, Ops); |
5760 | 2.37k | } |
5761 | 134k | InsertNode(N); |
5762 | 134k | return SDValue(N, 0); |
5763 | 134k | } |
5764 | | |
5765 | | /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a |
5766 | | /// MachinePointerInfo record from it. This is particularly useful because the |
5767 | | /// code generator has many cases where it doesn't bother passing in a |
5768 | | /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". |
5769 | | static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr, |
5770 | 38.3k | int64_t Offset = 0) { |
5771 | 38.3k | // If this is FI+Offset, we can model it. |
5772 | 38.3k | if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) |
5773 | 12.4k | return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), |
5774 | 12.4k | FI->getIndex(), Offset); |
5775 | 25.8k | |
5776 | 25.8k | // If this is (FI+Offset1)+Offset2, we can model it. |
5777 | 25.8k | if (25.8k Ptr.getOpcode() != ISD::ADD || |
5778 | 14.5k | !isa<ConstantSDNode>(Ptr.getOperand(1)) || |
5779 | 13.5k | !isa<FrameIndexSDNode>(Ptr.getOperand(0))) |
5780 | 23.1k | return MachinePointerInfo(); |
5781 | 2.76k | |
5782 | 2.76k | int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); |
5783 | 2.76k | return MachinePointerInfo::getFixedStack( |
5784 | 2.76k | DAG.getMachineFunction(), FI, |
5785 | 2.76k | Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); |
5786 | 2.76k | } |
5787 | | |
5788 | | /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a |
5789 | | /// MachinePointerInfo record from it. This is particularly useful because the |
5790 | | /// code generator has many cases where it doesn't bother passing in a |
5791 | | /// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". |
5792 | | static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr, |
5793 | 17.6k | SDValue OffsetOp) { |
5794 | 17.6k | // If the 'Offset' value isn't a constant, we can't handle this. |
5795 | 17.6k | if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) |
5796 | 15 | return InferPointerInfo(DAG, Ptr, OffsetNode->getSExtValue()); |
5797 | 17.6k | if (17.6k OffsetOp.isUndef()17.6k ) |
5798 | 17.6k | return InferPointerInfo(DAG, Ptr); |
5799 | 0 | return MachinePointerInfo(); |
5800 | 0 | } |
5801 | | |
5802 | | SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, |
5803 | | EVT VT, const SDLoc &dl, SDValue Chain, |
5804 | | SDValue Ptr, SDValue Offset, |
5805 | | MachinePointerInfo PtrInfo, EVT MemVT, |
5806 | | unsigned Alignment, |
5807 | | MachineMemOperand::Flags MMOFlags, |
5808 | 2.39M | const AAMDNodes &AAInfo, const MDNode *Ranges) { |
5809 | 2.39M | assert(Chain.getValueType() == MVT::Other && |
5810 | 2.39M | "Invalid chain type"); |
5811 | 2.39M | if (Alignment == 0) // Ensure that codegen never sees alignment 0 |
5812 | 130k | Alignment = getEVTAlignment(MemVT); |
5813 | 2.39M | |
5814 | 2.39M | MMOFlags |= MachineMemOperand::MOLoad; |
5815 | 2.39M | assert((MMOFlags & MachineMemOperand::MOStore) == 0); |
5816 | 2.39M | // If we don't have a PtrInfo, infer the trivial frame index case to simplify |
5817 | 2.39M | // clients. |
5818 | 2.39M | if (PtrInfo.V.isNull()) |
5819 | 17.6k | PtrInfo = InferPointerInfo(*this, Ptr, Offset); |
5820 | 2.39M | |
5821 | 2.39M | MachineFunction &MF = getMachineFunction(); |
5822 | 2.39M | MachineMemOperand *MMO = MF.getMachineMemOperand( |
5823 | 2.39M | PtrInfo, MMOFlags, MemVT.getStoreSize(), Alignment, AAInfo, Ranges); |
5824 | 2.39M | return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); |
5825 | 2.39M | } |
5826 | | |
5827 | | SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, |
5828 | | EVT VT, const SDLoc &dl, SDValue Chain, |
5829 | | SDValue Ptr, SDValue Offset, EVT MemVT, |
5830 | 3.38M | MachineMemOperand *MMO) { |
5831 | 3.38M | if (VT == MemVT3.38M ) { |
5832 | 2.64M | ExtType = ISD::NON_EXTLOAD; |
5833 | 3.38M | } else if (746k ExtType == ISD::NON_EXTLOAD746k ) { |
5834 | 0 | assert(VT == MemVT && "Non-extending load from different memory type!"); |
5835 | 746k | } else { |
5836 | 746k | // Extending load. |
5837 | 746k | assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && |
5838 | 746k | "Should only be an extending load, not truncating!"); |
5839 | 746k | assert(VT.isInteger() == MemVT.isInteger() && |
5840 | 746k | "Cannot convert from FP to Int or Int -> FP!"); |
5841 | 746k | assert(VT.isVector() == MemVT.isVector() && |
5842 | 746k | "Cannot use an ext load to convert to or from a vector!"); |
5843 | 746k | assert((!VT.isVector() || |
5844 | 746k | VT.getVectorNumElements() == MemVT.getVectorNumElements()) && |
5845 | 746k | "Cannot use an ext load to change the number of vector elements!"); |
5846 | 746k | } |
5847 | 3.38M | |
5848 | 3.38M | bool Indexed = AM != ISD::UNINDEXED; |
5849 | 3.38M | assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!"); |
5850 | 3.38M | |
5851 | 3.38M | SDVTList VTs = Indexed ? |
5852 | 3.38M | getVTList(VT, Ptr.getValueType(), MVT::Other)64.8k : getVTList(VT, MVT::Other)3.32M ; |
5853 | 3.38M | SDValue Ops[] = { Chain, Ptr, Offset }; |
5854 | 3.38M | FoldingSetNodeID ID; |
5855 | 3.38M | AddNodeIDNode(ID, ISD::LOAD, VTs, Ops); |
5856 | 3.38M | ID.AddInteger(MemVT.getRawBits()); |
5857 | 3.38M | ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>( |
5858 | 3.38M | dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO)); |
5859 | 3.38M | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
5860 | 3.38M | void *IP = nullptr; |
5861 | 3.38M | if (SDNode *E3.38M = FindNodeOrInsertPos(ID, dl, IP)) { |
5862 | 54.2k | cast<LoadSDNode>(E)->refineAlignment(MMO); |
5863 | 54.2k | return SDValue(E, 0); |
5864 | 54.2k | } |
5865 | 3.33M | auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
5866 | 3.33M | ExtType, MemVT, MMO); |
5867 | 3.33M | createOperands(N, Ops); |
5868 | 3.33M | |
5869 | 3.33M | CSEMap.InsertNode(N, IP); |
5870 | 3.33M | InsertNode(N); |
5871 | 3.33M | return SDValue(N, 0); |
5872 | 3.33M | } |
5873 | | |
5874 | | SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
5875 | | SDValue Ptr, MachinePointerInfo PtrInfo, |
5876 | | unsigned Alignment, |
5877 | | MachineMemOperand::Flags MMOFlags, |
5878 | 2.18M | const AAMDNodes &AAInfo, const MDNode *Ranges) { |
5879 | 2.18M | SDValue Undef = getUNDEF(Ptr.getValueType()); |
5880 | 2.18M | return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
5881 | 2.18M | PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges); |
5882 | 2.18M | } |
5883 | | |
5884 | | SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
5885 | 295k | SDValue Ptr, MachineMemOperand *MMO) { |
5886 | 295k | SDValue Undef = getUNDEF(Ptr.getValueType()); |
5887 | 295k | return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, |
5888 | 295k | VT, MMO); |
5889 | 295k | } |
5890 | | |
5891 | | SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, |
5892 | | EVT VT, SDValue Chain, SDValue Ptr, |
5893 | | MachinePointerInfo PtrInfo, EVT MemVT, |
5894 | | unsigned Alignment, |
5895 | | MachineMemOperand::Flags MMOFlags, |
5896 | 122k | const AAMDNodes &AAInfo) { |
5897 | 122k | SDValue Undef = getUNDEF(Ptr.getValueType()); |
5898 | 122k | return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo, |
5899 | 122k | MemVT, Alignment, MMOFlags, AAInfo); |
5900 | 122k | } |
5901 | | |
5902 | | SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, |
5903 | | EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT, |
5904 | 518k | MachineMemOperand *MMO) { |
5905 | 518k | SDValue Undef = getUNDEF(Ptr.getValueType()); |
5906 | 518k | return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, |
5907 | 518k | MemVT, MMO); |
5908 | 518k | } |
5909 | | |
5910 | | SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, |
5911 | | SDValue Base, SDValue Offset, |
5912 | 64.8k | ISD::MemIndexedMode AM) { |
5913 | 64.8k | LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); |
5914 | 64.8k | assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); |
5915 | 64.8k | // Don't propagate the invariant or dereferenceable flags. |
5916 | 64.8k | auto MMOFlags = |
5917 | 64.8k | LD->getMemOperand()->getFlags() & |
5918 | 64.8k | ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); |
5919 | 64.8k | return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, |
5920 | 64.8k | LD->getChain(), Base, Offset, LD->getPointerInfo(), |
5921 | 64.8k | LD->getMemoryVT(), LD->getAlignment(), MMOFlags, |
5922 | 64.8k | LD->getAAInfo()); |
5923 | 64.8k | } |
5924 | | |
5925 | | SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
5926 | | SDValue Ptr, MachinePointerInfo PtrInfo, |
5927 | | unsigned Alignment, |
5928 | | MachineMemOperand::Flags MMOFlags, |
5929 | 2.11M | const AAMDNodes &AAInfo) { |
5930 | 2.11M | assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); |
5931 | 2.11M | if (Alignment == 0) // Ensure that codegen never sees alignment 0 |
5932 | 178k | Alignment = getEVTAlignment(Val.getValueType()); |
5933 | 2.11M | |
5934 | 2.11M | MMOFlags |= MachineMemOperand::MOStore; |
5935 | 2.11M | assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
5936 | 2.11M | |
5937 | 2.11M | if (PtrInfo.V.isNull()) |
5938 | 16.4k | PtrInfo = InferPointerInfo(*this, Ptr); |
5939 | 2.11M | |
5940 | 2.11M | MachineFunction &MF = getMachineFunction(); |
5941 | 2.11M | MachineMemOperand *MMO = MF.getMachineMemOperand( |
5942 | 2.11M | PtrInfo, MMOFlags, Val.getValueType().getStoreSize(), Alignment, AAInfo); |
5943 | 2.11M | return getStore(Chain, dl, Val, Ptr, MMO); |
5944 | 2.11M | } |
5945 | | |
5946 | | SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
5947 | 3.12M | SDValue Ptr, MachineMemOperand *MMO) { |
5948 | 3.12M | assert(Chain.getValueType() == MVT::Other && |
5949 | 3.12M | "Invalid chain type"); |
5950 | 3.12M | EVT VT = Val.getValueType(); |
5951 | 3.12M | SDVTList VTs = getVTList(MVT::Other); |
5952 | 3.12M | SDValue Undef = getUNDEF(Ptr.getValueType()); |
5953 | 3.12M | SDValue Ops[] = { Chain, Val, Ptr, Undef }; |
5954 | 3.12M | FoldingSetNodeID ID; |
5955 | 3.12M | AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
5956 | 3.12M | ID.AddInteger(VT.getRawBits()); |
5957 | 3.12M | ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>( |
5958 | 3.12M | dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO)); |
5959 | 3.12M | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
5960 | 3.12M | void *IP = nullptr; |
5961 | 3.12M | if (SDNode *E3.12M = FindNodeOrInsertPos(ID, dl, IP)) { |
5962 | 28.8k | cast<StoreSDNode>(E)->refineAlignment(MMO); |
5963 | 28.8k | return SDValue(E, 0); |
5964 | 28.8k | } |
5965 | 3.10M | auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, |
5966 | 3.10M | ISD::UNINDEXED, false, VT, MMO); |
5967 | 3.10M | createOperands(N, Ops); |
5968 | 3.10M | |
5969 | 3.10M | CSEMap.InsertNode(N, IP); |
5970 | 3.10M | InsertNode(N); |
5971 | 3.10M | return SDValue(N, 0); |
5972 | 3.10M | } |
5973 | | |
5974 | | SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
5975 | | SDValue Ptr, MachinePointerInfo PtrInfo, |
5976 | | EVT SVT, unsigned Alignment, |
5977 | | MachineMemOperand::Flags MMOFlags, |
5978 | 104k | const AAMDNodes &AAInfo) { |
5979 | 104k | assert(Chain.getValueType() == MVT::Other && |
5980 | 104k | "Invalid chain type"); |
5981 | 104k | if (Alignment == 0) // Ensure that codegen never sees alignment 0 |
5982 | 1.13k | Alignment = getEVTAlignment(SVT); |
5983 | 104k | |
5984 | 104k | MMOFlags |= MachineMemOperand::MOStore; |
5985 | 104k | assert((MMOFlags & MachineMemOperand::MOLoad) == 0); |
5986 | 104k | |
5987 | 104k | if (PtrInfo.V.isNull()) |
5988 | 4.23k | PtrInfo = InferPointerInfo(*this, Ptr); |
5989 | 104k | |
5990 | 104k | MachineFunction &MF = getMachineFunction(); |
5991 | 104k | MachineMemOperand *MMO = MF.getMachineMemOperand( |
5992 | 104k | PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo); |
5993 | 104k | return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); |
5994 | 104k | } |
5995 | | |
5996 | | SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, |
5997 | | SDValue Ptr, EVT SVT, |
5998 | 450k | MachineMemOperand *MMO) { |
5999 | 450k | EVT VT = Val.getValueType(); |
6000 | 450k | |
6001 | 450k | assert(Chain.getValueType() == MVT::Other && |
6002 | 450k | "Invalid chain type"); |
6003 | 450k | if (VT == SVT) |
6004 | 59.0k | return getStore(Chain, dl, Val, Ptr, MMO); |
6005 | 391k | |
6006 | 450k | assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && |
6007 | 391k | "Should only be a truncating store, not extending!"); |
6008 | 391k | assert(VT.isInteger() == SVT.isInteger() && |
6009 | 391k | "Can't do FP-INT conversion!"); |
6010 | 391k | assert(VT.isVector() == SVT.isVector() && |
6011 | 391k | "Cannot use trunc store to convert to or from a vector!"); |
6012 | 391k | assert((!VT.isVector() || |
6013 | 391k | VT.getVectorNumElements() == SVT.getVectorNumElements()) && |
6014 | 391k | "Cannot use trunc store to change the number of vector elements!"); |
6015 | 391k | |
6016 | 391k | SDVTList VTs = getVTList(MVT::Other); |
6017 | 391k | SDValue Undef = getUNDEF(Ptr.getValueType()); |
6018 | 391k | SDValue Ops[] = { Chain, Val, Ptr, Undef }; |
6019 | 391k | FoldingSetNodeID ID; |
6020 | 391k | AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
6021 | 391k | ID.AddInteger(SVT.getRawBits()); |
6022 | 391k | ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>( |
6023 | 391k | dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO)); |
6024 | 391k | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
6025 | 391k | void *IP = nullptr; |
6026 | 391k | if (SDNode *E391k = FindNodeOrInsertPos(ID, dl, IP)) { |
6027 | 1.15k | cast<StoreSDNode>(E)->refineAlignment(MMO); |
6028 | 1.15k | return SDValue(E, 0); |
6029 | 1.15k | } |
6030 | 390k | auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, |
6031 | 390k | ISD::UNINDEXED, true, SVT, MMO); |
6032 | 390k | createOperands(N, Ops); |
6033 | 390k | |
6034 | 390k | CSEMap.InsertNode(N, IP); |
6035 | 390k | InsertNode(N); |
6036 | 390k | return SDValue(N, 0); |
6037 | 390k | } |
6038 | | |
6039 | | SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl, |
6040 | | SDValue Base, SDValue Offset, |
6041 | 23.5k | ISD::MemIndexedMode AM) { |
6042 | 23.5k | StoreSDNode *ST = cast<StoreSDNode>(OrigStore); |
6043 | 23.5k | assert(ST->getOffset().isUndef() && "Store is already a indexed store!"); |
6044 | 23.5k | SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); |
6045 | 23.5k | SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; |
6046 | 23.5k | FoldingSetNodeID ID; |
6047 | 23.5k | AddNodeIDNode(ID, ISD::STORE, VTs, Ops); |
6048 | 23.5k | ID.AddInteger(ST->getMemoryVT().getRawBits()); |
6049 | 23.5k | ID.AddInteger(ST->getRawSubclassData()); |
6050 | 23.5k | ID.AddInteger(ST->getPointerInfo().getAddrSpace()); |
6051 | 23.5k | void *IP = nullptr; |
6052 | 23.5k | if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) |
6053 | 0 | return SDValue(E, 0); |
6054 | 23.5k | |
6055 | 23.5k | auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, |
6056 | 23.5k | ST->isTruncatingStore(), ST->getMemoryVT(), |
6057 | 23.5k | ST->getMemOperand()); |
6058 | 23.5k | createOperands(N, Ops); |
6059 | 23.5k | |
6060 | 23.5k | CSEMap.InsertNode(N, IP); |
6061 | 23.5k | InsertNode(N); |
6062 | 23.5k | return SDValue(N, 0); |
6063 | 23.5k | } |
6064 | | |
6065 | | SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, |
6066 | | SDValue Ptr, SDValue Mask, SDValue Src0, |
6067 | | EVT MemVT, MachineMemOperand *MMO, |
6068 | 316 | ISD::LoadExtType ExtTy, bool isExpanding) { |
6069 | 316 | SDVTList VTs = getVTList(VT, MVT::Other); |
6070 | 316 | SDValue Ops[] = { Chain, Ptr, Mask, Src0 }; |
6071 | 316 | FoldingSetNodeID ID; |
6072 | 316 | AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); |
6073 | 316 | ID.AddInteger(VT.getRawBits()); |
6074 | 316 | ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>( |
6075 | 316 | dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO)); |
6076 | 316 | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
6077 | 316 | void *IP = nullptr; |
6078 | 316 | if (SDNode *E316 = FindNodeOrInsertPos(ID, dl, IP)) { |
6079 | 0 | cast<MaskedLoadSDNode>(E)->refineAlignment(MMO); |
6080 | 0 | return SDValue(E, 0); |
6081 | 0 | } |
6082 | 316 | auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, |
6083 | 316 | ExtTy, isExpanding, MemVT, MMO); |
6084 | 316 | createOperands(N, Ops); |
6085 | 316 | |
6086 | 316 | CSEMap.InsertNode(N, IP); |
6087 | 316 | InsertNode(N); |
6088 | 316 | return SDValue(N, 0); |
6089 | 316 | } |
6090 | | |
6091 | | SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, |
6092 | | SDValue Val, SDValue Ptr, SDValue Mask, |
6093 | | EVT MemVT, MachineMemOperand *MMO, |
6094 | 199 | bool IsTruncating, bool IsCompressing) { |
6095 | 199 | assert(Chain.getValueType() == MVT::Other && |
6096 | 199 | "Invalid chain type"); |
6097 | 199 | EVT VT = Val.getValueType(); |
6098 | 199 | SDVTList VTs = getVTList(MVT::Other); |
6099 | 199 | SDValue Ops[] = { Chain, Ptr, Mask, Val }; |
6100 | 199 | FoldingSetNodeID ID; |
6101 | 199 | AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); |
6102 | 199 | ID.AddInteger(VT.getRawBits()); |
6103 | 199 | ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>( |
6104 | 199 | dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO)); |
6105 | 199 | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
6106 | 199 | void *IP = nullptr; |
6107 | 199 | if (SDNode *E199 = FindNodeOrInsertPos(ID, dl, IP)) { |
6108 | 0 | cast<MaskedStoreSDNode>(E)->refineAlignment(MMO); |
6109 | 0 | return SDValue(E, 0); |
6110 | 0 | } |
6111 | 199 | auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, |
6112 | 199 | IsTruncating, IsCompressing, MemVT, MMO); |
6113 | 199 | createOperands(N, Ops); |
6114 | 199 | |
6115 | 199 | CSEMap.InsertNode(N, IP); |
6116 | 199 | InsertNode(N); |
6117 | 199 | return SDValue(N, 0); |
6118 | 199 | } |
6119 | | |
6120 | | SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl, |
6121 | | ArrayRef<SDValue> Ops, |
6122 | 290 | MachineMemOperand *MMO) { |
6123 | 290 | assert(Ops.size() == 5 && "Incompatible number of operands"); |
6124 | 290 | |
6125 | 290 | FoldingSetNodeID ID; |
6126 | 290 | AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); |
6127 | 290 | ID.AddInteger(VT.getRawBits()); |
6128 | 290 | ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>( |
6129 | 290 | dl.getIROrder(), VTs, VT, MMO)); |
6130 | 290 | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
6131 | 290 | void *IP = nullptr; |
6132 | 290 | if (SDNode *E290 = FindNodeOrInsertPos(ID, dl, IP)) { |
6133 | 0 | cast<MaskedGatherSDNode>(E)->refineAlignment(MMO); |
6134 | 0 | return SDValue(E, 0); |
6135 | 0 | } |
6136 | 290 | |
6137 | 290 | auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), |
6138 | 290 | VTs, VT, MMO); |
6139 | 290 | createOperands(N, Ops); |
6140 | 290 | |
6141 | 290 | assert(N->getValue().getValueType() == N->getValueType(0) && |
6142 | 290 | "Incompatible type of the PassThru value in MaskedGatherSDNode"); |
6143 | 290 | assert(N->getMask().getValueType().getVectorNumElements() == |
6144 | 290 | N->getValueType(0).getVectorNumElements() && |
6145 | 290 | "Vector width mismatch between mask and data"); |
6146 | 290 | assert(N->getIndex().getValueType().getVectorNumElements() == |
6147 | 290 | N->getValueType(0).getVectorNumElements() && |
6148 | 290 | "Vector width mismatch between index and data"); |
6149 | 290 | |
6150 | 290 | CSEMap.InsertNode(N, IP); |
6151 | 290 | InsertNode(N); |
6152 | 290 | return SDValue(N, 0); |
6153 | 290 | } |
6154 | | |
6155 | | SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl, |
6156 | | ArrayRef<SDValue> Ops, |
6157 | 178 | MachineMemOperand *MMO) { |
6158 | 178 | assert(Ops.size() == 5 && "Incompatible number of operands"); |
6159 | 178 | |
6160 | 178 | FoldingSetNodeID ID; |
6161 | 178 | AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); |
6162 | 178 | ID.AddInteger(VT.getRawBits()); |
6163 | 178 | ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>( |
6164 | 178 | dl.getIROrder(), VTs, VT, MMO)); |
6165 | 178 | ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); |
6166 | 178 | void *IP = nullptr; |
6167 | 178 | if (SDNode *E178 = FindNodeOrInsertPos(ID, dl, IP)) { |
6168 | 0 | cast<MaskedScatterSDNode>(E)->refineAlignment(MMO); |
6169 | 0 | return SDValue(E, 0); |
6170 | 0 | } |
6171 | 178 | auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), |
6172 | 178 | VTs, VT, MMO); |
6173 | 178 | createOperands(N, Ops); |
6174 | 178 | |
6175 | 178 | assert(N->getMask().getValueType().getVectorNumElements() == |
6176 | 178 | N->getValue().getValueType().getVectorNumElements() && |
6177 | 178 | "Vector width mismatch between mask and data"); |
6178 | 178 | assert(N->getIndex().getValueType().getVectorNumElements() == |
6179 | 178 | N->getValue().getValueType().getVectorNumElements() && |
6180 | 178 | "Vector width mismatch between index and data"); |
6181 | 178 | |
6182 | 178 | CSEMap.InsertNode(N, IP); |
6183 | 178 | InsertNode(N); |
6184 | 178 | return SDValue(N, 0); |
6185 | 178 | } |
6186 | | |
6187 | | SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, |
6188 | 421 | SDValue Ptr, SDValue SV, unsigned Align) { |
6189 | 421 | SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) }; |
6190 | 421 | return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops); |
6191 | 421 | } |
6192 | | |
6193 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
6194 | 6.77k | ArrayRef<SDUse> Ops) { |
6195 | 6.77k | switch (Ops.size()) { |
6196 | 0 | case 0: return getNode(Opcode, DL, VT); |
6197 | 142 | case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0])); |
6198 | 3.23k | case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); |
6199 | 45 | case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); |
6200 | 3.34k | default: break; |
6201 | 3.34k | } |
6202 | 3.34k | |
6203 | 3.34k | // Copy from an SDUse array into an SDValue array for use with |
6204 | 3.34k | // the regular getNode logic. |
6205 | 3.34k | SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end()); |
6206 | 3.34k | return getNode(Opcode, DL, VT, NewOps); |
6207 | 3.34k | } |
6208 | | |
6209 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, |
6210 | 20.3M | ArrayRef<SDValue> Ops, const SDNodeFlags Flags) { |
6211 | 20.3M | unsigned NumOps = Ops.size(); |
6212 | 20.3M | switch (NumOps) { |
6213 | 0 | case 0: return getNode(Opcode, DL, VT); |
6214 | 12.8M | case 1: return getNode(Opcode, DL, VT, Ops[0], Flags); |
6215 | 2.15M | case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags); |
6216 | 1.00M | case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); |
6217 | 4.39M | default: break; |
6218 | 4.39M | } |
6219 | 4.39M | |
6220 | 4.39M | switch (Opcode) { |
6221 | 2.33M | default: break; |
6222 | 6.78k | case ISD::CONCAT_VECTORS: |
6223 | 6.78k | // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. |
6224 | 6.78k | if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) |
6225 | 233 | return V; |
6226 | 6.55k | break; |
6227 | 144k | case ISD::SELECT_CC: |
6228 | 144k | assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); |
6229 | 144k | assert(Ops[0].getValueType() == Ops[1].getValueType() && |
6230 | 144k | "LHS and RHS of condition must have same type!"); |
6231 | 144k | assert(Ops[2].getValueType() == Ops[3].getValueType() && |
6232 | 144k | "True and False arms of SelectCC must have same type!"); |
6233 | 144k | assert(Ops[2].getValueType() == VT && |
6234 | 144k | "select_cc node must be of same type as true and false value!"); |
6235 | 144k | break; |
6236 | 1.90M | case ISD::BR_CC: |
6237 | 1.90M | assert(NumOps == 5 && "BR_CC takes 5 operands!"); |
6238 | 1.90M | assert(Ops[2].getValueType() == Ops[3].getValueType() && |
6239 | 1.90M | "LHS/RHS of comparison should match types!"); |
6240 | 1.90M | break; |
6241 | 4.38M | } |
6242 | 4.38M | |
6243 | 4.38M | // Memoize nodes. |
6244 | 4.38M | SDNode *N; |
6245 | 4.38M | SDVTList VTs = getVTList(VT); |
6246 | 4.38M | |
6247 | 4.38M | if (VT != MVT::Glue4.38M ) { |
6248 | 4.38M | FoldingSetNodeID ID; |
6249 | 4.38M | AddNodeIDNode(ID, Opcode, VTs, Ops); |
6250 | 4.38M | void *IP = nullptr; |
6251 | 4.38M | |
6252 | 4.38M | if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
6253 | 326k | return SDValue(E, 0); |
6254 | 4.06M | |
6255 | 4.06M | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
6256 | 4.06M | createOperands(N, Ops); |
6257 | 4.06M | |
6258 | 4.06M | CSEMap.InsertNode(N, IP); |
6259 | 4.38M | } else { |
6260 | 1 | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
6261 | 1 | createOperands(N, Ops); |
6262 | 1 | } |
6263 | 4.38M | |
6264 | 4.06M | InsertNode(N); |
6265 | 4.06M | return SDValue(N, 0); |
6266 | 20.3M | } |
6267 | | |
6268 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, |
6269 | 12.9k | ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) { |
6270 | 12.9k | return getNode(Opcode, DL, getVTList(ResultTys), Ops); |
6271 | 12.9k | } |
6272 | | |
6273 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
6274 | 30.3M | ArrayRef<SDValue> Ops) { |
6275 | 30.3M | if (VTList.NumVTs == 1) |
6276 | 8.84M | return getNode(Opcode, DL, VTList.VTs[0], Ops); |
6277 | 21.4M | |
6278 | | #if 0 |
6279 | | switch (Opcode) { |
6280 | | // FIXME: figure out how to safely handle things like |
6281 | | // int foo(int x) { return 1 << (x & 255); } |
6282 | | // int bar() { return foo(256); } |
6283 | | case ISD::SRA_PARTS: |
6284 | | case ISD::SRL_PARTS: |
6285 | | case ISD::SHL_PARTS: |
6286 | | if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && |
6287 | | cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) |
6288 | | return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); |
6289 | | else if (N3.getOpcode() == ISD::AND) |
6290 | | if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { |
6291 | | // If the and is only masking out bits that cannot effect the shift, |
6292 | | // eliminate the and. |
6293 | | unsigned NumBits = VT.getScalarSizeInBits()*2; |
6294 | | if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) |
6295 | | return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); |
6296 | | } |
6297 | | break; |
6298 | | } |
6299 | | #endif |
6300 | | |
6301 | 21.4M | // Memoize the node unless it returns a flag. |
6302 | 21.4M | SDNode *N; |
6303 | 21.4M | if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue21.4M ) { |
6304 | 10.2M | FoldingSetNodeID ID; |
6305 | 10.2M | AddNodeIDNode(ID, Opcode, VTList, Ops); |
6306 | 10.2M | void *IP = nullptr; |
6307 | 10.2M | if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) |
6308 | 1.36M | return SDValue(E, 0); |
6309 | 8.85M | |
6310 | 8.85M | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); |
6311 | 8.85M | createOperands(N, Ops); |
6312 | 8.85M | CSEMap.InsertNode(N, IP); |
6313 | 21.4M | } else { |
6314 | 11.2M | N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); |
6315 | 11.2M | createOperands(N, Ops); |
6316 | 11.2M | } |
6317 | 20.1M | InsertNode(N); |
6318 | 20.1M | return SDValue(N, 0); |
6319 | 30.3M | } |
6320 | | |
6321 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, |
6322 | 0 | SDVTList VTList) { |
6323 | 0 | return getNode(Opcode, DL, VTList, None); |
6324 | 0 | } |
6325 | | |
6326 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
6327 | 5.14k | SDValue N1) { |
6328 | 5.14k | SDValue Ops[] = { N1 }; |
6329 | 5.14k | return getNode(Opcode, DL, VTList, Ops); |
6330 | 5.14k | } |
6331 | | |
6332 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
6333 | 946k | SDValue N1, SDValue N2) { |
6334 | 946k | SDValue Ops[] = { N1, N2 }; |
6335 | 946k | return getNode(Opcode, DL, VTList, Ops); |
6336 | 946k | } |
6337 | | |
6338 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
6339 | 13.4k | SDValue N1, SDValue N2, SDValue N3) { |
6340 | 13.4k | SDValue Ops[] = { N1, N2, N3 }; |
6341 | 13.4k | return getNode(Opcode, DL, VTList, Ops); |
6342 | 13.4k | } |
6343 | | |
6344 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
6345 | 2.50k | SDValue N1, SDValue N2, SDValue N3, SDValue N4) { |
6346 | 2.50k | SDValue Ops[] = { N1, N2, N3, N4 }; |
6347 | 2.50k | return getNode(Opcode, DL, VTList, Ops); |
6348 | 2.50k | } |
6349 | | |
6350 | | SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, |
6351 | | SDValue N1, SDValue N2, SDValue N3, SDValue N4, |
6352 | 335 | SDValue N5) { |
6353 | 335 | SDValue Ops[] = { N1, N2, N3, N4, N5 }; |
6354 | 335 | return getNode(Opcode, DL, VTList, Ops); |
6355 | 335 | } |
6356 | | |
6357 | 138M | SDVTList SelectionDAG::getVTList(EVT VT) { |
6358 | 138M | return makeVTList(SDNode::getValueTypeList(VT), 1); |
6359 | 138M | } |
6360 | | |
6361 | 27.4M | SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { |
6362 | 27.4M | FoldingSetNodeID ID; |
6363 | 27.4M | ID.AddInteger(2U); |
6364 | 27.4M | ID.AddInteger(VT1.getRawBits()); |
6365 | 27.4M | ID.AddInteger(VT2.getRawBits()); |
6366 | 27.4M | |
6367 | 27.4M | void *IP = nullptr; |
6368 | 27.4M | SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
6369 | 27.4M | if (!Result27.4M ) { |
6370 | 179k | EVT *Array = Allocator.Allocate<EVT>(2); |
6371 | 179k | Array[0] = VT1; |
6372 | 179k | Array[1] = VT2; |
6373 | 179k | Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); |
6374 | 179k | VTListMap.InsertNode(Result, IP); |
6375 | 179k | } |
6376 | 27.4M | return Result->getSDVTList(); |
6377 | 27.4M | } |
6378 | | |
6379 | 1.19M | SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { |
6380 | 1.19M | FoldingSetNodeID ID; |
6381 | 1.19M | ID.AddInteger(3U); |
6382 | 1.19M | ID.AddInteger(VT1.getRawBits()); |
6383 | 1.19M | ID.AddInteger(VT2.getRawBits()); |
6384 | 1.19M | ID.AddInteger(VT3.getRawBits()); |
6385 | 1.19M | |
6386 | 1.19M | void *IP = nullptr; |
6387 | 1.19M | SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
6388 | 1.19M | if (!Result1.19M ) { |
6389 | 36.0k | EVT *Array = Allocator.Allocate<EVT>(3); |
6390 | 36.0k | Array[0] = VT1; |
6391 | 36.0k | Array[1] = VT2; |
6392 | 36.0k | Array[2] = VT3; |
6393 | 36.0k | Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); |
6394 | 36.0k | VTListMap.InsertNode(Result, IP); |
6395 | 36.0k | } |
6396 | 1.19M | return Result->getSDVTList(); |
6397 | 1.19M | } |
6398 | | |
6399 | 64 | SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { |
6400 | 64 | FoldingSetNodeID ID; |
6401 | 64 | ID.AddInteger(4U); |
6402 | 64 | ID.AddInteger(VT1.getRawBits()); |
6403 | 64 | ID.AddInteger(VT2.getRawBits()); |
6404 | 64 | ID.AddInteger(VT3.getRawBits()); |
6405 | 64 | ID.AddInteger(VT4.getRawBits()); |
6406 | 64 | |
6407 | 64 | void *IP = nullptr; |
6408 | 64 | SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
6409 | 64 | if (!Result64 ) { |
6410 | 20 | EVT *Array = Allocator.Allocate<EVT>(4); |
6411 | 20 | Array[0] = VT1; |
6412 | 20 | Array[1] = VT2; |
6413 | 20 | Array[2] = VT3; |
6414 | 20 | Array[3] = VT4; |
6415 | 20 | Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); |
6416 | 20 | VTListMap.InsertNode(Result, IP); |
6417 | 20 | } |
6418 | 64 | return Result->getSDVTList(); |
6419 | 64 | } |
6420 | | |
6421 | 13.7M | SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) { |
6422 | 13.7M | unsigned NumVTs = VTs.size(); |
6423 | 13.7M | FoldingSetNodeID ID; |
6424 | 13.7M | ID.AddInteger(NumVTs); |
6425 | 37.2M | for (unsigned index = 0; index < NumVTs37.2M ; index++23.4M ) { |
6426 | 23.4M | ID.AddInteger(VTs[index].getRawBits()); |
6427 | 23.4M | } |
6428 | 13.7M | |
6429 | 13.7M | void *IP = nullptr; |
6430 | 13.7M | SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); |
6431 | 13.7M | if (!Result13.7M ) { |
6432 | 104k | EVT *Array = Allocator.Allocate<EVT>(NumVTs); |
6433 | 104k | std::copy(VTs.begin(), VTs.end(), Array); |
6434 | 104k | Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); |
6435 | 104k | VTListMap.InsertNode(Result, IP); |
6436 | 104k | } |
6437 | 13.7M | return Result->getSDVTList(); |
6438 | 13.7M | } |
6439 | | |
6440 | | |
6441 | | /// UpdateNodeOperands - *Mutate* the specified node in-place to have the |
6442 | | /// specified operands. If the resultant node already exists in the DAG, |
6443 | | /// this does not modify the specified node, instead it returns the node that |
6444 | | /// already exists. If the resultant node does not exist in the DAG, the |
6445 | | /// input node is returned. As a degenerate case, if you specify the same |
6446 | | /// input operands as the node already has, the input node is returned. |
6447 | 21.0k | SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { |
6448 | 21.0k | assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); |
6449 | 21.0k | |
6450 | 21.0k | // Check to see if there is no change. |
6451 | 21.0k | if (Op == N->getOperand(0)21.0k ) return N0 ; |
6452 | 21.0k | |
6453 | 21.0k | // See if the modified node already exists. |
6454 | 21.0k | void *InsertPos = nullptr; |
6455 | 21.0k | if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) |
6456 | 16 | return Existing; |
6457 | 21.0k | |
6458 | 21.0k | // Nope it doesn't. Remove the node from its current place in the maps. |
6459 | 21.0k | if (21.0k InsertPos21.0k ) |
6460 | 21.0k | if (21.0k !RemoveNodeFromCSEMaps(N)21.0k ) |
6461 | 0 | InsertPos = nullptr; |
6462 | 21.0k | |
6463 | 21.0k | // Now we update the operands. |
6464 | 21.0k | N->OperandList[0].set(Op); |
6465 | 21.0k | |
6466 | 21.0k | // If this gets put into a CSE map, add it. |
6467 | 21.0k | if (InsertPos21.0k ) CSEMap.InsertNode(N, InsertPos)21.0k ; |
6468 | 21.0k | return N; |
6469 | 21.0k | } |
6470 | | |
6471 | 32.2k | SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { |
6472 | 32.2k | assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); |
6473 | 32.2k | |
6474 | 32.2k | // Check to see if there is no change. |
6475 | 32.2k | if (Op1 == N->getOperand(0) && 32.2k Op2 == N->getOperand(1)17.1k ) |
6476 | 68 | return N; // No operands changed, just return the input node. |
6477 | 32.1k | |
6478 | 32.1k | // See if the modified node already exists. |
6479 | 32.1k | void *InsertPos = nullptr; |
6480 | 32.1k | if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) |
6481 | 853 | return Existing; |
6482 | 31.3k | |
6483 | 31.3k | // Nope it doesn't. Remove the node from its current place in the maps. |
6484 | 31.3k | if (31.3k InsertPos31.3k ) |
6485 | 31.3k | if (31.3k !RemoveNodeFromCSEMaps(N)31.3k ) |
6486 | 0 | InsertPos = nullptr; |
6487 | 31.3k | |
6488 | 31.3k | // Now we update the operands. |
6489 | 31.3k | if (N->OperandList[0] != Op1) |
6490 | 14.3k | N->OperandList[0].set(Op1); |
6491 | 31.3k | if (N->OperandList[1] != Op2) |
6492 | 23.7k | N->OperandList[1].set(Op2); |
6493 | 31.3k | |
6494 | 31.3k | // If this gets put into a CSE map, add it. |
6495 | 31.3k | if (InsertPos31.3k ) CSEMap.InsertNode(N, InsertPos)31.3k ; |
6496 | 32.2k | return N; |
6497 | 32.2k | } |
6498 | | |
6499 | | SDNode *SelectionDAG:: |
6500 | 558k | UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { |
6501 | 558k | SDValue Ops[] = { Op1, Op2, Op3 }; |
6502 | 558k | return UpdateNodeOperands(N, Ops); |
6503 | 558k | } |
6504 | | |
6505 | | SDNode *SelectionDAG:: |
6506 | | UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
6507 | 315 | SDValue Op3, SDValue Op4) { |
6508 | 315 | SDValue Ops[] = { Op1, Op2, Op3, Op4 }; |
6509 | 315 | return UpdateNodeOperands(N, Ops); |
6510 | 315 | } |
6511 | | |
6512 | | SDNode *SelectionDAG:: |
6513 | | UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, |
6514 | 26.5k | SDValue Op3, SDValue Op4, SDValue Op5) { |
6515 | 26.5k | SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; |
6516 | 26.5k | return UpdateNodeOperands(N, Ops); |
6517 | 26.5k | } |
6518 | | |
6519 | | SDNode *SelectionDAG:: |
6520 | 5.81M | UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) { |
6521 | 5.81M | unsigned NumOps = Ops.size(); |
6522 | 5.81M | assert(N->getNumOperands() == NumOps && |
6523 | 5.81M | "Update with wrong number of operands"); |
6524 | 5.81M | |
6525 | 5.81M | // If no operands changed just return the input node. |
6526 | 5.81M | if (std::equal(Ops.begin(), Ops.end(), N->op_begin())) |
6527 | 5.09M | return N; |
6528 | 720k | |
6529 | 720k | // See if the modified node already exists. |
6530 | 720k | void *InsertPos = nullptr; |
6531 | 720k | if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos)) |
6532 | 596 | return Existing; |
6533 | 719k | |
6534 | 719k | // Nope it doesn't. Remove the node from its current place in the maps. |
6535 | 719k | if (719k InsertPos719k ) |
6536 | 708k | if (708k !RemoveNodeFromCSEMaps(N)708k ) |
6537 | 0 | InsertPos = nullptr; |
6538 | 719k | |
6539 | 719k | // Now we update the operands. |
6540 | 2.93M | for (unsigned i = 0; i != NumOps2.93M ; ++i2.21M ) |
6541 | 2.21M | if (2.21M N->OperandList[i] != Ops[i]2.21M ) |
6542 | 1.12M | N->OperandList[i].set(Ops[i]); |
6543 | 719k | |
6544 | 719k | // If this gets put into a CSE map, add it. |
6545 | 719k | if (InsertPos719k ) CSEMap.InsertNode(N, InsertPos)708k ; |
6546 | 5.81M | return N; |
6547 | 5.81M | } |
6548 | | |
6549 | | /// DropOperands - Release the operands and set this node to have |
6550 | | /// zero operands. |
6551 | 60.8M | void SDNode::DropOperands() { |
6552 | 60.8M | // Unlike the code in MorphNodeTo that does this, we don't need to |
6553 | 60.8M | // watch for dead nodes here. |
6554 | 146M | for (op_iterator I = op_begin(), E = op_end(); I != E146M ; ) { |
6555 | 85.2M | SDUse &Use = *I++; |
6556 | 85.2M | Use.set(SDValue()); |
6557 | 85.2M | } |
6558 | 60.8M | } |
6559 | | |
6560 | | /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a |
6561 | | /// machine opcode. |
6562 | | /// |
6563 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6564 | 7.32k | EVT VT) { |
6565 | 7.32k | SDVTList VTs = getVTList(VT); |
6566 | 7.32k | return SelectNodeTo(N, MachineOpc, VTs, None); |
6567 | 7.32k | } |
6568 | | |
6569 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6570 | 205 | EVT VT, SDValue Op1) { |
6571 | 205 | SDVTList VTs = getVTList(VT); |
6572 | 205 | SDValue Ops[] = { Op1 }; |
6573 | 205 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
6574 | 205 | } |
6575 | | |
6576 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6577 | | EVT VT, SDValue Op1, |
6578 | 1.74k | SDValue Op2) { |
6579 | 1.74k | SDVTList VTs = getVTList(VT); |
6580 | 1.74k | SDValue Ops[] = { Op1, Op2 }; |
6581 | 1.74k | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
6582 | 1.74k | } |
6583 | | |
6584 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6585 | | EVT VT, SDValue Op1, |
6586 | 66 | SDValue Op2, SDValue Op3) { |
6587 | 66 | SDVTList VTs = getVTList(VT); |
6588 | 66 | SDValue Ops[] = { Op1, Op2, Op3 }; |
6589 | 66 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
6590 | 66 | } |
6591 | | |
6592 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6593 | 184k | EVT VT, ArrayRef<SDValue> Ops) { |
6594 | 184k | SDVTList VTs = getVTList(VT); |
6595 | 184k | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
6596 | 184k | } |
6597 | | |
6598 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6599 | 2 | EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) { |
6600 | 2 | SDVTList VTs = getVTList(VT1, VT2); |
6601 | 2 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
6602 | 2 | } |
6603 | | |
6604 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6605 | 4 | EVT VT1, EVT VT2) { |
6606 | 4 | SDVTList VTs = getVTList(VT1, VT2); |
6607 | 4 | return SelectNodeTo(N, MachineOpc, VTs, None); |
6608 | 4 | } |
6609 | | |
6610 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6611 | | EVT VT1, EVT VT2, EVT VT3, |
6612 | 5 | ArrayRef<SDValue> Ops) { |
6613 | 5 | SDVTList VTs = getVTList(VT1, VT2, VT3); |
6614 | 5 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
6615 | 5 | } |
6616 | | |
6617 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6618 | | EVT VT1, EVT VT2, |
6619 | 0 | SDValue Op1, SDValue Op2) { |
6620 | 0 | SDVTList VTs = getVTList(VT1, VT2); |
6621 | 0 | SDValue Ops[] = { Op1, Op2 }; |
6622 | 0 | return SelectNodeTo(N, MachineOpc, VTs, Ops); |
6623 | 0 | } |
6624 | | |
6625 | | SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, |
6626 | 212k | SDVTList VTs,ArrayRef<SDValue> Ops) { |
6627 | 212k | SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops); |
6628 | 212k | // Reset the NodeID to -1. |
6629 | 212k | New->setNodeId(-1); |
6630 | 212k | if (New != N212k ) { |
6631 | 86 | ReplaceAllUsesWith(N, New); |
6632 | 86 | RemoveDeadNode(N); |
6633 | 86 | } |
6634 | 212k | return New; |
6635 | 212k | } |
6636 | | |
6637 | | /// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away |
6638 | | /// the line number information on the merged node since it is not possible to |
6639 | | /// preserve the information that operation is associated with multiple lines. |
6640 | | /// This will make the debugger working better at -O0, were there is a higher |
6641 | | /// probability having other instructions associated with that line. |
6642 | | /// |
6643 | | /// For IROrder, we keep the smaller of the two |
6644 | 218k | SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) { |
6645 | 218k | DebugLoc NLoc = N->getDebugLoc(); |
6646 | 218k | if (NLoc && 218k OptLevel == CodeGenOpt::None1.75k && OLoc.getDebugLoc() != NLoc14 ) { |
6647 | 0 | N->setDebugLoc(DebugLoc()); |
6648 | 0 | } |
6649 | 218k | unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); |
6650 | 218k | N->setIROrder(Order); |
6651 | 218k | return N; |
6652 | 218k | } |
6653 | | |
6654 | | /// MorphNodeTo - This *mutates* the specified node to have the specified |
6655 | | /// return type, opcode, and operands. |
6656 | | /// |
6657 | | /// Note that MorphNodeTo returns the resultant node. If there is already a |
6658 | | /// node of the specified opcode and operands, it returns that node instead of |
6659 | | /// the current one. Note that the SDLoc need not be the same. |
6660 | | /// |
6661 | | /// Using MorphNodeTo is faster than creating a new node and swapping it in |
6662 | | /// with ReplaceAllUsesWith both because it often avoids allocating a new |
6663 | | /// node, and because it doesn't require CSE recalculation for any of |
6664 | | /// the node's users. |
6665 | | /// |
6666 | | /// However, note that MorphNodeTo recursively deletes dead nodes from the DAG. |
6667 | | /// As a consequence it isn't appropriate to use from within the DAG combiner or |
6668 | | /// the legalizer which maintain worklists that would need to be updated when |
6669 | | /// deleting things. |
6670 | | SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, |
6671 | 20.1M | SDVTList VTs, ArrayRef<SDValue> Ops) { |
6672 | 20.1M | // If an identical node already exists, use it. |
6673 | 20.1M | void *IP = nullptr; |
6674 | 20.1M | if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue20.1M ) { |
6675 | 15.3M | FoldingSetNodeID ID; |
6676 | 15.3M | AddNodeIDNode(ID, Opc, VTs, Ops); |
6677 | 15.3M | if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP)) |
6678 | 25.2k | return UpdateSDLocOnMergeSDNode(ON, SDLoc(N)); |
6679 | 20.1M | } |
6680 | 20.1M | |
6681 | 20.1M | if (20.1M !RemoveNodeFromCSEMaps(N)20.1M ) |
6682 | 4.96M | IP = nullptr; |
6683 | 20.1M | |
6684 | 20.1M | // Start the morphing. |
6685 | 20.1M | N->NodeType = Opc; |
6686 | 20.1M | N->ValueList = VTs.VTs; |
6687 | 20.1M | N->NumValues = VTs.NumVTs; |
6688 | 20.1M | |
6689 | 20.1M | // Clear the operands list, updating used nodes to remove this from their |
6690 | 20.1M | // use list. Keep track of any operands that become dead as a result. |
6691 | 20.1M | SmallPtrSet<SDNode*, 16> DeadNodeSet; |
6692 | 78.0M | for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E78.0M ; ) { |
6693 | 57.9M | SDUse &Use = *I++; |
6694 | 57.9M | SDNode *Used = Use.getNode(); |
6695 | 57.9M | Use.set(SDValue()); |
6696 | 57.9M | if (Used->use_empty()) |
6697 | 28.1M | DeadNodeSet.insert(Used); |
6698 | 57.9M | } |
6699 | 20.1M | |
6700 | 20.1M | // For MachineNode, initialize the memory references information. |
6701 | 20.1M | if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) |
6702 | 20.1M | MN->setMemRefs(nullptr, nullptr); |
6703 | 20.1M | |
6704 | 20.1M | // Swap for an appropriately sized array from the recycler. |
6705 | 20.1M | removeOperands(N); |
6706 | 20.1M | createOperands(N, Ops); |
6707 | 20.1M | |
6708 | 20.1M | // Delete any nodes that are still dead after adding the uses for the |
6709 | 20.1M | // new operands. |
6710 | 20.1M | if (!DeadNodeSet.empty()20.1M ) { |
6711 | 16.8M | SmallVector<SDNode *, 16> DeadNodes; |
6712 | 16.8M | for (SDNode *N : DeadNodeSet) |
6713 | 28.1M | if (28.1M N->use_empty()28.1M ) |
6714 | 7.71M | DeadNodes.push_back(N); |
6715 | 16.8M | RemoveDeadNodes(DeadNodes); |
6716 | 16.8M | } |
6717 | 20.1M | |
6718 | 20.1M | if (IP) |
6719 | 15.1M | CSEMap.InsertNode(N, IP); // Memoize the new node. |
6720 | 20.1M | return N; |
6721 | 20.1M | } |
6722 | | |
6723 | 40 | SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { |
6724 | 40 | unsigned OrigOpc = Node->getOpcode(); |
6725 | 40 | unsigned NewOpc; |
6726 | 40 | bool IsUnary = false; |
6727 | 40 | bool IsTernary = false; |
6728 | 40 | switch (OrigOpc) { |
6729 | 0 | default: |
6730 | 0 | llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!"); |
6731 | 2 | case ISD::STRICT_FADD: NewOpc = ISD::FADD; break; |
6732 | 6 | case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break; |
6733 | 2 | case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break; |
6734 | 2 | case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break; |
6735 | 0 | case ISD::STRICT_FREM: NewOpc = ISD::FREM; break; |
6736 | 4 | case ISD::STRICT_FMA: NewOpc = ISD::FMA; IsTernary = true; break; |
6737 | 2 | case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; IsUnary = true; break; |
6738 | 2 | case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break; |
6739 | 2 | case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break; |
6740 | 2 | case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; IsUnary = true; break; |
6741 | 2 | case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; IsUnary = true; break; |
6742 | 2 | case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; IsUnary = true; break; |
6743 | 2 | case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; IsUnary = true; break; |
6744 | 2 | case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; IsUnary = true; break; |
6745 | 2 | case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; IsUnary = true; break; |
6746 | 2 | case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; IsUnary = true; break; |
6747 | 2 | case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; IsUnary = true; break; |
6748 | 2 | case ISD::STRICT_FNEARBYINT: |
6749 | 2 | NewOpc = ISD::FNEARBYINT; |
6750 | 2 | IsUnary = true; |
6751 | 2 | break; |
6752 | 40 | } |
6753 | 40 | |
6754 | 40 | // We're taking this node out of the chain, so we need to re-link things. |
6755 | 40 | SDValue InputChain = Node->getOperand(0); |
6756 | 40 | SDValue OutputChain = SDValue(Node, 1); |
6757 | 40 | ReplaceAllUsesOfValueWith(OutputChain, InputChain); |
6758 | 40 | |
6759 | 40 | SDVTList VTs = getVTList(Node->getOperand(1).getValueType()); |
6760 | 40 | SDNode *Res = nullptr; |
6761 | 40 | if (IsUnary) |
6762 | 20 | Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1) }); |
6763 | 20 | else if (20 IsTernary20 ) |
6764 | 4 | Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1), |
6765 | 4 | Node->getOperand(2), |
6766 | 4 | Node->getOperand(3)}); |
6767 | 20 | else |
6768 | 16 | Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1), |
6769 | 16 | Node->getOperand(2) }); |
6770 | 40 | |
6771 | 40 | // MorphNodeTo can operate in two ways: if an existing node with the |
6772 | 40 | // specified operands exists, it can just return it. Otherwise, it |
6773 | 40 | // updates the node in place to have the requested operands. |
6774 | 40 | if (Res == Node40 ) { |
6775 | 40 | // If we updated the node in place, reset the node ID. To the isel, |
6776 | 40 | // this should be just like a newly allocated machine node. |
6777 | 40 | Res->setNodeId(-1); |
6778 | 40 | } else { |
6779 | 0 | ReplaceAllUsesWith(Node, Res); |
6780 | 0 | RemoveDeadNode(Node); |
6781 | 0 | } |
6782 | 40 | |
6783 | 40 | return Res; |
6784 | 40 | } |
6785 | | |
6786 | | /// getMachineNode - These are used for target selectors to create a new node |
6787 | | /// with specified return type(s), MachineInstr opcode, and operands. |
6788 | | /// |
6789 | | /// Note that getMachineNode returns the resultant node. If there is already a |
6790 | | /// node of the specified opcode and operands, it returns that node instead of |
6791 | | /// the current one. |
6792 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6793 | 2.25k | EVT VT) { |
6794 | 2.25k | SDVTList VTs = getVTList(VT); |
6795 | 2.25k | return getMachineNode(Opcode, dl, VTs, None); |
6796 | 2.25k | } |
6797 | | |
6798 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6799 | 59.9k | EVT VT, SDValue Op1) { |
6800 | 59.9k | SDVTList VTs = getVTList(VT); |
6801 | 59.9k | SDValue Ops[] = { Op1 }; |
6802 | 59.9k | return getMachineNode(Opcode, dl, VTs, Ops); |
6803 | 59.9k | } |
6804 | | |
6805 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6806 | 120k | EVT VT, SDValue Op1, SDValue Op2) { |
6807 | 120k | SDVTList VTs = getVTList(VT); |
6808 | 120k | SDValue Ops[] = { Op1, Op2 }; |
6809 | 120k | return getMachineNode(Opcode, dl, VTs, Ops); |
6810 | 120k | } |
6811 | | |
6812 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6813 | | EVT VT, SDValue Op1, SDValue Op2, |
6814 | 5.42k | SDValue Op3) { |
6815 | 5.42k | SDVTList VTs = getVTList(VT); |
6816 | 5.42k | SDValue Ops[] = { Op1, Op2, Op3 }; |
6817 | 5.42k | return getMachineNode(Opcode, dl, VTs, Ops); |
6818 | 5.42k | } |
6819 | | |
6820 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6821 | 101k | EVT VT, ArrayRef<SDValue> Ops) { |
6822 | 101k | SDVTList VTs = getVTList(VT); |
6823 | 101k | return getMachineNode(Opcode, dl, VTs, Ops); |
6824 | 101k | } |
6825 | | |
6826 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6827 | | EVT VT1, EVT VT2, SDValue Op1, |
6828 | 7.62k | SDValue Op2) { |
6829 | 7.62k | SDVTList VTs = getVTList(VT1, VT2); |
6830 | 7.62k | SDValue Ops[] = { Op1, Op2 }; |
6831 | 7.62k | return getMachineNode(Opcode, dl, VTs, Ops); |
6832 | 7.62k | } |
6833 | | |
6834 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6835 | | EVT VT1, EVT VT2, SDValue Op1, |
6836 | 16 | SDValue Op2, SDValue Op3) { |
6837 | 16 | SDVTList VTs = getVTList(VT1, VT2); |
6838 | 16 | SDValue Ops[] = { Op1, Op2, Op3 }; |
6839 | 16 | return getMachineNode(Opcode, dl, VTs, Ops); |
6840 | 16 | } |
6841 | | |
6842 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6843 | | EVT VT1, EVT VT2, |
6844 | 27.5k | ArrayRef<SDValue> Ops) { |
6845 | 27.5k | SDVTList VTs = getVTList(VT1, VT2); |
6846 | 27.5k | return getMachineNode(Opcode, dl, VTs, Ops); |
6847 | 27.5k | } |
6848 | | |
6849 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6850 | | EVT VT1, EVT VT2, EVT VT3, |
6851 | 16 | SDValue Op1, SDValue Op2) { |
6852 | 16 | SDVTList VTs = getVTList(VT1, VT2, VT3); |
6853 | 16 | SDValue Ops[] = { Op1, Op2 }; |
6854 | 16 | return getMachineNode(Opcode, dl, VTs, Ops); |
6855 | 16 | } |
6856 | | |
6857 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6858 | | EVT VT1, EVT VT2, EVT VT3, |
6859 | | SDValue Op1, SDValue Op2, |
6860 | 53 | SDValue Op3) { |
6861 | 53 | SDVTList VTs = getVTList(VT1, VT2, VT3); |
6862 | 53 | SDValue Ops[] = { Op1, Op2, Op3 }; |
6863 | 53 | return getMachineNode(Opcode, dl, VTs, Ops); |
6864 | 53 | } |
6865 | | |
6866 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6867 | | EVT VT1, EVT VT2, EVT VT3, |
6868 | 64.8k | ArrayRef<SDValue> Ops) { |
6869 | 64.8k | SDVTList VTs = getVTList(VT1, VT2, VT3); |
6870 | 64.8k | return getMachineNode(Opcode, dl, VTs, Ops); |
6871 | 64.8k | } |
6872 | | |
6873 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl, |
6874 | | ArrayRef<EVT> ResultTys, |
6875 | 4.23k | ArrayRef<SDValue> Ops) { |
6876 | 4.23k | SDVTList VTs = getVTList(ResultTys); |
6877 | 4.23k | return getMachineNode(Opcode, dl, VTs, Ops); |
6878 | 4.23k | } |
6879 | | |
6880 | | MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL, |
6881 | | SDVTList VTs, |
6882 | 1.37M | ArrayRef<SDValue> Ops) { |
6883 | 1.37M | bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; |
6884 | 1.37M | MachineSDNode *N; |
6885 | 1.37M | void *IP = nullptr; |
6886 | 1.37M | |
6887 | 1.37M | if (DoCSE1.37M ) { |
6888 | 1.32M | FoldingSetNodeID ID; |
6889 | 1.32M | AddNodeIDNode(ID, ~Opcode, VTs, Ops); |
6890 | 1.32M | IP = nullptr; |
6891 | 1.32M | if (SDNode *E1.32M = FindNodeOrInsertPos(ID, DL, IP)) { |
6892 | 192k | return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL)); |
6893 | 192k | } |
6894 | 1.18M | } |
6895 | 1.18M | |
6896 | 1.18M | // Allocate a new MachineSDNode. |
6897 | 1.18M | N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); |
6898 | 1.18M | createOperands(N, Ops); |
6899 | 1.18M | |
6900 | 1.18M | if (DoCSE) |
6901 | 1.13M | CSEMap.InsertNode(N, IP); |
6902 | 1.37M | |
6903 | 1.37M | InsertNode(N); |
6904 | 1.37M | return N; |
6905 | 1.37M | } |
6906 | | |
6907 | | /// getTargetExtractSubreg - A convenience function for creating |
6908 | | /// TargetOpcode::EXTRACT_SUBREG nodes. |
6909 | | SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
6910 | 69.9k | SDValue Operand) { |
6911 | 69.9k | SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); |
6912 | 69.9k | SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, |
6913 | 69.9k | VT, Operand, SRIdxVal); |
6914 | 69.9k | return SDValue(Subreg, 0); |
6915 | 69.9k | } |
6916 | | |
6917 | | /// getTargetInsertSubreg - A convenience function for creating |
6918 | | /// TargetOpcode::INSERT_SUBREG nodes. |
6919 | | SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, |
6920 | 1.26k | SDValue Operand, SDValue Subreg) { |
6921 | 1.26k | SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32); |
6922 | 1.26k | SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, |
6923 | 1.26k | VT, Operand, Subreg, SRIdxVal); |
6924 | 1.26k | return SDValue(Result, 0); |
6925 | 1.26k | } |
6926 | | |
6927 | | /// getNodeIfExists - Get the specified node if it's already available, or |
6928 | | /// else return NULL. |
6929 | | SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, |
6930 | | ArrayRef<SDValue> Ops, |
6931 | 2.37M | const SDNodeFlags Flags) { |
6932 | 2.37M | if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue2.37M ) { |
6933 | 2.37M | FoldingSetNodeID ID; |
6934 | 2.37M | AddNodeIDNode(ID, Opcode, VTList, Ops); |
6935 | 2.37M | void *IP = nullptr; |
6936 | 2.37M | if (SDNode *E2.37M = FindNodeOrInsertPos(ID, SDLoc(), IP)) { |
6937 | 378 | E->intersectFlagsWith(Flags); |
6938 | 378 | return E; |
6939 | 378 | } |
6940 | 2.37M | } |
6941 | 2.37M | return nullptr; |
6942 | 2.37M | } |
6943 | | |
6944 | | /// getDbgValue - Creates a SDDbgValue node. |
6945 | | /// |
6946 | | /// SDNode |
6947 | | SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr, |
6948 | | SDNode *N, unsigned R, bool IsIndirect, |
6949 | 304 | const DebugLoc &DL, unsigned O) { |
6950 | 304 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
6951 | 304 | "Expected inlined-at fields to agree"); |
6952 | 304 | return new (DbgInfo->getAlloc()) |
6953 | 304 | SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O); |
6954 | 304 | } |
6955 | | |
6956 | | /// Constant |
6957 | | SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var, |
6958 | | DIExpression *Expr, |
6959 | | const Value *C, |
6960 | 95 | const DebugLoc &DL, unsigned O) { |
6961 | 95 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
6962 | 95 | "Expected inlined-at fields to agree"); |
6963 | 95 | return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O); |
6964 | 95 | } |
6965 | | |
6966 | | /// FrameIndex |
6967 | | SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var, |
6968 | | DIExpression *Expr, unsigned FI, |
6969 | | const DebugLoc &DL, |
6970 | 46 | unsigned O) { |
6971 | 46 | assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && |
6972 | 46 | "Expected inlined-at fields to agree"); |
6973 | 46 | return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, FI, DL, O); |
6974 | 46 | } |
6975 | | |
6976 | | namespace { |
6977 | | |
6978 | | /// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node |
6979 | | /// pointed to by a use iterator is deleted, increment the use iterator |
6980 | | /// so that it doesn't dangle. |
6981 | | /// |
6982 | | class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { |
6983 | | SDNode::use_iterator &UI; |
6984 | | SDNode::use_iterator &UE; |
6985 | | |
6986 | 74.2k | void NodeDeleted(SDNode *N, SDNode *E) override { |
6987 | 74.2k | // Increment the iterator as needed. |
6988 | 74.2k | while (UI != UE && 74.2k N == *UI1.08k ) |
6989 | 0 | ++UI; |
6990 | 74.2k | } |
6991 | | |
6992 | | public: |
6993 | | RAUWUpdateListener(SelectionDAG &d, |
6994 | | SDNode::use_iterator &ui, |
6995 | | SDNode::use_iterator &ue) |
6996 | 24.8M | : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} |
6997 | | }; |
6998 | | |
6999 | | } // end anonymous namespace |
7000 | | |
7001 | | /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
7002 | | /// This can cause recursive merging of nodes in the DAG. |
7003 | | /// |
7004 | | /// This version assumes From has a single result value. |
7005 | | /// |
7006 | 8.01M | void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { |
7007 | 8.01M | SDNode *From = FromN.getNode(); |
7008 | 8.01M | assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && |
7009 | 8.01M | "Cannot replace with this method!"); |
7010 | 8.01M | assert(From != To.getNode() && "Cannot replace uses of with self"); |
7011 | 8.01M | |
7012 | 8.01M | // Preserve Debug Values |
7013 | 8.01M | TransferDbgValues(FromN, To); |
7014 | 8.01M | |
7015 | 8.01M | // Iterate over all the existing uses of From. New uses will be added |
7016 | 8.01M | // to the beginning of the use list, which we avoid visiting. |
7017 | 8.01M | // This specifically avoids visiting uses of From that arise while the |
7018 | 8.01M | // replacement is happening, because any such uses would be the result |
7019 | 8.01M | // of CSE: If an existing node looks like From after one of its operands |
7020 | 8.01M | // is replaced by To, we don't want to replace of all its users with To |
7021 | 8.01M | // too. See PR3018 for more info. |
7022 | 8.01M | SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
7023 | 8.01M | RAUWUpdateListener Listener(*this, UI, UE); |
7024 | 16.9M | while (UI != UE16.9M ) { |
7025 | 8.90M | SDNode *User = *UI; |
7026 | 8.90M | |
7027 | 8.90M | // This node is about to morph, remove its old self from the CSE maps. |
7028 | 8.90M | RemoveNodeFromCSEMaps(User); |
7029 | 8.90M | |
7030 | 8.90M | // A user can appear in a use list multiple times, and when this |
7031 | 8.90M | // happens the uses are usually next to each other in the list. |
7032 | 8.90M | // To help reduce the number of CSE recomputations, process all |
7033 | 8.90M | // the uses of this user that we can find this way. |
7034 | 8.92M | do { |
7035 | 8.92M | SDUse &Use = UI.getUse(); |
7036 | 8.92M | ++UI; |
7037 | 8.92M | Use.set(To); |
7038 | 8.92M | } while (UI != UE && 8.92M *UI == User955k ); |
7039 | 8.90M | |
7040 | 8.90M | // Now that we have modified User, add it back to the CSE maps. If it |
7041 | 8.90M | // already exists there, recursively merge the results together. |
7042 | 8.90M | AddModifiedNodeToCSEMaps(User); |
7043 | 8.90M | } |
7044 | 8.01M | |
7045 | 8.01M | // If we just RAUW'd the root, take note. |
7046 | 8.01M | if (FromN == getRoot()) |
7047 | 35.4k | setRoot(To); |
7048 | 8.01M | } |
7049 | | |
7050 | | /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
7051 | | /// This can cause recursive merging of nodes in the DAG. |
7052 | | /// |
7053 | | /// This version assumes that for each value of From, there is a |
7054 | | /// corresponding value in To in the same position with the same type. |
7055 | | /// |
7056 | 5.79M | void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { |
7057 | | #ifndef NDEBUG |
7058 | | for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) |
7059 | | assert((!From->hasAnyUseOfValue(i) || |
7060 | | From->getValueType(i) == To->getValueType(i)) && |
7061 | | "Cannot use this version of ReplaceAllUsesWith!"); |
7062 | | #endif |
7063 | | |
7064 | 5.79M | // Handle the trivial case. |
7065 | 5.79M | if (From == To) |
7066 | 0 | return; |
7067 | 5.79M | |
7068 | 5.79M | // Preserve Debug Info. Only do this if there's a use. |
7069 | 11.6M | for (unsigned i = 0, e = From->getNumValues(); 5.79M i != e11.6M ; ++i5.82M ) |
7070 | 5.82M | if (5.82M From->hasAnyUseOfValue(i)5.82M ) { |
7071 | 5.80M | assert((i < To->getNumValues()) && "Invalid To location"); |
7072 | 5.80M | TransferDbgValues(SDValue(From, i), SDValue(To, i)); |
7073 | 5.80M | } |
7074 | 5.79M | |
7075 | 5.79M | // Iterate over just the existing users of From. See the comments in |
7076 | 5.79M | // the ReplaceAllUsesWith above. |
7077 | 5.79M | SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
7078 | 5.79M | RAUWUpdateListener Listener(*this, UI, UE); |
7079 | 12.1M | while (UI != UE12.1M ) { |
7080 | 6.38M | SDNode *User = *UI; |
7081 | 6.38M | |
7082 | 6.38M | // This node is about to morph, remove its old self from the CSE maps. |
7083 | 6.38M | RemoveNodeFromCSEMaps(User); |
7084 | 6.38M | |
7085 | 6.38M | // A user can appear in a use list multiple times, and when this |
7086 | 6.38M | // happens the uses are usually next to each other in the list. |
7087 | 6.38M | // To help reduce the number of CSE recomputations, process all |
7088 | 6.38M | // the uses of this user that we can find this way. |
7089 | 6.41M | do { |
7090 | 6.41M | SDUse &Use = UI.getUse(); |
7091 | 6.41M | ++UI; |
7092 | 6.41M | Use.setNode(To); |
7093 | 6.41M | } while (UI != UE && 6.41M *UI == User626k ); |
7094 | 6.38M | |
7095 | 6.38M | // Now that we have modified User, add it back to the CSE maps. If it |
7096 | 6.38M | // already exists there, recursively merge the results together. |
7097 | 6.38M | AddModifiedNodeToCSEMaps(User); |
7098 | 6.38M | } |
7099 | 5.79M | |
7100 | 5.79M | // If we just RAUW'd the root, take note. |
7101 | 5.79M | if (From == getRoot().getNode()) |
7102 | 49.0k | setRoot(SDValue(To, getRoot().getResNo())); |
7103 | 5.79M | } |
7104 | | |
7105 | | /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. |
7106 | | /// This can cause recursive merging of nodes in the DAG. |
7107 | | /// |
7108 | | /// This version can replace From with any result values. To must match the |
7109 | | /// number and types of values returned by From. |
7110 | 2.48M | void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { |
7111 | 2.48M | if (From->getNumValues() == 1) // Handle the simple case efficiently. |
7112 | 1.91M | return ReplaceAllUsesWith(SDValue(From, 0), To[0]); |
7113 | 570k | |
7114 | 570k | // Preserve Debug Info. |
7115 | 1.71M | for (unsigned i = 0, e = From->getNumValues(); 570k i != e1.71M ; ++i1.14M ) |
7116 | 1.14M | TransferDbgValues(SDValue(From, i), *To); |
7117 | 570k | |
7118 | 570k | // Iterate over just the existing users of From. See the comments in |
7119 | 570k | // the ReplaceAllUsesWith above. |
7120 | 570k | SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); |
7121 | 570k | RAUWUpdateListener Listener(*this, UI, UE); |
7122 | 1.67M | while (UI != UE1.67M ) { |
7123 | 1.10M | SDNode *User = *UI; |
7124 | 1.10M | |
7125 | 1.10M | // This node is about to morph, remove its old self from the CSE maps. |
7126 | 1.10M | RemoveNodeFromCSEMaps(User); |
7127 | 1.10M | |
7128 | 1.10M | // A user can appear in a use list multiple times, and when this |
7129 | 1.10M | // happens the uses are usually next to each other in the list. |
7130 | 1.10M | // To help reduce the number of CSE recomputations, process all |
7131 | 1.10M | // the uses of this user that we can find this way. |
7132 | 1.20M | do { |
7133 | 1.20M | SDUse &Use = UI.getUse(); |
7134 | 1.20M | const SDValue &ToOp = To[Use.getResNo()]; |
7135 | 1.20M | ++UI; |
7136 | 1.20M | Use.set(ToOp); |
7137 | 1.20M | } while (UI != UE && 1.20M *UI == User637k ); |
7138 | 1.10M | |
7139 | 1.10M | // Now that we have modified User, add it back to the CSE maps. If it |
7140 | 1.10M | // already exists there, recursively merge the results together. |
7141 | 1.10M | AddModifiedNodeToCSEMaps(User); |
7142 | 1.10M | } |
7143 | 570k | |
7144 | 570k | // If we just RAUW'd the root, take note. |
7145 | 570k | if (From == getRoot().getNode()) |
7146 | 13 | setRoot(SDValue(To[getRoot().getResNo()])); |
7147 | 2.48M | } |
7148 | | |
7149 | | /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving |
7150 | | /// uses of other values produced by From.getNode() alone. The Deleted |
7151 | | /// vector is handled the same way as for ReplaceAllUsesWith. |
7152 | 12.8M | void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ |
7153 | 12.8M | // Handle the really simple, really trivial case efficiently. |
7154 | 12.8M | if (From == To12.8M ) return6.09k ; |
7155 | 12.8M | |
7156 | 12.8M | // Handle the simple, trivial, case efficiently. |
7157 | 12.8M | if (12.8M From.getNode()->getNumValues() == 112.8M ) { |
7158 | 2.40M | ReplaceAllUsesWith(From, To); |
7159 | 2.40M | return; |
7160 | 2.40M | } |
7161 | 10.4M | |
7162 | 10.4M | // Preserve Debug Info. |
7163 | 10.4M | TransferDbgValues(From, To); |
7164 | 10.4M | |
7165 | 10.4M | // Iterate over just the existing users of From. See the comments in |
7166 | 10.4M | // the ReplaceAllUsesWith above. |
7167 | 10.4M | SDNode::use_iterator UI = From.getNode()->use_begin(), |
7168 | 10.4M | UE = From.getNode()->use_end(); |
7169 | 10.4M | RAUWUpdateListener Listener(*this, UI, UE); |
7170 | 22.7M | while (UI != UE22.7M ) { |
7171 | 12.2M | SDNode *User = *UI; |
7172 | 12.2M | bool UserRemovedFromCSEMaps = false; |
7173 | 12.2M | |
7174 | 12.2M | // A user can appear in a use list multiple times, and when this |
7175 | 12.2M | // happens the uses are usually next to each other in the list. |
7176 | 12.2M | // To help reduce the number of CSE recomputations, process all |
7177 | 12.2M | // the uses of this user that we can find this way. |
7178 | 17.3M | do { |
7179 | 17.3M | SDUse &Use = UI.getUse(); |
7180 | 17.3M | |
7181 | 17.3M | // Skip uses of different values from the same node. |
7182 | 17.3M | if (Use.getResNo() != From.getResNo()17.3M ) { |
7183 | 8.57M | ++UI; |
7184 | 8.57M | continue; |
7185 | 8.57M | } |
7186 | 8.73M | |
7187 | 8.73M | // If this node hasn't been modified yet, it's still in the CSE maps, |
7188 | 8.73M | // so remove its old self from the CSE maps. |
7189 | 8.73M | if (8.73M !UserRemovedFromCSEMaps8.73M ) { |
7190 | 8.73M | RemoveNodeFromCSEMaps(User); |
7191 | 8.73M | UserRemovedFromCSEMaps = true; |
7192 | 8.73M | } |
7193 | 17.3M | |
7194 | 17.3M | ++UI; |
7195 | 17.3M | Use.set(To); |
7196 | 17.3M | } while (UI != UE && 17.3M *UI == User7.04M ); |
7197 | 12.2M | |
7198 | 12.2M | // We are iterating over all uses of the From node, so if a use |
7199 | 12.2M | // doesn't use the specific value, no changes are made. |
7200 | 12.2M | if (!UserRemovedFromCSEMaps) |
7201 | 3.49M | continue; |
7202 | 8.73M | |
7203 | 8.73M | // Now that we have modified User, add it back to the CSE maps. If it |
7204 | 8.73M | // already exists there, recursively merge the results together. |
7205 | 8.73M | AddModifiedNodeToCSEMaps(User); |
7206 | 8.73M | } |
7207 | 10.4M | |
7208 | 10.4M | // If we just RAUW'd the root, take note. |
7209 | 10.4M | if (From == getRoot()) |
7210 | 97.3k | setRoot(To); |
7211 | 12.8M | } |
7212 | | |
7213 | | namespace { |
7214 | | |
7215 | | /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith |
7216 | | /// to record information about a use. |
7217 | | struct UseMemo { |
7218 | | SDNode *User; |
7219 | | unsigned Index; |
7220 | | SDUse *Use; |
7221 | | }; |
7222 | | |
7223 | | /// operator< - Sort Memos by User. |
7224 | 643 | bool operator<(const UseMemo &L, const UseMemo &R) { |
7225 | 643 | return (intptr_t)L.User < (intptr_t)R.User; |
7226 | 643 | } |
7227 | | |
7228 | | } // end anonymous namespace |
7229 | | |
7230 | | /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving |
7231 | | /// uses of other values produced by From.getNode() alone. The same value |
7232 | | /// may appear in both the From and To list. The Deleted vector is |
7233 | | /// handled the same way as for ReplaceAllUsesWith. |
7234 | | void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, |
7235 | | const SDValue *To, |
7236 | 605 | unsigned Num){ |
7237 | 605 | // Handle the simple, trivial case efficiently. |
7238 | 605 | if (Num == 1) |
7239 | 0 | return ReplaceAllUsesOfValueWith(*From, *To); |
7240 | 605 | |
7241 | 605 | TransferDbgValues(*From, *To); |
7242 | 605 | |
7243 | 605 | // Read up all the uses and make records of them. This helps |
7244 | 605 | // processing new uses that are introduced during the |
7245 | 605 | // replacement process. |
7246 | 605 | SmallVector<UseMemo, 4> Uses; |
7247 | 1.93k | for (unsigned i = 0; i != Num1.93k ; ++i1.33k ) { |
7248 | 1.33k | unsigned FromResNo = From[i].getResNo(); |
7249 | 1.33k | SDNode *FromNode = From[i].getNode(); |
7250 | 1.33k | for (SDNode::use_iterator UI = FromNode->use_begin(), |
7251 | 3.47k | E = FromNode->use_end(); UI != E3.47k ; ++UI2.14k ) { |
7252 | 2.14k | SDUse &Use = UI.getUse(); |
7253 | 2.14k | if (Use.getResNo() == FromResNo2.14k ) { |
7254 | 1.19k | UseMemo Memo = { *UI, i, &Use }; |
7255 | 1.19k | Uses.push_back(Memo); |
7256 | 1.19k | } |
7257 | 2.14k | } |
7258 | 1.33k | } |
7259 | 605 | |
7260 | 605 | // Sort the uses, so that all the uses from a given User are together. |
7261 | 605 | std::sort(Uses.begin(), Uses.end()); |
7262 | 605 | |
7263 | 605 | for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); |
7264 | 1.72k | UseIndex != UseIndexEnd1.72k ; ) { |
7265 | 1.12k | // We know that this user uses some value of From. If it is the right |
7266 | 1.12k | // value, update it. |
7267 | 1.12k | SDNode *User = Uses[UseIndex].User; |
7268 | 1.12k | |
7269 | 1.12k | // This node is about to morph, remove its old self from the CSE maps. |
7270 | 1.12k | RemoveNodeFromCSEMaps(User); |
7271 | 1.12k | |
7272 | 1.12k | // The Uses array is sorted, so all the uses for a given User |
7273 | 1.12k | // are next to each other in the list. |
7274 | 1.12k | // To help reduce the number of CSE recomputations, process all |
7275 | 1.12k | // the uses of this user that we can find this way. |
7276 | 1.19k | do { |
7277 | 1.19k | unsigned i = Uses[UseIndex].Index; |
7278 | 1.19k | SDUse &Use = *Uses[UseIndex].Use; |
7279 | 1.19k | ++UseIndex; |
7280 | 1.19k | |
7281 | 1.19k | Use.set(To[i]); |
7282 | 1.19k | } while (UseIndex != UseIndexEnd && 1.19k Uses[UseIndex].User == User585 ); |
7283 | 1.12k | |
7284 | 1.12k | // Now that we have modified User, add it back to the CSE maps. If it |
7285 | 1.12k | // already exists there, recursively merge the results together. |
7286 | 1.12k | AddModifiedNodeToCSEMaps(User); |
7287 | 1.12k | } |
7288 | 605 | } |
7289 | | |
7290 | | /// AssignTopologicalOrder - Assign a unique node id for each node in the DAG |
7291 | | /// based on their topological order. It returns the maximum id and a vector |
7292 | | /// of the SDNodes* in assigned order by reference. |
7293 | 6.98M | unsigned SelectionDAG::AssignTopologicalOrder() { |
7294 | 6.98M | unsigned DAGSize = 0; |
7295 | 6.98M | |
7296 | 6.98M | // SortedPos tracks the progress of the algorithm. Nodes before it are |
7297 | 6.98M | // sorted, nodes after it are unsorted. When the algorithm completes |
7298 | 6.98M | // it is at the end of the list. |
7299 | 6.98M | allnodes_iterator SortedPos = allnodes_begin(); |
7300 | 6.98M | |
7301 | 6.98M | // Visit all the nodes. Move nodes with no operands to the front of |
7302 | 6.98M | // the list immediately. Annotate nodes that do have operands with their |
7303 | 6.98M | // operand count. Before we do this, the Node Id fields of the nodes |
7304 | 6.98M | // may contain arbitrary values. After, the Node Id fields for nodes |
7305 | 6.98M | // before SortedPos will contain the topological sort index, and the |
7306 | 6.98M | // Node Id fields for nodes At SortedPos and after will contain the |
7307 | 6.98M | // count of outstanding operands. |
7308 | 152M | for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E152M ; ) { |
7309 | 145M | SDNode *N = &*I++; |
7310 | 145M | checkForCycles(N, this); |
7311 | 145M | unsigned Degree = N->getNumOperands(); |
7312 | 145M | if (Degree == 0145M ) { |
7313 | 69.3M | // A node with no uses, add it to the result array immediately. |
7314 | 69.3M | N->setNodeId(DAGSize++); |
7315 | 69.3M | allnodes_iterator Q(N); |
7316 | 69.3M | if (Q != SortedPos) |
7317 | 29.6M | SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); |
7318 | 69.3M | assert(SortedPos != AllNodes.end() && "Overran node list"); |
7319 | 69.3M | ++SortedPos; |
7320 | 145M | } else { |
7321 | 75.6M | // Temporarily use the Node Id as scratch space for the degree count. |
7322 | 75.6M | N->setNodeId(Degree); |
7323 | 75.6M | } |
7324 | 145M | } |
7325 | 6.98M | |
7326 | 6.98M | // Visit all the nodes. As we iterate, move nodes into sorted order, |
7327 | 6.98M | // such that by the time the end is reached all nodes will be sorted. |
7328 | 145M | for (SDNode &Node : allnodes()) { |
7329 | 145M | SDNode *N = &Node; |
7330 | 145M | checkForCycles(N, this); |
7331 | 145M | // N is in sorted position, so all its uses have one less operand |
7332 | 145M | // that needs to be sorted. |
7333 | 145M | for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); |
7334 | 365M | UI != UE365M ; ++UI220M ) { |
7335 | 220M | SDNode *P = *UI; |
7336 | 220M | unsigned Degree = P->getNodeId(); |
7337 | 220M | assert(Degree != 0 && "Invalid node degree"); |
7338 | 220M | --Degree; |
7339 | 220M | if (Degree == 0220M ) { |
7340 | 75.6M | // All of P's operands are sorted, so P may sorted now. |
7341 | 75.6M | P->setNodeId(DAGSize++); |
7342 | 75.6M | if (P->getIterator() != SortedPos) |
7343 | 18.3M | SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); |
7344 | 75.6M | assert(SortedPos != AllNodes.end() && "Overran node list"); |
7345 | 75.6M | ++SortedPos; |
7346 | 220M | } else { |
7347 | 144M | // Update P's outstanding operand count. |
7348 | 144M | P->setNodeId(Degree); |
7349 | 144M | } |
7350 | 220M | } |
7351 | 145M | if (Node.getIterator() == SortedPos145M ) { |
7352 | | #ifndef NDEBUG |
7353 | | allnodes_iterator I(N); |
7354 | | SDNode *S = &*++I; |
7355 | | dbgs() << "Overran sorted position:\n"; |
7356 | | S->dumprFull(this); dbgs() << "\n"; |
7357 | | dbgs() << "Checking if this is due to cycles\n"; |
7358 | | checkForCycles(this, true); |
7359 | | #endif |
7360 | 0 | llvm_unreachable(nullptr); |
7361 | 0 | } |
7362 | 6.98M | } |
7363 | 6.98M | |
7364 | 6.98M | assert(SortedPos == AllNodes.end() && |
7365 | 6.98M | "Topological sort incomplete!"); |
7366 | 6.98M | assert(AllNodes.front().getOpcode() == ISD::EntryToken && |
7367 | 6.98M | "First node in topological sort is not the entry token!"); |
7368 | 6.98M | assert(AllNodes.front().getNodeId() == 0 && |
7369 | 6.98M | "First node in topological sort has non-zero id!"); |
7370 | 6.98M | assert(AllNodes.front().getNumOperands() == 0 && |
7371 | 6.98M | "First node in topological sort has operands!"); |
7372 | 6.98M | assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && |
7373 | 6.98M | "Last node in topologic sort has unexpected id!"); |
7374 | 6.98M | assert(AllNodes.back().use_empty() && |
7375 | 6.98M | "Last node in topologic sort has users!"); |
7376 | 6.98M | assert(DAGSize == allnodes_size() && "Node count mismatch!"); |
7377 | 6.98M | return DAGSize; |
7378 | 6.98M | } |
7379 | | |
7380 | | /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the |
7381 | | /// value is produced by SD. |
7382 | 445 | void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { |
7383 | 445 | if (SD445 ) { |
7384 | 350 | assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue()); |
7385 | 350 | SD->setHasDebugValue(true); |
7386 | 350 | } |
7387 | 445 | DbgInfo->add(DB, SD, isParameter); |
7388 | 445 | } |
7389 | | |
7390 | | /// TransferDbgValues - Transfer SDDbgValues. Called in replace nodes. |
7391 | 25.4M | void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { |
7392 | 25.4M | if (From == To || 25.4M !From.getNode()->getHasDebugValue()25.4M ) |
7393 | 25.4M | return; |
7394 | 76 | SDNode *FromNode = From.getNode(); |
7395 | 76 | SDNode *ToNode = To.getNode(); |
7396 | 76 | ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); |
7397 | 76 | SmallVector<SDDbgValue *, 2> ClonedDVs; |
7398 | 76 | for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); |
7399 | 152 | I != E152 ; ++I76 ) { |
7400 | 76 | SDDbgValue *Dbg = *I; |
7401 | 76 | // Only add Dbgvalues attached to same ResNo. |
7402 | 76 | if (Dbg->getKind() == SDDbgValue::SDNODE && |
7403 | 76 | Dbg->getSDNode() == From.getNode() && |
7404 | 76 | Dbg->getResNo() == From.getResNo()76 && !Dbg->isInvalidated()47 ) { |
7405 | 47 | assert(FromNode != ToNode && |
7406 | 47 | "Should not transfer Debug Values intranode"); |
7407 | 47 | SDDbgValue *Clone = getDbgValue(Dbg->getVariable(), Dbg->getExpression(), |
7408 | 47 | ToNode, To.getResNo(), Dbg->isIndirect(), |
7409 | 47 | Dbg->getDebugLoc(), Dbg->getOrder()); |
7410 | 47 | ClonedDVs.push_back(Clone); |
7411 | 47 | Dbg->setIsInvalidated(); |
7412 | 47 | } |
7413 | 76 | } |
7414 | 76 | for (SDDbgValue *I : ClonedDVs) |
7415 | 47 | AddDbgValue(I, ToNode, false); |
7416 | 25.4M | } |
7417 | | |
7418 | | SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad, |
7419 | 1.38k | SDValue NewMemOp) { |
7420 | 1.38k | assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node"); |
7421 | 1.38k | // The new memory operation must have the same position as the old load in |
7422 | 1.38k | // terms of memory dependency. Create a TokenFactor for the old load and new |
7423 | 1.38k | // memory operation and update uses of the old load's output chain to use that |
7424 | 1.38k | // TokenFactor. |
7425 | 1.38k | SDValue OldChain = SDValue(OldLoad, 1); |
7426 | 1.38k | SDValue NewChain = SDValue(NewMemOp.getNode(), 1); |
7427 | 1.38k | if (!OldLoad->hasAnyUseOfValue(1)) |
7428 | 1.17k | return NewChain; |
7429 | 212 | |
7430 | 212 | SDValue TokenFactor = |
7431 | 212 | getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain); |
7432 | 212 | ReplaceAllUsesOfValueWith(OldChain, TokenFactor); |
7433 | 212 | UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain); |
7434 | 212 | return TokenFactor; |
7435 | 212 | } |
7436 | | |
7437 | | //===----------------------------------------------------------------------===// |
7438 | | // SDNode Class |
7439 | | //===----------------------------------------------------------------------===// |
7440 | | |
7441 | 18.7M | bool llvm::isNullConstant(SDValue V) { |
7442 | 18.7M | ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); |
7443 | 10.9M | return Const != nullptr && Const->isNullValue(); |
7444 | 18.7M | } |
7445 | | |
7446 | 5.61M | bool llvm::isNullFPConstant(SDValue V) { |
7447 | 5.61M | ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V); |
7448 | 5.61M | return Const != nullptr && Const->isZero()40.3k && !Const->isNegative()20.8k ; |
7449 | 5.61M | } |
7450 | | |
7451 | 1.54M | bool llvm::isAllOnesConstant(SDValue V) { |
7452 | 1.54M | ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); |
7453 | 1.10M | return Const != nullptr && Const->isAllOnesValue(); |
7454 | 1.54M | } |
7455 | | |
7456 | 334k | bool llvm::isOneConstant(SDValue V) { |
7457 | 334k | ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V); |
7458 | 145k | return Const != nullptr && Const->isOne(); |
7459 | 334k | } |
7460 | | |
7461 | 522k | bool llvm::isBitwiseNot(SDValue V) { |
7462 | 7.64k | return V.getOpcode() == ISD::XOR && isAllOnesConstant(V.getOperand(1)); |
7463 | 522k | } |
7464 | | |
7465 | 13.6M | ConstantSDNode *llvm::isConstOrConstSplat(SDValue N) { |
7466 | 13.6M | if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N)) |
7467 | 10.3M | return CN; |
7468 | 3.29M | |
7469 | 3.29M | if (BuildVectorSDNode *3.29M BV3.29M = dyn_cast<BuildVectorSDNode>(N)) { |
7470 | 139k | BitVector UndefElements; |
7471 | 139k | ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements); |
7472 | 139k | |
7473 | 139k | // BuildVectors can truncate their operands. Ignore that case here. |
7474 | 139k | // FIXME: We blindly ignore splats which include undef which is overly |
7475 | 139k | // pessimistic. |
7476 | 139k | if (CN && 139k UndefElements.none()131k && |
7477 | 130k | CN->getValueType(0) == N.getValueType().getScalarType()) |
7478 | 116k | return CN; |
7479 | 3.18M | } |
7480 | 3.18M | |
7481 | 3.18M | return nullptr; |
7482 | 3.18M | } |
7483 | | |
7484 | 781k | ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N) { |
7485 | 781k | if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N)) |
7486 | 51.9k | return CN; |
7487 | 729k | |
7488 | 729k | if (BuildVectorSDNode *729k BV729k = dyn_cast<BuildVectorSDNode>(N)) { |
7489 | 12.1k | BitVector UndefElements; |
7490 | 12.1k | ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements); |
7491 | 12.1k | |
7492 | 12.1k | if (CN && 12.1k UndefElements.none()5.20k ) |
7493 | 5.18k | return CN; |
7494 | 724k | } |
7495 | 724k | |
7496 | 724k | return nullptr; |
7497 | 724k | } |
7498 | | |
7499 | 30.4M | HandleSDNode::~HandleSDNode() { |
7500 | 30.4M | DropOperands(); |
7501 | 30.4M | } |
7502 | | |
7503 | | GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, |
7504 | | const DebugLoc &DL, |
7505 | | const GlobalValue *GA, EVT VT, |
7506 | | int64_t o, unsigned char TF) |
7507 | 6.00M | : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { |
7508 | 6.00M | TheGlobal = GA; |
7509 | 6.00M | } |
7510 | | |
7511 | | AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, |
7512 | | EVT VT, unsigned SrcAS, |
7513 | | unsigned DestAS) |
7514 | | : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), |
7515 | 199 | SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} |
7516 | | |
7517 | | MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, |
7518 | | SDVTList VTs, EVT memvt, MachineMemOperand *mmo) |
7519 | 13.9M | : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { |
7520 | 13.9M | MemSDNodeBits.IsVolatile = MMO->isVolatile(); |
7521 | 13.9M | MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); |
7522 | 13.9M | MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); |
7523 | 13.9M | MemSDNodeBits.IsInvariant = MMO->isInvariant(); |
7524 | 13.9M | |
7525 | 13.9M | // We check here that the size of the memory operand fits within the size of |
7526 | 13.9M | // the MMO. This is because the MMO might indicate only a possible address |
7527 | 13.9M | // range instead of specifying the affected memory addresses precisely. |
7528 | 13.9M | assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!"); |
7529 | 13.9M | } |
7530 | | |
7531 | | /// Profile - Gather unique data for the node. |
7532 | | /// |
7533 | 108M | void SDNode::Profile(FoldingSetNodeID &ID) const { |
7534 | 108M | AddNodeIDNode(ID, this); |
7535 | 108M | } |
7536 | | |
7537 | | namespace { |
7538 | | |
7539 | | struct EVTArray { |
7540 | | std::vector<EVT> VTs; |
7541 | | |
7542 | 33.2k | EVTArray() { |
7543 | 33.2k | VTs.reserve(MVT::LAST_VALUETYPE); |
7544 | 3.76M | for (unsigned i = 0; i < MVT::LAST_VALUETYPE3.76M ; ++i3.72M ) |
7545 | 3.72M | VTs.push_back(MVT((MVT::SimpleValueType)i)); |
7546 | 33.2k | } |
7547 | | }; |
7548 | | |
7549 | | } // end anonymous namespace |
7550 | | |
7551 | | static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs; |
7552 | | static ManagedStatic<EVTArray> SimpleVTArray; |
7553 | | static ManagedStatic<sys::SmartMutex<true>> VTMutex; |
7554 | | |
7555 | | /// getValueTypeList - Return a pointer to the specified value type. |
7556 | | /// |
7557 | 221M | const EVT *SDNode::getValueTypeList(EVT VT) { |
7558 | 221M | if (VT.isExtended()221M ) { |
7559 | 88.4k | sys::SmartScopedLock<true> Lock(*VTMutex); |
7560 | 88.4k | return &(*EVTs->insert(VT).first); |
7561 | 0 | } else { |
7562 | 221M | assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && |
7563 | 221M | "Value type out of range!"); |
7564 | 221M | return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; |
7565 | 221M | } |
7566 | 0 | } |
7567 | | |
7568 | | /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the |
7569 | | /// indicated value. This method ignores uses of other values defined by this |
7570 | | /// operation. |
7571 | 37.6M | bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { |
7572 | 37.6M | assert(Value < getNumValues() && "Bad value!"); |
7573 | 37.6M | |
7574 | 37.6M | // TODO: Only iterate over uses of a given value of the node |
7575 | 80.3M | for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E80.3M ; ++UI42.7M ) { |
7576 | 54.9M | if (UI.getUse().getResNo() == Value54.9M ) { |
7577 | 49.6M | if (NUses == 0) |
7578 | 12.2M | return false; |
7579 | 37.3M | --NUses; |
7580 | 37.3M | } |
7581 | 54.9M | } |
7582 | 37.6M | |
7583 | 37.6M | // Found exactly the right number of uses? |
7584 | 25.3M | return NUses == 0; |
7585 | 37.6M | } |
7586 | | |
7587 | | /// hasAnyUseOfValue - Return true if there are any use of the indicated |
7588 | | /// value. This method ignores uses of other values defined by this operation. |
7589 | 44.0M | bool SDNode::hasAnyUseOfValue(unsigned Value) const { |
7590 | 44.0M | assert(Value < getNumValues() && "Bad value!"); |
7591 | 44.0M | |
7592 | 65.5M | for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E65.5M ; ++UI21.4M ) |
7593 | 52.7M | if (52.7M UI.getUse().getResNo() == Value52.7M ) |
7594 | 31.2M | return true; |
7595 | 44.0M | |
7596 | 12.8M | return false; |
7597 | 44.0M | } |
7598 | | |
7599 | | /// isOnlyUserOf - Return true if this node is the only use of N. |
7600 | 81.8k | bool SDNode::isOnlyUserOf(const SDNode *N) const { |
7601 | 81.8k | bool Seen = false; |
7602 | 173k | for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E173k ; ++I91.8k ) { |
7603 | 107k | SDNode *User = *I; |
7604 | 107k | if (User == this) |
7605 | 91.8k | Seen = true; |
7606 | 107k | else |
7607 | 15.9k | return false; |
7608 | 107k | } |
7609 | 81.8k | |
7610 | 65.8k | return Seen; |
7611 | 81.8k | } |
7612 | | |
7613 | | /// Return true if the only users of N are contained in Nodes. |
7614 | 76.2k | bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) { |
7615 | 76.2k | bool Seen = false; |
7616 | 162k | for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E162k ; ++I86.0k ) { |
7617 | 129k | SDNode *User = *I; |
7618 | 129k | if (llvm::any_of(Nodes, |
7619 | 665k | [&User](const SDNode *Node) { return User == Node; })) |
7620 | 86.0k | Seen = true; |
7621 | 129k | else |
7622 | 43.7k | return false; |
7623 | 129k | } |
7624 | 76.2k | |
7625 | 32.5k | return Seen; |
7626 | 76.2k | } |
7627 | | |
7628 | | /// isOperand - Return true if this node is an operand of N. |
7629 | 4.63M | bool SDValue::isOperandOf(const SDNode *N) const { |
7630 | 4.63M | for (const SDValue &Op : N->op_values()) |
7631 | 14.0M | if (14.0M *this == Op14.0M ) |
7632 | 163k | return true; |
7633 | 4.46M | return false; |
7634 | 4.46M | } |
7635 | | |
7636 | 4 | bool SDNode::isOperandOf(const SDNode *N) const { |
7637 | 4 | for (const SDValue &Op : N->op_values()) |
7638 | 11 | if (11 this == Op.getNode()11 ) |
7639 | 3 | return true; |
7640 | 1 | return false; |
7641 | 1 | } |
7642 | | |
7643 | | /// reachesChainWithoutSideEffects - Return true if this operand (which must |
7644 | | /// be a chain) reaches the specified operand without crossing any |
7645 | | /// side-effecting instructions on any chain path. In practice, this looks |
7646 | | /// through token factors and non-volatile loads. In order to remain efficient, |
7647 | | /// this only looks a couple of nodes in, it does not do an exhaustive search. |
7648 | | /// |
7649 | | /// Note that we only need to examine chains when we're searching for |
7650 | | /// side-effects; SelectionDAG requires that all side-effects are represented |
7651 | | /// by chains, even if another operand would force a specific ordering. This |
7652 | | /// constraint is necessary to allow transformations like splitting loads. |
7653 | | bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, |
7654 | 3.52k | unsigned Depth) const { |
7655 | 3.52k | if (*this == Dest3.52k ) return true1.76k ; |
7656 | 1.76k | |
7657 | 1.76k | // Don't search too deeply, we just want to be able to see through |
7658 | 1.76k | // TokenFactor's etc. |
7659 | 1.76k | if (1.76k Depth == 01.76k ) return false161 ; |
7660 | 1.60k | |
7661 | 1.60k | // If this is a token factor, all inputs to the TF happen in parallel. |
7662 | 1.60k | if (1.60k getOpcode() == ISD::TokenFactor1.60k ) { |
7663 | 328 | // First, try a shallow search. |
7664 | 328 | if (is_contained((*this)->ops(), Dest)328 ) { |
7665 | 241 | // We found the chain we want as an operand of this TokenFactor. |
7666 | 241 | // Essentially, we reach the chain without side-effects if we could |
7667 | 241 | // serialize the TokenFactor into a simple chain of operations with |
7668 | 241 | // Dest as the last operation. This is automatically true if the |
7669 | 241 | // chain has one use: there are no other ordering constraints. |
7670 | 241 | // If the chain has more than one use, we give up: some other |
7671 | 241 | // use of Dest might force a side-effect between Dest and the current |
7672 | 241 | // node. |
7673 | 241 | if (Dest.hasOneUse()) |
7674 | 139 | return true; |
7675 | 189 | } |
7676 | 189 | // Next, try a deep search: check whether every operand of the TokenFactor |
7677 | 189 | // reaches Dest. |
7678 | 189 | return llvm::all_of((*this)->ops(), [=](SDValue Op) 189 { |
7679 | 201 | return Op.reachesChainWithoutSideEffects(Dest, Depth - 1); |
7680 | 201 | }); |
7681 | 328 | } |
7682 | 1.28k | |
7683 | 1.28k | // Loads don't have side effects, look through them. |
7684 | 1.28k | if (LoadSDNode *1.28k Ld1.28k = dyn_cast<LoadSDNode>(*this)) { |
7685 | 226 | if (!Ld->isVolatile()) |
7686 | 226 | return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); |
7687 | 1.05k | } |
7688 | 1.05k | return false; |
7689 | 1.05k | } |
7690 | | |
7691 | 259k | bool SDNode::hasPredecessor(const SDNode *N) const { |
7692 | 259k | SmallPtrSet<const SDNode *, 32> Visited; |
7693 | 259k | SmallVector<const SDNode *, 16> Worklist; |
7694 | 259k | Worklist.push_back(this); |
7695 | 259k | return hasPredecessorHelper(N, Visited, Worklist); |
7696 | 259k | } |
7697 | | |
7698 | 1.58M | void SDNode::intersectFlagsWith(const SDNodeFlags Flags) { |
7699 | 1.58M | this->Flags.intersectWith(Flags); |
7700 | 1.58M | } |
7701 | | |
7702 | 6.35k | SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { |
7703 | 6.35k | assert(N->getNumValues() == 1 && |
7704 | 6.35k | "Can't unroll a vector with multiple results!"); |
7705 | 6.35k | |
7706 | 6.35k | EVT VT = N->getValueType(0); |
7707 | 6.35k | unsigned NE = VT.getVectorNumElements(); |
7708 | 6.35k | EVT EltVT = VT.getVectorElementType(); |
7709 | 6.35k | SDLoc dl(N); |
7710 | 6.35k | |
7711 | 6.35k | SmallVector<SDValue, 8> Scalars; |
7712 | 6.35k | SmallVector<SDValue, 4> Operands(N->getNumOperands()); |
7713 | 6.35k | |
7714 | 6.35k | // If ResNE is 0, fully unroll the vector op. |
7715 | 6.35k | if (ResNE == 0) |
7716 | 6.34k | ResNE = NE; |
7717 | 15 | else if (15 NE > ResNE15 ) |
7718 | 0 | NE = ResNE; |
7719 | 6.35k | |
7720 | 6.35k | unsigned i; |
7721 | 22.5k | for (i= 0; i != NE22.5k ; ++i16.1k ) { |
7722 | 44.4k | for (unsigned j = 0, e = N->getNumOperands(); j != e44.4k ; ++j28.2k ) { |
7723 | 28.2k | SDValue Operand = N->getOperand(j); |
7724 | 28.2k | EVT OperandVT = Operand.getValueType(); |
7725 | 28.2k | if (OperandVT.isVector()28.2k ) { |
7726 | 27.8k | // A vector operand; extract a single element. |
7727 | 27.8k | EVT OperandEltVT = OperandVT.getVectorElementType(); |
7728 | 27.8k | Operands[j] = |
7729 | 27.8k | getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand, |
7730 | 27.8k | getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout()))); |
7731 | 28.2k | } else { |
7732 | 440 | // A scalar operand; just use it as is. |
7733 | 440 | Operands[j] = Operand; |
7734 | 440 | } |
7735 | 28.2k | } |
7736 | 16.1k | |
7737 | 16.1k | switch (N->getOpcode()) { |
7738 | 15.6k | default: { |
7739 | 15.6k | Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands, |
7740 | 15.6k | N->getFlags())); |
7741 | 15.6k | break; |
7742 | 16.1k | } |
7743 | 46 | case ISD::VSELECT: |
7744 | 46 | Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands)); |
7745 | 46 | break; |
7746 | 400 | case ISD::SHL: |
7747 | 400 | case ISD::SRA: |
7748 | 400 | case ISD::SRL: |
7749 | 400 | case ISD::ROTL: |
7750 | 400 | case ISD::ROTR: |
7751 | 400 | Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], |
7752 | 400 | getShiftAmountOperand(Operands[0].getValueType(), |
7753 | 400 | Operands[1]))); |
7754 | 400 | break; |
7755 | 72 | case ISD::SIGN_EXTEND_INREG: |
7756 | 72 | case ISD::FP_ROUND_INREG: { |
7757 | 72 | EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); |
7758 | 72 | Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, |
7759 | 72 | Operands[0], |
7760 | 72 | getValueType(ExtVT))); |
7761 | 72 | } |
7762 | 16.1k | } |
7763 | 16.1k | } |
7764 | 6.35k | |
7765 | 6.36k | for (; 6.35k i < ResNE6.36k ; ++i7 ) |
7766 | 7 | Scalars.push_back(getUNDEF(EltVT)); |
7767 | 6.35k | |
7768 | 6.35k | EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE); |
7769 | 6.35k | return getBuildVector(VecVT, dl, Scalars); |
7770 | 6.35k | } |
7771 | | |
7772 | | bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, |
7773 | | LoadSDNode *Base, |
7774 | | unsigned Bytes, |
7775 | 4.83k | int Dist) const { |
7776 | 4.83k | if (LD->isVolatile() || 4.83k Base->isVolatile()4.78k ) |
7777 | 91 | return false; |
7778 | 4.74k | if (4.74k LD->isIndexed() || 4.74k Base->isIndexed()4.74k ) |
7779 | 0 | return false; |
7780 | 4.74k | if (4.74k LD->getChain() != Base->getChain()4.74k ) |
7781 | 333 | return false; |
7782 | 4.41k | EVT VT = LD->getValueType(0); |
7783 | 4.41k | if (VT.getSizeInBits() / 8 != Bytes) |
7784 | 0 | return false; |
7785 | 4.41k | |
7786 | 4.41k | SDValue Loc = LD->getOperand(1); |
7787 | 4.41k | SDValue BaseLoc = Base->getOperand(1); |
7788 | 4.41k | |
7789 | 4.41k | auto BaseLocDecomp = BaseIndexOffset::match(BaseLoc, *this); |
7790 | 4.41k | auto LocDecomp = BaseIndexOffset::match(Loc, *this); |
7791 | 4.41k | |
7792 | 4.41k | int64_t Offset = 0; |
7793 | 4.41k | if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset)) |
7794 | 4.21k | return (Dist * Bytes == Offset); |
7795 | 191 | return false; |
7796 | 191 | } |
7797 | | |
7798 | | /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if |
7799 | | /// it cannot be inferred. |
7800 | 18.9M | unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { |
7801 | 18.9M | // If this is a GlobalAddress + cst, return the alignment. |
7802 | 18.9M | const GlobalValue *GV; |
7803 | 18.9M | int64_t GVOffset = 0; |
7804 | 18.9M | if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)18.9M ) { |
7805 | 1.15M | unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType()); |
7806 | 1.15M | KnownBits Known(PtrWidth); |
7807 | 1.15M | llvm::computeKnownBits(GV, Known, getDataLayout()); |
7808 | 1.15M | unsigned AlignBits = Known.countMinTrailingZeros(); |
7809 | 1.15M | unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits)1.07M : 082.1k ; |
7810 | 1.15M | if (Align) |
7811 | 1.07M | return MinAlign(Align, GVOffset); |
7812 | 17.9M | } |
7813 | 17.9M | |
7814 | 17.9M | // If this is a direct reference to a stack slot, use information about the |
7815 | 17.9M | // stack slot's alignment. |
7816 | 17.9M | int FrameIdx = 1 << 31; |
7817 | 17.9M | int64_t FrameOffset = 0; |
7818 | 17.9M | if (FrameIndexSDNode *FI17.9M = dyn_cast<FrameIndexSDNode>(Ptr)) { |
7819 | 4.63M | FrameIdx = FI->getIndex(); |
7820 | 17.9M | } else if (13.2M isBaseWithConstantOffset(Ptr) && |
7821 | 13.2M | isa<FrameIndexSDNode>(Ptr.getOperand(0))10.5M ) { |
7822 | 1.43M | // Handle FI+Cst |
7823 | 1.43M | FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); |
7824 | 1.43M | FrameOffset = Ptr.getConstantOperandVal(1); |
7825 | 1.43M | } |
7826 | 17.9M | |
7827 | 17.9M | if (FrameIdx != (1 << 31)17.9M ) { |
7828 | 6.07M | const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); |
7829 | 6.07M | unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), |
7830 | 6.07M | FrameOffset); |
7831 | 6.07M | return FIInfoAlign; |
7832 | 6.07M | } |
7833 | 11.8M | |
7834 | 11.8M | return 0; |
7835 | 11.8M | } |
7836 | | |
7837 | | /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type |
7838 | | /// which is split (or expanded) into two not necessarily identical pieces. |
7839 | 143k | std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const { |
7840 | 143k | // Currently all types are split in half. |
7841 | 143k | EVT LoVT, HiVT; |
7842 | 143k | if (!VT.isVector()) |
7843 | 1.63k | LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT); |
7844 | 143k | else |
7845 | 141k | LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext()); |
7846 | 143k | |
7847 | 143k | return std::make_pair(LoVT, HiVT); |
7848 | 143k | } |
7849 | | |
7850 | | /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the |
7851 | | /// low/high part. |
7852 | | std::pair<SDValue, SDValue> |
7853 | | SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, |
7854 | 34.2k | const EVT &HiVT) { |
7855 | 34.2k | assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <= |
7856 | 34.2k | N.getValueType().getVectorNumElements() && |
7857 | 34.2k | "More vector elements requested than available!"); |
7858 | 34.2k | SDValue Lo, Hi; |
7859 | 34.2k | Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N, |
7860 | 34.2k | getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout()))); |
7861 | 34.2k | Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N, |
7862 | 34.2k | getConstant(LoVT.getVectorNumElements(), DL, |
7863 | 34.2k | TLI->getVectorIdxTy(getDataLayout()))); |
7864 | 34.2k | return std::make_pair(Lo, Hi); |
7865 | 34.2k | } |
7866 | | |
7867 | | void SelectionDAG::ExtractVectorElements(SDValue Op, |
7868 | | SmallVectorImpl<SDValue> &Args, |
7869 | 17.4k | unsigned Start, unsigned Count) { |
7870 | 17.4k | EVT VT = Op.getValueType(); |
7871 | 17.4k | if (Count == 0) |
7872 | 4.46k | Count = VT.getVectorNumElements(); |
7873 | 17.4k | |
7874 | 17.4k | EVT EltVT = VT.getVectorElementType(); |
7875 | 17.4k | EVT IdxTy = TLI->getVectorIdxTy(getDataLayout()); |
7876 | 17.4k | SDLoc SL(Op); |
7877 | 92.2k | for (unsigned i = Start, e = Start + Count; i != e92.2k ; ++i74.8k ) { |
7878 | 74.8k | Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, |
7879 | 74.8k | Op, getConstant(i, SL, IdxTy))); |
7880 | 74.8k | } |
7881 | 17.4k | } |
7882 | | |
7883 | | // getAddressSpace - Return the address space this GlobalAddress belongs to. |
7884 | 4.87k | unsigned GlobalAddressSDNode::getAddressSpace() const { |
7885 | 4.87k | return getGlobal()->getType()->getAddressSpace(); |
7886 | 4.87k | } |
7887 | | |
7888 | 175k | Type *ConstantPoolSDNode::getType() const { |
7889 | 175k | if (isMachineConstantPoolEntry()) |
7890 | 343 | return Val.MachineCPVal->getType(); |
7891 | 175k | return Val.ConstVal->getType(); |
7892 | 175k | } |
7893 | | |
7894 | | bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, |
7895 | | unsigned &SplatBitSize, |
7896 | | bool &HasAnyUndefs, |
7897 | | unsigned MinSplatBits, |
7898 | 205k | bool IsBigEndian) const { |
7899 | 205k | EVT VT = getValueType(0); |
7900 | 205k | assert(VT.isVector() && "Expected a vector type"); |
7901 | 205k | unsigned VecWidth = VT.getSizeInBits(); |
7902 | 205k | if (MinSplatBits > VecWidth) |
7903 | 0 | return false; |
7904 | 205k | |
7905 | 205k | // FIXME: The widths are based on this node's type, but build vectors can |
7906 | 205k | // truncate their operands. |
7907 | 205k | SplatValue = APInt(VecWidth, 0); |
7908 | 205k | SplatUndef = APInt(VecWidth, 0); |
7909 | 205k | |
7910 | 205k | // Get the bits. Bits with undefined values (when the corresponding element |
7911 | 205k | // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared |
7912 | 205k | // in SplatValue. If any of the values are not constant, give up and return |
7913 | 205k | // false. |
7914 | 205k | unsigned int NumOps = getNumOperands(); |
7915 | 205k | assert(NumOps > 0 && "isConstantSplat has 0-size build vector"); |
7916 | 205k | unsigned EltWidth = VT.getScalarSizeInBits(); |
7917 | 205k | |
7918 | 849k | for (unsigned j = 0; j < NumOps849k ; ++j643k ) { |
7919 | 675k | unsigned i = IsBigEndian ? NumOps - 1 - j28.4k : j647k ; |
7920 | 675k | SDValue OpVal = getOperand(i); |
7921 | 675k | unsigned BitPos = j * EltWidth; |
7922 | 675k | |
7923 | 675k | if (OpVal.isUndef()) |
7924 | 13.6k | SplatUndef.setBits(BitPos, BitPos + EltWidth); |
7925 | 662k | else if (auto *662k CN662k = dyn_cast<ConstantSDNode>(OpVal)) |
7926 | 592k | SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos); |
7927 | 69.1k | else if (auto *69.1k CN69.1k = dyn_cast<ConstantFPSDNode>(OpVal)) |
7928 | 36.9k | SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos); |
7929 | 69.1k | else |
7930 | 32.1k | return false; |
7931 | 675k | } |
7932 | 205k | |
7933 | 205k | // The build_vector is all constants or undefs. Find the smallest element |
7934 | 205k | // size that splats the vector. |
7935 | 173k | HasAnyUndefs = (SplatUndef != 0); |
7936 | 173k | |
7937 | 173k | // FIXME: This does not work for vectors with elements less than 8 bits. |
7938 | 382k | while (VecWidth > 8382k ) { |
7939 | 349k | unsigned HalfSize = VecWidth / 2; |
7940 | 349k | APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); |
7941 | 349k | APInt LowValue = SplatValue.trunc(HalfSize); |
7942 | 349k | APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); |
7943 | 349k | APInt LowUndef = SplatUndef.trunc(HalfSize); |
7944 | 349k | |
7945 | 349k | // If the two halves do not match (ignoring undef bits), stop here. |
7946 | 349k | if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || |
7947 | 209k | MinSplatBits > HalfSize) |
7948 | 140k | break; |
7949 | 208k | |
7950 | 208k | SplatValue = HighValue | LowValue; |
7951 | 208k | SplatUndef = HighUndef & LowUndef; |
7952 | 208k | |
7953 | 208k | VecWidth = HalfSize; |
7954 | 208k | } |
7955 | 173k | |
7956 | 173k | SplatBitSize = VecWidth; |
7957 | 173k | return true; |
7958 | 205k | } |
7959 | | |
7960 | 200k | SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { |
7961 | 200k | if (UndefElements200k ) { |
7962 | 178k | UndefElements->clear(); |
7963 | 178k | UndefElements->resize(getNumOperands()); |
7964 | 178k | } |
7965 | 200k | SDValue Splatted; |
7966 | 1.28M | for (unsigned i = 0, e = getNumOperands(); i != e1.28M ; ++i1.08M ) { |
7967 | 1.10M | SDValue Op = getOperand(i); |
7968 | 1.10M | if (Op.isUndef()1.10M ) { |
7969 | 8.73k | if (UndefElements) |
7970 | 8.49k | (*UndefElements)[i] = true; |
7971 | 1.10M | } else if (1.09M !Splatted1.09M ) { |
7972 | 200k | Splatted = Op; |
7973 | 1.09M | } else if (899k Splatted != Op899k ) { |
7974 | 25.1k | return SDValue(); |
7975 | 25.1k | } |
7976 | 1.10M | } |
7977 | 200k | |
7978 | 175k | if (175k !Splatted175k ) { |
7979 | 49 | assert(getOperand(0).isUndef() && |
7980 | 49 | "Can only have a splat without a constant for all undefs."); |
7981 | 49 | return getOperand(0); |
7982 | 49 | } |
7983 | 175k | |
7984 | 175k | return Splatted; |
7985 | 175k | } |
7986 | | |
7987 | | ConstantSDNode * |
7988 | 160k | BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { |
7989 | 160k | return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements)); |
7990 | 160k | } |
7991 | | |
7992 | | ConstantFPSDNode * |
7993 | 12.5k | BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { |
7994 | 12.5k | return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements)); |
7995 | 12.5k | } |
7996 | | |
7997 | | int32_t |
7998 | | BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, |
7999 | 117 | uint32_t BitWidth) const { |
8000 | 117 | if (ConstantFPSDNode *CN = |
8001 | 98 | dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) { |
8002 | 98 | bool IsExact; |
8003 | 98 | APSInt IntVal(BitWidth); |
8004 | 98 | const APFloat &APF = CN->getValueAPF(); |
8005 | 98 | if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) != |
8006 | 98 | APFloat::opOK || |
8007 | 92 | !IsExact) |
8008 | 6 | return -1; |
8009 | 92 | |
8010 | 92 | return IntVal.exactLogBase2(); |
8011 | 92 | } |
8012 | 19 | return -1; |
8013 | 19 | } |
8014 | | |
8015 | 241k | bool BuildVectorSDNode::isConstant() const { |
8016 | 584k | for (const SDValue &Op : op_values()) { |
8017 | 584k | unsigned Opc = Op.getOpcode(); |
8018 | 584k | if (Opc != ISD::UNDEF && 584k Opc != ISD::Constant580k && Opc != ISD::ConstantFP231k ) |
8019 | 181k | return false; |
8020 | 60.7k | } |
8021 | 60.7k | return true; |
8022 | 60.7k | } |
8023 | | |
8024 | 61.7k | bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { |
8025 | 61.7k | // Find the first non-undef value in the shuffle mask. |
8026 | 61.7k | unsigned i, e; |
8027 | 64.8k | for (i = 0, e = VT.getVectorNumElements(); i != e && 64.8k Mask[i] < 064.8k ; ++i3.10k ) |
8028 | 3.10k | /* search */; |
8029 | 61.7k | |
8030 | 61.7k | assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); |
8031 | 61.7k | |
8032 | 61.7k | // Make sure all remaining elements are either undef or the same as the first |
8033 | 61.7k | // non-undef value. |
8034 | 216k | for (int Idx = Mask[i]; i != e216k ; ++i154k ) |
8035 | 199k | if (199k Mask[i] >= 0 && 199k Mask[i] != Idx183k ) |
8036 | 44.8k | return false; |
8037 | 16.9k | return true; |
8038 | 61.7k | } |
8039 | | |
8040 | | // \brief Returns the SDNode if it is a constant integer BuildVector |
8041 | | // or constant integer. |
8042 | 16.1M | SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) { |
8043 | 16.1M | if (isa<ConstantSDNode>(N)) |
8044 | 1.47M | return N.getNode(); |
8045 | 14.6M | if (14.6M ISD::isBuildVectorOfConstantSDNodes(N.getNode())14.6M ) |
8046 | 7.49k | return N.getNode(); |
8047 | 14.6M | // Treat a GlobalAddress supporting constant offset folding as a |
8048 | 14.6M | // constant integer. |
8049 | 14.6M | if (GlobalAddressSDNode *14.6M GA14.6M = dyn_cast<GlobalAddressSDNode>(N)) |
8050 | 357k | if (357k GA->getOpcode() == ISD::GlobalAddress && |
8051 | 357k | TLI->isOffsetFoldingLegal(GA)) |
8052 | 885 | return GA; |
8053 | 14.6M | return nullptr; |
8054 | 14.6M | } |
8055 | | |
8056 | 5.43k | SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) { |
8057 | 5.43k | if (isa<ConstantFPSDNode>(N)) |
8058 | 226 | return N.getNode(); |
8059 | 5.20k | |
8060 | 5.20k | if (5.20k ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())5.20k ) |
8061 | 4 | return N.getNode(); |
8062 | 5.20k | |
8063 | 5.20k | return nullptr; |
8064 | 5.20k | } |
8065 | | |
8066 | | #ifndef NDEBUG |
8067 | | static void checkForCyclesHelper(const SDNode *N, |
8068 | | SmallPtrSetImpl<const SDNode*> &Visited, |
8069 | | SmallPtrSetImpl<const SDNode*> &Checked, |
8070 | | const llvm::SelectionDAG *DAG) { |
8071 | | // If this node has already been checked, don't check it again. |
8072 | | if (Checked.count(N)) |
8073 | | return; |
8074 | | |
8075 | | // If a node has already been visited on this depth-first walk, reject it as |
8076 | | // a cycle. |
8077 | | if (!Visited.insert(N).second) { |
8078 | | errs() << "Detected cycle in SelectionDAG\n"; |
8079 | | dbgs() << "Offending node:\n"; |
8080 | | N->dumprFull(DAG); dbgs() << "\n"; |
8081 | | abort(); |
8082 | | } |
8083 | | |
8084 | | for (const SDValue &Op : N->op_values()) |
8085 | | checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG); |
8086 | | |
8087 | | Checked.insert(N); |
8088 | | Visited.erase(N); |
8089 | | } |
8090 | | #endif |
8091 | | |
8092 | | void llvm::checkForCycles(const llvm::SDNode *N, |
8093 | | const llvm::SelectionDAG *DAG, |
8094 | 455M | bool force) { |
8095 | | #ifndef NDEBUG |
8096 | | bool check = force; |
8097 | | #ifdef EXPENSIVE_CHECKS |
8098 | | check = true; |
8099 | | #endif // EXPENSIVE_CHECKS |
8100 | | if (check) { |
8101 | | assert(N && "Checking nonexistent SDNode"); |
8102 | | SmallPtrSet<const SDNode*, 32> visited; |
8103 | | SmallPtrSet<const SDNode*, 32> checked; |
8104 | | checkForCyclesHelper(N, visited, checked, DAG); |
8105 | | } |
8106 | | #endif // !NDEBUG |
8107 | | } |
8108 | | |
8109 | 41.8M | void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) { |
8110 | 41.8M | checkForCycles(DAG->getRoot().getNode(), DAG, force); |
8111 | 41.8M | } |