/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Line | Count | Source (jump to first uncovered line) |
1 | | //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | // |
10 | | // This file declares the SDNode class and derived classes, which are used to |
11 | | // represent the nodes and operations present in a SelectionDAG. These nodes |
12 | | // and operations are machine code level operations, with some similarities to |
13 | | // the GCC RTL representation. |
14 | | // |
15 | | // Clients should include the SelectionDAG.h file instead of this file directly. |
16 | | // |
17 | | //===----------------------------------------------------------------------===// |
18 | | |
19 | | #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H |
20 | | #define LLVM_CODEGEN_SELECTIONDAGNODES_H |
21 | | |
22 | | #include "llvm/ADT/APFloat.h" |
23 | | #include "llvm/ADT/ArrayRef.h" |
24 | | #include "llvm/ADT/BitVector.h" |
25 | | #include "llvm/ADT/FoldingSet.h" |
26 | | #include "llvm/ADT/GraphTraits.h" |
27 | | #include "llvm/ADT/SmallPtrSet.h" |
28 | | #include "llvm/ADT/SmallVector.h" |
29 | | #include "llvm/ADT/ilist_node.h" |
30 | | #include "llvm/ADT/iterator.h" |
31 | | #include "llvm/ADT/iterator_range.h" |
32 | | #include "llvm/CodeGen/ISDOpcodes.h" |
33 | | #include "llvm/CodeGen/MachineMemOperand.h" |
34 | | #include "llvm/CodeGen/MachineValueType.h" |
35 | | #include "llvm/CodeGen/ValueTypes.h" |
36 | | #include "llvm/IR/Constants.h" |
37 | | #include "llvm/IR/DebugLoc.h" |
38 | | #include "llvm/IR/Instruction.h" |
39 | | #include "llvm/IR/Instructions.h" |
40 | | #include "llvm/IR/Metadata.h" |
41 | | #include "llvm/Support/AlignOf.h" |
42 | | #include "llvm/Support/AtomicOrdering.h" |
43 | | #include "llvm/Support/Casting.h" |
44 | | #include "llvm/Support/ErrorHandling.h" |
45 | | #include <algorithm> |
46 | | #include <cassert> |
47 | | #include <climits> |
48 | | #include <cstddef> |
49 | | #include <cstdint> |
50 | | #include <cstring> |
51 | | #include <iterator> |
52 | | #include <string> |
53 | | #include <tuple> |
54 | | |
55 | | namespace llvm { |
56 | | |
57 | | class APInt; |
58 | | class Constant; |
59 | | template <typename T> struct DenseMapInfo; |
60 | | class GlobalValue; |
61 | | class MachineBasicBlock; |
62 | | class MachineConstantPoolValue; |
63 | | class MCSymbol; |
64 | | class raw_ostream; |
65 | | class SDNode; |
66 | | class SelectionDAG; |
67 | | class Type; |
68 | | class Value; |
69 | | |
70 | | void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr, |
71 | | bool force = false); |
72 | | |
73 | | /// This represents a list of ValueType's that has been intern'd by |
74 | | /// a SelectionDAG. Instances of this simple value class are returned by |
75 | | /// SelectionDAG::getVTList(...). |
76 | | /// |
77 | | struct SDVTList { |
78 | | const EVT *VTs; |
79 | | unsigned int NumVTs; |
80 | | }; |
81 | | |
82 | | namespace ISD { |
83 | | |
84 | | /// Node predicates |
85 | | |
86 | | /// If N is a BUILD_VECTOR node whose elements are all the same constant or |
87 | | /// undefined, return true and return the constant value in \p SplatValue. |
88 | | bool isConstantSplatVector(const SDNode *N, APInt &SplatValue); |
89 | | |
90 | | /// Return true if the specified node is a BUILD_VECTOR where all of the |
91 | | /// elements are ~0 or undef. |
92 | | bool isBuildVectorAllOnes(const SDNode *N); |
93 | | |
94 | | /// Return true if the specified node is a BUILD_VECTOR where all of the |
95 | | /// elements are 0 or undef. |
96 | | bool isBuildVectorAllZeros(const SDNode *N); |
97 | | |
98 | | /// Return true if the specified node is a BUILD_VECTOR node of all |
99 | | /// ConstantSDNode or undef. |
100 | | bool isBuildVectorOfConstantSDNodes(const SDNode *N); |
101 | | |
102 | | /// Return true if the specified node is a BUILD_VECTOR node of all |
103 | | /// ConstantFPSDNode or undef. |
104 | | bool isBuildVectorOfConstantFPSDNodes(const SDNode *N); |
105 | | |
106 | | /// Return true if the node has at least one operand and all operands of the |
107 | | /// specified node are ISD::UNDEF. |
108 | | bool allOperandsUndef(const SDNode *N); |
109 | | |
110 | | } // end namespace ISD |
111 | | |
112 | | //===----------------------------------------------------------------------===// |
113 | | /// Unlike LLVM values, Selection DAG nodes may return multiple |
114 | | /// values as the result of a computation. Many nodes return multiple values, |
115 | | /// from loads (which define a token and a return value) to ADDC (which returns |
116 | | /// a result and a carry value), to calls (which may return an arbitrary number |
117 | | /// of values). |
118 | | /// |
119 | | /// As such, each use of a SelectionDAG computation must indicate the node that |
120 | | /// computes it as well as which return value to use from that node. This pair |
121 | | /// of information is represented with the SDValue value type. |
122 | | /// |
123 | | class SDValue { |
124 | | friend struct DenseMapInfo<SDValue>; |
125 | | |
126 | | SDNode *Node = nullptr; // The node defining the value we are using. |
127 | | unsigned ResNo = 0; // Which return value of the node we are using. |
128 | | |
129 | | public: |
130 | 1.37G | SDValue() = default; |
131 | | SDValue(SDNode *node, unsigned resno); |
132 | | |
133 | | /// get the index which selects a specific result in the SDNode |
134 | 610M | unsigned getResNo() const { return ResNo; } |
135 | | |
136 | | /// get the SDNode which holds the desired result |
137 | 5.91G | SDNode *getNode() const { return Node; } |
138 | | |
139 | | /// set the SDNode |
140 | 23.4M | void setNode(SDNode *N) { Node = N; } |
141 | | |
142 | 1.13G | inline SDNode *operator->() const { return Node; } |
143 | | |
144 | 1.35G | bool operator==(const SDValue &O) const { |
145 | 993M | return Node == O.Node && ResNo == O.ResNo; |
146 | 1.35G | } |
147 | 54.6M | bool operator!=(const SDValue &O) const { |
148 | 54.6M | return !operator==(O); |
149 | 54.6M | } |
150 | 265k | bool operator<(const SDValue &O) const { |
151 | 265k | return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo); |
152 | 265k | } |
153 | 158M | explicit operator bool() const { |
154 | 158M | return Node != nullptr; |
155 | 158M | } |
156 | | |
157 | 41.6M | SDValue getValue(unsigned R) const { |
158 | 41.6M | return SDValue(Node, R); |
159 | 41.6M | } |
160 | | |
161 | | /// Return true if this node is an operand of N. |
162 | | bool isOperandOf(const SDNode *N) const; |
163 | | |
164 | | /// Return the ValueType of the referenced return value. |
165 | | inline EVT getValueType() const; |
166 | | |
167 | | /// Return the simple ValueType of the referenced return value. |
168 | 23.5M | MVT getSimpleValueType() const { |
169 | 23.5M | return getValueType().getSimpleVT(); |
170 | 23.5M | } |
171 | | |
172 | | /// Returns the size of the value in bits. |
173 | 6.00M | unsigned getValueSizeInBits() const { |
174 | 6.00M | return getValueType().getSizeInBits(); |
175 | 6.00M | } |
176 | | |
177 | 130M | unsigned getScalarValueSizeInBits() const { |
178 | 130M | return getValueType().getScalarType().getSizeInBits(); |
179 | 130M | } |
180 | | |
181 | | // Forwarding methods - These forward to the corresponding methods in SDNode. |
182 | | inline unsigned getOpcode() const; |
183 | | inline unsigned getNumOperands() const; |
184 | | inline const SDValue &getOperand(unsigned i) const; |
185 | | inline uint64_t getConstantOperandVal(unsigned i) const; |
186 | | inline bool isTargetMemoryOpcode() const; |
187 | | inline bool isTargetOpcode() const; |
188 | | inline bool isMachineOpcode() const; |
189 | | inline bool isUndef() const; |
190 | | inline unsigned getMachineOpcode() const; |
191 | | inline const DebugLoc &getDebugLoc() const; |
192 | | inline void dump() const; |
193 | | inline void dumpr() const; |
194 | | |
195 | | /// Return true if this operand (which must be a chain) reaches the |
196 | | /// specified operand without crossing any side-effecting instructions. |
197 | | /// In practice, this looks through token factors and non-volatile loads. |
198 | | /// In order to remain efficient, this only |
199 | | /// looks a couple of nodes in, it does not do an exhaustive search. |
200 | | bool reachesChainWithoutSideEffects(SDValue Dest, |
201 | | unsigned Depth = 2) const; |
202 | | |
203 | | /// Return true if there are no nodes using value ResNo of Node. |
204 | | inline bool use_empty() const; |
205 | | |
206 | | /// Return true if there is exactly one node using value ResNo of Node. |
207 | | inline bool hasOneUse() const; |
208 | | }; |
209 | | |
210 | | template<> struct DenseMapInfo<SDValue> { |
211 | 194M | static inline SDValue getEmptyKey() { |
212 | 194M | SDValue V; |
213 | 194M | V.ResNo = -1U; |
214 | 194M | return V; |
215 | 194M | } |
216 | | |
217 | 129M | static inline SDValue getTombstoneKey() { |
218 | 129M | SDValue V; |
219 | 129M | V.ResNo = -2U; |
220 | 129M | return V; |
221 | 129M | } |
222 | | |
223 | 91.4M | static unsigned getHashValue(const SDValue &Val) { |
224 | 91.4M | return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^ |
225 | 91.4M | (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo(); |
226 | 91.4M | } |
227 | | |
228 | 999M | static bool isEqual(const SDValue &LHS, const SDValue &RHS) { |
229 | 999M | return LHS == RHS; |
230 | 999M | } |
231 | | }; |
232 | | template <> struct isPodLike<SDValue> { static const bool value = true; }; |
233 | | |
234 | | /// Allow casting operators to work directly on |
235 | | /// SDValues as if they were SDNode*'s. |
236 | | template<> struct simplify_type<SDValue> { |
237 | | using SimpleType = SDNode *; |
238 | | |
239 | 183M | static SimpleType getSimplifiedValue(SDValue &Val) { |
240 | 183M | return Val.getNode(); |
241 | 183M | } |
242 | | }; |
243 | | template<> struct simplify_type<const SDValue> { |
244 | | using SimpleType = /*const*/ SDNode *; |
245 | | |
246 | 1.11G | static SimpleType getSimplifiedValue(const SDValue &Val) { |
247 | 1.11G | return Val.getNode(); |
248 | 1.11G | } |
249 | | }; |
250 | | |
251 | | /// Represents a use of a SDNode. This class holds an SDValue, |
252 | | /// which records the SDNode being used and the result number, a |
253 | | /// pointer to the SDNode using the value, and Next and Prev pointers, |
254 | | /// which link together all the uses of an SDNode. |
255 | | /// |
256 | | class SDUse { |
257 | | /// Val - The value being used. |
258 | | SDValue Val; |
259 | | /// User - The user of this value. |
260 | | SDNode *User = nullptr; |
261 | | /// Prev, Next - Pointers to the uses list of the SDNode referred by |
262 | | /// this operand. |
263 | | SDUse **Prev = nullptr; |
264 | | SDUse *Next = nullptr; |
265 | | |
266 | | public: |
267 | 30.4M | SDUse() = default; |
268 | | SDUse(const SDUse &U) = delete; |
269 | | SDUse &operator=(const SDUse &) = delete; |
270 | | |
271 | | /// Normally SDUse will just implicitly convert to an SDValue that it holds. |
272 | 2.39G | operator const SDValue&() const { return Val; } |
273 | | |
274 | | /// If implicit conversion to SDValue doesn't work, the get() method returns |
275 | | /// the SDValue. |
276 | 783M | const SDValue &get() const { return Val; } |
277 | | |
278 | | /// This returns the SDNode that contains this Use. |
279 | 545M | SDNode *getUser() { return User; } |
280 | | |
281 | | /// Get the next SDUse in the use list. |
282 | 656M | SDUse *getNext() const { return Next; } |
283 | | |
284 | | /// Convenience function for get().getNode(). |
285 | 217M | SDNode *getNode() const { return Val.getNode(); } |
286 | | /// Convenience function for get().getResNo(). |
287 | 272M | unsigned getResNo() const { return Val.getResNo(); } |
288 | | /// Convenience function for get().getValueType(). |
289 | 24.9M | EVT getValueType() const { return Val.getValueType(); } |
290 | | |
291 | | /// Convenience function for get().operator== |
292 | 858 | bool operator==(const SDValue &V) const { |
293 | 858 | return Val == V; |
294 | 858 | } |
295 | | |
296 | | /// Convenience function for get().operator!= |
297 | 2.65M | bool operator!=(const SDValue &V) const { |
298 | 2.65M | return Val != V; |
299 | 2.65M | } |
300 | | |
301 | | /// Convenience function for get().operator< |
302 | 0 | bool operator<(const SDValue &V) const { |
303 | 0 | return Val < V; |
304 | 0 | } |
305 | | |
306 | | private: |
307 | | friend class SelectionDAG; |
308 | | friend class SDNode; |
309 | | // TODO: unfriend HandleSDNode once we fix its operand handling. |
310 | | friend class HandleSDNode; |
311 | | |
312 | 270M | void setUser(SDNode *p) { User = p; } |
313 | | |
314 | | /// Remove this use from its existing use list, assign it the |
315 | | /// given value, and add it to the new value's node's use list. |
316 | | inline void set(const SDValue &V); |
317 | | /// Like set, but only supports initializing a newly-allocated |
318 | | /// SDUse with a non-null value. |
319 | | inline void setInitial(const SDValue &V); |
320 | | /// Like set, but only sets the Node portion of the value, |
321 | | /// leaving the ResNo portion unmodified. |
322 | | inline void setNode(SDNode *N); |
323 | | |
324 | 297M | void addToList(SDUse **List) { |
325 | 297M | Next = *List; |
326 | 297M | if (Next297M ) Next->Prev = &Next148M ; |
327 | 297M | Prev = List; |
328 | 297M | *List = this; |
329 | 297M | } |
330 | | |
331 | 184M | void removeFromList() { |
332 | 184M | *Prev = Next; |
333 | 184M | if (Next184M ) Next->Prev = Prev52.7M ; |
334 | 184M | } |
335 | | }; |
336 | | |
337 | | /// simplify_type specializations - Allow casting operators to work directly on |
338 | | /// SDValues as if they were SDNode*'s. |
339 | | template<> struct simplify_type<SDUse> { |
340 | | using SimpleType = SDNode *; |
341 | | |
342 | 0 | static SimpleType getSimplifiedValue(SDUse &Val) { |
343 | 0 | return Val.getNode(); |
344 | 0 | } |
345 | | }; |
346 | | |
347 | | /// These are IR-level optimization flags that may be propagated to SDNodes. |
348 | | /// TODO: This data structure should be shared by the IR optimizer and the |
349 | | /// the backend. |
350 | | struct SDNodeFlags { |
351 | | private: |
352 | | // This bit is used to determine if the flags are in a defined state. |
353 | | // Flag bits can only be masked out during intersection if the masking flags |
354 | | // are defined. |
355 | | bool AnyDefined : 1; |
356 | | |
357 | | bool NoUnsignedWrap : 1; |
358 | | bool NoSignedWrap : 1; |
359 | | bool Exact : 1; |
360 | | bool UnsafeAlgebra : 1; |
361 | | bool NoNaNs : 1; |
362 | | bool NoInfs : 1; |
363 | | bool NoSignedZeros : 1; |
364 | | bool AllowReciprocal : 1; |
365 | | bool VectorReduction : 1; |
366 | | bool AllowContract : 1; |
367 | | |
368 | | public: |
369 | | /// Default constructor turns off all optimization flags. |
370 | | SDNodeFlags() |
371 | | : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false), |
372 | | Exact(false), UnsafeAlgebra(false), NoNaNs(false), NoInfs(false), |
373 | | NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false), |
374 | 203M | AllowContract(false) {} |
375 | | |
376 | | /// Sets the state of the flags to the defined state. |
377 | 20.6M | void setDefined() { AnyDefined = true; } |
378 | | /// Returns true if the flags are in a defined state. |
379 | 1.58M | bool isDefined() const { return AnyDefined; } |
380 | | |
381 | | // These are mutators for each flag. |
382 | 7.13M | void setNoUnsignedWrap(bool b) { |
383 | 7.13M | setDefined(); |
384 | 7.13M | NoUnsignedWrap = b; |
385 | 7.13M | } |
386 | 1.66M | void setNoSignedWrap(bool b) { |
387 | 1.66M | setDefined(); |
388 | 1.66M | NoSignedWrap = b; |
389 | 1.66M | } |
390 | 1.66M | void setExact(bool b) { |
391 | 1.66M | setDefined(); |
392 | 1.66M | Exact = b; |
393 | 1.66M | } |
394 | 1.45M | void setUnsafeAlgebra(bool b) { |
395 | 1.45M | setDefined(); |
396 | 1.45M | UnsafeAlgebra = b; |
397 | 1.45M | } |
398 | 1.45M | void setNoNaNs(bool b) { |
399 | 1.45M | setDefined(); |
400 | 1.45M | NoNaNs = b; |
401 | 1.45M | } |
402 | 1.45M | void setNoInfs(bool b) { |
403 | 1.45M | setDefined(); |
404 | 1.45M | NoInfs = b; |
405 | 1.45M | } |
406 | 1.45M | void setNoSignedZeros(bool b) { |
407 | 1.45M | setDefined(); |
408 | 1.45M | NoSignedZeros = b; |
409 | 1.45M | } |
410 | 1.45M | void setAllowReciprocal(bool b) { |
411 | 1.45M | setDefined(); |
412 | 1.45M | AllowReciprocal = b; |
413 | 1.45M | } |
414 | 1.45M | void setVectorReduction(bool b) { |
415 | 1.45M | setDefined(); |
416 | 1.45M | VectorReduction = b; |
417 | 1.45M | } |
418 | 1.45M | void setAllowContract(bool b) { |
419 | 1.45M | setDefined(); |
420 | 1.45M | AllowContract = b; |
421 | 1.45M | } |
422 | | |
423 | | // These are accessors for each flag. |
424 | 86.2k | bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } |
425 | 51.6k | bool hasNoSignedWrap() const { return NoSignedWrap; } |
426 | 667k | bool hasExact() const { return Exact; } |
427 | 141k | bool hasUnsafeAlgebra() const { return UnsafeAlgebra; } |
428 | 123k | bool hasNoNaNs() const { return NoNaNs; } |
429 | 0 | bool hasNoInfs() const { return NoInfs; } |
430 | 180k | bool hasNoSignedZeros() const { return NoSignedZeros; } |
431 | 120k | bool hasAllowReciprocal() const { return AllowReciprocal; } |
432 | 261k | bool hasVectorReduction() const { return VectorReduction; } |
433 | 141k | bool hasAllowContract() const { return AllowContract; } |
434 | | |
435 | | /// Clear any flags in this flag set that aren't also set in Flags. |
436 | | /// If the given Flags are undefined then don't do anything. |
437 | 1.58M | void intersectWith(const SDNodeFlags Flags) { |
438 | 1.58M | if (!Flags.isDefined()) |
439 | 1.21M | return; |
440 | 365k | NoUnsignedWrap &= Flags.NoUnsignedWrap; |
441 | 365k | NoSignedWrap &= Flags.NoSignedWrap; |
442 | 365k | Exact &= Flags.Exact; |
443 | 365k | UnsafeAlgebra &= Flags.UnsafeAlgebra; |
444 | 365k | NoNaNs &= Flags.NoNaNs; |
445 | 365k | NoInfs &= Flags.NoInfs; |
446 | 365k | NoSignedZeros &= Flags.NoSignedZeros; |
447 | 365k | AllowReciprocal &= Flags.AllowReciprocal; |
448 | 365k | VectorReduction &= Flags.VectorReduction; |
449 | 365k | AllowContract &= Flags.AllowContract; |
450 | 365k | } |
451 | | }; |
452 | | |
453 | | /// Represents one node in the SelectionDAG. |
454 | | /// |
455 | | class SDNode : public FoldingSetNode, public ilist_node<SDNode> { |
456 | | private: |
457 | | /// The operation that this node performs. |
458 | | int16_t NodeType; |
459 | | |
460 | | protected: |
461 | | // We define a set of mini-helper classes to help us interpret the bits in our |
462 | | // SubclassData. These are designed to fit within a uint16_t so they pack |
463 | | // with NodeType. |
464 | | |
465 | | class SDNodeBitfields { |
466 | | friend class SDNode; |
467 | | friend class MemIntrinsicSDNode; |
468 | | friend class MemSDNode; |
469 | | |
470 | | uint16_t HasDebugValue : 1; |
471 | | uint16_t IsMemIntrinsic : 1; |
472 | | }; |
473 | | enum { NumSDNodeBits = 2 }; |
474 | | |
475 | | class ConstantSDNodeBitfields { |
476 | | friend class ConstantSDNode; |
477 | | |
478 | | uint16_t : NumSDNodeBits; |
479 | | |
480 | | uint16_t IsOpaque : 1; |
481 | | }; |
482 | | |
483 | | class MemSDNodeBitfields { |
484 | | friend class MemSDNode; |
485 | | friend class MemIntrinsicSDNode; |
486 | | friend class AtomicSDNode; |
487 | | |
488 | | uint16_t : NumSDNodeBits; |
489 | | |
490 | | uint16_t IsVolatile : 1; |
491 | | uint16_t IsNonTemporal : 1; |
492 | | uint16_t IsDereferenceable : 1; |
493 | | uint16_t IsInvariant : 1; |
494 | | }; |
495 | | enum { NumMemSDNodeBits = NumSDNodeBits + 4 }; |
496 | | |
497 | | class LSBaseSDNodeBitfields { |
498 | | friend class LSBaseSDNode; |
499 | | |
500 | | uint16_t : NumMemSDNodeBits; |
501 | | |
502 | | uint16_t AddressingMode : 3; // enum ISD::MemIndexedMode |
503 | | }; |
504 | | enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 }; |
505 | | |
506 | | class LoadSDNodeBitfields { |
507 | | friend class LoadSDNode; |
508 | | friend class MaskedLoadSDNode; |
509 | | |
510 | | uint16_t : NumLSBaseSDNodeBits; |
511 | | |
512 | | uint16_t ExtTy : 2; // enum ISD::LoadExtType |
513 | | uint16_t IsExpanding : 1; |
514 | | }; |
515 | | |
516 | | class StoreSDNodeBitfields { |
517 | | friend class StoreSDNode; |
518 | | friend class MaskedStoreSDNode; |
519 | | |
520 | | uint16_t : NumLSBaseSDNodeBits; |
521 | | |
522 | | uint16_t IsTruncating : 1; |
523 | | uint16_t IsCompressing : 1; |
524 | | }; |
525 | | |
526 | | union { |
527 | | char RawSDNodeBits[sizeof(uint16_t)]; |
528 | | SDNodeBitfields SDNodeBits; |
529 | | ConstantSDNodeBitfields ConstantSDNodeBits; |
530 | | MemSDNodeBitfields MemSDNodeBits; |
531 | | LSBaseSDNodeBitfields LSBaseSDNodeBits; |
532 | | LoadSDNodeBitfields LoadSDNodeBits; |
533 | | StoreSDNodeBitfields StoreSDNodeBits; |
534 | | }; |
535 | | |
536 | | // RawSDNodeBits must cover the entirety of the union. This means that all of |
537 | | // the union's members must have size <= RawSDNodeBits. We write the RHS as |
538 | | // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter. |
539 | | static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide"); |
540 | | static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide"); |
541 | | static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide"); |
542 | | static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide"); |
543 | | static_assert(sizeof(LoadSDNodeBitfields) <= 4, "field too wide"); |
544 | | static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide"); |
545 | | |
546 | | private: |
547 | | friend class SelectionDAG; |
548 | | // TODO: unfriend HandleSDNode once we fix its operand handling. |
549 | | friend class HandleSDNode; |
550 | | |
551 | | /// Unique id per SDNode in the DAG. |
552 | | int NodeId = -1; |
553 | | |
554 | | /// The values that are used by this operation. |
555 | | SDUse *OperandList = nullptr; |
556 | | |
557 | | /// The types of the values this node defines. SDNode's may |
558 | | /// define multiple values simultaneously. |
559 | | const EVT *ValueList; |
560 | | |
561 | | /// List of uses for this SDNode. |
562 | | SDUse *UseList = nullptr; |
563 | | |
564 | | /// The number of entries in the Operand/Value list. |
565 | | unsigned short NumOperands = 0; |
566 | | unsigned short NumValues; |
567 | | |
568 | | // The ordering of the SDNodes. It roughly corresponds to the ordering of the |
569 | | // original LLVM instructions. |
570 | | // This is used for turning off scheduling, because we'll forgo |
571 | | // the normal scheduling algorithms and output the instructions according to |
572 | | // this ordering. |
573 | | unsigned IROrder; |
574 | | |
575 | | /// Source line information. |
576 | | DebugLoc debugLoc; |
577 | | |
578 | | /// Return a pointer to the specified value type. |
579 | | static const EVT *getValueTypeList(EVT VT); |
580 | | |
581 | | SDNodeFlags Flags; |
582 | | |
583 | | public: |
584 | | /// Unique and persistent id per SDNode in the DAG. |
585 | | /// Used for debug printing. |
586 | | uint16_t PersistentId; |
587 | | |
588 | | //===--------------------------------------------------------------------===// |
589 | | // Accessors |
590 | | // |
591 | | |
592 | | /// Return the SelectionDAG opcode value for this node. For |
593 | | /// pre-isel nodes (those for which isMachineOpcode returns false), these |
594 | | /// are the opcode values in the ISD and <target>ISD namespaces. For |
595 | | /// post-isel opcodes, see getMachineOpcode. |
596 | 10.3G | unsigned getOpcode() const { return (unsigned short)NodeType; } |
597 | | |
598 | | /// Test if this node has a target-specific opcode (in the |
599 | | /// \<target\>ISD namespace). |
600 | 872k | bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; } |
601 | | |
602 | | /// Test if this node has a target-specific |
603 | | /// memory-referencing opcode (in the \<target\>ISD namespace and |
604 | | /// greater than FIRST_TARGET_MEMORY_OPCODE). |
605 | 111M | bool isTargetMemoryOpcode() const { |
606 | 111M | return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE; |
607 | 111M | } |
608 | | |
609 | | /// Return true if the type of the node type undefined. |
610 | 104M | bool isUndef() const { return NodeType == ISD::UNDEF; } |
611 | | |
612 | | /// Test if this node is a memory intrinsic (with valid pointer information). |
613 | | /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for |
614 | | /// non-memory intrinsics (with chains) that are not really instances of |
615 | | /// MemSDNode. For such nodes, we need some extra state to determine the |
616 | | /// proper classof relationship. |
617 | 2.28M | bool isMemIntrinsic() const { |
618 | 2.28M | return (NodeType == ISD::INTRINSIC_W_CHAIN || |
619 | 2.16M | NodeType == ISD::INTRINSIC_VOID) && |
620 | 251k | SDNodeBits.IsMemIntrinsic; |
621 | 2.28M | } |
622 | | |
623 | | /// Test if this node is a strict floating point pseudo-op. |
624 | 60.5M | bool isStrictFPOpcode() { |
625 | 60.5M | switch (NodeType) { |
626 | 60.5M | default: |
627 | 60.5M | return false; |
628 | 40 | case ISD::STRICT_FADD: |
629 | 40 | case ISD::STRICT_FSUB: |
630 | 40 | case ISD::STRICT_FMUL: |
631 | 40 | case ISD::STRICT_FDIV: |
632 | 40 | case ISD::STRICT_FREM: |
633 | 40 | case ISD::STRICT_FMA: |
634 | 40 | case ISD::STRICT_FSQRT: |
635 | 40 | case ISD::STRICT_FPOW: |
636 | 40 | case ISD::STRICT_FPOWI: |
637 | 40 | case ISD::STRICT_FSIN: |
638 | 40 | case ISD::STRICT_FCOS: |
639 | 40 | case ISD::STRICT_FEXP: |
640 | 40 | case ISD::STRICT_FEXP2: |
641 | 40 | case ISD::STRICT_FLOG: |
642 | 40 | case ISD::STRICT_FLOG10: |
643 | 40 | case ISD::STRICT_FLOG2: |
644 | 40 | case ISD::STRICT_FRINT: |
645 | 40 | case ISD::STRICT_FNEARBYINT: |
646 | 40 | return true; |
647 | 60.5M | } |
648 | 60.5M | } |
649 | | |
650 | | /// Test if this node has a post-isel opcode, directly |
651 | | /// corresponding to a MachineInstr opcode. |
652 | 478M | bool isMachineOpcode() const { return NodeType < 0; } |
653 | | |
654 | | /// This may only be called if isMachineOpcode returns |
655 | | /// true. It returns the MachineInstr opcode value that the node's opcode |
656 | | /// corresponds to. |
657 | 213M | unsigned getMachineOpcode() const { |
658 | 213M | assert(isMachineOpcode() && "Not a MachineInstr opcode!"); |
659 | 213M | return ~NodeType; |
660 | 213M | } |
661 | | |
662 | 25.4M | bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; } |
663 | 350 | void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; } |
664 | | |
665 | | /// Return true if there are no uses of this node. |
666 | 1.07G | bool use_empty() const { return UseList == nullptr; } |
667 | | |
668 | | /// Return true if there is exactly one use of this node. |
669 | 83.3M | bool hasOneUse() const { |
670 | 83.3M | return !use_empty() && std::next(use_begin()) == use_end(); |
671 | 83.3M | } |
672 | | |
673 | | /// Return the number of uses of this node. This method takes |
674 | | /// time proportional to the number of uses. |
675 | 1.25k | size_t use_size() const { return std::distance(use_begin(), use_end()); } |
676 | | |
677 | | /// Return the unique node id. |
678 | 521M | int getNodeId() const { return NodeId; } |
679 | | |
680 | | /// Set unique node id. |
681 | 805M | void setNodeId(int Id) { NodeId = Id; } |
682 | | |
683 | | /// Return the node ordering. |
684 | 280M | unsigned getIROrder() const { return IROrder; } |
685 | | |
686 | | /// Set the node ordering. |
687 | 220k | void setIROrder(unsigned Order) { IROrder = Order; } |
688 | | |
689 | | /// Return the source location info. |
690 | 210M | const DebugLoc &getDebugLoc() const { return debugLoc; } |
691 | | |
692 | | /// Set source location info. Try to avoid this, putting |
693 | | /// it in the constructor is preferable. |
694 | 395k | void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); } |
695 | | |
696 | | /// This class provides iterator support for SDUse |
697 | | /// operands that use a specific SDNode. |
698 | | class use_iterator |
699 | | : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> { |
700 | | friend class SDNode; |
701 | | |
702 | | SDUse *Op = nullptr; |
703 | | |
704 | 945M | explicit use_iterator(SDUse *op) : Op(op) {} |
705 | | |
706 | | public: |
707 | | using reference = std::iterator<std::forward_iterator_tag, |
708 | | SDUse, ptrdiff_t>::reference; |
709 | | using pointer = std::iterator<std::forward_iterator_tag, |
710 | | SDUse, ptrdiff_t>::pointer; |
711 | | |
712 | | use_iterator() = default; |
713 | 235M | use_iterator(const use_iterator &I) : Op(I.Op) {} |
714 | | |
715 | 1.07G | bool operator==(const use_iterator &x) const { |
716 | 1.07G | return Op == x.Op; |
717 | 1.07G | } |
718 | 990M | bool operator!=(const use_iterator &x) const { |
719 | 990M | return !operator==(x); |
720 | 990M | } |
721 | | |
722 | | /// Return true if this iterator is at the end of uses list. |
723 | 0 | bool atEnd() const { return Op == nullptr; } |
724 | | |
725 | | // Iterator traversal: forward iteration only. |
726 | 656M | use_iterator &operator++() { // Preincrement |
727 | 656M | assert(Op && "Cannot increment end iterator!"); |
728 | 656M | Op = Op->getNext(); |
729 | 656M | return *this; |
730 | 656M | } |
731 | | |
732 | 3.28k | use_iterator operator++(int) { // Postincrement |
733 | 3.28k | use_iterator tmp = *this; ++*this; return tmp; |
734 | 3.28k | } |
735 | | |
736 | | /// Retrieve a pointer to the current user node. |
737 | 504M | SDNode *operator*() const { |
738 | 504M | assert(Op && "Cannot dereference end iterator!"); |
739 | 504M | return Op->getUser(); |
740 | 504M | } |
741 | | |
742 | 870k | SDNode *operator->() const { return operator*(); } |
743 | | |
744 | 184M | SDUse &getUse() const { return *Op; } |
745 | | |
746 | | /// Retrieve the operand # of this use in its user. |
747 | 39.5M | unsigned getOperandNo() const { |
748 | 39.5M | assert(Op && "Cannot dereference end iterator!"); |
749 | 39.5M | return (unsigned)(Op - Op->getUser()->OperandList); |
750 | 39.5M | } |
751 | | }; |
752 | | |
753 | | /// Provide iteration support to walk over all uses of an SDNode. |
754 | 473M | use_iterator use_begin() const { |
755 | 473M | return use_iterator(UseList); |
756 | 473M | } |
757 | | |
758 | 472M | static use_iterator use_end() { return use_iterator(nullptr); } |
759 | | |
760 | 22.6M | inline iterator_range<use_iterator> uses() { |
761 | 22.6M | return make_range(use_begin(), use_end()); |
762 | 22.6M | } |
763 | 2.66M | inline iterator_range<use_iterator> uses() const { |
764 | 2.66M | return make_range(use_begin(), use_end()); |
765 | 2.66M | } |
766 | | |
767 | | /// Return true if there are exactly NUSES uses of the indicated value. |
768 | | /// This method ignores uses of other values defined by this operation. |
769 | | bool hasNUsesOfValue(unsigned NUses, unsigned Value) const; |
770 | | |
771 | | /// Return true if there are any use of the indicated value. |
772 | | /// This method ignores uses of other values defined by this operation. |
773 | | bool hasAnyUseOfValue(unsigned Value) const; |
774 | | |
775 | | /// Return true if this node is the only use of N. |
776 | | bool isOnlyUserOf(const SDNode *N) const; |
777 | | |
778 | | /// Return true if this node is an operand of N. |
779 | | bool isOperandOf(const SDNode *N) const; |
780 | | |
781 | | /// Return true if this node is a predecessor of N. |
782 | | /// NOTE: Implemented on top of hasPredecessor and every bit as |
783 | | /// expensive. Use carefully. |
784 | 257k | bool isPredecessorOf(const SDNode *N) const { |
785 | 257k | return N->hasPredecessor(this); |
786 | 257k | } |
787 | | |
788 | | /// Return true if N is a predecessor of this node. |
789 | | /// N is either an operand of this node, or can be reached by recursively |
790 | | /// traversing up the operands. |
791 | | /// NOTE: This is an expensive method. Use it carefully. |
792 | | bool hasPredecessor(const SDNode *N) const; |
793 | | |
794 | | /// Returns true if N is a predecessor of any node in Worklist. This |
795 | | /// helper keeps Visited and Worklist sets externally to allow unions |
796 | | /// searches to be performed in parallel, caching of results across |
797 | | /// queries and incremental addition to Worklist. Stops early if N is |
798 | | /// found but will resume. Remember to clear Visited and Worklists |
799 | | /// if DAG changes. |
800 | | static bool hasPredecessorHelper(const SDNode *N, |
801 | | SmallPtrSetImpl<const SDNode *> &Visited, |
802 | | SmallVectorImpl<const SDNode *> &Worklist, |
803 | 19.9M | unsigned int MaxSteps = 0) { |
804 | 19.9M | if (Visited.count(N)) |
805 | 90.2k | return true; |
806 | 142M | while (19.8M !Worklist.empty()142M ) { |
807 | 122M | const SDNode *M = Worklist.pop_back_val(); |
808 | 122M | bool Found = false; |
809 | 160M | for (const SDValue &OpV : M->op_values()) { |
810 | 160M | SDNode *Op = OpV.getNode(); |
811 | 160M | if (Visited.insert(Op).second) |
812 | 65.0M | Worklist.push_back(Op); |
813 | 160M | if (Op == N) |
814 | 143k | Found = true; |
815 | 160M | } |
816 | 122M | if (Found) |
817 | 137k | return true; |
818 | 122M | if (122M MaxSteps != 0 && 122M Visited.size() >= MaxSteps113M ) |
819 | 0 | return false; |
820 | 122M | } |
821 | 19.7M | return false; |
822 | 19.9M | } |
823 | | |
824 | | /// Return true if all the users of N are contained in Nodes. |
825 | | /// NOTE: Requires at least one match, but doesn't require them all. |
826 | | static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N); |
827 | | |
828 | | /// Return the number of values used by this operation. |
829 | 1.08G | unsigned getNumOperands() const { return NumOperands; } |
830 | | |
831 | | /// Helper method returns the integer value of a ConstantSDNode operand. |
832 | | inline uint64_t getConstantOperandVal(unsigned Num) const; |
833 | | |
834 | 2.34G | const SDValue &getOperand(unsigned Num) const { |
835 | 2.34G | assert(Num < NumOperands && "Invalid child # of SDNode!"); |
836 | 2.34G | return OperandList[Num]; |
837 | 2.34G | } |
838 | | |
839 | | using op_iterator = SDUse *; |
840 | | |
841 | 667M | op_iterator op_begin() const { return OperandList; } |
842 | 661M | op_iterator op_end() const { return OperandList+NumOperands; } |
843 | 111M | ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); } |
844 | | |
845 | | /// Iterator for directly iterating over the operand SDValue's. |
846 | | struct value_op_iterator |
847 | | : iterator_adaptor_base<value_op_iterator, op_iterator, |
848 | | std::random_access_iterator_tag, SDValue, |
849 | | ptrdiff_t, value_op_iterator *, |
850 | | value_op_iterator *> { |
851 | | explicit value_op_iterator(SDUse *U = nullptr) |
852 | 904M | : iterator_adaptor_base(U) {} |
853 | | |
854 | 768M | const SDValue &operator*() const { return I->get(); } |
855 | | }; |
856 | | |
857 | 452M | iterator_range<value_op_iterator> op_values() const { |
858 | 452M | return make_range(value_op_iterator(op_begin()), |
859 | 452M | value_op_iterator(op_end())); |
860 | 452M | } |
861 | | |
862 | 111M | SDVTList getVTList() const { |
863 | 111M | SDVTList X = { ValueList, NumValues }; |
864 | 111M | return X; |
865 | 111M | } |
866 | | |
867 | | /// If this node has a glue operand, return the node |
868 | | /// to which the glue operand points. Otherwise return NULL. |
869 | 156M | SDNode *getGluedNode() const { |
870 | 156M | if (getNumOperands() != 0 && |
871 | 155M | getOperand(getNumOperands()-1).getValueType() == MVT::Glue) |
872 | 33.2M | return getOperand(getNumOperands()-1).getNode(); |
873 | 122M | return nullptr; |
874 | 156M | } |
875 | | |
876 | | /// If this node has a glue value with a user, return |
877 | | /// the user (there is at most one). Otherwise return NULL. |
878 | 8.58M | SDNode *getGluedUser() const { |
879 | 20.1M | for (use_iterator UI = use_begin(), UE = use_end(); UI != UE20.1M ; ++UI11.5M ) |
880 | 15.3M | if (15.3M UI.getUse().get().getValueType() == MVT::Glue15.3M ) |
881 | 3.78M | return *UI; |
882 | 4.80M | return nullptr; |
883 | 8.58M | } |
884 | | |
885 | 4.64M | const SDNodeFlags getFlags() const { return Flags; } |
886 | 19.4M | void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; } |
887 | | |
888 | | /// Clear any flags in this node that aren't also set in Flags. |
889 | | /// If Flags is not in a defined state then this has no effect. |
890 | | void intersectFlagsWith(const SDNodeFlags Flags); |
891 | | |
892 | | /// Return the number of values defined/returned by this operator. |
893 | 322M | unsigned getNumValues() const { return NumValues; } |
894 | | |
895 | | /// Return the type of a specified result. |
896 | 1.62G | EVT getValueType(unsigned ResNo) const { |
897 | 1.62G | assert(ResNo < NumValues && "Illegal result number!"); |
898 | 1.62G | return ValueList[ResNo]; |
899 | 1.62G | } |
900 | | |
901 | | /// Return the type of a specified result as a simple type. |
902 | 50.5M | MVT getSimpleValueType(unsigned ResNo) const { |
903 | 50.5M | return getValueType(ResNo).getSimpleVT(); |
904 | 50.5M | } |
905 | | |
906 | | /// Returns MVT::getSizeInBits(getValueType(ResNo)). |
907 | 2.23M | unsigned getValueSizeInBits(unsigned ResNo) const { |
908 | 2.23M | return getValueType(ResNo).getSizeInBits(); |
909 | 2.23M | } |
910 | | |
911 | | using value_iterator = const EVT *; |
912 | | |
913 | 70.7M | value_iterator value_begin() const { return ValueList; } |
914 | 70.7M | value_iterator value_end() const { return ValueList+NumValues; } |
915 | | |
916 | | /// Return the opcode of this operation for printing. |
917 | | std::string getOperationName(const SelectionDAG *G = nullptr) const; |
918 | | static const char* getIndexedModeName(ISD::MemIndexedMode AM); |
919 | | void print_types(raw_ostream &OS, const SelectionDAG *G) const; |
920 | | void print_details(raw_ostream &OS, const SelectionDAG *G) const; |
921 | | void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const; |
922 | | void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const; |
923 | | |
924 | | /// Print a SelectionDAG node and all children down to |
925 | | /// the leaves. The given SelectionDAG allows target-specific nodes |
926 | | /// to be printed in human-readable form. Unlike printr, this will |
927 | | /// print the whole DAG, including children that appear multiple |
928 | | /// times. |
929 | | /// |
930 | | void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const; |
931 | | |
932 | | /// Print a SelectionDAG node and children up to |
933 | | /// depth "depth." The given SelectionDAG allows target-specific |
934 | | /// nodes to be printed in human-readable form. Unlike printr, this |
935 | | /// will print children that appear multiple times wherever they are |
936 | | /// used. |
937 | | /// |
938 | | void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr, |
939 | | unsigned depth = 100) const; |
940 | | |
941 | | /// Dump this node, for debugging. |
942 | | void dump() const; |
943 | | |
944 | | /// Dump (recursively) this node and its use-def subgraph. |
945 | | void dumpr() const; |
946 | | |
947 | | /// Dump this node, for debugging. |
948 | | /// The given SelectionDAG allows target-specific nodes to be printed |
949 | | /// in human-readable form. |
950 | | void dump(const SelectionDAG *G) const; |
951 | | |
952 | | /// Dump (recursively) this node and its use-def subgraph. |
953 | | /// The given SelectionDAG allows target-specific nodes to be printed |
954 | | /// in human-readable form. |
955 | | void dumpr(const SelectionDAG *G) const; |
956 | | |
957 | | /// printrFull to dbgs(). The given SelectionDAG allows |
958 | | /// target-specific nodes to be printed in human-readable form. |
959 | | /// Unlike dumpr, this will print the whole DAG, including children |
960 | | /// that appear multiple times. |
961 | | void dumprFull(const SelectionDAG *G = nullptr) const; |
962 | | |
963 | | /// printrWithDepth to dbgs(). The given |
964 | | /// SelectionDAG allows target-specific nodes to be printed in |
965 | | /// human-readable form. Unlike dumpr, this will print children |
966 | | /// that appear multiple times wherever they are used. |
967 | | /// |
968 | | void dumprWithDepth(const SelectionDAG *G = nullptr, |
969 | | unsigned depth = 100) const; |
970 | | |
971 | | /// Gather unique data for the node. |
972 | | void Profile(FoldingSetNodeID &ID) const; |
973 | | |
974 | | /// This method should only be used by the SDUse class. |
975 | 297M | void addUse(SDUse &U) { U.addToList(&UseList); } |
976 | | |
977 | | protected: |
978 | 83.0M | static SDVTList getSDVTList(EVT VT) { |
979 | 83.0M | SDVTList Ret = { getValueTypeList(VT), 1 }; |
980 | 83.0M | return Ret; |
981 | 83.0M | } |
982 | | |
983 | | /// Create an SDNode. |
984 | | /// |
985 | | /// SDNodes are created without any operands, and never own the operand |
986 | | /// storage. To add operands, see SelectionDAG::createOperands. |
987 | | SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) |
988 | | : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs), |
989 | 153M | IROrder(Order), debugLoc(std::move(dl)) { |
990 | 153M | memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits)); |
991 | 153M | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); |
992 | 153M | assert(NumValues == VTs.NumVTs && |
993 | 153M | "NumValues wasn't wide enough for its operands!"); |
994 | 153M | } |
995 | | |
996 | | /// Release the operands and set this node to have zero operands. |
997 | | void DropOperands(); |
998 | | }; |
999 | | |
1000 | | /// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed |
1001 | | /// into SDNode creation functions. |
1002 | | /// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted |
1003 | | /// from the original Instruction, and IROrder is the ordinal position of |
1004 | | /// the instruction. |
1005 | | /// When an SDNode is created after the DAG is being built, both DebugLoc and |
1006 | | /// the IROrder are propagated from the original SDNode. |
1007 | | /// So SDLoc class provides two constructors besides the default one, one to |
1008 | | /// be used by the DAGBuilder, the other to be used by others. |
1009 | | class SDLoc { |
1010 | | private: |
1011 | | DebugLoc DL; |
1012 | | int IROrder = 0; |
1013 | | |
1014 | | public: |
1015 | 11.3M | SDLoc() = default; |
1016 | 173M | SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {} |
1017 | 69.5M | SDLoc(const SDValue V) : SDLoc(V.getNode()) {} Unexecuted instantiation: llvm::SDLoc::SDLoc(llvm::SDValue) llvm::SDLoc::SDLoc(llvm::SDValue) Line | Count | Source | 1017 | 69.5M | SDLoc(const SDValue V) : SDLoc(V.getNode()) {} |
|
1018 | 37.6M | SDLoc(const Instruction *I, int Order) : IROrder(Order) { |
1019 | 37.6M | assert(Order >= 0 && "bad IROrder"); |
1020 | 37.6M | if (I) |
1021 | 34.5M | DL = I->getDebugLoc(); |
1022 | 37.6M | } |
1023 | | |
1024 | 110M | unsigned getIROrder() const { return IROrder; } |
1025 | 98.1M | const DebugLoc &getDebugLoc() const { return DL; } |
1026 | | }; |
1027 | | |
1028 | | // Define inline functions from the SDValue class. |
1029 | | |
1030 | | inline SDValue::SDValue(SDNode *node, unsigned resno) |
1031 | 431M | : Node(node), ResNo(resno) { |
1032 | 431M | // Explicitly check for !ResNo to avoid use-after-free, because there are |
1033 | 431M | // callers that use SDValue(N, 0) with a deleted N to indicate successful |
1034 | 431M | // combines. |
1035 | 431M | assert((!Node || !ResNo || ResNo < Node->getNumValues()) && |
1036 | 431M | "Invalid result number for the given node!"); |
1037 | 431M | assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."); |
1038 | 431M | } |
1039 | | |
1040 | 1.06G | inline unsigned SDValue::getOpcode() const { |
1041 | 1.06G | return Node->getOpcode(); |
1042 | 1.06G | } |
1043 | | |
1044 | 1.04G | inline EVT SDValue::getValueType() const { |
1045 | 1.04G | return Node->getValueType(ResNo); |
1046 | 1.04G | } |
1047 | | |
1048 | 106M | inline unsigned SDValue::getNumOperands() const { |
1049 | 106M | return Node->getNumOperands(); |
1050 | 106M | } |
1051 | | |
1052 | 337M | inline const SDValue &SDValue::getOperand(unsigned i) const { |
1053 | 337M | return Node->getOperand(i); |
1054 | 337M | } |
1055 | | |
1056 | 1.65M | inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { |
1057 | 1.65M | return Node->getConstantOperandVal(i); |
1058 | 1.65M | } |
1059 | | |
1060 | 0 | inline bool SDValue::isTargetOpcode() const { |
1061 | 0 | return Node->isTargetOpcode(); |
1062 | 0 | } |
1063 | | |
1064 | 0 | inline bool SDValue::isTargetMemoryOpcode() const { |
1065 | 0 | return Node->isTargetMemoryOpcode(); |
1066 | 0 | } |
1067 | | |
1068 | 68.9M | inline bool SDValue::isMachineOpcode() const { |
1069 | 68.9M | return Node->isMachineOpcode(); |
1070 | 68.9M | } |
1071 | | |
1072 | 13.1M | inline unsigned SDValue::getMachineOpcode() const { |
1073 | 13.1M | return Node->getMachineOpcode(); |
1074 | 13.1M | } |
1075 | | |
1076 | 104M | inline bool SDValue::isUndef() const { |
1077 | 104M | return Node->isUndef(); |
1078 | 104M | } |
1079 | | |
1080 | 1.36M | inline bool SDValue::use_empty() const { |
1081 | 1.36M | return !Node->hasAnyUseOfValue(ResNo); |
1082 | 1.36M | } |
1083 | | |
1084 | 36.5M | inline bool SDValue::hasOneUse() const { |
1085 | 36.5M | return Node->hasNUsesOfValue(1, ResNo); |
1086 | 36.5M | } |
1087 | | |
1088 | 205k | inline const DebugLoc &SDValue::getDebugLoc() const { |
1089 | 205k | return Node->getDebugLoc(); |
1090 | 205k | } |
1091 | | |
1092 | 0 | inline void SDValue::dump() const { |
1093 | 0 | return Node->dump(); |
1094 | 0 | } |
1095 | | |
1096 | 0 | inline void SDValue::dumpr() const { |
1097 | 0 | return Node->dumpr(); |
1098 | 0 | } |
1099 | | |
1100 | | // Define inline functions from the SDUse class. |
1101 | | |
1102 | 178M | inline void SDUse::set(const SDValue &V) { |
1103 | 178M | if (Val.getNode()178M ) removeFromList()178M ; |
1104 | 178M | Val = V; |
1105 | 178M | if (V.getNode()178M ) V.getNode()->addUse(*this)20.0M ; |
1106 | 178M | } |
1107 | | |
1108 | 270M | inline void SDUse::setInitial(const SDValue &V) { |
1109 | 270M | Val = V; |
1110 | 270M | V.getNode()->addUse(*this); |
1111 | 270M | } |
1112 | | |
1113 | 6.41M | inline void SDUse::setNode(SDNode *N) { |
1114 | 6.41M | if (Val.getNode()6.41M ) removeFromList()6.41M ; |
1115 | 6.41M | Val.setNode(N); |
1116 | 6.41M | if (N6.41M ) N->addUse(*this)6.41M ; |
1117 | 6.41M | } |
1118 | | |
1119 | | /// This class is used to form a handle around another node that |
1120 | | /// is persistent and is updated across invocations of replaceAllUsesWith on its |
1121 | | /// operand. This node should be directly created by end-users and not added to |
1122 | | /// the AllNodes list. |
1123 | | class HandleSDNode : public SDNode { |
1124 | | SDUse Op; |
1125 | | |
1126 | | public: |
1127 | | explicit HandleSDNode(SDValue X) |
1128 | 30.4M | : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) { |
1129 | 30.4M | // HandleSDNodes are never inserted into the DAG, so they won't be |
1130 | 30.4M | // auto-numbered. Use ID 65535 as a sentinel. |
1131 | 30.4M | PersistentId = 0xffff; |
1132 | 30.4M | |
1133 | 30.4M | // Manually set up the operand list. This node type is special in that it's |
1134 | 30.4M | // always stack allocated and SelectionDAG does not manage its operands. |
1135 | 30.4M | // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not |
1136 | 30.4M | // be so special. |
1137 | 30.4M | Op.setUser(this); |
1138 | 30.4M | Op.setInitial(X); |
1139 | 30.4M | NumOperands = 1; |
1140 | 30.4M | OperandList = &Op; |
1141 | 30.4M | } |
1142 | | ~HandleSDNode(); |
1143 | | |
1144 | 28.9M | const SDValue &getValue() const { return Op; } |
1145 | | }; |
1146 | | |
1147 | | class AddrSpaceCastSDNode : public SDNode { |
1148 | | private: |
1149 | | unsigned SrcAddrSpace; |
1150 | | unsigned DestAddrSpace; |
1151 | | |
1152 | | public: |
1153 | | AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, |
1154 | | unsigned SrcAS, unsigned DestAS); |
1155 | | |
1156 | 435 | unsigned getSrcAddressSpace() const { return SrcAddrSpace; } |
1157 | 363 | unsigned getDestAddressSpace() const { return DestAddrSpace; } |
1158 | | |
1159 | 1.18k | static bool classof(const SDNode *N) { |
1160 | 1.18k | return N->getOpcode() == ISD::ADDRSPACECAST; |
1161 | 1.18k | } |
1162 | | }; |
1163 | | |
1164 | | /// This is an abstract virtual class for memory operations. |
1165 | | class MemSDNode : public SDNode { |
1166 | | private: |
1167 | | // VT of in-memory value. |
1168 | | EVT MemoryVT; |
1169 | | |
1170 | | protected: |
1171 | | /// Memory reference information. |
1172 | | MachineMemOperand *MMO; |
1173 | | |
1174 | | public: |
1175 | | MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, |
1176 | | EVT MemoryVT, MachineMemOperand *MMO); |
1177 | | |
1178 | 6.35k | bool readMem() const { return MMO->isLoad(); } |
1179 | 29.0k | bool writeMem() const { return MMO->isStore(); } |
1180 | | |
1181 | | /// Returns alignment and volatility of the memory access |
1182 | 11.8M | unsigned getOriginalAlignment() const { |
1183 | 11.8M | return MMO->getBaseAlignment(); |
1184 | 11.8M | } |
1185 | 37.0M | unsigned getAlignment() const { |
1186 | 37.0M | return MMO->getAlignment(); |
1187 | 37.0M | } |
1188 | | |
1189 | | /// Return the SubclassData value, without HasDebugValue. This contains an |
1190 | | /// encoding of the volatile flag, as well as bits used by subclasses. This |
1191 | | /// function should only be used to compute a FoldingSetNodeID value. |
1192 | | /// The HasDebugValue bit is masked out because CSE map needs to match |
1193 | | /// nodes with debug info with nodes without debug info. |
1194 | 15.7M | unsigned getRawSubclassData() const { |
1195 | 15.7M | uint16_t Data; |
1196 | 15.7M | union { |
1197 | 15.7M | char RawSDNodeBits[sizeof(uint16_t)]; |
1198 | 15.7M | SDNodeBitfields SDNodeBits; |
1199 | 15.7M | }; |
1200 | 15.7M | memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits)); |
1201 | 15.7M | SDNodeBits.HasDebugValue = 0; |
1202 | 15.7M | memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits)); |
1203 | 15.7M | return Data; |
1204 | 15.7M | } |
1205 | | |
1206 | 122M | bool isVolatile() const { return MemSDNodeBits.IsVolatile; } |
1207 | 972k | bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; } |
1208 | 21.2k | bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; } |
1209 | 69.5M | bool isInvariant() const { return MemSDNodeBits.IsInvariant; } |
1210 | | |
1211 | | // Returns the offset from the location of the access. |
1212 | 11.7M | int64_t getSrcValueOffset() const { return MMO->getOffset(); } |
1213 | | |
1214 | | /// Returns the AA info that describes the dereference. |
1215 | 8.09M | AAMDNodes getAAInfo() const { return MMO->getAAInfo(); } |
1216 | | |
1217 | | /// Returns the Ranges that describes the dereference. |
1218 | 8.33M | const MDNode *getRanges() const { return MMO->getRanges(); } |
1219 | | |
1220 | | /// Returns the synchronization scope ID for this memory operation. |
1221 | 0 | SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); } |
1222 | | |
1223 | | /// Return the atomic ordering requirements for this memory operation. For |
1224 | | /// cmpxchg atomic operations, return the atomic ordering requirements when |
1225 | | /// store occurs. |
1226 | 203k | AtomicOrdering getOrdering() const { return MMO->getOrdering(); } |
1227 | | |
1228 | | /// Return the type of the in-memory value. |
1229 | 153M | EVT getMemoryVT() const { return MemoryVT; } |
1230 | | |
1231 | | /// Return a MachineMemOperand object describing the memory |
1232 | | /// reference performed by operation. |
1233 | 15.6M | MachineMemOperand *getMemOperand() const { return MMO; } |
1234 | | |
1235 | 21.3M | const MachinePointerInfo &getPointerInfo() const { |
1236 | 21.3M | return MMO->getPointerInfo(); |
1237 | 21.3M | } |
1238 | | |
1239 | | /// Return the address space for the associated pointer |
1240 | 11.5M | unsigned getAddressSpace() const { |
1241 | 11.5M | return getPointerInfo().getAddrSpace(); |
1242 | 11.5M | } |
1243 | | |
1244 | | /// Update this MemSDNode's MachineMemOperand information |
1245 | | /// to reflect the alignment of NewMMO, if it has a greater alignment. |
1246 | | /// This must only be used when the new alignment applies to all users of |
1247 | | /// this MachineMemOperand. |
1248 | 84.3k | void refineAlignment(const MachineMemOperand *NewMMO) { |
1249 | 84.3k | MMO->refineAlignment(NewMMO); |
1250 | 84.3k | } |
1251 | | |
1252 | 103M | const SDValue &getChain() const { return getOperand(0); } |
1253 | 153M | const SDValue &getBasePtr() const { |
1254 | 153M | return getOperand(getOpcode() == ISD::STORE ? 2134M : 118.2M ); |
1255 | 153M | } |
1256 | | |
1257 | | // Methods to support isa and dyn_cast |
1258 | 25.4M | static bool classof(const SDNode *N) { |
1259 | 25.4M | // For some targets, we lower some target intrinsics to a MemIntrinsicNode |
1260 | 25.4M | // with either an intrinsic or a target opcode. |
1261 | 25.4M | return N->getOpcode() == ISD::LOAD || |
1262 | 19.1M | N->getOpcode() == ISD::STORE || |
1263 | 2.29M | N->getOpcode() == ISD::PREFETCH || |
1264 | 2.29M | N->getOpcode() == ISD::ATOMIC_CMP_SWAP || |
1265 | 2.29M | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || |
1266 | 2.29M | N->getOpcode() == ISD::ATOMIC_SWAP || |
1267 | 2.29M | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || |
1268 | 2.29M | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || |
1269 | 2.29M | N->getOpcode() == ISD::ATOMIC_LOAD_AND || |
1270 | 2.29M | N->getOpcode() == ISD::ATOMIC_LOAD_OR || |
1271 | 2.28M | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || |
1272 | 2.28M | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || |
1273 | 2.28M | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || |
1274 | 2.28M | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || |
1275 | 2.28M | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || |
1276 | 2.28M | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || |
1277 | 2.28M | N->getOpcode() == ISD::ATOMIC_LOAD || |
1278 | 2.28M | N->getOpcode() == ISD::ATOMIC_STORE || |
1279 | 2.28M | N->getOpcode() == ISD::MLOAD || |
1280 | 2.28M | N->getOpcode() == ISD::MSTORE || |
1281 | 2.28M | N->getOpcode() == ISD::MGATHER || |
1282 | 2.28M | N->getOpcode() == ISD::MSCATTER || |
1283 | 2.28M | N->isMemIntrinsic() || |
1284 | 2.04M | N->isTargetMemoryOpcode(); |
1285 | 25.4M | } |
1286 | | }; |
1287 | | |
1288 | | /// This is an SDNode representing atomic operations. |
1289 | | class AtomicSDNode : public MemSDNode { |
1290 | | public: |
1291 | | AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, |
1292 | | EVT MemVT, MachineMemOperand *MMO) |
1293 | 26.5k | : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {} |
1294 | | |
1295 | 3.72k | const SDValue &getBasePtr() const { return getOperand(1); } |
1296 | 338 | const SDValue &getVal() const { return getOperand(2); } |
1297 | | |
1298 | | /// Returns true if this SDNode represents cmpxchg atomic operation, false |
1299 | | /// otherwise. |
1300 | 0 | bool isCompareAndSwap() const { |
1301 | 0 | unsigned Op = getOpcode(); |
1302 | 0 | return Op == ISD::ATOMIC_CMP_SWAP || |
1303 | 0 | Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS; |
1304 | 0 | } |
1305 | | |
1306 | | /// For cmpxchg atomic operations, return the atomic ordering requirements |
1307 | | /// when store does not occur. |
1308 | 0 | AtomicOrdering getFailureOrdering() const { |
1309 | 0 | assert(isCompareAndSwap() && "Must be cmpxchg operation"); |
1310 | 0 | return MMO->getFailureOrdering(); |
1311 | 0 | } |
1312 | | |
1313 | | // Methods to support isa and dyn_cast |
1314 | 424k | static bool classof(const SDNode *N) { |
1315 | 424k | return N->getOpcode() == ISD::ATOMIC_CMP_SWAP || |
1316 | 424k | N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS || |
1317 | 424k | N->getOpcode() == ISD::ATOMIC_SWAP || |
1318 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_ADD || |
1319 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_SUB || |
1320 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_AND || |
1321 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_OR || |
1322 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_XOR || |
1323 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_NAND || |
1324 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_MIN || |
1325 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_MAX || |
1326 | 423k | N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || |
1327 | 422k | N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || |
1328 | 422k | N->getOpcode() == ISD::ATOMIC_LOAD || |
1329 | 422k | N->getOpcode() == ISD::ATOMIC_STORE; |
1330 | 424k | } |
1331 | | }; |
1332 | | |
1333 | | /// This SDNode is used for target intrinsics that touch |
1334 | | /// memory and need an associated MachineMemOperand. Its opcode may be |
1335 | | /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode |
1336 | | /// with a value not less than FIRST_TARGET_MEMORY_OPCODE. |
1337 | | class MemIntrinsicSDNode : public MemSDNode { |
1338 | | public: |
1339 | | MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, |
1340 | | SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO) |
1341 | 134k | : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { |
1342 | 134k | SDNodeBits.IsMemIntrinsic = true; |
1343 | 134k | } |
1344 | | |
1345 | | // Methods to support isa and dyn_cast |
1346 | 125 | static bool classof(const SDNode *N) { |
1347 | 125 | // We lower some target intrinsics to their target opcode |
1348 | 125 | // early a node with a target opcode can be of this class |
1349 | 125 | return N->isMemIntrinsic() || |
1350 | 0 | N->getOpcode() == ISD::PREFETCH || |
1351 | 0 | N->isTargetMemoryOpcode(); |
1352 | 125 | } |
1353 | | }; |
1354 | | |
1355 | | /// This SDNode is used to implement the code generator |
1356 | | /// support for the llvm IR shufflevector instruction. It combines elements |
1357 | | /// from two input vectors into a new input vector, with the selection and |
1358 | | /// ordering of elements determined by an array of integers, referred to as |
1359 | | /// the shuffle mask. For input vectors of width N, mask indices of 0..N-1 |
1360 | | /// refer to elements from the LHS input, and indices from N to 2N-1 the RHS. |
1361 | | /// An index of -1 is treated as undef, such that the code generator may put |
1362 | | /// any value in the corresponding element of the result. |
1363 | | class ShuffleVectorSDNode : public SDNode { |
1364 | | // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and |
1365 | | // is freed when the SelectionDAG object is destroyed. |
1366 | | const int *Mask; |
1367 | | |
1368 | | protected: |
1369 | | friend class SelectionDAG; |
1370 | | |
1371 | | ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M) |
1372 | 70.7k | : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} |
1373 | | |
1374 | | public: |
1375 | 169k | ArrayRef<int> getMask() const { |
1376 | 169k | EVT VT = getValueType(0); |
1377 | 169k | return makeArrayRef(Mask, VT.getVectorNumElements()); |
1378 | 169k | } |
1379 | | |
1380 | 1.90M | int getMaskElt(unsigned Idx) const { |
1381 | 1.90M | assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!"); |
1382 | 1.90M | return Mask[Idx]; |
1383 | 1.90M | } |
1384 | | |
1385 | 61.6k | bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } |
1386 | | |
1387 | 22.4k | int getSplatIndex() const { |
1388 | 22.4k | assert(isSplat() && "Cannot get splat index for non-splat!"); |
1389 | 22.4k | EVT VT = getValueType(0); |
1390 | 22.7k | for (unsigned i = 0, e = VT.getVectorNumElements(); i != e22.7k ; ++i359 ) { |
1391 | 22.7k | if (Mask[i] >= 0) |
1392 | 22.4k | return Mask[i]; |
1393 | 22.7k | } |
1394 | 0 | llvm_unreachable0 ("Splat with all undef indices?"); |
1395 | 22.4k | } |
1396 | | |
1397 | | static bool isSplatMask(const int *Mask, EVT VT); |
1398 | | |
1399 | | /// Change values in a shuffle permute mask assuming |
1400 | | /// the two vector operands have swapped position. |
1401 | 88.3k | static void commuteMask(MutableArrayRef<int> Mask) { |
1402 | 88.3k | unsigned NumElems = Mask.size(); |
1403 | 998k | for (unsigned i = 0; i != NumElems998k ; ++i910k ) { |
1404 | 910k | int idx = Mask[i]; |
1405 | 910k | if (idx < 0) |
1406 | 6.23k | continue; |
1407 | 904k | else if (904k idx < (int)NumElems904k ) |
1408 | 438k | Mask[i] = idx + NumElems; |
1409 | 904k | else |
1410 | 466k | Mask[i] = idx - NumElems; |
1411 | 910k | } |
1412 | 88.3k | } |
1413 | | |
1414 | 668k | static bool classof(const SDNode *N) { |
1415 | 668k | return N->getOpcode() == ISD::VECTOR_SHUFFLE; |
1416 | 668k | } |
1417 | | }; |
1418 | | |
1419 | | class ConstantSDNode : public SDNode { |
1420 | | friend class SelectionDAG; |
1421 | | |
1422 | | const ConstantInt *Value; |
1423 | | |
1424 | | ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, |
1425 | | const DebugLoc &DL, EVT VT) |
1426 | | : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DL, |
1427 | | getSDVTList(VT)), |
1428 | 21.9M | Value(val) { |
1429 | 21.9M | ConstantSDNodeBits.IsOpaque = isOpaque; |
1430 | 21.9M | } |
1431 | | |
1432 | | public: |
1433 | 30.7M | const ConstantInt *getConstantIntValue() const { return Value; } |
1434 | 57.4M | const APInt &getAPIntValue() const { return Value->getValue(); } |
1435 | 23.6M | uint64_t getZExtValue() const { return Value->getZExtValue(); } |
1436 | 200M | int64_t getSExtValue() const { return Value->getSExtValue(); } |
1437 | 593 | uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX) { |
1438 | 593 | return Value->getLimitedValue(Limit); |
1439 | 593 | } |
1440 | | |
1441 | 3.93M | bool isOne() const { return Value->isOne(); } |
1442 | 28.7M | bool isNullValue() const { return Value->isZero(); } |
1443 | 2.66M | bool isAllOnesValue() const { return Value->isMinusOne(); } |
1444 | | |
1445 | 52.2M | bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; } |
1446 | | |
1447 | 676M | static bool classof(const SDNode *N) { |
1448 | 676M | return N->getOpcode() == ISD::Constant || |
1449 | 381M | N->getOpcode() == ISD::TargetConstant; |
1450 | 676M | } |
1451 | | }; |
1452 | | |
1453 | 1.94M | uint64_t SDNode::getConstantOperandVal(unsigned Num) const { |
1454 | 1.94M | return cast<ConstantSDNode>(getOperand(Num))->getZExtValue(); |
1455 | 1.94M | } |
1456 | | |
1457 | | class ConstantFPSDNode : public SDNode { |
1458 | | friend class SelectionDAG; |
1459 | | |
1460 | | const ConstantFP *Value; |
1461 | | |
1462 | | ConstantFPSDNode(bool isTarget, const ConstantFP *val, const DebugLoc &DL, |
1463 | | EVT VT) |
1464 | | : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0, DL, |
1465 | | getSDVTList(VT)), |
1466 | 158k | Value(val) {} |
1467 | | |
1468 | | public: |
1469 | 555k | const APFloat& getValueAPF() const { return Value->getValueAPF(); } |
1470 | 303k | const ConstantFP *getConstantFPValue() const { return Value; } |
1471 | | |
1472 | | /// Return true if the value is positive or negative zero. |
1473 | 46.4k | bool isZero() const { return Value->isZero(); } |
1474 | | |
1475 | | /// Return true if the value is a NaN. |
1476 | 4.99k | bool isNaN() const { return Value->isNaN(); } |
1477 | | |
1478 | | /// Return true if the value is an infinity |
1479 | 4 | bool isInfinity() const { return Value->isInfinity(); } |
1480 | | |
1481 | | /// Return true if the value is negative. |
1482 | 20.9k | bool isNegative() const { return Value->isNegative(); } |
1483 | | |
1484 | | /// We don't rely on operator== working on double values, as |
1485 | | /// it returns true for things that are clearly not equal, like -0.0 and 0.0. |
1486 | | /// As such, this method can be used to do an exact bit-for-bit comparison of |
1487 | | /// two floating point values. |
1488 | | |
1489 | | /// We leave the version with the double argument here because it's just so |
1490 | | /// convenient to write "2.0" and the like. Without this function we'd |
1491 | | /// have to duplicate its logic everywhere it's called. |
1492 | 172k | bool isExactlyValue(double V) const { |
1493 | 172k | bool ignored; |
1494 | 172k | APFloat Tmp(V); |
1495 | 172k | Tmp.convert(Value->getValueAPF().getSemantics(), |
1496 | 172k | APFloat::rmNearestTiesToEven, &ignored); |
1497 | 172k | return isExactlyValue(Tmp); |
1498 | 172k | } |
1499 | | bool isExactlyValue(const APFloat& V) const; |
1500 | | |
1501 | | static bool isValueValidForType(EVT VT, const APFloat& Val); |
1502 | | |
1503 | 285M | static bool classof(const SDNode *N) { |
1504 | 285M | return N->getOpcode() == ISD::ConstantFP || |
1505 | 285M | N->getOpcode() == ISD::TargetConstantFP; |
1506 | 285M | } |
1507 | | }; |
1508 | | |
1509 | | /// Returns true if \p V is a constant integer zero. |
1510 | | bool isNullConstant(SDValue V); |
1511 | | |
1512 | | /// Returns true if \p V is an FP constant with a value of positive zero. |
1513 | | bool isNullFPConstant(SDValue V); |
1514 | | |
1515 | | /// Returns true if \p V is an integer constant with all bits set. |
1516 | | bool isAllOnesConstant(SDValue V); |
1517 | | |
1518 | | /// Returns true if \p V is a constant integer one. |
1519 | | bool isOneConstant(SDValue V); |
1520 | | |
1521 | | /// Returns true if \p V is a bitwise not operation. Assumes that an all ones |
1522 | | /// constant is canonicalized to be operand 1. |
1523 | | bool isBitwiseNot(SDValue V); |
1524 | | |
1525 | | /// Returns the SDNode if it is a constant splat BuildVector or constant int. |
1526 | | ConstantSDNode *isConstOrConstSplat(SDValue V); |
1527 | | |
1528 | | /// Returns the SDNode if it is a constant splat BuildVector or constant float. |
1529 | | ConstantFPSDNode *isConstOrConstSplatFP(SDValue V); |
1530 | | |
1531 | | class GlobalAddressSDNode : public SDNode { |
1532 | | friend class SelectionDAG; |
1533 | | |
1534 | | const GlobalValue *TheGlobal; |
1535 | | int64_t Offset; |
1536 | | unsigned char TargetFlags; |
1537 | | |
1538 | | GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, |
1539 | | const GlobalValue *GA, EVT VT, int64_t o, |
1540 | | unsigned char TargetFlags); |
1541 | | |
1542 | | public: |
1543 | 15.9M | const GlobalValue *getGlobal() const { return TheGlobal; } |
1544 | 8.44M | int64_t getOffset() const { return Offset; } |
1545 | 6.09M | unsigned char getTargetFlags() const { return TargetFlags; } |
1546 | | // Return the address space this GlobalAddress belongs to. |
1547 | | unsigned getAddressSpace() const; |
1548 | | |
1549 | 250M | static bool classof(const SDNode *N) { |
1550 | 250M | return N->getOpcode() == ISD::GlobalAddress || |
1551 | 244M | N->getOpcode() == ISD::TargetGlobalAddress || |
1552 | 233M | N->getOpcode() == ISD::GlobalTLSAddress || |
1553 | 233M | N->getOpcode() == ISD::TargetGlobalTLSAddress; |
1554 | 250M | } |
1555 | | }; |
1556 | | |
1557 | | class FrameIndexSDNode : public SDNode { |
1558 | | friend class SelectionDAG; |
1559 | | |
1560 | | int FI; |
1561 | | |
1562 | | FrameIndexSDNode(int fi, EVT VT, bool isTarg) |
1563 | | : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex, |
1564 | 1.00M | 0, DebugLoc(), getSDVTList(VT)), FI(fi) { |
1565 | 1.00M | } |
1566 | | |
1567 | | public: |
1568 | 8.93M | int getIndex() const { return FI; } |
1569 | | |
1570 | 170M | static bool classof(const SDNode *N) { |
1571 | 170M | return N->getOpcode() == ISD::FrameIndex || |
1572 | 160M | N->getOpcode() == ISD::TargetFrameIndex; |
1573 | 170M | } |
1574 | | }; |
1575 | | |
1576 | | class JumpTableSDNode : public SDNode { |
1577 | | friend class SelectionDAG; |
1578 | | |
1579 | | int JTI; |
1580 | | unsigned char TargetFlags; |
1581 | | |
1582 | | JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned char TF) |
1583 | | : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable, |
1584 | 14.8k | 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) { |
1585 | 14.8k | } |
1586 | | |
1587 | | public: |
1588 | 24.2k | int getIndex() const { return JTI; } |
1589 | 14.8k | unsigned char getTargetFlags() const { return TargetFlags; } |
1590 | | |
1591 | 112M | static bool classof(const SDNode *N) { |
1592 | 112M | return N->getOpcode() == ISD::JumpTable || |
1593 | 112M | N->getOpcode() == ISD::TargetJumpTable; |
1594 | 112M | } |
1595 | | }; |
1596 | | |
1597 | | class ConstantPoolSDNode : public SDNode { |
1598 | | friend class SelectionDAG; |
1599 | | |
1600 | | union { |
1601 | | const Constant *ConstVal; |
1602 | | MachineConstantPoolValue *MachineCPVal; |
1603 | | } Val; |
1604 | | int Offset; // It's a MachineConstantPoolValue if top bit is set. |
1605 | | unsigned Alignment; // Minimum alignment requirement of CP (not log2 value). |
1606 | | unsigned char TargetFlags; |
1607 | | |
1608 | | ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o, |
1609 | | unsigned Align, unsigned char TF) |
1610 | | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, |
1611 | | DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align), |
1612 | 284k | TargetFlags(TF) { |
1613 | 284k | assert(Offset >= 0 && "Offset is too large"); |
1614 | 284k | Val.ConstVal = c; |
1615 | 284k | } |
1616 | | |
1617 | | ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, |
1618 | | EVT VT, int o, unsigned Align, unsigned char TF) |
1619 | | : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0, |
1620 | | DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align), |
1621 | 356 | TargetFlags(TF) { |
1622 | 356 | assert(Offset >= 0 && "Offset is too large"); |
1623 | 356 | Val.MachineCPVal = v; |
1624 | 356 | Offset |= 1 << (sizeof(unsigned)*CHAR_BIT-1); |
1625 | 356 | } |
1626 | | |
1627 | | public: |
1628 | 578k | bool isMachineConstantPoolEntry() const { |
1629 | 578k | return Offset < 0; |
1630 | 578k | } |
1631 | | |
1632 | 594k | const Constant *getConstVal() const { |
1633 | 594k | assert(!isMachineConstantPoolEntry() && "Wrong constantpool type"); |
1634 | 594k | return Val.ConstVal; |
1635 | 594k | } |
1636 | | |
1637 | 463 | MachineConstantPoolValue *getMachineCPVal() const { |
1638 | 463 | assert(isMachineConstantPoolEntry() && "Wrong constantpool type"); |
1639 | 463 | return Val.MachineCPVal; |
1640 | 463 | } |
1641 | | |
1642 | 484k | int getOffset() const { |
1643 | 484k | return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT-1)); |
1644 | 484k | } |
1645 | | |
1646 | | // Return the alignment of this constant pool object, which is either 0 (for |
1647 | | // default alignment) or the desired value. |
1648 | 583k | unsigned getAlignment() const { return Alignment; } |
1649 | 312k | unsigned char getTargetFlags() const { return TargetFlags; } |
1650 | | |
1651 | | Type *getType() const; |
1652 | | |
1653 | 124M | static bool classof(const SDNode *N) { |
1654 | 124M | return N->getOpcode() == ISD::ConstantPool || |
1655 | 124M | N->getOpcode() == ISD::TargetConstantPool; |
1656 | 124M | } |
1657 | | }; |
1658 | | |
1659 | | /// Completely target-dependent object reference. |
1660 | | class TargetIndexSDNode : public SDNode { |
1661 | | friend class SelectionDAG; |
1662 | | |
1663 | | unsigned char TargetFlags; |
1664 | | int Index; |
1665 | | int64_t Offset; |
1666 | | |
1667 | | public: |
1668 | | TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned char TF) |
1669 | | : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)), |
1670 | 0 | TargetFlags(TF), Index(Idx), Offset(Ofs) {} |
1671 | | |
1672 | 0 | unsigned char getTargetFlags() const { return TargetFlags; } |
1673 | 0 | int getIndex() const { return Index; } |
1674 | 0 | int64_t getOffset() const { return Offset; } |
1675 | | |
1676 | 112M | static bool classof(const SDNode *N) { |
1677 | 112M | return N->getOpcode() == ISD::TargetIndex; |
1678 | 112M | } |
1679 | | }; |
1680 | | |
1681 | | class BasicBlockSDNode : public SDNode { |
1682 | | friend class SelectionDAG; |
1683 | | |
1684 | | MachineBasicBlock *MBB; |
1685 | | |
1686 | | /// Debug info is meaningful and potentially useful here, but we create |
1687 | | /// blocks out of order when they're jumped to, which makes it a bit |
1688 | | /// harder. Let's see if we need it first. |
1689 | | explicit BasicBlockSDNode(MachineBasicBlock *mbb) |
1690 | | : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb) |
1691 | 3.91M | {} |
1692 | | |
1693 | | public: |
1694 | 5.11M | MachineBasicBlock *getBasicBlock() const { return MBB; } |
1695 | | |
1696 | 126M | static bool classof(const SDNode *N) { |
1697 | 126M | return N->getOpcode() == ISD::BasicBlock; |
1698 | 126M | } |
1699 | | }; |
1700 | | |
1701 | | /// A "pseudo-class" with methods for operating on BUILD_VECTORs. |
1702 | | class BuildVectorSDNode : public SDNode { |
1703 | | public: |
1704 | | // These are constructed as SDNodes and then cast to BuildVectorSDNodes. |
1705 | | explicit BuildVectorSDNode() = delete; |
1706 | | |
1707 | | /// Check if this is a constant splat, and if so, find the |
1708 | | /// smallest element size that splats the vector. If MinSplatBits is |
1709 | | /// nonzero, the element size must be at least that large. Note that the |
1710 | | /// splat element may be the entire vector (i.e., a one element vector). |
1711 | | /// Returns the splat element value in SplatValue. Any undefined bits in |
1712 | | /// that value are zero, and the corresponding bits in the SplatUndef mask |
1713 | | /// are set. The SplatBitSize value is set to the splat element size in |
1714 | | /// bits. HasAnyUndefs is set to true if any bits in the vector are |
1715 | | /// undefined. isBigEndian describes the endianness of the target. |
1716 | | bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, |
1717 | | unsigned &SplatBitSize, bool &HasAnyUndefs, |
1718 | | unsigned MinSplatBits = 0, |
1719 | | bool isBigEndian = false) const; |
1720 | | |
1721 | | /// \brief Returns the splatted value or a null value if this is not a splat. |
1722 | | /// |
1723 | | /// If passed a non-null UndefElements bitvector, it will resize it to match |
1724 | | /// the vector width and set the bits where elements are undef. |
1725 | | SDValue getSplatValue(BitVector *UndefElements = nullptr) const; |
1726 | | |
1727 | | /// \brief Returns the splatted constant or null if this is not a constant |
1728 | | /// splat. |
1729 | | /// |
1730 | | /// If passed a non-null UndefElements bitvector, it will resize it to match |
1731 | | /// the vector width and set the bits where elements are undef. |
1732 | | ConstantSDNode * |
1733 | | getConstantSplatNode(BitVector *UndefElements = nullptr) const; |
1734 | | |
1735 | | /// \brief Returns the splatted constant FP or null if this is not a constant |
1736 | | /// FP splat. |
1737 | | /// |
1738 | | /// If passed a non-null UndefElements bitvector, it will resize it to match |
1739 | | /// the vector width and set the bits where elements are undef. |
1740 | | ConstantFPSDNode * |
1741 | | getConstantFPSplatNode(BitVector *UndefElements = nullptr) const; |
1742 | | |
1743 | | /// \brief If this is a constant FP splat and the splatted constant FP is an |
1744 | | /// exact power or 2, return the log base 2 integer value. Otherwise, |
1745 | | /// return -1. |
1746 | | /// |
1747 | | /// The BitWidth specifies the necessary bit precision. |
1748 | | int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, |
1749 | | uint32_t BitWidth) const; |
1750 | | |
1751 | | bool isConstant() const; |
1752 | | |
1753 | 54.7M | static bool classof(const SDNode *N) { |
1754 | 54.7M | return N->getOpcode() == ISD::BUILD_VECTOR; |
1755 | 54.7M | } |
1756 | | }; |
1757 | | |
1758 | | /// An SDNode that holds an arbitrary LLVM IR Value. This is |
1759 | | /// used when the SelectionDAG needs to make a simple reference to something |
1760 | | /// in the LLVM IR representation. |
1761 | | /// |
1762 | | class SrcValueSDNode : public SDNode { |
1763 | | friend class SelectionDAG; |
1764 | | |
1765 | | const Value *V; |
1766 | | |
1767 | | /// Create a SrcValue for a general value. |
1768 | | explicit SrcValueSDNode(const Value *v) |
1769 | 849 | : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {} |
1770 | | |
1771 | | public: |
1772 | | /// Return the contained Value. |
1773 | 1.89k | const Value *getValue() const { return V; } |
1774 | | |
1775 | 15 | static bool classof(const SDNode *N) { |
1776 | 15 | return N->getOpcode() == ISD::SRCVALUE; |
1777 | 15 | } |
1778 | | }; |
1779 | | |
1780 | | class MDNodeSDNode : public SDNode { |
1781 | | friend class SelectionDAG; |
1782 | | |
1783 | | const MDNode *MD; |
1784 | | |
1785 | | explicit MDNodeSDNode(const MDNode *md) |
1786 | | : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md) |
1787 | 11.8k | {} |
1788 | | |
1789 | | public: |
1790 | 11.7k | const MDNode *getMD() const { return MD; } |
1791 | | |
1792 | 90.8M | static bool classof(const SDNode *N) { |
1793 | 90.8M | return N->getOpcode() == ISD::MDNODE_SDNODE; |
1794 | 90.8M | } |
1795 | | }; |
1796 | | |
1797 | | class RegisterSDNode : public SDNode { |
1798 | | friend class SelectionDAG; |
1799 | | |
1800 | | unsigned Reg; |
1801 | | |
1802 | | RegisterSDNode(unsigned reg, EVT VT) |
1803 | 14.1M | : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {} |
1804 | | |
1805 | | public: |
1806 | 67.0M | unsigned getReg() const { return Reg; } |
1807 | | |
1808 | 201M | static bool classof(const SDNode *N) { |
1809 | 201M | return N->getOpcode() == ISD::Register; |
1810 | 201M | } |
1811 | | }; |
1812 | | |
1813 | | class RegisterMaskSDNode : public SDNode { |
1814 | | friend class SelectionDAG; |
1815 | | |
1816 | | // The memory for RegMask is not owned by the node. |
1817 | | const uint32_t *RegMask; |
1818 | | |
1819 | | RegisterMaskSDNode(const uint32_t *mask) |
1820 | | : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)), |
1821 | 1.09M | RegMask(mask) {} |
1822 | | |
1823 | | public: |
1824 | 2.93M | const uint32_t *getRegMask() const { return RegMask; } |
1825 | | |
1826 | 158M | static bool classof(const SDNode *N) { |
1827 | 158M | return N->getOpcode() == ISD::RegisterMask; |
1828 | 158M | } |
1829 | | }; |
1830 | | |
1831 | | class BlockAddressSDNode : public SDNode { |
1832 | | friend class SelectionDAG; |
1833 | | |
1834 | | const BlockAddress *BA; |
1835 | | int64_t Offset; |
1836 | | unsigned char TargetFlags; |
1837 | | |
1838 | | BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba, |
1839 | | int64_t o, unsigned char Flags) |
1840 | | : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)), |
1841 | 244 | BA(ba), Offset(o), TargetFlags(Flags) {} |
1842 | | |
1843 | | public: |
1844 | 374 | const BlockAddress *getBlockAddress() const { return BA; } |
1845 | 245 | int64_t getOffset() const { return Offset; } |
1846 | 219 | unsigned char getTargetFlags() const { return TargetFlags; } |
1847 | | |
1848 | 112M | static bool classof(const SDNode *N) { |
1849 | 112M | return N->getOpcode() == ISD::BlockAddress || |
1850 | 112M | N->getOpcode() == ISD::TargetBlockAddress; |
1851 | 112M | } |
1852 | | }; |
1853 | | |
1854 | | class LabelSDNode : public SDNode { |
1855 | | friend class SelectionDAG; |
1856 | | |
1857 | | MCSymbol *Label; |
1858 | | |
1859 | | LabelSDNode(unsigned Order, const DebugLoc &dl, MCSymbol *L) |
1860 | 37.0k | : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {} |
1861 | | |
1862 | | public: |
1863 | 36.9k | MCSymbol *getLabel() const { return Label; } |
1864 | | |
1865 | 0 | static bool classof(const SDNode *N) { |
1866 | 0 | return N->getOpcode() == ISD::EH_LABEL || |
1867 | 0 | N->getOpcode() == ISD::ANNOTATION_LABEL; |
1868 | 0 | } |
1869 | | }; |
1870 | | |
1871 | | class ExternalSymbolSDNode : public SDNode { |
1872 | | friend class SelectionDAG; |
1873 | | |
1874 | | const char *Symbol; |
1875 | | unsigned char TargetFlags; |
1876 | | |
1877 | | ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned char TF, EVT VT) |
1878 | | : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, |
1879 | 67.6k | 0, DebugLoc(), getSDVTList(VT)), Symbol(Sym), TargetFlags(TF) {} |
1880 | | |
1881 | | public: |
1882 | 113k | const char *getSymbol() const { return Symbol; } |
1883 | 35.9k | unsigned char getTargetFlags() const { return TargetFlags; } |
1884 | | |
1885 | 112M | static bool classof(const SDNode *N) { |
1886 | 112M | return N->getOpcode() == ISD::ExternalSymbol || |
1887 | 112M | N->getOpcode() == ISD::TargetExternalSymbol; |
1888 | 112M | } |
1889 | | }; |
1890 | | |
1891 | | class MCSymbolSDNode : public SDNode { |
1892 | | friend class SelectionDAG; |
1893 | | |
1894 | | MCSymbol *Symbol; |
1895 | | |
1896 | | MCSymbolSDNode(MCSymbol *Symbol, EVT VT) |
1897 | 49 | : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {} |
1898 | | |
1899 | | public: |
1900 | 69 | MCSymbol *getMCSymbol() const { return Symbol; } |
1901 | | |
1902 | 112M | static bool classof(const SDNode *N) { |
1903 | 112M | return N->getOpcode() == ISD::MCSymbol; |
1904 | 112M | } |
1905 | | }; |
1906 | | |
1907 | | class CondCodeSDNode : public SDNode { |
1908 | | friend class SelectionDAG; |
1909 | | |
1910 | | ISD::CondCode Condition; |
1911 | | |
1912 | | explicit CondCodeSDNode(ISD::CondCode Cond) |
1913 | | : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)), |
1914 | 2.91M | Condition(Cond) {} |
1915 | | |
1916 | | public: |
1917 | 17.9M | ISD::CondCode get() const { return Condition; } |
1918 | | |
1919 | 15 | static bool classof(const SDNode *N) { |
1920 | 15 | return N->getOpcode() == ISD::CONDCODE; |
1921 | 15 | } |
1922 | | }; |
1923 | | |
1924 | | /// This class is used to represent EVT's, which are used |
1925 | | /// to parameterize some operations. |
1926 | | class VTSDNode : public SDNode { |
1927 | | friend class SelectionDAG; |
1928 | | |
1929 | | EVT ValueType; |
1930 | | |
1931 | | explicit VTSDNode(EVT VT) |
1932 | | : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)), |
1933 | 876k | ValueType(VT) {} |
1934 | | |
1935 | | public: |
1936 | 4.77M | EVT getVT() const { return ValueType; } |
1937 | | |
1938 | 16 | static bool classof(const SDNode *N) { |
1939 | 16 | return N->getOpcode() == ISD::VALUETYPE; |
1940 | 16 | } |
1941 | | }; |
1942 | | |
1943 | | /// Base class for LoadSDNode and StoreSDNode |
1944 | | class LSBaseSDNode : public MemSDNode { |
1945 | | public: |
1946 | | LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl, |
1947 | | SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, |
1948 | | MachineMemOperand *MMO) |
1949 | 13.7M | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { |
1950 | 13.7M | LSBaseSDNodeBits.AddressingMode = AM; |
1951 | 13.7M | assert(getAddressingMode() == AM && "Value truncated"); |
1952 | 13.7M | } |
1953 | | |
1954 | 0 | const SDValue &getOffset() const { |
1955 | 0 | return getOperand(getOpcode() == ISD::LOAD ? 2 : 3); |
1956 | 0 | } |
1957 | | |
1958 | | /// Return the addressing mode for this load or store: |
1959 | | /// unindexed, pre-inc, pre-dec, post-inc, or post-dec. |
1960 | 119M | ISD::MemIndexedMode getAddressingMode() const { |
1961 | 119M | return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode); |
1962 | 119M | } |
1963 | | |
1964 | | /// Return true if this is a pre/post inc/dec load/store. |
1965 | 62.9M | bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; } |
1966 | | |
1967 | | /// Return true if this is NOT a pre/post inc/dec load/store. |
1968 | 33.6M | bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; } |
1969 | | |
1970 | 8.16k | static bool classof(const SDNode *N) { |
1971 | 8.16k | return N->getOpcode() == ISD::LOAD || |
1972 | 4.03k | N->getOpcode() == ISD::STORE; |
1973 | 8.16k | } |
1974 | | }; |
1975 | | |
1976 | | /// This class is used to represent ISD::LOAD nodes. |
1977 | | class LoadSDNode : public LSBaseSDNode { |
1978 | | friend class SelectionDAG; |
1979 | | |
1980 | | LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
1981 | | ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, |
1982 | | MachineMemOperand *MMO) |
1983 | 6.72M | : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { |
1984 | 6.72M | LoadSDNodeBits.ExtTy = ETy; |
1985 | 6.72M | assert(readMem() && "Load MachineMemOperand is not a load!"); |
1986 | 6.72M | assert(!writeMem() && "Load MachineMemOperand is a store!"); |
1987 | 6.72M | } |
1988 | | |
1989 | | public: |
1990 | | /// Return whether this is a plain node, |
1991 | | /// or one of the varieties of value-extending loads. |
1992 | 47.1M | ISD::LoadExtType getExtensionType() const { |
1993 | 47.1M | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); |
1994 | 47.1M | } |
1995 | | |
1996 | 19.3M | const SDValue &getBasePtr() const { return getOperand(1); } |
1997 | 289k | const SDValue &getOffset() const { return getOperand(2); } |
1998 | | |
1999 | 138M | static bool classof(const SDNode *N) { |
2000 | 138M | return N->getOpcode() == ISD::LOAD; |
2001 | 138M | } |
2002 | | }; |
2003 | | |
2004 | | /// This class is used to represent ISD::STORE nodes. |
2005 | | class StoreSDNode : public LSBaseSDNode { |
2006 | | friend class SelectionDAG; |
2007 | | |
2008 | | StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2009 | | ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, |
2010 | | MachineMemOperand *MMO) |
2011 | 7.03M | : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { |
2012 | 7.03M | StoreSDNodeBits.IsTruncating = isTrunc; |
2013 | 7.03M | assert(!readMem() && "Store MachineMemOperand is a load!"); |
2014 | 7.03M | assert(writeMem() && "Store MachineMemOperand is not a store!"); |
2015 | 7.03M | } |
2016 | | |
2017 | | public: |
2018 | | /// Return true if the op does a truncation before store. |
2019 | | /// For integers this is the same as doing a TRUNCATE and storing the result. |
2020 | | /// For floats, it is the same as doing an FP_ROUND and storing the result. |
2021 | 55.9M | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } |
2022 | | |
2023 | 119M | const SDValue &getValue() const { return getOperand(1); } |
2024 | 75.6M | const SDValue &getBasePtr() const { return getOperand(2); } |
2025 | 1.46k | const SDValue &getOffset() const { return getOperand(3); } |
2026 | | |
2027 | 78.6M | static bool classof(const SDNode *N) { |
2028 | 78.6M | return N->getOpcode() == ISD::STORE; |
2029 | 78.6M | } |
2030 | | }; |
2031 | | |
2032 | | /// This base class is used to represent MLOAD and MSTORE nodes |
2033 | | class MaskedLoadStoreSDNode : public MemSDNode { |
2034 | | public: |
2035 | | friend class SelectionDAG; |
2036 | | |
2037 | | MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, |
2038 | | const DebugLoc &dl, SDVTList VTs, EVT MemVT, |
2039 | | MachineMemOperand *MMO) |
2040 | 1.03k | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} |
2041 | | |
2042 | | // In the both nodes address is Op1, mask is Op2: |
2043 | | // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value |
2044 | | // MaskedStoreSDNode (Chain, ptr, mask, data) |
2045 | | // Mask is a vector of i1 elements |
2046 | 163 | const SDValue &getBasePtr() const { return getOperand(1); } |
2047 | 1.86k | const SDValue &getMask() const { return getOperand(2); } |
2048 | | |
2049 | 0 | static bool classof(const SDNode *N) { |
2050 | 0 | return N->getOpcode() == ISD::MLOAD || |
2051 | 0 | N->getOpcode() == ISD::MSTORE; |
2052 | 0 | } |
2053 | | }; |
2054 | | |
2055 | | /// This class is used to represent an MLOAD node |
2056 | | class MaskedLoadSDNode : public MaskedLoadStoreSDNode { |
2057 | | public: |
2058 | | friend class SelectionDAG; |
2059 | | |
2060 | | MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2061 | | ISD::LoadExtType ETy, bool IsExpanding, EVT MemVT, |
2062 | | MachineMemOperand *MMO) |
2063 | 632 | : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) { |
2064 | 632 | LoadSDNodeBits.ExtTy = ETy; |
2065 | 632 | LoadSDNodeBits.IsExpanding = IsExpanding; |
2066 | 632 | } |
2067 | | |
2068 | 1.74k | ISD::LoadExtType getExtensionType() const { |
2069 | 1.74k | return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy); |
2070 | 1.74k | } |
2071 | | |
2072 | 129 | const SDValue &getSrc0() const { return getOperand(3); } |
2073 | 262 | static bool classof(const SDNode *N) { |
2074 | 262 | return N->getOpcode() == ISD::MLOAD; |
2075 | 262 | } |
2076 | | |
2077 | 1.50k | bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; } |
2078 | | }; |
2079 | | |
2080 | | /// This class is used to represent an MSTORE node |
2081 | | class MaskedStoreSDNode : public MaskedLoadStoreSDNode { |
2082 | | public: |
2083 | | friend class SelectionDAG; |
2084 | | |
2085 | | MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2086 | | bool isTrunc, bool isCompressing, EVT MemVT, |
2087 | | MachineMemOperand *MMO) |
2088 | 398 | : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) { |
2089 | 398 | StoreSDNodeBits.IsTruncating = isTrunc; |
2090 | 398 | StoreSDNodeBits.IsCompressing = isCompressing; |
2091 | 398 | } |
2092 | | |
2093 | | /// Return true if the op does a truncation before store. |
2094 | | /// For integers this is the same as doing a TRUNCATE and storing the result. |
2095 | | /// For floats, it is the same as doing an FP_ROUND and storing the result. |
2096 | 532 | bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; } |
2097 | | |
2098 | | /// Returns true if the op does a compression to the vector before storing. |
2099 | | /// The node contiguously stores the active elements (integers or floats) |
2100 | | /// in src (those with their respective bit set in writemask k) to unaligned |
2101 | | /// memory at base_addr. |
2102 | 631 | bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; } |
2103 | | |
2104 | 374 | const SDValue &getValue() const { return getOperand(3); } |
2105 | | |
2106 | 130 | static bool classof(const SDNode *N) { |
2107 | 130 | return N->getOpcode() == ISD::MSTORE; |
2108 | 130 | } |
2109 | | }; |
2110 | | |
2111 | | /// This is a base class used to represent |
2112 | | /// MGATHER and MSCATTER nodes |
2113 | | /// |
2114 | | class MaskedGatherScatterSDNode : public MemSDNode { |
2115 | | public: |
2116 | | friend class SelectionDAG; |
2117 | | |
2118 | | MaskedGatherScatterSDNode(unsigned NodeTy, unsigned Order, |
2119 | | const DebugLoc &dl, SDVTList VTs, EVT MemVT, |
2120 | | MachineMemOperand *MMO) |
2121 | 963 | : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} |
2122 | | |
2123 | | // In the both nodes address is Op1, mask is Op2: |
2124 | | // MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value |
2125 | | // MaskedScatterSDNode (Chain, value, mask, base, index) |
2126 | | // Mask is a vector of i1 elements |
2127 | 550 | const SDValue &getBasePtr() const { return getOperand(3); } |
2128 | 1.03k | const SDValue &getIndex() const { return getOperand(4); } |
2129 | 670 | const SDValue &getMask() const { return getOperand(2); } |
2130 | 894 | const SDValue &getValue() const { return getOperand(1); } |
2131 | | |
2132 | 277 | static bool classof(const SDNode *N) { |
2133 | 277 | return N->getOpcode() == ISD::MGATHER || |
2134 | 87 | N->getOpcode() == ISD::MSCATTER; |
2135 | 277 | } |
2136 | | }; |
2137 | | |
2138 | | /// This class is used to represent an MGATHER node |
2139 | | /// |
2140 | | class MaskedGatherSDNode : public MaskedGatherScatterSDNode { |
2141 | | public: |
2142 | | friend class SelectionDAG; |
2143 | | |
2144 | | MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2145 | | EVT MemVT, MachineMemOperand *MMO) |
2146 | 580 | : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {} |
2147 | | |
2148 | 460 | static bool classof(const SDNode *N) { |
2149 | 460 | return N->getOpcode() == ISD::MGATHER; |
2150 | 460 | } |
2151 | | }; |
2152 | | |
2153 | | /// This class is used to represent an MSCATTER node |
2154 | | /// |
2155 | | class MaskedScatterSDNode : public MaskedGatherScatterSDNode { |
2156 | | public: |
2157 | | friend class SelectionDAG; |
2158 | | |
2159 | | MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, |
2160 | | EVT MemVT, MachineMemOperand *MMO) |
2161 | 356 | : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {} |
2162 | | |
2163 | 98 | static bool classof(const SDNode *N) { |
2164 | 98 | return N->getOpcode() == ISD::MSCATTER; |
2165 | 98 | } |
2166 | | }; |
2167 | | |
2168 | | /// An SDNode that represents everything that will be needed |
2169 | | /// to construct a MachineInstr. These nodes are created during the |
2170 | | /// instruction selection proper phase. |
2171 | | class MachineSDNode : public SDNode { |
2172 | | public: |
2173 | | using mmo_iterator = MachineMemOperand **; |
2174 | | |
2175 | | private: |
2176 | | friend class SelectionDAG; |
2177 | | |
2178 | | MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) |
2179 | 1.18M | : SDNode(Opc, Order, DL, VTs) {} |
2180 | | |
2181 | | /// Memory reference descriptions for this instruction. |
2182 | | mmo_iterator MemRefs = nullptr; |
2183 | | mmo_iterator MemRefsEnd = nullptr; |
2184 | | |
2185 | | public: |
2186 | 19.9M | mmo_iterator memoperands_begin() const { return MemRefs; } |
2187 | 19.8M | mmo_iterator memoperands_end() const { return MemRefsEnd; } |
2188 | 130k | bool memoperands_empty() const { return MemRefsEnd == MemRefs; } |
2189 | | |
2190 | | /// Assign this MachineSDNodes's memory reference descriptor |
2191 | | /// list. This does not transfer ownership. |
2192 | 23.9M | void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { |
2193 | 27.8M | for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME27.8M ; ++MMI3.88M ) |
2194 | 23.9M | assert(*MMI && "Null mem ref detected!"); |
2195 | 23.9M | MemRefs = NewMemRefs; |
2196 | 23.9M | MemRefsEnd = NewMemRefsEnd; |
2197 | 23.9M | } |
2198 | | |
2199 | 21.3M | static bool classof(const SDNode *N) { |
2200 | 21.3M | return N->isMachineOpcode(); |
2201 | 21.3M | } |
2202 | | }; |
2203 | | |
2204 | | class SDNodeIterator : public std::iterator<std::forward_iterator_tag, |
2205 | | SDNode, ptrdiff_t> { |
2206 | | const SDNode *Node; |
2207 | | unsigned Operand; |
2208 | | |
2209 | 10.4k | SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {} |
2210 | | |
2211 | | public: |
2212 | 14.3k | bool operator==(const SDNodeIterator& x) const { |
2213 | 14.3k | return Operand == x.Operand; |
2214 | 14.3k | } |
2215 | 14.3k | bool operator!=(const SDNodeIterator& x) const { return !operator==(x); } |
2216 | | |
2217 | 9.17k | pointer operator*() const { |
2218 | 9.17k | return Node->getOperand(Operand).getNode(); |
2219 | 9.17k | } |
2220 | 0 | pointer operator->() const { return operator*(); } |
2221 | | |
2222 | 9.17k | SDNodeIterator& operator++() { // Preincrement |
2223 | 9.17k | ++Operand; |
2224 | 9.17k | return *this; |
2225 | 9.17k | } |
2226 | 0 | SDNodeIterator operator++(int) { // Postincrement |
2227 | 0 | SDNodeIterator tmp = *this; ++*this; return tmp; |
2228 | 0 | } |
2229 | 0 | size_t operator-(SDNodeIterator Other) const { |
2230 | 0 | assert(Node == Other.Node && |
2231 | 0 | "Cannot compare iterators of two different nodes!"); |
2232 | 0 | return Operand - Other.Operand; |
2233 | 0 | } |
2234 | | |
2235 | 5.21k | static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); } |
2236 | 5.21k | static SDNodeIterator end (const SDNode *N) { |
2237 | 5.21k | return SDNodeIterator(N, N->getNumOperands()); |
2238 | 5.21k | } |
2239 | | |
2240 | 0 | unsigned getOperand() const { return Operand; } |
2241 | 0 | const SDNode *getNode() const { return Node; } |
2242 | | }; |
2243 | | |
2244 | | template <> struct GraphTraits<SDNode*> { |
2245 | | using NodeRef = SDNode *; |
2246 | | using ChildIteratorType = SDNodeIterator; |
2247 | | |
2248 | 0 | static NodeRef getEntryNode(SDNode *N) { return N; } |
2249 | | |
2250 | 0 | static ChildIteratorType child_begin(NodeRef N) { |
2251 | 0 | return SDNodeIterator::begin(N); |
2252 | 0 | } |
2253 | | |
2254 | 0 | static ChildIteratorType child_end(NodeRef N) { |
2255 | 0 | return SDNodeIterator::end(N); |
2256 | 0 | } |
2257 | | }; |
2258 | | |
2259 | | /// A representation of the largest SDNode, for use in sizeof(). |
2260 | | /// |
2261 | | /// This needs to be a union because the largest node differs on 32 bit systems |
2262 | | /// with 4 and 8 byte pointer alignment, respectively. |
2263 | | using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode, |
2264 | | BlockAddressSDNode, |
2265 | | GlobalAddressSDNode>; |
2266 | | |
2267 | | /// The SDNode class with the greatest alignment requirement. |
2268 | | using MostAlignedSDNode = GlobalAddressSDNode; |
2269 | | |
2270 | | namespace ISD { |
2271 | | |
2272 | | /// Returns true if the specified node is a non-extending and unindexed load. |
2273 | 17.0M | inline bool isNormalLoad(const SDNode *N) { |
2274 | 17.0M | const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N); |
2275 | 9.30M | return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD && |
2276 | 7.28M | Ld->getAddressingMode() == ISD::UNINDEXED; |
2277 | 17.0M | } |
2278 | | |
2279 | | /// Returns true if the specified node is a non-extending load. |
2280 | 1.30M | inline bool isNON_EXTLoad(const SDNode *N) { |
2281 | 1.30M | return isa<LoadSDNode>(N) && |
2282 | 513k | cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD; |
2283 | 1.30M | } |
2284 | | |
2285 | | /// Returns true if the specified node is a EXTLOAD. |
2286 | 1.23M | inline bool isEXTLoad(const SDNode *N) { |
2287 | 1.23M | return isa<LoadSDNode>(N) && |
2288 | 109k | cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD; |
2289 | 1.23M | } |
2290 | | |
2291 | | /// Returns true if the specified node is a SEXTLOAD. |
2292 | 889k | inline bool isSEXTLoad(const SDNode *N) { |
2293 | 889k | return isa<LoadSDNode>(N) && |
2294 | 88.0k | cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD; |
2295 | 889k | } |
2296 | | |
2297 | | /// Returns true if the specified node is a ZEXTLOAD. |
2298 | 9.82M | inline bool isZEXTLoad(const SDNode *N) { |
2299 | 9.82M | return isa<LoadSDNode>(N) && |
2300 | 9.49M | cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD; |
2301 | 9.82M | } |
2302 | | |
2303 | | /// Returns true if the specified node is an unindexed load. |
2304 | 3.06M | inline bool isUNINDEXEDLoad(const SDNode *N) { |
2305 | 3.06M | return isa<LoadSDNode>(N) && |
2306 | 2.55M | cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; |
2307 | 3.06M | } |
2308 | | |
2309 | | /// Returns true if the specified node is a non-truncating |
2310 | | /// and unindexed store. |
2311 | 9.09M | inline bool isNormalStore(const SDNode *N) { |
2312 | 9.09M | const StoreSDNode *St = dyn_cast<StoreSDNode>(N); |
2313 | 9.09M | return St && !St->isTruncatingStore() && |
2314 | 8.11M | St->getAddressingMode() == ISD::UNINDEXED; |
2315 | 9.09M | } |
2316 | | |
2317 | | /// Returns true if the specified node is a non-truncating store. |
2318 | 4.49M | inline bool isNON_TRUNCStore(const SDNode *N) { |
2319 | 474k | return isa<StoreSDNode>(N) && !cast<StoreSDNode>(N)->isTruncatingStore(); |
2320 | 4.49M | } |
2321 | | |
2322 | | /// Returns true if the specified node is a truncating store. |
2323 | 0 | inline bool isTRUNCStore(const SDNode *N) { |
2324 | 0 | return isa<StoreSDNode>(N) && cast<StoreSDNode>(N)->isTruncatingStore(); |
2325 | 0 | } |
2326 | | |
2327 | | /// Returns true if the specified node is an unindexed store. |
2328 | 0 | inline bool isUNINDEXEDStore(const SDNode *N) { |
2329 | 0 | return isa<StoreSDNode>(N) && |
2330 | 0 | cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED; |
2331 | 0 | } |
2332 | | |
2333 | | } // end namespace ISD |
2334 | | |
2335 | | } // end namespace llvm |
2336 | | |
2337 | | #endif // LLVM_CODEGEN_SELECTIONDAGNODES_H |