/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
Line | Count | Source |
1 | | //===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This file defines the DAGTypeLegalizer class. This is a private interface |
10 | | // shared between the code that implements the SelectionDAG::LegalizeTypes |
11 | | // method. |
12 | | // |
13 | | //===----------------------------------------------------------------------===// |
14 | | |
15 | | #ifndef LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H |
16 | | #define LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H |
17 | | |
18 | | #include "llvm/ADT/DenseMap.h" |
19 | | #include "llvm/CodeGen/SelectionDAG.h" |
20 | | #include "llvm/CodeGen/TargetLowering.h" |
21 | | #include "llvm/Support/Compiler.h" |
22 | | #include "llvm/Support/Debug.h" |
23 | | |
24 | | namespace llvm { |
25 | | |
26 | | //===----------------------------------------------------------------------===// |
27 | | /// This takes an arbitrary SelectionDAG as input and hacks on it until only |
28 | | /// value types the target machine can handle are left. This involves promoting |
29 | | /// small sizes to large sizes or splitting up large values into small values. |
30 | | /// |
31 | | class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { |
32 | | const TargetLowering &TLI; |
33 | | SelectionDAG &DAG; |
34 | | public: |
35 | | /// This pass uses the NodeId on the SDNodes to hold information about the |
36 | | /// state of the node. The enum has all the values. |
37 | | enum NodeIdFlags { |
38 | | /// All operands have been processed, so this node is ready to be handled. |
39 | | ReadyToProcess = 0, |
40 | | |
41 | | /// This is a new node, not before seen, that was created in the process of |
42 | | /// legalizing some other node. |
43 | | NewNode = -1, |
44 | | |
45 | | /// This node's ID needs to be set to the number of its unprocessed |
46 | | /// operands. |
47 | | Unanalyzed = -2, |
48 | | |
49 | | /// This is a node that has already been processed. |
50 | | Processed = -3 |
51 | | |
52 | | // 1+ - This is a node which has this many unprocessed operands. |
53 | | }; |
54 | | private: |
55 | | |
56 | | /// This is a bitvector that contains two bits for each simple value type, |
57 | | /// where the two bits correspond to the LegalizeAction enum from |
58 | | /// TargetLowering. This can be queried with "getTypeAction(VT)". |
59 | | TargetLowering::ValueTypeActionImpl ValueTypeActions; |
60 | | |
61 | | /// Return how we should legalize values of this type. |
62 | 67.1M | TargetLowering::LegalizeTypeAction getTypeAction(EVT VT) const { |
63 | 67.1M | return TLI.getTypeAction(*DAG.getContext(), VT); |
64 | 67.1M | } |
65 | | |
66 | | /// Return true if this type is legal on this target. |
67 | 136k | bool isTypeLegal(EVT VT) const { |
68 | 136k | return TLI.getTypeAction(*DAG.getContext(), VT) == TargetLowering::TypeLegal; |
69 | 136k | } |
70 | | |
71 | | /// Return true if this is a simple legal type. |
72 | 6.25k | bool isSimpleLegalType(EVT VT) const { |
73 | 6.25k | return VT.isSimple() && TLI.isTypeLegal(VT)6.24k ; |
74 | 6.25k | } |
75 | | |
76 | | /// Return true if this type can be passed in registers. |
77 | | /// For example, x86_64's f128, should to be legally in registers |
78 | | /// and only some operations converted to library calls or integer |
79 | | /// bitwise operations. |
80 | 11.6k | bool isLegalInHWReg(EVT VT) const { |
81 | 11.6k | EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); |
82 | 11.6k | return VT == NVT && isSimpleLegalType(VT)6.12k ; |
83 | 11.6k | } |
84 | | |
85 | 549k | EVT getSetCCResultType(EVT VT) const { |
86 | 549k | return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); |
87 | 549k | } |
88 | | |
89 | | /// Pretend all of this node's results are legal. |
90 | 73.1M | bool IgnoreNodeResults(SDNode *N) const { |
91 | 73.1M | return N->getOpcode() == ISD::TargetConstant || |
92 | 73.1M | N->getOpcode() == ISD::Register70.4M ; |
93 | 73.1M | } |
94 | | |
95 | | // Bijection from SDValue to unique id. As each created node gets a |
96 | | // new id we do not need to worry about reuse expunging. Should we |
97 | | // run out of ids, we can do a one time expensive compactifcation. |
98 | | typedef unsigned TableId; |
99 | | |
100 | | TableId NextValueId = 1; |
101 | | |
102 | | SmallDenseMap<SDValue, TableId, 8> ValueToIdMap; |
103 | | SmallDenseMap<TableId, SDValue, 8> IdToValueMap; |
104 | | |
105 | | /// For integer nodes that are below legal width, this map indicates what |
106 | | /// promoted value to use. |
107 | | SmallDenseMap<TableId, TableId, 8> PromotedIntegers; |
108 | | |
109 | | /// For integer nodes that need to be expanded this map indicates which |
110 | | /// operands are the expanded version of the input. |
111 | | SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedIntegers; |
112 | | |
113 | | /// For floating-point nodes converted to integers of the same size, this map |
114 | | /// indicates the converted value to use. |
115 | | SmallDenseMap<TableId, TableId, 8> SoftenedFloats; |
116 | | |
117 | | /// For floating-point nodes that have a smaller precision than the smallest |
118 | | /// supported precision, this map indicates what promoted value to use. |
119 | | SmallDenseMap<TableId, TableId, 8> PromotedFloats; |
120 | | |
121 | | /// For float nodes that need to be expanded this map indicates which operands |
122 | | /// are the expanded version of the input. |
123 | | SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedFloats; |
124 | | |
125 | | /// For nodes that are <1 x ty>, this map indicates the scalar value of type |
126 | | /// 'ty' to use. |
127 | | SmallDenseMap<TableId, TableId, 8> ScalarizedVectors; |
128 | | |
129 | | /// For nodes that need to be split this map indicates which operands are the |
130 | | /// expanded version of the input. |
131 | | SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> SplitVectors; |
132 | | |
133 | | /// For vector nodes that need to be widened, indicates the widened value to |
134 | | /// use. |
135 | | SmallDenseMap<TableId, TableId, 8> WidenedVectors; |
136 | | |
137 | | /// For values that have been replaced with another, indicates the replacement |
138 | | /// value to use. |
139 | | SmallDenseMap<TableId, TableId, 8> ReplacedValues; |
140 | | |
141 | | /// This defines a worklist of nodes to process. In order to be pushed onto |
142 | | /// this worklist, all operands of a node must have already been processed. |
143 | | SmallVector<SDNode*, 128> Worklist; |
144 | | |
145 | 13.3M | TableId getTableId(SDValue V) { |
146 | 13.3M | assert(V.getNode() && "Getting TableId on SDValue()"); |
147 | 13.3M | |
148 | 13.3M | auto I = ValueToIdMap.find(V); |
149 | 13.3M | if (I != ValueToIdMap.end()) { |
150 | 7.48M | // replace if there's been a shift. |
151 | 7.48M | RemapId(I->second); |
152 | 7.48M | assert(I->second && "All Ids should be nonzero"); |
153 | 7.48M | return I->second; |
154 | 7.48M | } |
155 | 5.87M | // Add if it's not there. |
156 | 5.87M | ValueToIdMap.insert(std::make_pair(V, NextValueId)); |
157 | 5.87M | IdToValueMap.insert(std::make_pair(NextValueId, V)); |
158 | 5.87M | ++NextValueId; |
159 | 5.87M | assert(NextValueId != 0 && |
160 | 5.87M | "Ran out of Ids. Increase id type size or add compactification"); |
161 | 5.87M | return NextValueId - 1; |
162 | 5.87M | } |
163 | | |
164 | 9.06M | const SDValue &getSDValue(TableId &Id) { |
165 | 9.06M | RemapId(Id); |
166 | 9.06M | assert(Id && "TableId should be non-zero"); |
167 | 9.06M | return IdToValueMap[Id]; |
168 | 9.06M | } |
169 | | |
170 | | public: |
171 | | explicit DAGTypeLegalizer(SelectionDAG &dag) |
172 | | : TLI(dag.getTargetLoweringInfo()), DAG(dag), |
173 | 1.28M | ValueTypeActions(TLI.getValueTypeActions()) { |
174 | 1.28M | static_assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE, |
175 | 1.28M | "Too many value types for ValueTypeActions to hold!"); |
176 | 1.28M | } |
177 | | |
178 | | /// This is the main entry point for the type legalizer. This does a |
179 | | /// top-down traversal of the dag, legalizing types as it goes. Returns |
180 | | /// "true" if it made any changes. |
181 | | bool run(); |
182 | | |
183 | 573 | void NoteDeletion(SDNode *Old, SDNode *New) { |
184 | 1.19k | for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i622 ) { |
185 | 622 | TableId NewId = getTableId(SDValue(New, i)); |
186 | 622 | TableId OldId = getTableId(SDValue(Old, i)); |
187 | 622 | |
188 | 622 | if (OldId != NewId) |
189 | 622 | ReplacedValues[OldId] = NewId; |
190 | 622 | |
191 | 622 | // Delete Node from tables. |
192 | 622 | ValueToIdMap.erase(SDValue(Old, i)); |
193 | 622 | IdToValueMap.erase(OldId); |
194 | 622 | PromotedIntegers.erase(OldId); |
195 | 622 | ExpandedIntegers.erase(OldId); |
196 | 622 | SoftenedFloats.erase(OldId); |
197 | 622 | PromotedFloats.erase(OldId); |
198 | 622 | ExpandedFloats.erase(OldId); |
199 | 622 | ScalarizedVectors.erase(OldId); |
200 | 622 | SplitVectors.erase(OldId); |
201 | 622 | WidenedVectors.erase(OldId); |
202 | 622 | } |
203 | 573 | } |
204 | | |
205 | 989k | SelectionDAG &getDAG() const { return DAG; } |
206 | | |
207 | | private: |
208 | | SDNode *AnalyzeNewNode(SDNode *N); |
209 | | void AnalyzeNewValue(SDValue &Val); |
210 | | void PerformExpensiveChecks(); |
211 | | void RemapId(TableId &Id); |
212 | | void RemapValue(SDValue &V); |
213 | | |
214 | | // Common routines. |
215 | | SDValue BitConvertToInteger(SDValue Op); |
216 | | SDValue BitConvertVectorToIntegerVector(SDValue Op); |
217 | | SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT); |
218 | | bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult); |
219 | | bool CustomWidenLowerNode(SDNode *N, EVT VT); |
220 | | |
221 | | /// Replace each result of the given MERGE_VALUES node with the corresponding |
222 | | /// input operand, except for the result 'ResNo', for which the corresponding |
223 | | /// input operand is returned. |
224 | | SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo); |
225 | | |
226 | | SDValue JoinIntegers(SDValue Lo, SDValue Hi); |
227 | | SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned); |
228 | | |
229 | | std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, |
230 | | SDNode *Node, bool isSigned); |
231 | | std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node); |
232 | | |
233 | | SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT); |
234 | | |
235 | | void ReplaceValueWith(SDValue From, SDValue To); |
236 | | void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi); |
237 | | void SplitInteger(SDValue Op, EVT LoVT, EVT HiVT, |
238 | | SDValue &Lo, SDValue &Hi); |
239 | | |
240 | 12.6M | void AddToWorklist(SDNode *N) { |
241 | 12.6M | N->setNodeId(ReadyToProcess); |
242 | 12.6M | Worklist.push_back(N); |
243 | 12.6M | } |
244 | | |
245 | | //===--------------------------------------------------------------------===// |
246 | | // Integer Promotion Support: LegalizeIntegerTypes.cpp |
247 | | //===--------------------------------------------------------------------===// |
248 | | |
249 | | /// Given a processed operand Op which was promoted to a larger integer type, |
250 | | /// this returns the promoted value. The low bits of the promoted value |
251 | | /// corresponding to the original type are exactly equal to Op. |
252 | | /// The extra bits contain rubbish, so the promoted value may need to be zero- |
253 | | /// or sign-extended from the original type before it is usable (the helpers |
254 | | /// SExtPromotedInteger and ZExtPromotedInteger can do this for you). |
255 | | /// For example, if Op is an i16 and was promoted to an i32, then this method |
256 | | /// returns an i32, the lower 16 bits of which coincide with Op, and the upper |
257 | | /// 16 bits of which contain rubbish. |
258 | 1.08M | SDValue GetPromotedInteger(SDValue Op) { |
259 | 1.08M | TableId &PromotedId = PromotedIntegers[getTableId(Op)]; |
260 | 1.08M | SDValue PromotedOp = getSDValue(PromotedId); |
261 | 1.08M | assert(PromotedOp.getNode() && "Operand wasn't promoted?"); |
262 | 1.08M | return PromotedOp; |
263 | 1.08M | } |
264 | | void SetPromotedInteger(SDValue Op, SDValue Result); |
265 | | |
266 | | /// Get a promoted operand and sign extend it to the final size. |
267 | 19.5k | SDValue SExtPromotedInteger(SDValue Op) { |
268 | 19.5k | EVT OldVT = Op.getValueType(); |
269 | 19.5k | SDLoc dl(Op); |
270 | 19.5k | Op = GetPromotedInteger(Op); |
271 | 19.5k | return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op, |
272 | 19.5k | DAG.getValueType(OldVT)); |
273 | 19.5k | } |
274 | | |
275 | | /// Get a promoted operand and zero extend it to the final size. |
276 | 23.1k | SDValue ZExtPromotedInteger(SDValue Op) { |
277 | 23.1k | EVT OldVT = Op.getValueType(); |
278 | 23.1k | SDLoc dl(Op); |
279 | 23.1k | Op = GetPromotedInteger(Op); |
280 | 23.1k | return DAG.getZeroExtendInReg(Op, dl, OldVT.getScalarType()); |
281 | 23.1k | } |
282 | | |
283 | | // Get a promoted operand and sign or zero extend it to the final size |
284 | | // (depending on TargetLoweringInfo::isSExtCheaperThanZExt). For a given |
285 | | // subtarget and type, the choice of sign or zero-extension will be |
286 | | // consistent. |
287 | 175k | SDValue SExtOrZExtPromotedInteger(SDValue Op) { |
288 | 175k | EVT OldVT = Op.getValueType(); |
289 | 175k | SDLoc DL(Op); |
290 | 175k | Op = GetPromotedInteger(Op); |
291 | 175k | if (TLI.isSExtCheaperThanZExt(OldVT, Op.getValueType())) |
292 | 36 | return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), Op, |
293 | 36 | DAG.getValueType(OldVT)); |
294 | 175k | return DAG.getZeroExtendInReg(Op, DL, OldVT.getScalarType()); |
295 | 175k | } |
296 | | |
297 | | // Integer Result Promotion. |
298 | | void PromoteIntegerResult(SDNode *N, unsigned ResNo); |
299 | | SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo); |
300 | | SDValue PromoteIntRes_AssertSext(SDNode *N); |
301 | | SDValue PromoteIntRes_AssertZext(SDNode *N); |
302 | | SDValue PromoteIntRes_Atomic0(AtomicSDNode *N); |
303 | | SDValue PromoteIntRes_Atomic1(AtomicSDNode *N); |
304 | | SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo); |
305 | | SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N); |
306 | | SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N); |
307 | | SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N); |
308 | | SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N); |
309 | | SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N); |
310 | | SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N); |
311 | | SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N); |
312 | | SDValue PromoteIntRes_BITCAST(SDNode *N); |
313 | | SDValue PromoteIntRes_BSWAP(SDNode *N); |
314 | | SDValue PromoteIntRes_BITREVERSE(SDNode *N); |
315 | | SDValue PromoteIntRes_BUILD_PAIR(SDNode *N); |
316 | | SDValue PromoteIntRes_Constant(SDNode *N); |
317 | | SDValue PromoteIntRes_CTLZ(SDNode *N); |
318 | | SDValue PromoteIntRes_CTPOP(SDNode *N); |
319 | | SDValue PromoteIntRes_CTTZ(SDNode *N); |
320 | | SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N); |
321 | | SDValue PromoteIntRes_FP_TO_XINT(SDNode *N); |
322 | | SDValue PromoteIntRes_FP_TO_FP16(SDNode *N); |
323 | | SDValue PromoteIntRes_INT_EXTEND(SDNode *N); |
324 | | SDValue PromoteIntRes_LOAD(LoadSDNode *N); |
325 | | SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N); |
326 | | SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N); |
327 | | SDValue PromoteIntRes_Overflow(SDNode *N); |
328 | | SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo); |
329 | | SDValue PromoteIntRes_SELECT(SDNode *N); |
330 | | SDValue PromoteIntRes_VSELECT(SDNode *N); |
331 | | SDValue PromoteIntRes_SELECT_CC(SDNode *N); |
332 | | SDValue PromoteIntRes_SETCC(SDNode *N); |
333 | | SDValue PromoteIntRes_SHL(SDNode *N); |
334 | | SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N); |
335 | | SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N); |
336 | | SDValue PromoteIntRes_SExtIntBinOp(SDNode *N); |
337 | | SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N); |
338 | | SDValue PromoteIntRes_SRA(SDNode *N); |
339 | | SDValue PromoteIntRes_SRL(SDNode *N); |
340 | | SDValue PromoteIntRes_TRUNCATE(SDNode *N); |
341 | | SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo); |
342 | | SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo); |
343 | | SDValue PromoteIntRes_UNDEF(SDNode *N); |
344 | | SDValue PromoteIntRes_VAARG(SDNode *N); |
345 | | SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo); |
346 | | SDValue PromoteIntRes_ADDSUBSAT(SDNode *N); |
347 | | SDValue PromoteIntRes_MULFIX(SDNode *N); |
348 | | SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N); |
349 | | SDValue PromoteIntRes_VECREDUCE(SDNode *N); |
350 | | SDValue PromoteIntRes_ABS(SDNode *N); |
351 | | |
352 | | // Integer Operand Promotion. |
353 | | bool PromoteIntegerOperand(SDNode *N, unsigned OpNo); |
354 | | SDValue PromoteIntOp_ANY_EXTEND(SDNode *N); |
355 | | SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N); |
356 | | SDValue PromoteIntOp_BITCAST(SDNode *N); |
357 | | SDValue PromoteIntOp_BUILD_PAIR(SDNode *N); |
358 | | SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo); |
359 | | SDValue PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo); |
360 | | SDValue PromoteIntOp_BUILD_VECTOR(SDNode *N); |
361 | | SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo); |
362 | | SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N); |
363 | | SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N); |
364 | | SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N); |
365 | | SDValue PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N); |
366 | | SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo); |
367 | | SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo); |
368 | | SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo); |
369 | | SDValue PromoteIntOp_Shift(SDNode *N); |
370 | | SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N); |
371 | | SDValue PromoteIntOp_SINT_TO_FP(SDNode *N); |
372 | | SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo); |
373 | | SDValue PromoteIntOp_TRUNCATE(SDNode *N); |
374 | | SDValue PromoteIntOp_UINT_TO_FP(SDNode *N); |
375 | | SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N); |
376 | | SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo); |
377 | | SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo); |
378 | | SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo); |
379 | | SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo); |
380 | | SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo); |
381 | | SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N); |
382 | | SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo); |
383 | | SDValue PromoteIntOp_MULFIX(SDNode *N); |
384 | | SDValue PromoteIntOp_FPOWI(SDNode *N); |
385 | | SDValue PromoteIntOp_VECREDUCE(SDNode *N); |
386 | | |
387 | | void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code); |
388 | | |
389 | | //===--------------------------------------------------------------------===// |
390 | | // Integer Expansion Support: LegalizeIntegerTypes.cpp |
391 | | //===--------------------------------------------------------------------===// |
392 | | |
393 | | /// Given a processed operand Op which was expanded into two integers of half |
394 | | /// the size, this returns the two halves. The low bits of Op are exactly |
395 | | /// equal to the bits of Lo; the high bits exactly equal Hi. |
396 | | /// For example, if Op is an i64 which was expanded into two i32's, then this |
397 | | /// method returns the two i32's, with Lo being equal to the lower 32 bits of |
398 | | /// Op, and Hi being equal to the upper 32 bits. |
399 | | void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi); |
400 | | void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi); |
401 | | |
402 | | // Integer Result Expansion. |
403 | | void ExpandIntegerResult(SDNode *N, unsigned ResNo); |
404 | | void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi); |
405 | | void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi); |
406 | | void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi); |
407 | | void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi); |
408 | | void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi); |
409 | | void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi); |
410 | | void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi); |
411 | | void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi); |
412 | | void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi); |
413 | | void ExpandIntRes_READCYCLECOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi); |
414 | | void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi); |
415 | | void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi); |
416 | | void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi); |
417 | | void ExpandIntRes_ZERO_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi); |
418 | | void ExpandIntRes_FLT_ROUNDS (SDNode *N, SDValue &Lo, SDValue &Hi); |
419 | | void ExpandIntRes_FP_TO_SINT (SDNode *N, SDValue &Lo, SDValue &Hi); |
420 | | void ExpandIntRes_FP_TO_UINT (SDNode *N, SDValue &Lo, SDValue &Hi); |
421 | | void ExpandIntRes_LLROUND (SDNode *N, SDValue &Lo, SDValue &Hi); |
422 | | void ExpandIntRes_LLRINT (SDNode *N, SDValue &Lo, SDValue &Hi); |
423 | | |
424 | | void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi); |
425 | | void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi); |
426 | | void ExpandIntRes_ADDSUBC (SDNode *N, SDValue &Lo, SDValue &Hi); |
427 | | void ExpandIntRes_ADDSUBE (SDNode *N, SDValue &Lo, SDValue &Hi); |
428 | | void ExpandIntRes_ADDSUBCARRY (SDNode *N, SDValue &Lo, SDValue &Hi); |
429 | | void ExpandIntRes_BITREVERSE (SDNode *N, SDValue &Lo, SDValue &Hi); |
430 | | void ExpandIntRes_BSWAP (SDNode *N, SDValue &Lo, SDValue &Hi); |
431 | | void ExpandIntRes_MUL (SDNode *N, SDValue &Lo, SDValue &Hi); |
432 | | void ExpandIntRes_SDIV (SDNode *N, SDValue &Lo, SDValue &Hi); |
433 | | void ExpandIntRes_SREM (SDNode *N, SDValue &Lo, SDValue &Hi); |
434 | | void ExpandIntRes_UDIV (SDNode *N, SDValue &Lo, SDValue &Hi); |
435 | | void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi); |
436 | | void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi); |
437 | | |
438 | | void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi); |
439 | | |
440 | | void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); |
441 | | void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); |
442 | | void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi); |
443 | | void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi); |
444 | | void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi); |
445 | | |
446 | | void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi); |
447 | | void ExpandIntRes_VECREDUCE (SDNode *N, SDValue &Lo, SDValue &Hi); |
448 | | |
449 | | void ExpandShiftByConstant(SDNode *N, const APInt &Amt, |
450 | | SDValue &Lo, SDValue &Hi); |
451 | | bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi); |
452 | | bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi); |
453 | | |
454 | | // Integer Operand Expansion. |
455 | | bool ExpandIntegerOperand(SDNode *N, unsigned OpNo); |
456 | | SDValue ExpandIntOp_BR_CC(SDNode *N); |
457 | | SDValue ExpandIntOp_SELECT_CC(SDNode *N); |
458 | | SDValue ExpandIntOp_SETCC(SDNode *N); |
459 | | SDValue ExpandIntOp_SETCCCARRY(SDNode *N); |
460 | | SDValue ExpandIntOp_Shift(SDNode *N); |
461 | | SDValue ExpandIntOp_SINT_TO_FP(SDNode *N); |
462 | | SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo); |
463 | | SDValue ExpandIntOp_TRUNCATE(SDNode *N); |
464 | | SDValue ExpandIntOp_UINT_TO_FP(SDNode *N); |
465 | | SDValue ExpandIntOp_RETURNADDR(SDNode *N); |
466 | | SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N); |
467 | | |
468 | | void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS, |
469 | | ISD::CondCode &CCCode, const SDLoc &dl); |
470 | | |
471 | | //===--------------------------------------------------------------------===// |
472 | | // Float to Integer Conversion Support: LegalizeFloatTypes.cpp |
473 | | //===--------------------------------------------------------------------===// |
474 | | |
475 | | /// Given an operand Op of Float type, returns the integer if the Op is not |
476 | | /// supported in target HW and converted to the integer. |
477 | | /// The integer contains exactly the same bits as Op - only the type changed. |
478 | | /// For example, if Op is an f32 which was softened to an i32, then this |
479 | | /// method returns an i32, the bits of which coincide with those of Op. |
480 | | /// If the Op can be efficiently supported in target HW or the operand must |
481 | | /// stay in a register, the Op is not converted to an integer. |
482 | | /// In that case, the given op is returned. |
483 | 9.27k | SDValue GetSoftenedFloat(SDValue Op) { |
484 | 9.27k | TableId Id = getTableId(Op); |
485 | 9.27k | auto Iter = SoftenedFloats.find(Id); |
486 | 9.27k | if (Iter == SoftenedFloats.end()) { |
487 | 977 | assert(isSimpleLegalType(Op.getValueType()) && |
488 | 977 | "Operand wasn't converted to integer?"); |
489 | 977 | return Op; |
490 | 977 | } |
491 | 8.30k | SDValue SoftenedOp = getSDValue(Iter->second); |
492 | 8.30k | assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?"); |
493 | 8.30k | return SoftenedOp; |
494 | 8.30k | } |
495 | | void SetSoftenedFloat(SDValue Op, SDValue Result); |
496 | | |
497 | | // Convert Float Results to Integer for Non-HW-supported Operations. |
498 | | bool SoftenFloatResult(SDNode *N, unsigned ResNo); |
499 | | SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo); |
500 | | SDValue SoftenFloatRes_BITCAST(SDNode *N, unsigned ResNo); |
501 | | SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N); |
502 | | SDValue SoftenFloatRes_ConstantFP(SDNode *N, unsigned ResNo); |
503 | | SDValue SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo); |
504 | | SDValue SoftenFloatRes_FABS(SDNode *N, unsigned ResNo); |
505 | | SDValue SoftenFloatRes_FMINNUM(SDNode *N); |
506 | | SDValue SoftenFloatRes_FMAXNUM(SDNode *N); |
507 | | SDValue SoftenFloatRes_FADD(SDNode *N); |
508 | | SDValue SoftenFloatRes_FCEIL(SDNode *N); |
509 | | SDValue SoftenFloatRes_FCOPYSIGN(SDNode *N, unsigned ResNo); |
510 | | SDValue SoftenFloatRes_FCOS(SDNode *N); |
511 | | SDValue SoftenFloatRes_FDIV(SDNode *N); |
512 | | SDValue SoftenFloatRes_FEXP(SDNode *N); |
513 | | SDValue SoftenFloatRes_FEXP2(SDNode *N); |
514 | | SDValue SoftenFloatRes_FFLOOR(SDNode *N); |
515 | | SDValue SoftenFloatRes_FLOG(SDNode *N); |
516 | | SDValue SoftenFloatRes_FLOG2(SDNode *N); |
517 | | SDValue SoftenFloatRes_FLOG10(SDNode *N); |
518 | | SDValue SoftenFloatRes_FMA(SDNode *N); |
519 | | SDValue SoftenFloatRes_FMUL(SDNode *N); |
520 | | SDValue SoftenFloatRes_FNEARBYINT(SDNode *N); |
521 | | SDValue SoftenFloatRes_FNEG(SDNode *N, unsigned ResNo); |
522 | | SDValue SoftenFloatRes_FP_EXTEND(SDNode *N); |
523 | | SDValue SoftenFloatRes_FP16_TO_FP(SDNode *N); |
524 | | SDValue SoftenFloatRes_FP_ROUND(SDNode *N); |
525 | | SDValue SoftenFloatRes_FPOW(SDNode *N); |
526 | | SDValue SoftenFloatRes_FPOWI(SDNode *N); |
527 | | SDValue SoftenFloatRes_FREM(SDNode *N); |
528 | | SDValue SoftenFloatRes_FRINT(SDNode *N); |
529 | | SDValue SoftenFloatRes_FROUND(SDNode *N); |
530 | | SDValue SoftenFloatRes_FSIN(SDNode *N); |
531 | | SDValue SoftenFloatRes_FSQRT(SDNode *N); |
532 | | SDValue SoftenFloatRes_FSUB(SDNode *N); |
533 | | SDValue SoftenFloatRes_FTRUNC(SDNode *N); |
534 | | SDValue SoftenFloatRes_LOAD(SDNode *N, unsigned ResNo); |
535 | | SDValue SoftenFloatRes_SELECT(SDNode *N, unsigned ResNo); |
536 | | SDValue SoftenFloatRes_SELECT_CC(SDNode *N, unsigned ResNo); |
537 | | SDValue SoftenFloatRes_UNDEF(SDNode *N); |
538 | | SDValue SoftenFloatRes_VAARG(SDNode *N); |
539 | | SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N); |
540 | | |
541 | | // Return true if we can skip softening the given operand or SDNode because |
542 | | // either it was soften before by SoftenFloatResult and references to the |
543 | | // operand were replaced by ReplaceValueWith or it's value type is legal in HW |
544 | | // registers and the operand can be left unchanged. |
545 | | bool CanSkipSoftenFloatOperand(SDNode *N, unsigned OpNo); |
546 | | |
547 | | // Convert Float Operand to Integer for Non-HW-supported Operations. |
548 | | bool SoftenFloatOperand(SDNode *N, unsigned OpNo); |
549 | | SDValue SoftenFloatOp_BITCAST(SDNode *N); |
550 | | SDValue SoftenFloatOp_COPY_TO_REG(SDNode *N); |
551 | | SDValue SoftenFloatOp_BR_CC(SDNode *N); |
552 | | SDValue SoftenFloatOp_FABS(SDNode *N); |
553 | | SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N); |
554 | | SDValue SoftenFloatOp_FNEG(SDNode *N); |
555 | | SDValue SoftenFloatOp_FP_EXTEND(SDNode *N); |
556 | | SDValue SoftenFloatOp_FP_ROUND(SDNode *N); |
557 | | SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N); |
558 | | SDValue SoftenFloatOp_LROUND(SDNode *N); |
559 | | SDValue SoftenFloatOp_LLROUND(SDNode *N); |
560 | | SDValue SoftenFloatOp_LRINT(SDNode *N); |
561 | | SDValue SoftenFloatOp_LLRINT(SDNode *N); |
562 | | SDValue SoftenFloatOp_SELECT(SDNode *N); |
563 | | SDValue SoftenFloatOp_SELECT_CC(SDNode *N); |
564 | | SDValue SoftenFloatOp_SETCC(SDNode *N); |
565 | | SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo); |
566 | | |
567 | | //===--------------------------------------------------------------------===// |
568 | | // Float Expansion Support: LegalizeFloatTypes.cpp |
569 | | //===--------------------------------------------------------------------===// |
570 | | |
571 | | /// Given a processed operand Op which was expanded into two floating-point |
572 | | /// values of half the size, this returns the two halves. |
573 | | /// The low bits of Op are exactly equal to the bits of Lo; the high bits |
574 | | /// exactly equal Hi. For example, if Op is a ppcf128 which was expanded |
575 | | /// into two f64's, then this method returns the two f64's, with Lo being |
576 | | /// equal to the lower 64 bits of Op, and Hi to the upper 64 bits. |
577 | | void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi); |
578 | | void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi); |
579 | | |
580 | | // Float Result Expansion. |
581 | | void ExpandFloatResult(SDNode *N, unsigned ResNo); |
582 | | void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi); |
583 | | void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi); |
584 | | void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi); |
585 | | void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi); |
586 | | void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi); |
587 | | void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi); |
588 | | void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi); |
589 | | void ExpandFloatRes_FCOS (SDNode *N, SDValue &Lo, SDValue &Hi); |
590 | | void ExpandFloatRes_FDIV (SDNode *N, SDValue &Lo, SDValue &Hi); |
591 | | void ExpandFloatRes_FEXP (SDNode *N, SDValue &Lo, SDValue &Hi); |
592 | | void ExpandFloatRes_FEXP2 (SDNode *N, SDValue &Lo, SDValue &Hi); |
593 | | void ExpandFloatRes_FFLOOR (SDNode *N, SDValue &Lo, SDValue &Hi); |
594 | | void ExpandFloatRes_FLOG (SDNode *N, SDValue &Lo, SDValue &Hi); |
595 | | void ExpandFloatRes_FLOG2 (SDNode *N, SDValue &Lo, SDValue &Hi); |
596 | | void ExpandFloatRes_FLOG10 (SDNode *N, SDValue &Lo, SDValue &Hi); |
597 | | void ExpandFloatRes_FMA (SDNode *N, SDValue &Lo, SDValue &Hi); |
598 | | void ExpandFloatRes_FMUL (SDNode *N, SDValue &Lo, SDValue &Hi); |
599 | | void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi); |
600 | | void ExpandFloatRes_FNEG (SDNode *N, SDValue &Lo, SDValue &Hi); |
601 | | void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi); |
602 | | void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi); |
603 | | void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi); |
604 | | void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi); |
605 | | void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi); |
606 | | void ExpandFloatRes_FROUND (SDNode *N, SDValue &Lo, SDValue &Hi); |
607 | | void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi); |
608 | | void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi); |
609 | | void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi); |
610 | | void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi); |
611 | | void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi); |
612 | | void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi); |
613 | | |
614 | | // Float Operand Expansion. |
615 | | bool ExpandFloatOperand(SDNode *N, unsigned OpNo); |
616 | | SDValue ExpandFloatOp_BR_CC(SDNode *N); |
617 | | SDValue ExpandFloatOp_FCOPYSIGN(SDNode *N); |
618 | | SDValue ExpandFloatOp_FP_ROUND(SDNode *N); |
619 | | SDValue ExpandFloatOp_FP_TO_SINT(SDNode *N); |
620 | | SDValue ExpandFloatOp_FP_TO_UINT(SDNode *N); |
621 | | SDValue ExpandFloatOp_LROUND(SDNode *N); |
622 | | SDValue ExpandFloatOp_LLROUND(SDNode *N); |
623 | | SDValue ExpandFloatOp_LRINT(SDNode *N); |
624 | | SDValue ExpandFloatOp_LLRINT(SDNode *N); |
625 | | SDValue ExpandFloatOp_SELECT_CC(SDNode *N); |
626 | | SDValue ExpandFloatOp_SETCC(SDNode *N); |
627 | | SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo); |
628 | | |
629 | | void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS, |
630 | | ISD::CondCode &CCCode, const SDLoc &dl); |
631 | | |
632 | | //===--------------------------------------------------------------------===// |
633 | | // Float promotion support: LegalizeFloatTypes.cpp |
634 | | //===--------------------------------------------------------------------===// |
635 | | |
636 | 6.69k | SDValue GetPromotedFloat(SDValue Op) { |
637 | 6.69k | TableId &PromotedId = PromotedFloats[getTableId(Op)]; |
638 | 6.69k | SDValue PromotedOp = getSDValue(PromotedId); |
639 | 6.69k | assert(PromotedOp.getNode() && "Operand wasn't promoted?"); |
640 | 6.69k | return PromotedOp; |
641 | 6.69k | } |
642 | | void SetPromotedFloat(SDValue Op, SDValue Result); |
643 | | |
644 | | void PromoteFloatResult(SDNode *N, unsigned ResNo); |
645 | | SDValue PromoteFloatRes_BITCAST(SDNode *N); |
646 | | SDValue PromoteFloatRes_BinOp(SDNode *N); |
647 | | SDValue PromoteFloatRes_ConstantFP(SDNode *N); |
648 | | SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N); |
649 | | SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N); |
650 | | SDValue PromoteFloatRes_FMAD(SDNode *N); |
651 | | SDValue PromoteFloatRes_FPOWI(SDNode *N); |
652 | | SDValue PromoteFloatRes_FP_ROUND(SDNode *N); |
653 | | SDValue PromoteFloatRes_LOAD(SDNode *N); |
654 | | SDValue PromoteFloatRes_SELECT(SDNode *N); |
655 | | SDValue PromoteFloatRes_SELECT_CC(SDNode *N); |
656 | | SDValue PromoteFloatRes_UnaryOp(SDNode *N); |
657 | | SDValue PromoteFloatRes_UNDEF(SDNode *N); |
658 | | SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N); |
659 | | SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N); |
660 | | |
661 | | bool PromoteFloatOperand(SDNode *N, unsigned OpNo); |
662 | | SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo); |
663 | | SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo); |
664 | | SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo); |
665 | | SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo); |
666 | | SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo); |
667 | | SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo); |
668 | | SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo); |
669 | | |
670 | | //===--------------------------------------------------------------------===// |
671 | | // Scalarization Support: LegalizeVectorTypes.cpp |
672 | | //===--------------------------------------------------------------------===// |
673 | | |
674 | | /// Given a processed one-element vector Op which was scalarized to its |
675 | | /// element type, this returns the element. For example, if Op is a v1i32, |
676 | | /// Op = < i32 val >, this method returns val, an i32. |
677 | 74.8k | SDValue GetScalarizedVector(SDValue Op) { |
678 | 74.8k | TableId &ScalarizedId = ScalarizedVectors[getTableId(Op)]; |
679 | 74.8k | SDValue ScalarizedOp = getSDValue(ScalarizedId); |
680 | 74.8k | assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?"); |
681 | 74.8k | return ScalarizedOp; |
682 | 74.8k | } |
683 | | void SetScalarizedVector(SDValue Op, SDValue Result); |
684 | | |
685 | | // Vector Result Scalarization: <1 x ty> -> ty. |
686 | | void ScalarizeVectorResult(SDNode *N, unsigned ResNo); |
687 | | SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo); |
688 | | SDValue ScalarizeVecRes_BinOp(SDNode *N); |
689 | | SDValue ScalarizeVecRes_TernaryOp(SDNode *N); |
690 | | SDValue ScalarizeVecRes_UnaryOp(SDNode *N); |
691 | | SDValue ScalarizeVecRes_StrictFPOp(SDNode *N); |
692 | | SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo); |
693 | | SDValue ScalarizeVecRes_InregOp(SDNode *N); |
694 | | SDValue ScalarizeVecRes_VecInregOp(SDNode *N); |
695 | | |
696 | | SDValue ScalarizeVecRes_BITCAST(SDNode *N); |
697 | | SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N); |
698 | | SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N); |
699 | | SDValue ScalarizeVecRes_FP_ROUND(SDNode *N); |
700 | | SDValue ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N); |
701 | | SDValue ScalarizeVecRes_FPOWI(SDNode *N); |
702 | | SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N); |
703 | | SDValue ScalarizeVecRes_LOAD(LoadSDNode *N); |
704 | | SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N); |
705 | | SDValue ScalarizeVecRes_VSELECT(SDNode *N); |
706 | | SDValue ScalarizeVecRes_SELECT(SDNode *N); |
707 | | SDValue ScalarizeVecRes_SELECT_CC(SDNode *N); |
708 | | SDValue ScalarizeVecRes_SETCC(SDNode *N); |
709 | | SDValue ScalarizeVecRes_UNDEF(SDNode *N); |
710 | | SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N); |
711 | | |
712 | | SDValue ScalarizeVecRes_MULFIX(SDNode *N); |
713 | | |
714 | | // Vector Operand Scalarization: <1 x ty> -> ty. |
715 | | bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo); |
716 | | SDValue ScalarizeVecOp_BITCAST(SDNode *N); |
717 | | SDValue ScalarizeVecOp_UnaryOp(SDNode *N); |
718 | | SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N); |
719 | | SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N); |
720 | | SDValue ScalarizeVecOp_VSELECT(SDNode *N); |
721 | | SDValue ScalarizeVecOp_VSETCC(SDNode *N); |
722 | | SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo); |
723 | | SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo); |
724 | | SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo); |
725 | | SDValue ScalarizeVecOp_VECREDUCE(SDNode *N); |
726 | | |
727 | | //===--------------------------------------------------------------------===// |
728 | | // Vector Splitting Support: LegalizeVectorTypes.cpp |
729 | | //===--------------------------------------------------------------------===// |
730 | | |
731 | | /// Given a processed vector Op which was split into vectors of half the size, |
732 | | /// this method returns the halves. The first elements of Op coincide with the |
733 | | /// elements of Lo; the remaining elements of Op coincide with the elements of |
734 | | /// Hi: Op is what you would get by concatenating Lo and Hi. |
735 | | /// For example, if Op is a v8i32 that was split into two v4i32's, then this |
736 | | /// method returns the two v4i32's, with Lo corresponding to the first 4 |
737 | | /// elements of Op, and Hi to the last 4 elements. |
738 | | void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi); |
739 | | void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi); |
740 | | |
741 | | // Vector Result Splitting: <128 x ty> -> 2 x <64 x ty>. |
742 | | void SplitVectorResult(SDNode *N, unsigned ResNo); |
743 | | void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi); |
744 | | void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi); |
745 | | void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi); |
746 | | void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi); |
747 | | void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi); |
748 | | void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi); |
749 | | void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi); |
750 | | void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, |
751 | | SDValue &Lo, SDValue &Hi); |
752 | | |
753 | | void SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi); |
754 | | |
755 | | void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi); |
756 | | void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi); |
757 | | void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi); |
758 | | void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi); |
759 | | void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi); |
760 | | void SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, SDValue &Hi); |
761 | | void SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, SDValue &Hi); |
762 | | void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi); |
763 | | void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi); |
764 | | void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi); |
765 | | void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi); |
766 | | void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi); |
767 | | void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi); |
768 | | void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo, |
769 | | SDValue &Hi); |
770 | | void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi); |
771 | | |
772 | | // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>. |
773 | | bool SplitVectorOperand(SDNode *N, unsigned OpNo); |
774 | | SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo); |
775 | | SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo); |
776 | | SDValue SplitVecOp_UnaryOp(SDNode *N); |
777 | | SDValue SplitVecOp_TruncateHelper(SDNode *N); |
778 | | |
779 | | SDValue SplitVecOp_BITCAST(SDNode *N); |
780 | | SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N); |
781 | | SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N); |
782 | | SDValue SplitVecOp_ExtVecInRegOp(SDNode *N); |
783 | | SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo); |
784 | | SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo); |
785 | | SDValue SplitVecOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo); |
786 | | SDValue SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, unsigned OpNo); |
787 | | SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N); |
788 | | SDValue SplitVecOp_VSETCC(SDNode *N); |
789 | | SDValue SplitVecOp_FP_ROUND(SDNode *N); |
790 | | SDValue SplitVecOp_FCOPYSIGN(SDNode *N); |
791 | | |
792 | | //===--------------------------------------------------------------------===// |
793 | | // Vector Widening Support: LegalizeVectorTypes.cpp |
794 | | //===--------------------------------------------------------------------===// |
795 | | |
796 | | /// Given a processed vector Op which was widened into a larger vector, this |
797 | | /// method returns the larger vector. The elements of the returned vector |
798 | | /// consist of the elements of Op followed by elements containing rubbish. |
799 | | /// For example, if Op is a v2i32 that was widened to a v4i32, then this |
800 | | /// method returns a v4i32 for which the first two elements are the same as |
801 | | /// those of Op, while the last two elements contain rubbish. |
802 | 32.8k | SDValue GetWidenedVector(SDValue Op) { |
803 | 32.8k | TableId &WidenedId = WidenedVectors[getTableId(Op)]; |
804 | 32.8k | SDValue WidenedOp = getSDValue(WidenedId); |
805 | 32.8k | assert(WidenedOp.getNode() && "Operand wasn't widened?"); |
806 | 32.8k | return WidenedOp; |
807 | 32.8k | } |
808 | | void SetWidenedVector(SDValue Op, SDValue Result); |
809 | | |
810 | | // Widen Vector Result Promotion. |
811 | | void WidenVectorResult(SDNode *N, unsigned ResNo); |
812 | | SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo); |
813 | | SDValue WidenVecRes_BITCAST(SDNode* N); |
814 | | SDValue WidenVecRes_BUILD_VECTOR(SDNode* N); |
815 | | SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N); |
816 | | SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N); |
817 | | SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N); |
818 | | SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N); |
819 | | SDValue WidenVecRes_LOAD(SDNode* N); |
820 | | SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N); |
821 | | SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N); |
822 | | SDValue WidenVecRes_SCALAR_TO_VECTOR(SDNode* N); |
823 | | SDValue WidenVecRes_SELECT(SDNode* N); |
824 | | SDValue WidenVSELECTAndMask(SDNode *N); |
825 | | SDValue WidenVecRes_SELECT_CC(SDNode* N); |
826 | | SDValue WidenVecRes_SETCC(SDNode* N); |
827 | | SDValue WidenVecRes_UNDEF(SDNode *N); |
828 | | SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N); |
829 | | |
830 | | SDValue WidenVecRes_Ternary(SDNode *N); |
831 | | SDValue WidenVecRes_Binary(SDNode *N); |
832 | | SDValue WidenVecRes_BinaryCanTrap(SDNode *N); |
833 | | SDValue WidenVecRes_StrictFP(SDNode *N); |
834 | | SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo); |
835 | | SDValue WidenVecRes_Convert(SDNode *N); |
836 | | SDValue WidenVecRes_Convert_StrictFP(SDNode *N); |
837 | | SDValue WidenVecRes_FCOPYSIGN(SDNode *N); |
838 | | SDValue WidenVecRes_POWI(SDNode *N); |
839 | | SDValue WidenVecRes_Shift(SDNode *N); |
840 | | SDValue WidenVecRes_Unary(SDNode *N); |
841 | | SDValue WidenVecRes_InregOp(SDNode *N); |
842 | | |
843 | | // Widen Vector Operand. |
844 | | bool WidenVectorOperand(SDNode *N, unsigned OpNo); |
845 | | SDValue WidenVecOp_BITCAST(SDNode *N); |
846 | | SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N); |
847 | | SDValue WidenVecOp_EXTEND(SDNode *N); |
848 | | SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N); |
849 | | SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N); |
850 | | SDValue WidenVecOp_STORE(SDNode* N); |
851 | | SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo); |
852 | | SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo); |
853 | | SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo); |
854 | | SDValue WidenVecOp_SETCC(SDNode* N); |
855 | | SDValue WidenVecOp_VSELECT(SDNode *N); |
856 | | |
857 | | SDValue WidenVecOp_Convert(SDNode *N); |
858 | | SDValue WidenVecOp_FCOPYSIGN(SDNode *N); |
859 | | SDValue WidenVecOp_VECREDUCE(SDNode *N); |
860 | | |
861 | | /// Helper function to generate a set of operations to perform |
862 | | /// a vector operation for a wider type. |
863 | | /// |
864 | | SDValue UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE); |
865 | | |
866 | | //===--------------------------------------------------------------------===// |
867 | | // Vector Widening Utilities Support: LegalizeVectorTypes.cpp |
868 | | //===--------------------------------------------------------------------===// |
869 | | |
870 | | /// Helper function to generate a set of loads to load a vector with a |
871 | | /// resulting wider type. It takes: |
872 | | /// LdChain: list of chains for the load to be generated. |
873 | | /// Ld: load to widen |
874 | | SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain, |
875 | | LoadSDNode *LD); |
876 | | |
877 | | /// Helper function to generate a set of extension loads to load a vector with |
878 | | /// a resulting wider type. It takes: |
879 | | /// LdChain: list of chains for the load to be generated. |
880 | | /// Ld: load to widen |
881 | | /// ExtType: extension element type |
882 | | SDValue GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain, |
883 | | LoadSDNode *LD, ISD::LoadExtType ExtType); |
884 | | |
885 | | /// Helper function to generate a set of stores to store a widen vector into |
886 | | /// non-widen memory. |
887 | | /// StChain: list of chains for the stores we have generated |
888 | | /// ST: store of a widen value |
889 | | void GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, StoreSDNode *ST); |
890 | | |
891 | | /// Helper function to generate a set of stores to store a truncate widen |
892 | | /// vector into non-widen memory. |
893 | | /// StChain: list of chains for the stores we have generated |
894 | | /// ST: store of a widen value |
895 | | void GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain, |
896 | | StoreSDNode *ST); |
897 | | |
898 | | /// Modifies a vector input (widen or narrows) to a vector of NVT. The |
899 | | /// input vector must have the same element type as NVT. |
900 | | /// When FillWithZeroes is "on" the vector will be widened with zeroes. |
901 | | /// By default, the vector will be widened with undefined values. |
902 | | SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false); |
903 | | |
904 | | /// Return a mask of vector type MaskVT to replace InMask. Also adjust |
905 | | /// MaskVT to ToMaskVT if needed with vector extension or truncation. |
906 | | SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT); |
907 | | |
908 | | //===--------------------------------------------------------------------===// |
909 | | // Generic Splitting: LegalizeTypesGeneric.cpp |
910 | | //===--------------------------------------------------------------------===// |
911 | | |
912 | | // Legalization methods which only use that the illegal type is split into two |
913 | | // not necessarily identical types. As such they can be used for splitting |
914 | | // vectors and expanding integers and floats. |
915 | | |
916 | 6.29k | void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) { |
917 | 6.29k | if (Op.getValueType().isVector()) |
918 | 2.29k | GetSplitVector(Op, Lo, Hi); |
919 | 4.00k | else if (Op.getValueType().isInteger()) |
920 | 3.99k | GetExpandedInteger(Op, Lo, Hi); |
921 | 14 | else |
922 | 14 | GetExpandedFloat(Op, Lo, Hi); |
923 | 6.29k | } |
924 | | |
925 | | /// Use ISD::EXTRACT_ELEMENT nodes to extract the low and high parts of the |
926 | | /// given value. |
927 | | void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi); |
928 | | |
929 | | // Generic Result Splitting. |
930 | | void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo, |
931 | | SDValue &Lo, SDValue &Hi); |
932 | | void SplitRes_SELECT (SDNode *N, SDValue &Lo, SDValue &Hi); |
933 | | void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi); |
934 | | void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi); |
935 | | |
936 | | //===--------------------------------------------------------------------===// |
937 | | // Generic Expansion: LegalizeTypesGeneric.cpp |
938 | | //===--------------------------------------------------------------------===// |
939 | | |
940 | | // Legalization methods which only use that the illegal type is split into two |
941 | | // identical types of half the size, and that the Lo/Hi part is stored first |
942 | | // in memory on little/big-endian machines, followed by the Hi/Lo part. As |
943 | | // such they can be used for expanding integers and floats. |
944 | | |
945 | 59.0k | void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) { |
946 | 59.0k | if (Op.getValueType().isInteger()) |
947 | 58.8k | GetExpandedInteger(Op, Lo, Hi); |
948 | 224 | else |
949 | 224 | GetExpandedFloat(Op, Lo, Hi); |
950 | 59.0k | } |
951 | | |
952 | | |
953 | | /// This function will split the integer \p Op into \p NumElements |
954 | | /// operations of type \p EltVT and store them in \p Ops. |
955 | | void IntegerToVector(SDValue Op, unsigned NumElements, |
956 | | SmallVectorImpl<SDValue> &Ops, EVT EltVT); |
957 | | |
958 | | // Generic Result Expansion. |
959 | | void ExpandRes_MERGE_VALUES (SDNode *N, unsigned ResNo, |
960 | | SDValue &Lo, SDValue &Hi); |
961 | | void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi); |
962 | | void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi); |
963 | | void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi); |
964 | | void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi); |
965 | | void ExpandRes_NormalLoad (SDNode *N, SDValue &Lo, SDValue &Hi); |
966 | | void ExpandRes_VAARG (SDNode *N, SDValue &Lo, SDValue &Hi); |
967 | | |
968 | | // Generic Operand Expansion. |
969 | | SDValue ExpandOp_BITCAST (SDNode *N); |
970 | | SDValue ExpandOp_BUILD_VECTOR (SDNode *N); |
971 | | SDValue ExpandOp_EXTRACT_ELEMENT (SDNode *N); |
972 | | SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N); |
973 | | SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N); |
974 | | SDValue ExpandOp_NormalStore (SDNode *N, unsigned OpNo); |
975 | | }; |
976 | | |
977 | | } // end namespace llvm. |
978 | | |
979 | | #endif |