Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
Line
Count
Source
1
//===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file defines the DAGTypeLegalizer class.  This is a private interface
10
// shared between the code that implements the SelectionDAG::LegalizeTypes
11
// method.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#ifndef LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
16
#define LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
17
18
#include "llvm/ADT/DenseMap.h"
19
#include "llvm/CodeGen/SelectionDAG.h"
20
#include "llvm/CodeGen/TargetLowering.h"
21
#include "llvm/Support/Compiler.h"
22
#include "llvm/Support/Debug.h"
23
24
namespace llvm {
25
26
//===----------------------------------------------------------------------===//
27
/// This takes an arbitrary SelectionDAG as input and hacks on it until only
28
/// value types the target machine can handle are left. This involves promoting
29
/// small sizes to large sizes or splitting up large values into small values.
30
///
31
class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
32
  const TargetLowering &TLI;
33
  SelectionDAG &DAG;
34
public:
35
  /// This pass uses the NodeId on the SDNodes to hold information about the
36
  /// state of the node. The enum has all the values.
37
  enum NodeIdFlags {
38
    /// All operands have been processed, so this node is ready to be handled.
39
    ReadyToProcess = 0,
40
41
    /// This is a new node, not before seen, that was created in the process of
42
    /// legalizing some other node.
43
    NewNode = -1,
44
45
    /// This node's ID needs to be set to the number of its unprocessed
46
    /// operands.
47
    Unanalyzed = -2,
48
49
    /// This is a node that has already been processed.
50
    Processed = -3
51
52
    // 1+ - This is a node which has this many unprocessed operands.
53
  };
54
private:
55
56
  /// This is a bitvector that contains two bits for each simple value type,
57
  /// where the two bits correspond to the LegalizeAction enum from
58
  /// TargetLowering. This can be queried with "getTypeAction(VT)".
59
  TargetLowering::ValueTypeActionImpl ValueTypeActions;
60
61
  /// Return how we should legalize values of this type.
62
67.1M
  TargetLowering::LegalizeTypeAction getTypeAction(EVT VT) const {
63
67.1M
    return TLI.getTypeAction(*DAG.getContext(), VT);
64
67.1M
  }
65
66
  /// Return true if this type is legal on this target.
67
136k
  bool isTypeLegal(EVT VT) const {
68
136k
    return TLI.getTypeAction(*DAG.getContext(), VT) == TargetLowering::TypeLegal;
69
136k
  }
70
71
  /// Return true if this is a simple legal type.
72
6.25k
  bool isSimpleLegalType(EVT VT) const {
73
6.25k
    return VT.isSimple() && 
TLI.isTypeLegal(VT)6.24k
;
74
6.25k
  }
75
76
  /// Return true if this type can be passed in registers.
77
  /// For example, x86_64's f128, should to be legally in registers
78
  /// and only some operations converted to library calls or integer
79
  /// bitwise operations.
80
11.6k
  bool isLegalInHWReg(EVT VT) const {
81
11.6k
    EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
82
11.6k
    return VT == NVT && 
isSimpleLegalType(VT)6.12k
;
83
11.6k
  }
84
85
549k
  EVT getSetCCResultType(EVT VT) const {
86
549k
    return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
87
549k
  }
88
89
  /// Pretend all of this node's results are legal.
90
73.1M
  bool IgnoreNodeResults(SDNode *N) const {
91
73.1M
    return N->getOpcode() == ISD::TargetConstant ||
92
73.1M
           
N->getOpcode() == ISD::Register70.4M
;
93
73.1M
  }
94
95
  // Bijection from SDValue to unique id. As each created node gets a
96
  // new id we do not need to worry about reuse expunging.  Should we
97
  // run out of ids, we can do a one time expensive compactifcation.
98
  typedef unsigned TableId;
99
100
  TableId NextValueId = 1;
101
102
  SmallDenseMap<SDValue, TableId, 8> ValueToIdMap;
103
  SmallDenseMap<TableId, SDValue, 8> IdToValueMap;
104
105
  /// For integer nodes that are below legal width, this map indicates what
106
  /// promoted value to use.
107
  SmallDenseMap<TableId, TableId, 8> PromotedIntegers;
108
109
  /// For integer nodes that need to be expanded this map indicates which
110
  /// operands are the expanded version of the input.
111
  SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedIntegers;
112
113
  /// For floating-point nodes converted to integers of the same size, this map
114
  /// indicates the converted value to use.
115
  SmallDenseMap<TableId, TableId, 8> SoftenedFloats;
116
117
  /// For floating-point nodes that have a smaller precision than the smallest
118
  /// supported precision, this map indicates what promoted value to use.
119
  SmallDenseMap<TableId, TableId, 8> PromotedFloats;
120
121
  /// For float nodes that need to be expanded this map indicates which operands
122
  /// are the expanded version of the input.
123
  SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedFloats;
124
125
  /// For nodes that are <1 x ty>, this map indicates the scalar value of type
126
  /// 'ty' to use.
127
  SmallDenseMap<TableId, TableId, 8> ScalarizedVectors;
128
129
  /// For nodes that need to be split this map indicates which operands are the
130
  /// expanded version of the input.
131
  SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> SplitVectors;
132
133
  /// For vector nodes that need to be widened, indicates the widened value to
134
  /// use.
135
  SmallDenseMap<TableId, TableId, 8> WidenedVectors;
136
137
  /// For values that have been replaced with another, indicates the replacement
138
  /// value to use.
139
  SmallDenseMap<TableId, TableId, 8> ReplacedValues;
140
141
  /// This defines a worklist of nodes to process. In order to be pushed onto
142
  /// this worklist, all operands of a node must have already been processed.
143
  SmallVector<SDNode*, 128> Worklist;
144
145
13.3M
  TableId getTableId(SDValue V) {
146
13.3M
    assert(V.getNode() && "Getting TableId on SDValue()");
147
13.3M
148
13.3M
    auto I = ValueToIdMap.find(V);
149
13.3M
    if (I != ValueToIdMap.end()) {
150
7.48M
      // replace if there's been a shift.
151
7.48M
      RemapId(I->second);
152
7.48M
      assert(I->second && "All Ids should be nonzero");
153
7.48M
      return I->second;
154
7.48M
    }
155
5.87M
    // Add if it's not there.
156
5.87M
    ValueToIdMap.insert(std::make_pair(V, NextValueId));
157
5.87M
    IdToValueMap.insert(std::make_pair(NextValueId, V));
158
5.87M
    ++NextValueId;
159
5.87M
    assert(NextValueId != 0 &&
160
5.87M
           "Ran out of Ids. Increase id type size or add compactification");
161
5.87M
    return NextValueId - 1;
162
5.87M
  }
163
164
9.06M
  const SDValue &getSDValue(TableId &Id) {
165
9.06M
    RemapId(Id);
166
9.06M
    assert(Id && "TableId should be non-zero");
167
9.06M
    return IdToValueMap[Id];
168
9.06M
  }
169
170
public:
171
  explicit DAGTypeLegalizer(SelectionDAG &dag)
172
    : TLI(dag.getTargetLoweringInfo()), DAG(dag),
173
1.28M
    ValueTypeActions(TLI.getValueTypeActions()) {
174
1.28M
    static_assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE,
175
1.28M
                  "Too many value types for ValueTypeActions to hold!");
176
1.28M
  }
177
178
  /// This is the main entry point for the type legalizer.  This does a
179
  /// top-down traversal of the dag, legalizing types as it goes.  Returns
180
  /// "true" if it made any changes.
181
  bool run();
182
183
573
  void NoteDeletion(SDNode *Old, SDNode *New) {
184
1.19k
    for (unsigned i = 0, e = Old->getNumValues(); i != e; 
++i622
) {
185
622
      TableId NewId = getTableId(SDValue(New, i));
186
622
      TableId OldId = getTableId(SDValue(Old, i));
187
622
188
622
      if (OldId != NewId)
189
622
        ReplacedValues[OldId] = NewId;
190
622
191
622
      // Delete Node from tables.
192
622
      ValueToIdMap.erase(SDValue(Old, i));
193
622
      IdToValueMap.erase(OldId);
194
622
      PromotedIntegers.erase(OldId);
195
622
      ExpandedIntegers.erase(OldId);
196
622
      SoftenedFloats.erase(OldId);
197
622
      PromotedFloats.erase(OldId);
198
622
      ExpandedFloats.erase(OldId);
199
622
      ScalarizedVectors.erase(OldId);
200
622
      SplitVectors.erase(OldId);
201
622
      WidenedVectors.erase(OldId);
202
622
    }
203
573
  }
204
205
989k
  SelectionDAG &getDAG() const { return DAG; }
206
207
private:
208
  SDNode *AnalyzeNewNode(SDNode *N);
209
  void AnalyzeNewValue(SDValue &Val);
210
  void PerformExpensiveChecks();
211
  void RemapId(TableId &Id);
212
  void RemapValue(SDValue &V);
213
214
  // Common routines.
215
  SDValue BitConvertToInteger(SDValue Op);
216
  SDValue BitConvertVectorToIntegerVector(SDValue Op);
217
  SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
218
  bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);
219
  bool CustomWidenLowerNode(SDNode *N, EVT VT);
220
221
  /// Replace each result of the given MERGE_VALUES node with the corresponding
222
  /// input operand, except for the result 'ResNo', for which the corresponding
223
  /// input operand is returned.
224
  SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);
225
226
  SDValue JoinIntegers(SDValue Lo, SDValue Hi);
227
  SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned);
228
229
  std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
230
                                                 SDNode *Node, bool isSigned);
231
  std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
232
233
  SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT);
234
235
  void ReplaceValueWith(SDValue From, SDValue To);
236
  void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
237
  void SplitInteger(SDValue Op, EVT LoVT, EVT HiVT,
238
                    SDValue &Lo, SDValue &Hi);
239
240
12.6M
  void AddToWorklist(SDNode *N) {
241
12.6M
    N->setNodeId(ReadyToProcess);
242
12.6M
    Worklist.push_back(N);
243
12.6M
  }
244
245
  //===--------------------------------------------------------------------===//
246
  // Integer Promotion Support: LegalizeIntegerTypes.cpp
247
  //===--------------------------------------------------------------------===//
248
249
  /// Given a processed operand Op which was promoted to a larger integer type,
250
  /// this returns the promoted value. The low bits of the promoted value
251
  /// corresponding to the original type are exactly equal to Op.
252
  /// The extra bits contain rubbish, so the promoted value may need to be zero-
253
  /// or sign-extended from the original type before it is usable (the helpers
254
  /// SExtPromotedInteger and ZExtPromotedInteger can do this for you).
255
  /// For example, if Op is an i16 and was promoted to an i32, then this method
256
  /// returns an i32, the lower 16 bits of which coincide with Op, and the upper
257
  /// 16 bits of which contain rubbish.
258
1.08M
  SDValue GetPromotedInteger(SDValue Op) {
259
1.08M
    TableId &PromotedId = PromotedIntegers[getTableId(Op)];
260
1.08M
    SDValue PromotedOp = getSDValue(PromotedId);
261
1.08M
    assert(PromotedOp.getNode() && "Operand wasn't promoted?");
262
1.08M
    return PromotedOp;
263
1.08M
  }
264
  void SetPromotedInteger(SDValue Op, SDValue Result);
265
266
  /// Get a promoted operand and sign extend it to the final size.
267
19.5k
  SDValue SExtPromotedInteger(SDValue Op) {
268
19.5k
    EVT OldVT = Op.getValueType();
269
19.5k
    SDLoc dl(Op);
270
19.5k
    Op = GetPromotedInteger(Op);
271
19.5k
    return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op,
272
19.5k
                       DAG.getValueType(OldVT));
273
19.5k
  }
274
275
  /// Get a promoted operand and zero extend it to the final size.
276
23.1k
  SDValue ZExtPromotedInteger(SDValue Op) {
277
23.1k
    EVT OldVT = Op.getValueType();
278
23.1k
    SDLoc dl(Op);
279
23.1k
    Op = GetPromotedInteger(Op);
280
23.1k
    return DAG.getZeroExtendInReg(Op, dl, OldVT.getScalarType());
281
23.1k
  }
282
283
  // Get a promoted operand and sign or zero extend it to the final size
284
  // (depending on TargetLoweringInfo::isSExtCheaperThanZExt). For a given
285
  // subtarget and type, the choice of sign or zero-extension will be
286
  // consistent.
287
175k
  SDValue SExtOrZExtPromotedInteger(SDValue Op) {
288
175k
    EVT OldVT = Op.getValueType();
289
175k
    SDLoc DL(Op);
290
175k
    Op = GetPromotedInteger(Op);
291
175k
    if (TLI.isSExtCheaperThanZExt(OldVT, Op.getValueType()))
292
36
      return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), Op,
293
36
                         DAG.getValueType(OldVT));
294
175k
    return DAG.getZeroExtendInReg(Op, DL, OldVT.getScalarType());
295
175k
  }
296
297
  // Integer Result Promotion.
298
  void PromoteIntegerResult(SDNode *N, unsigned ResNo);
299
  SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
300
  SDValue PromoteIntRes_AssertSext(SDNode *N);
301
  SDValue PromoteIntRes_AssertZext(SDNode *N);
302
  SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);
303
  SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);
304
  SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo);
305
  SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);
306
  SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
307
  SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
308
  SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N);
309
  SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N);
310
  SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);
311
  SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);
312
  SDValue PromoteIntRes_BITCAST(SDNode *N);
313
  SDValue PromoteIntRes_BSWAP(SDNode *N);
314
  SDValue PromoteIntRes_BITREVERSE(SDNode *N);
315
  SDValue PromoteIntRes_BUILD_PAIR(SDNode *N);
316
  SDValue PromoteIntRes_Constant(SDNode *N);
317
  SDValue PromoteIntRes_CTLZ(SDNode *N);
318
  SDValue PromoteIntRes_CTPOP(SDNode *N);
319
  SDValue PromoteIntRes_CTTZ(SDNode *N);
320
  SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N);
321
  SDValue PromoteIntRes_FP_TO_XINT(SDNode *N);
322
  SDValue PromoteIntRes_FP_TO_FP16(SDNode *N);
323
  SDValue PromoteIntRes_INT_EXTEND(SDNode *N);
324
  SDValue PromoteIntRes_LOAD(LoadSDNode *N);
325
  SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
326
  SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
327
  SDValue PromoteIntRes_Overflow(SDNode *N);
328
  SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
329
  SDValue PromoteIntRes_SELECT(SDNode *N);
330
  SDValue PromoteIntRes_VSELECT(SDNode *N);
331
  SDValue PromoteIntRes_SELECT_CC(SDNode *N);
332
  SDValue PromoteIntRes_SETCC(SDNode *N);
333
  SDValue PromoteIntRes_SHL(SDNode *N);
334
  SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N);
335
  SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N);
336
  SDValue PromoteIntRes_SExtIntBinOp(SDNode *N);
337
  SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N);
338
  SDValue PromoteIntRes_SRA(SDNode *N);
339
  SDValue PromoteIntRes_SRL(SDNode *N);
340
  SDValue PromoteIntRes_TRUNCATE(SDNode *N);
341
  SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);
342
  SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo);
343
  SDValue PromoteIntRes_UNDEF(SDNode *N);
344
  SDValue PromoteIntRes_VAARG(SDNode *N);
345
  SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
346
  SDValue PromoteIntRes_ADDSUBSAT(SDNode *N);
347
  SDValue PromoteIntRes_MULFIX(SDNode *N);
348
  SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N);
349
  SDValue PromoteIntRes_VECREDUCE(SDNode *N);
350
  SDValue PromoteIntRes_ABS(SDNode *N);
351
352
  // Integer Operand Promotion.
353
  bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);
354
  SDValue PromoteIntOp_ANY_EXTEND(SDNode *N);
355
  SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N);
356
  SDValue PromoteIntOp_BITCAST(SDNode *N);
357
  SDValue PromoteIntOp_BUILD_PAIR(SDNode *N);
358
  SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo);
359
  SDValue PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo);
360
  SDValue PromoteIntOp_BUILD_VECTOR(SDNode *N);
361
  SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo);
362
  SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N);
363
  SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N);
364
  SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
365
  SDValue PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N);
366
  SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
367
  SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
368
  SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
369
  SDValue PromoteIntOp_Shift(SDNode *N);
370
  SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);
371
  SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);
372
  SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo);
373
  SDValue PromoteIntOp_TRUNCATE(SDNode *N);
374
  SDValue PromoteIntOp_UINT_TO_FP(SDNode *N);
375
  SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N);
376
  SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
377
  SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
378
  SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
379
  SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
380
  SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
381
  SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
382
  SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
383
  SDValue PromoteIntOp_MULFIX(SDNode *N);
384
  SDValue PromoteIntOp_FPOWI(SDNode *N);
385
  SDValue PromoteIntOp_VECREDUCE(SDNode *N);
386
387
  void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
388
389
  //===--------------------------------------------------------------------===//
390
  // Integer Expansion Support: LegalizeIntegerTypes.cpp
391
  //===--------------------------------------------------------------------===//
392
393
  /// Given a processed operand Op which was expanded into two integers of half
394
  /// the size, this returns the two halves. The low bits of Op are exactly
395
  /// equal to the bits of Lo; the high bits exactly equal Hi.
396
  /// For example, if Op is an i64 which was expanded into two i32's, then this
397
  /// method returns the two i32's, with Lo being equal to the lower 32 bits of
398
  /// Op, and Hi being equal to the upper 32 bits.
399
  void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
400
  void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi);
401
402
  // Integer Result Expansion.
403
  void ExpandIntegerResult(SDNode *N, unsigned ResNo);
404
  void ExpandIntRes_ANY_EXTEND        (SDNode *N, SDValue &Lo, SDValue &Hi);
405
  void ExpandIntRes_AssertSext        (SDNode *N, SDValue &Lo, SDValue &Hi);
406
  void ExpandIntRes_AssertZext        (SDNode *N, SDValue &Lo, SDValue &Hi);
407
  void ExpandIntRes_Constant          (SDNode *N, SDValue &Lo, SDValue &Hi);
408
  void ExpandIntRes_ABS               (SDNode *N, SDValue &Lo, SDValue &Hi);
409
  void ExpandIntRes_CTLZ              (SDNode *N, SDValue &Lo, SDValue &Hi);
410
  void ExpandIntRes_CTPOP             (SDNode *N, SDValue &Lo, SDValue &Hi);
411
  void ExpandIntRes_CTTZ              (SDNode *N, SDValue &Lo, SDValue &Hi);
412
  void ExpandIntRes_LOAD          (LoadSDNode *N, SDValue &Lo, SDValue &Hi);
413
  void ExpandIntRes_READCYCLECOUNTER  (SDNode *N, SDValue &Lo, SDValue &Hi);
414
  void ExpandIntRes_SIGN_EXTEND       (SDNode *N, SDValue &Lo, SDValue &Hi);
415
  void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi);
416
  void ExpandIntRes_TRUNCATE          (SDNode *N, SDValue &Lo, SDValue &Hi);
417
  void ExpandIntRes_ZERO_EXTEND       (SDNode *N, SDValue &Lo, SDValue &Hi);
418
  void ExpandIntRes_FLT_ROUNDS        (SDNode *N, SDValue &Lo, SDValue &Hi);
419
  void ExpandIntRes_FP_TO_SINT        (SDNode *N, SDValue &Lo, SDValue &Hi);
420
  void ExpandIntRes_FP_TO_UINT        (SDNode *N, SDValue &Lo, SDValue &Hi);
421
  void ExpandIntRes_LLROUND           (SDNode *N, SDValue &Lo, SDValue &Hi);
422
  void ExpandIntRes_LLRINT            (SDNode *N, SDValue &Lo, SDValue &Hi);
423
424
  void ExpandIntRes_Logical           (SDNode *N, SDValue &Lo, SDValue &Hi);
425
  void ExpandIntRes_ADDSUB            (SDNode *N, SDValue &Lo, SDValue &Hi);
426
  void ExpandIntRes_ADDSUBC           (SDNode *N, SDValue &Lo, SDValue &Hi);
427
  void ExpandIntRes_ADDSUBE           (SDNode *N, SDValue &Lo, SDValue &Hi);
428
  void ExpandIntRes_ADDSUBCARRY       (SDNode *N, SDValue &Lo, SDValue &Hi);
429
  void ExpandIntRes_BITREVERSE        (SDNode *N, SDValue &Lo, SDValue &Hi);
430
  void ExpandIntRes_BSWAP             (SDNode *N, SDValue &Lo, SDValue &Hi);
431
  void ExpandIntRes_MUL               (SDNode *N, SDValue &Lo, SDValue &Hi);
432
  void ExpandIntRes_SDIV              (SDNode *N, SDValue &Lo, SDValue &Hi);
433
  void ExpandIntRes_SREM              (SDNode *N, SDValue &Lo, SDValue &Hi);
434
  void ExpandIntRes_UDIV              (SDNode *N, SDValue &Lo, SDValue &Hi);
435
  void ExpandIntRes_UREM              (SDNode *N, SDValue &Lo, SDValue &Hi);
436
  void ExpandIntRes_Shift             (SDNode *N, SDValue &Lo, SDValue &Hi);
437
438
  void ExpandIntRes_MINMAX            (SDNode *N, SDValue &Lo, SDValue &Hi);
439
440
  void ExpandIntRes_SADDSUBO          (SDNode *N, SDValue &Lo, SDValue &Hi);
441
  void ExpandIntRes_UADDSUBO          (SDNode *N, SDValue &Lo, SDValue &Hi);
442
  void ExpandIntRes_XMULO             (SDNode *N, SDValue &Lo, SDValue &Hi);
443
  void ExpandIntRes_ADDSUBSAT         (SDNode *N, SDValue &Lo, SDValue &Hi);
444
  void ExpandIntRes_MULFIX            (SDNode *N, SDValue &Lo, SDValue &Hi);
445
446
  void ExpandIntRes_ATOMIC_LOAD       (SDNode *N, SDValue &Lo, SDValue &Hi);
447
  void ExpandIntRes_VECREDUCE         (SDNode *N, SDValue &Lo, SDValue &Hi);
448
449
  void ExpandShiftByConstant(SDNode *N, const APInt &Amt,
450
                             SDValue &Lo, SDValue &Hi);
451
  bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
452
  bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
453
454
  // Integer Operand Expansion.
455
  bool ExpandIntegerOperand(SDNode *N, unsigned OpNo);
456
  SDValue ExpandIntOp_BR_CC(SDNode *N);
457
  SDValue ExpandIntOp_SELECT_CC(SDNode *N);
458
  SDValue ExpandIntOp_SETCC(SDNode *N);
459
  SDValue ExpandIntOp_SETCCCARRY(SDNode *N);
460
  SDValue ExpandIntOp_Shift(SDNode *N);
461
  SDValue ExpandIntOp_SINT_TO_FP(SDNode *N);
462
  SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);
463
  SDValue ExpandIntOp_TRUNCATE(SDNode *N);
464
  SDValue ExpandIntOp_UINT_TO_FP(SDNode *N);
465
  SDValue ExpandIntOp_RETURNADDR(SDNode *N);
466
  SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
467
468
  void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
469
                                  ISD::CondCode &CCCode, const SDLoc &dl);
470
471
  //===--------------------------------------------------------------------===//
472
  // Float to Integer Conversion Support: LegalizeFloatTypes.cpp
473
  //===--------------------------------------------------------------------===//
474
475
  /// Given an operand Op of Float type, returns the integer if the Op is not
476
  /// supported in target HW and converted to the integer.
477
  /// The integer contains exactly the same bits as Op - only the type changed.
478
  /// For example, if Op is an f32 which was softened to an i32, then this
479
  /// method returns an i32, the bits of which coincide with those of Op.
480
  /// If the Op can be efficiently supported in target HW or the operand must
481
  /// stay in a register, the Op is not converted to an integer.
482
  /// In that case, the given op is returned.
483
9.27k
  SDValue GetSoftenedFloat(SDValue Op) {
484
9.27k
    TableId Id = getTableId(Op);
485
9.27k
    auto Iter = SoftenedFloats.find(Id);
486
9.27k
    if (Iter == SoftenedFloats.end()) {
487
977
      assert(isSimpleLegalType(Op.getValueType()) &&
488
977
             "Operand wasn't converted to integer?");
489
977
      return Op;
490
977
    }
491
8.30k
    SDValue SoftenedOp = getSDValue(Iter->second);
492
8.30k
    assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?");
493
8.30k
    return SoftenedOp;
494
8.30k
  }
495
  void SetSoftenedFloat(SDValue Op, SDValue Result);
496
497
  // Convert Float Results to Integer for Non-HW-supported Operations.
498
  bool SoftenFloatResult(SDNode *N, unsigned ResNo);
499
  SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
500
  SDValue SoftenFloatRes_BITCAST(SDNode *N, unsigned ResNo);
501
  SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N);
502
  SDValue SoftenFloatRes_ConstantFP(SDNode *N, unsigned ResNo);
503
  SDValue SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo);
504
  SDValue SoftenFloatRes_FABS(SDNode *N, unsigned ResNo);
505
  SDValue SoftenFloatRes_FMINNUM(SDNode *N);
506
  SDValue SoftenFloatRes_FMAXNUM(SDNode *N);
507
  SDValue SoftenFloatRes_FADD(SDNode *N);
508
  SDValue SoftenFloatRes_FCEIL(SDNode *N);
509
  SDValue SoftenFloatRes_FCOPYSIGN(SDNode *N, unsigned ResNo);
510
  SDValue SoftenFloatRes_FCOS(SDNode *N);
511
  SDValue SoftenFloatRes_FDIV(SDNode *N);
512
  SDValue SoftenFloatRes_FEXP(SDNode *N);
513
  SDValue SoftenFloatRes_FEXP2(SDNode *N);
514
  SDValue SoftenFloatRes_FFLOOR(SDNode *N);
515
  SDValue SoftenFloatRes_FLOG(SDNode *N);
516
  SDValue SoftenFloatRes_FLOG2(SDNode *N);
517
  SDValue SoftenFloatRes_FLOG10(SDNode *N);
518
  SDValue SoftenFloatRes_FMA(SDNode *N);
519
  SDValue SoftenFloatRes_FMUL(SDNode *N);
520
  SDValue SoftenFloatRes_FNEARBYINT(SDNode *N);
521
  SDValue SoftenFloatRes_FNEG(SDNode *N, unsigned ResNo);
522
  SDValue SoftenFloatRes_FP_EXTEND(SDNode *N);
523
  SDValue SoftenFloatRes_FP16_TO_FP(SDNode *N);
524
  SDValue SoftenFloatRes_FP_ROUND(SDNode *N);
525
  SDValue SoftenFloatRes_FPOW(SDNode *N);
526
  SDValue SoftenFloatRes_FPOWI(SDNode *N);
527
  SDValue SoftenFloatRes_FREM(SDNode *N);
528
  SDValue SoftenFloatRes_FRINT(SDNode *N);
529
  SDValue SoftenFloatRes_FROUND(SDNode *N);
530
  SDValue SoftenFloatRes_FSIN(SDNode *N);
531
  SDValue SoftenFloatRes_FSQRT(SDNode *N);
532
  SDValue SoftenFloatRes_FSUB(SDNode *N);
533
  SDValue SoftenFloatRes_FTRUNC(SDNode *N);
534
  SDValue SoftenFloatRes_LOAD(SDNode *N, unsigned ResNo);
535
  SDValue SoftenFloatRes_SELECT(SDNode *N, unsigned ResNo);
536
  SDValue SoftenFloatRes_SELECT_CC(SDNode *N, unsigned ResNo);
537
  SDValue SoftenFloatRes_UNDEF(SDNode *N);
538
  SDValue SoftenFloatRes_VAARG(SDNode *N);
539
  SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
540
541
  // Return true if we can skip softening the given operand or SDNode because
542
  // either it was soften before by SoftenFloatResult and references to the
543
  // operand were replaced by ReplaceValueWith or it's value type is legal in HW
544
  // registers and the operand can be left unchanged.
545
  bool CanSkipSoftenFloatOperand(SDNode *N, unsigned OpNo);
546
547
  // Convert Float Operand to Integer for Non-HW-supported Operations.
548
  bool SoftenFloatOperand(SDNode *N, unsigned OpNo);
549
  SDValue SoftenFloatOp_BITCAST(SDNode *N);
550
  SDValue SoftenFloatOp_COPY_TO_REG(SDNode *N);
551
  SDValue SoftenFloatOp_BR_CC(SDNode *N);
552
  SDValue SoftenFloatOp_FABS(SDNode *N);
553
  SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N);
554
  SDValue SoftenFloatOp_FNEG(SDNode *N);
555
  SDValue SoftenFloatOp_FP_EXTEND(SDNode *N);
556
  SDValue SoftenFloatOp_FP_ROUND(SDNode *N);
557
  SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N);
558
  SDValue SoftenFloatOp_LROUND(SDNode *N);
559
  SDValue SoftenFloatOp_LLROUND(SDNode *N);
560
  SDValue SoftenFloatOp_LRINT(SDNode *N);
561
  SDValue SoftenFloatOp_LLRINT(SDNode *N);
562
  SDValue SoftenFloatOp_SELECT(SDNode *N);
563
  SDValue SoftenFloatOp_SELECT_CC(SDNode *N);
564
  SDValue SoftenFloatOp_SETCC(SDNode *N);
565
  SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo);
566
567
  //===--------------------------------------------------------------------===//
568
  // Float Expansion Support: LegalizeFloatTypes.cpp
569
  //===--------------------------------------------------------------------===//
570
571
  /// Given a processed operand Op which was expanded into two floating-point
572
  /// values of half the size, this returns the two halves.
573
  /// The low bits of Op are exactly equal to the bits of Lo; the high bits
574
  /// exactly equal Hi.  For example, if Op is a ppcf128 which was expanded
575
  /// into two f64's, then this method returns the two f64's, with Lo being
576
  /// equal to the lower 64 bits of Op, and Hi to the upper 64 bits.
577
  void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi);
578
  void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi);
579
580
  // Float Result Expansion.
581
  void ExpandFloatResult(SDNode *N, unsigned ResNo);
582
  void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi);
583
  void ExpandFloatRes_FABS      (SDNode *N, SDValue &Lo, SDValue &Hi);
584
  void ExpandFloatRes_FMINNUM   (SDNode *N, SDValue &Lo, SDValue &Hi);
585
  void ExpandFloatRes_FMAXNUM   (SDNode *N, SDValue &Lo, SDValue &Hi);
586
  void ExpandFloatRes_FADD      (SDNode *N, SDValue &Lo, SDValue &Hi);
587
  void ExpandFloatRes_FCEIL     (SDNode *N, SDValue &Lo, SDValue &Hi);
588
  void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi);
589
  void ExpandFloatRes_FCOS      (SDNode *N, SDValue &Lo, SDValue &Hi);
590
  void ExpandFloatRes_FDIV      (SDNode *N, SDValue &Lo, SDValue &Hi);
591
  void ExpandFloatRes_FEXP      (SDNode *N, SDValue &Lo, SDValue &Hi);
592
  void ExpandFloatRes_FEXP2     (SDNode *N, SDValue &Lo, SDValue &Hi);
593
  void ExpandFloatRes_FFLOOR    (SDNode *N, SDValue &Lo, SDValue &Hi);
594
  void ExpandFloatRes_FLOG      (SDNode *N, SDValue &Lo, SDValue &Hi);
595
  void ExpandFloatRes_FLOG2     (SDNode *N, SDValue &Lo, SDValue &Hi);
596
  void ExpandFloatRes_FLOG10    (SDNode *N, SDValue &Lo, SDValue &Hi);
597
  void ExpandFloatRes_FMA       (SDNode *N, SDValue &Lo, SDValue &Hi);
598
  void ExpandFloatRes_FMUL      (SDNode *N, SDValue &Lo, SDValue &Hi);
599
  void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi);
600
  void ExpandFloatRes_FNEG      (SDNode *N, SDValue &Lo, SDValue &Hi);
601
  void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
602
  void ExpandFloatRes_FPOW      (SDNode *N, SDValue &Lo, SDValue &Hi);
603
  void ExpandFloatRes_FPOWI     (SDNode *N, SDValue &Lo, SDValue &Hi);
604
  void ExpandFloatRes_FREM      (SDNode *N, SDValue &Lo, SDValue &Hi);
605
  void ExpandFloatRes_FRINT     (SDNode *N, SDValue &Lo, SDValue &Hi);
606
  void ExpandFloatRes_FROUND    (SDNode *N, SDValue &Lo, SDValue &Hi);
607
  void ExpandFloatRes_FSIN      (SDNode *N, SDValue &Lo, SDValue &Hi);
608
  void ExpandFloatRes_FSQRT     (SDNode *N, SDValue &Lo, SDValue &Hi);
609
  void ExpandFloatRes_FSUB      (SDNode *N, SDValue &Lo, SDValue &Hi);
610
  void ExpandFloatRes_FTRUNC    (SDNode *N, SDValue &Lo, SDValue &Hi);
611
  void ExpandFloatRes_LOAD      (SDNode *N, SDValue &Lo, SDValue &Hi);
612
  void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);
613
614
  // Float Operand Expansion.
615
  bool ExpandFloatOperand(SDNode *N, unsigned OpNo);
616
  SDValue ExpandFloatOp_BR_CC(SDNode *N);
617
  SDValue ExpandFloatOp_FCOPYSIGN(SDNode *N);
618
  SDValue ExpandFloatOp_FP_ROUND(SDNode *N);
619
  SDValue ExpandFloatOp_FP_TO_SINT(SDNode *N);
620
  SDValue ExpandFloatOp_FP_TO_UINT(SDNode *N);
621
  SDValue ExpandFloatOp_LROUND(SDNode *N);
622
  SDValue ExpandFloatOp_LLROUND(SDNode *N);
623
  SDValue ExpandFloatOp_LRINT(SDNode *N);
624
  SDValue ExpandFloatOp_LLRINT(SDNode *N);
625
  SDValue ExpandFloatOp_SELECT_CC(SDNode *N);
626
  SDValue ExpandFloatOp_SETCC(SDNode *N);
627
  SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo);
628
629
  void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
630
                                ISD::CondCode &CCCode, const SDLoc &dl);
631
632
  //===--------------------------------------------------------------------===//
633
  // Float promotion support: LegalizeFloatTypes.cpp
634
  //===--------------------------------------------------------------------===//
635
636
6.69k
  SDValue GetPromotedFloat(SDValue Op) {
637
6.69k
    TableId &PromotedId = PromotedFloats[getTableId(Op)];
638
6.69k
    SDValue PromotedOp = getSDValue(PromotedId);
639
6.69k
    assert(PromotedOp.getNode() && "Operand wasn't promoted?");
640
6.69k
    return PromotedOp;
641
6.69k
  }
642
  void SetPromotedFloat(SDValue Op, SDValue Result);
643
644
  void PromoteFloatResult(SDNode *N, unsigned ResNo);
645
  SDValue PromoteFloatRes_BITCAST(SDNode *N);
646
  SDValue PromoteFloatRes_BinOp(SDNode *N);
647
  SDValue PromoteFloatRes_ConstantFP(SDNode *N);
648
  SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N);
649
  SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N);
650
  SDValue PromoteFloatRes_FMAD(SDNode *N);
651
  SDValue PromoteFloatRes_FPOWI(SDNode *N);
652
  SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
653
  SDValue PromoteFloatRes_LOAD(SDNode *N);
654
  SDValue PromoteFloatRes_SELECT(SDNode *N);
655
  SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
656
  SDValue PromoteFloatRes_UnaryOp(SDNode *N);
657
  SDValue PromoteFloatRes_UNDEF(SDNode *N);
658
  SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);
659
  SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
660
661
  bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
662
  SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
663
  SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
664
  SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
665
  SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo);
666
  SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
667
  SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
668
  SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
669
670
  //===--------------------------------------------------------------------===//
671
  // Scalarization Support: LegalizeVectorTypes.cpp
672
  //===--------------------------------------------------------------------===//
673
674
  /// Given a processed one-element vector Op which was scalarized to its
675
  /// element type, this returns the element. For example, if Op is a v1i32,
676
  /// Op = < i32 val >, this method returns val, an i32.
677
74.8k
  SDValue GetScalarizedVector(SDValue Op) {
678
74.8k
    TableId &ScalarizedId = ScalarizedVectors[getTableId(Op)];
679
74.8k
    SDValue ScalarizedOp = getSDValue(ScalarizedId);
680
74.8k
    assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?");
681
74.8k
    return ScalarizedOp;
682
74.8k
  }
683
  void SetScalarizedVector(SDValue Op, SDValue Result);
684
685
  // Vector Result Scalarization: <1 x ty> -> ty.
686
  void ScalarizeVectorResult(SDNode *N, unsigned ResNo);
687
  SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
688
  SDValue ScalarizeVecRes_BinOp(SDNode *N);
689
  SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
690
  SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
691
  SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
692
  SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
693
  SDValue ScalarizeVecRes_InregOp(SDNode *N);
694
  SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
695
696
  SDValue ScalarizeVecRes_BITCAST(SDNode *N);
697
  SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
698
  SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
699
  SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
700
  SDValue ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N);
701
  SDValue ScalarizeVecRes_FPOWI(SDNode *N);
702
  SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N);
703
  SDValue ScalarizeVecRes_LOAD(LoadSDNode *N);
704
  SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
705
  SDValue ScalarizeVecRes_VSELECT(SDNode *N);
706
  SDValue ScalarizeVecRes_SELECT(SDNode *N);
707
  SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
708
  SDValue ScalarizeVecRes_SETCC(SDNode *N);
709
  SDValue ScalarizeVecRes_UNDEF(SDNode *N);
710
  SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N);
711
712
  SDValue ScalarizeVecRes_MULFIX(SDNode *N);
713
714
  // Vector Operand Scalarization: <1 x ty> -> ty.
715
  bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo);
716
  SDValue ScalarizeVecOp_BITCAST(SDNode *N);
717
  SDValue ScalarizeVecOp_UnaryOp(SDNode *N);
718
  SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);
719
  SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
720
  SDValue ScalarizeVecOp_VSELECT(SDNode *N);
721
  SDValue ScalarizeVecOp_VSETCC(SDNode *N);
722
  SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo);
723
  SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo);
724
  SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo);
725
  SDValue ScalarizeVecOp_VECREDUCE(SDNode *N);
726
727
  //===--------------------------------------------------------------------===//
728
  // Vector Splitting Support: LegalizeVectorTypes.cpp
729
  //===--------------------------------------------------------------------===//
730
731
  /// Given a processed vector Op which was split into vectors of half the size,
732
  /// this method returns the halves. The first elements of Op coincide with the
733
  /// elements of Lo; the remaining elements of Op coincide with the elements of
734
  /// Hi: Op is what you would get by concatenating Lo and Hi.
735
  /// For example, if Op is a v8i32 that was split into two v4i32's, then this
736
  /// method returns the two v4i32's, with Lo corresponding to the first 4
737
  /// elements of Op, and Hi to the last 4 elements.
738
  void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi);
739
  void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi);
740
741
  // Vector Result Splitting: <128 x ty> -> 2 x <64 x ty>.
742
  void SplitVectorResult(SDNode *N, unsigned ResNo);
743
  void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);
744
  void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
745
  void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
746
  void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
747
  void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
748
  void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
749
  void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
750
  void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
751
                              SDValue &Lo, SDValue &Hi);
752
753
  void SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi);
754
755
  void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
756
  void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
757
  void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);
758
  void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
759
  void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
760
  void SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, SDValue &Hi);
761
  void SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, SDValue &Hi);
762
  void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
763
  void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);
764
  void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
765
  void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi);
766
  void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
767
  void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
768
  void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
769
                                  SDValue &Hi);
770
  void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi);
771
772
  // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>.
773
  bool SplitVectorOperand(SDNode *N, unsigned OpNo);
774
  SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
775
  SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);
776
  SDValue SplitVecOp_UnaryOp(SDNode *N);
777
  SDValue SplitVecOp_TruncateHelper(SDNode *N);
778
779
  SDValue SplitVecOp_BITCAST(SDNode *N);
780
  SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);
781
  SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
782
  SDValue SplitVecOp_ExtVecInRegOp(SDNode *N);
783
  SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);
784
  SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
785
  SDValue SplitVecOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
786
  SDValue SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, unsigned OpNo);
787
  SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);
788
  SDValue SplitVecOp_VSETCC(SDNode *N);
789
  SDValue SplitVecOp_FP_ROUND(SDNode *N);
790
  SDValue SplitVecOp_FCOPYSIGN(SDNode *N);
791
792
  //===--------------------------------------------------------------------===//
793
  // Vector Widening Support: LegalizeVectorTypes.cpp
794
  //===--------------------------------------------------------------------===//
795
796
  /// Given a processed vector Op which was widened into a larger vector, this
797
  /// method returns the larger vector. The elements of the returned vector
798
  /// consist of the elements of Op followed by elements containing rubbish.
799
  /// For example, if Op is a v2i32 that was widened to a v4i32, then this
800
  /// method returns a v4i32 for which the first two elements are the same as
801
  /// those of Op, while the last two elements contain rubbish.
802
32.8k
  SDValue GetWidenedVector(SDValue Op) {
803
32.8k
    TableId &WidenedId = WidenedVectors[getTableId(Op)];
804
32.8k
    SDValue WidenedOp = getSDValue(WidenedId);
805
32.8k
    assert(WidenedOp.getNode() && "Operand wasn't widened?");
806
32.8k
    return WidenedOp;
807
32.8k
  }
808
  void SetWidenedVector(SDValue Op, SDValue Result);
809
810
  // Widen Vector Result Promotion.
811
  void WidenVectorResult(SDNode *N, unsigned ResNo);
812
  SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
813
  SDValue WidenVecRes_BITCAST(SDNode* N);
814
  SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
815
  SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
816
  SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N);
817
  SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
818
  SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
819
  SDValue WidenVecRes_LOAD(SDNode* N);
820
  SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
821
  SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
822
  SDValue WidenVecRes_SCALAR_TO_VECTOR(SDNode* N);
823
  SDValue WidenVecRes_SELECT(SDNode* N);
824
  SDValue WidenVSELECTAndMask(SDNode *N);
825
  SDValue WidenVecRes_SELECT_CC(SDNode* N);
826
  SDValue WidenVecRes_SETCC(SDNode* N);
827
  SDValue WidenVecRes_UNDEF(SDNode *N);
828
  SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
829
830
  SDValue WidenVecRes_Ternary(SDNode *N);
831
  SDValue WidenVecRes_Binary(SDNode *N);
832
  SDValue WidenVecRes_BinaryCanTrap(SDNode *N);
833
  SDValue WidenVecRes_StrictFP(SDNode *N);
834
  SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo);
835
  SDValue WidenVecRes_Convert(SDNode *N);
836
  SDValue WidenVecRes_Convert_StrictFP(SDNode *N);
837
  SDValue WidenVecRes_FCOPYSIGN(SDNode *N);
838
  SDValue WidenVecRes_POWI(SDNode *N);
839
  SDValue WidenVecRes_Shift(SDNode *N);
840
  SDValue WidenVecRes_Unary(SDNode *N);
841
  SDValue WidenVecRes_InregOp(SDNode *N);
842
843
  // Widen Vector Operand.
844
  bool WidenVectorOperand(SDNode *N, unsigned OpNo);
845
  SDValue WidenVecOp_BITCAST(SDNode *N);
846
  SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
847
  SDValue WidenVecOp_EXTEND(SDNode *N);
848
  SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
849
  SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
850
  SDValue WidenVecOp_STORE(SDNode* N);
851
  SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
852
  SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
853
  SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
854
  SDValue WidenVecOp_SETCC(SDNode* N);
855
  SDValue WidenVecOp_VSELECT(SDNode *N);
856
857
  SDValue WidenVecOp_Convert(SDNode *N);
858
  SDValue WidenVecOp_FCOPYSIGN(SDNode *N);
859
  SDValue WidenVecOp_VECREDUCE(SDNode *N);
860
861
  /// Helper function to generate a set of operations to perform
862
  /// a vector operation for a wider type.
863
  ///
864
  SDValue UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE);
865
866
  //===--------------------------------------------------------------------===//
867
  // Vector Widening Utilities Support: LegalizeVectorTypes.cpp
868
  //===--------------------------------------------------------------------===//
869
870
  /// Helper function to generate a set of loads to load a vector with a
871
  /// resulting wider type. It takes:
872
  ///   LdChain: list of chains for the load to be generated.
873
  ///   Ld:      load to widen
874
  SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
875
                              LoadSDNode *LD);
876
877
  /// Helper function to generate a set of extension loads to load a vector with
878
  /// a resulting wider type. It takes:
879
  ///   LdChain: list of chains for the load to be generated.
880
  ///   Ld:      load to widen
881
  ///   ExtType: extension element type
882
  SDValue GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
883
                                 LoadSDNode *LD, ISD::LoadExtType ExtType);
884
885
  /// Helper function to generate a set of stores to store a widen vector into
886
  /// non-widen memory.
887
  ///   StChain: list of chains for the stores we have generated
888
  ///   ST:      store of a widen value
889
  void GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, StoreSDNode *ST);
890
891
  /// Helper function to generate a set of stores to store a truncate widen
892
  /// vector into non-widen memory.
893
  ///   StChain: list of chains for the stores we have generated
894
  ///   ST:      store of a widen value
895
  void GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
896
                                 StoreSDNode *ST);
897
898
  /// Modifies a vector input (widen or narrows) to a vector of NVT.  The
899
  /// input vector must have the same element type as NVT.
900
  /// When FillWithZeroes is "on" the vector will be widened with zeroes.
901
  /// By default, the vector will be widened with undefined values.
902
  SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false);
903
904
  /// Return a mask of vector type MaskVT to replace InMask. Also adjust
905
  /// MaskVT to ToMaskVT if needed with vector extension or truncation.
906
  SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT);
907
908
  //===--------------------------------------------------------------------===//
909
  // Generic Splitting: LegalizeTypesGeneric.cpp
910
  //===--------------------------------------------------------------------===//
911
912
  // Legalization methods which only use that the illegal type is split into two
913
  // not necessarily identical types.  As such they can be used for splitting
914
  // vectors and expanding integers and floats.
915
916
6.29k
  void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
917
6.29k
    if (Op.getValueType().isVector())
918
2.29k
      GetSplitVector(Op, Lo, Hi);
919
4.00k
    else if (Op.getValueType().isInteger())
920
3.99k
      GetExpandedInteger(Op, Lo, Hi);
921
14
    else
922
14
      GetExpandedFloat(Op, Lo, Hi);
923
6.29k
  }
924
925
  /// Use ISD::EXTRACT_ELEMENT nodes to extract the low and high parts of the
926
  /// given value.
927
  void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
928
929
  // Generic Result Splitting.
930
  void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
931
                             SDValue &Lo, SDValue &Hi);
932
  void SplitRes_SELECT      (SDNode *N, SDValue &Lo, SDValue &Hi);
933
  void SplitRes_SELECT_CC   (SDNode *N, SDValue &Lo, SDValue &Hi);
934
  void SplitRes_UNDEF       (SDNode *N, SDValue &Lo, SDValue &Hi);
935
936
  //===--------------------------------------------------------------------===//
937
  // Generic Expansion: LegalizeTypesGeneric.cpp
938
  //===--------------------------------------------------------------------===//
939
940
  // Legalization methods which only use that the illegal type is split into two
941
  // identical types of half the size, and that the Lo/Hi part is stored first
942
  // in memory on little/big-endian machines, followed by the Hi/Lo part.  As
943
  // such they can be used for expanding integers and floats.
944
945
59.0k
  void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
946
59.0k
    if (Op.getValueType().isInteger())
947
58.8k
      GetExpandedInteger(Op, Lo, Hi);
948
224
    else
949
224
      GetExpandedFloat(Op, Lo, Hi);
950
59.0k
  }
951
952
953
  /// This function will split the integer \p Op into \p NumElements
954
  /// operations of type \p EltVT and store them in \p Ops.
955
  void IntegerToVector(SDValue Op, unsigned NumElements,
956
                       SmallVectorImpl<SDValue> &Ops, EVT EltVT);
957
958
  // Generic Result Expansion.
959
  void ExpandRes_MERGE_VALUES      (SDNode *N, unsigned ResNo,
960
                                    SDValue &Lo, SDValue &Hi);
961
  void ExpandRes_BITCAST           (SDNode *N, SDValue &Lo, SDValue &Hi);
962
  void ExpandRes_BUILD_PAIR        (SDNode *N, SDValue &Lo, SDValue &Hi);
963
  void ExpandRes_EXTRACT_ELEMENT   (SDNode *N, SDValue &Lo, SDValue &Hi);
964
  void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
965
  void ExpandRes_NormalLoad        (SDNode *N, SDValue &Lo, SDValue &Hi);
966
  void ExpandRes_VAARG             (SDNode *N, SDValue &Lo, SDValue &Hi);
967
968
  // Generic Operand Expansion.
969
  SDValue ExpandOp_BITCAST          (SDNode *N);
970
  SDValue ExpandOp_BUILD_VECTOR     (SDNode *N);
971
  SDValue ExpandOp_EXTRACT_ELEMENT  (SDNode *N);
972
  SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N);
973
  SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N);
974
  SDValue ExpandOp_NormalStore      (SDNode *N, unsigned OpNo);
975
};
976
977
} // end namespace llvm.
978
979
#endif