Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// This file implements the SelectionDAG::Legalize method.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "llvm/ADT/APFloat.h"
15
#include "llvm/ADT/APInt.h"
16
#include "llvm/ADT/ArrayRef.h"
17
#include "llvm/ADT/SetVector.h"
18
#include "llvm/ADT/SmallPtrSet.h"
19
#include "llvm/ADT/SmallSet.h"
20
#include "llvm/ADT/SmallVector.h"
21
#include "llvm/CodeGen/ISDOpcodes.h"
22
#include "llvm/CodeGen/MachineFunction.h"
23
#include "llvm/CodeGen/MachineJumpTableInfo.h"
24
#include "llvm/CodeGen/MachineMemOperand.h"
25
#include "llvm/CodeGen/MachineValueType.h"
26
#include "llvm/CodeGen/RuntimeLibcalls.h"
27
#include "llvm/CodeGen/SelectionDAG.h"
28
#include "llvm/CodeGen/SelectionDAGNodes.h"
29
#include "llvm/CodeGen/ValueTypes.h"
30
#include "llvm/IR/CallingConv.h"
31
#include "llvm/IR/Constants.h"
32
#include "llvm/IR/DataLayout.h"
33
#include "llvm/IR/DerivedTypes.h"
34
#include "llvm/IR/Function.h"
35
#include "llvm/IR/Metadata.h"
36
#include "llvm/IR/Type.h"
37
#include "llvm/Support/Casting.h"
38
#include "llvm/Support/Compiler.h"
39
#include "llvm/Support/Debug.h"
40
#include "llvm/Support/ErrorHandling.h"
41
#include "llvm/Support/MathExtras.h"
42
#include "llvm/Support/raw_ostream.h"
43
#include "llvm/Target/TargetFrameLowering.h"
44
#include "llvm/Target/TargetLowering.h"
45
#include "llvm/Target/TargetMachine.h"
46
#include "llvm/Target/TargetOptions.h"
47
#include "llvm/Target/TargetSubtargetInfo.h"
48
#include <algorithm>
49
#include <cassert>
50
#include <cstdint>
51
#include <tuple>
52
#include <utility>
53
54
using namespace llvm;
55
56
#define DEBUG_TYPE "legalizedag"
57
58
namespace {
59
60
/// Keeps track of state when getting the sign of a floating-point value as an
61
/// integer.
62
struct FloatSignAsInt {
63
  EVT FloatVT;
64
  SDValue Chain;
65
  SDValue FloatPtr;
66
  SDValue IntPtr;
67
  MachinePointerInfo IntPointerInfo;
68
  MachinePointerInfo FloatPointerInfo;
69
  SDValue IntValue;
70
  APInt SignMask;
71
  uint8_t SignBit;
72
};
73
74
//===----------------------------------------------------------------------===//
75
/// This takes an arbitrary SelectionDAG as input and
76
/// hacks on it until the target machine can handle it.  This involves
77
/// eliminating value sizes the machine cannot handle (promoting small sizes to
78
/// large sizes or splitting up large values into small values) as well as
79
/// eliminating operations the machine cannot handle.
80
///
81
/// This code also does a small amount of optimization and recognition of idioms
82
/// as part of its processing.  For example, if a target does not support a
83
/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84
/// will attempt merge setcc and brc instructions into brcc's.
85
class SelectionDAGLegalize {
86
  const TargetMachine &TM;
87
  const TargetLowering &TLI;
88
  SelectionDAG &DAG;
89
90
  /// \brief The set of nodes which have already been legalized. We hold a
91
  /// reference to it in order to update as necessary on node deletion.
92
  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93
94
  /// \brief A set of all the nodes updated during legalization.
95
  SmallSetVector<SDNode *, 16> *UpdatedNodes;
96
97
664
  EVT getSetCCResultType(EVT VT) const {
98
664
    return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99
664
  }
100
101
  // Libcall insertion helpers.
102
103
public:
104
  SelectionDAGLegalize(SelectionDAG &DAG,
105
                       SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106
                       SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107
      : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108
76.2M
        LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109
110
  /// \brief Legalizes the given operation.
111
  void LegalizeOp(SDNode *Node);
112
113
private:
114
  SDValue OptimizeFloatStore(StoreSDNode *ST);
115
116
  void LegalizeLoadOps(SDNode *Node);
117
  void LegalizeStoreOps(SDNode *Node);
118
119
  /// Some targets cannot handle a variable
120
  /// insertion index for the INSERT_VECTOR_ELT instruction.  In this case, it
121
  /// is necessary to spill the vector being inserted into to memory, perform
122
  /// the insert there, and then read the result back.
123
  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124
                                         const SDLoc &dl);
125
  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126
                                  const SDLoc &dl);
127
128
  /// Return a vector shuffle operation which
129
  /// performs the same shuffe in terms of order or result bytes, but on a type
130
  /// whose vector element type is narrower than the original shuffle type.
131
  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132
  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133
                                     SDValue N1, SDValue N2,
134
                                     ArrayRef<int> Mask) const;
135
136
  bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
137
                             bool &NeedInvert, const SDLoc &dl);
138
139
  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140
  SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops,
141
                        unsigned NumOps, bool isSigned, const SDLoc &dl);
142
143
  std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
144
                                                 SDNode *Node, bool isSigned);
145
  SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146
                          RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147
                          RTLIB::Libcall Call_F128,
148
                          RTLIB::Libcall Call_PPCF128);
149
  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150
                           RTLIB::Libcall Call_I8,
151
                           RTLIB::Libcall Call_I16,
152
                           RTLIB::Libcall Call_I32,
153
                           RTLIB::Libcall Call_I64,
154
                           RTLIB::Libcall Call_I128);
155
  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156
  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
158
  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
159
                           const SDLoc &dl);
160
  SDValue ExpandBUILD_VECTOR(SDNode *Node);
161
  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
162
  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
163
                                SmallVectorImpl<SDValue> &Results);
164
  void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
165
                         SDValue Value) const;
166
  SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
167
                          SDValue NewIntValue) const;
168
  SDValue ExpandFCOPYSIGN(SDNode *Node) const;
169
  SDValue ExpandFABS(SDNode *Node) const;
170
  SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue LegalOp, EVT DestVT,
171
                               const SDLoc &dl);
172
  SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
173
                                const SDLoc &dl);
174
  SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
175
                                const SDLoc &dl);
176
177
  SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
178
  SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
179
  SDValue ExpandBitCount(unsigned Opc, SDValue Op, const SDLoc &dl);
180
181
  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
182
  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
183
  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
184
185
  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
186
  SDValue ExpandConstant(ConstantSDNode *CP);
187
188
  // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
189
  bool ExpandNode(SDNode *Node);
190
  void ConvertNodeToLibcall(SDNode *Node);
191
  void PromoteNode(SDNode *Node);
192
193
public:
194
  // Node replacement helpers
195
196
3.99M
  void ReplacedNode(SDNode *N) {
197
3.99M
    LegalizedNodes.erase(N);
198
3.99M
    if (UpdatedNodes)
199
7.46k
      UpdatedNodes->insert(N);
200
3.99M
  }
201
202
157
  void ReplaceNode(SDNode *Old, SDNode *New) {
203
157
    DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
204
157
          dbgs() << "     with:      "; New->dump(&DAG));
205
157
206
157
    assert(Old->getNumValues() == New->getNumValues() &&
207
157
           "Replacing one node with another that produces a different number "
208
157
           "of values!");
209
157
    DAG.ReplaceAllUsesWith(Old, New);
210
157
    if (UpdatedNodes)
211
1
      UpdatedNodes->insert(New);
212
157
    ReplacedNode(Old);
213
157
  }
214
215
3.69M
  void ReplaceNode(SDValue Old, SDValue New) {
216
3.69M
    DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
217
3.69M
          dbgs() << "     with:      "; New->dump(&DAG));
218
3.69M
219
3.69M
    DAG.ReplaceAllUsesWith(Old, New);
220
3.69M
    if (UpdatedNodes)
221
3.73k
      UpdatedNodes->insert(New.getNode());
222
3.69M
    ReplacedNode(Old.getNode());
223
3.69M
  }
224
225
213k
  void ReplaceNode(SDNode *Old, const SDValue *New) {
226
213k
    DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
227
213k
228
213k
    DAG.ReplaceAllUsesWith(Old, New);
229
474k
    for (unsigned i = 0, e = Old->getNumValues(); 
i != e474k
;
++i261k
) {
230
261k
      DEBUG(dbgs() << (i == 0 ? "     with:      "
231
261k
                              : "      and:      ");
232
261k
            New[i]->dump(&DAG));
233
261k
      if (UpdatedNodes)
234
1.97k
        UpdatedNodes->insert(New[i].getNode());
235
261k
    }
236
213k
    ReplacedNode(Old);
237
213k
  }
238
};
239
240
} // end anonymous namespace
241
242
/// Return a vector shuffle operation which
243
/// performs the same shuffe in terms of order or result bytes, but on a type
244
/// whose vector element type is narrower than the original shuffle type.
245
/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
246
SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
247
    EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
248
402
    ArrayRef<int> Mask) const {
249
402
  unsigned NumMaskElts = VT.getVectorNumElements();
250
402
  unsigned NumDestElts = NVT.getVectorNumElements();
251
402
  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
252
402
253
402
  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
254
402
255
402
  if (NumEltsGrowth == 1)
256
0
    return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
257
402
258
402
  SmallVector<int, 8> NewMask;
259
2.35k
  for (unsigned i = 0; 
i != NumMaskElts2.35k
;
++i1.95k
) {
260
1.95k
    int Idx = Mask[i];
261
8.38k
    for (unsigned j = 0; 
j != NumEltsGrowth8.38k
;
++j6.43k
) {
262
6.43k
      if (Idx < 0)
263
712
        NewMask.push_back(-1);
264
6.43k
      else
265
5.72k
        NewMask.push_back(Idx * NumEltsGrowth + j);
266
6.43k
    }
267
1.95k
  }
268
402
  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
269
402
  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
270
402
  return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
271
402
}
272
273
/// Expands the ConstantFP node to an integer constant or
274
/// a load from the constant pool.
275
SDValue
276
34.1k
SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
277
34.1k
  bool Extend = false;
278
34.1k
  SDLoc dl(CFP);
279
34.1k
280
34.1k
  // If a FP immediate is precise when represented as a float and if the
281
34.1k
  // target can do an extending load from float to double, we put it into
282
34.1k
  // the constant pool as a float, even if it's is statically typed as a
283
34.1k
  // double.  This shrinks FP constants and canonicalizes them for targets where
284
34.1k
  // an FP extending load is the same cost as a normal load (such as on the x87
285
34.1k
  // fp stack or PPC FP unit).
286
34.1k
  EVT VT = CFP->getValueType(0);
287
34.1k
  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
288
34.1k
  if (
!UseCP34.1k
) {
289
0
    assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
290
0
    return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
291
0
                           (VT == MVT::f64) ? 
MVT::i640
:
MVT::i320
);
292
0
  }
293
34.1k
294
34.1k
  APFloat APF = CFP->getValueAPF();
295
34.1k
  EVT OrigVT = VT;
296
34.1k
  EVT SVT = VT;
297
34.1k
298
34.1k
  // We don't want to shrink SNaNs. Converting the SNaN back to its real type
299
34.1k
  // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
300
34.1k
  if (
!APF.isSignaling()34.1k
) {
301
55.4k
    while (
SVT != MVT::f32 && 55.4k
SVT != MVT::f1621.3k
) {
302
21.3k
      SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
303
21.3k
      if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
304
21.3k
          // Only do this if the target has a native EXTLOAD instruction from
305
21.3k
          // smaller type.
306
9.99k
          TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
307
21.3k
          
TLI.ShouldShrinkFPConstant(OrigVT)1.78k
) {
308
1.07k
        Type *SType = SVT.getTypeForEVT(*DAG.getContext());
309
1.07k
        LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
310
1.07k
        VT = SVT;
311
1.07k
        Extend = true;
312
1.07k
      }
313
21.3k
    }
314
34.1k
  }
315
34.1k
316
34.1k
  SDValue CPIdx =
317
34.1k
      DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
318
34.1k
  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
319
34.1k
  if (
Extend34.1k
) {
320
733
    SDValue Result = DAG.getExtLoad(
321
733
        ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
322
733
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
323
733
        Alignment);
324
733
    return Result;
325
733
  }
326
33.3k
  SDValue Result = DAG.getLoad(
327
33.3k
      OrigVT, dl, DAG.getEntryNode(), CPIdx,
328
33.3k
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
329
33.3k
  return Result;
330
33.3k
}
331
332
/// Expands the Constant node to a load from the constant pool.
333
0
SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
334
0
  SDLoc dl(CP);
335
0
  EVT VT = CP->getValueType(0);
336
0
  SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
337
0
                                      TLI.getPointerTy(DAG.getDataLayout()));
338
0
  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
339
0
  SDValue Result = DAG.getLoad(
340
0
      VT, dl, DAG.getEntryNode(), CPIdx,
341
0
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
342
0
  return Result;
343
0
}
344
345
/// Some target cannot handle a variable insertion index for the
346
/// INSERT_VECTOR_ELT instruction.  In this case, it
347
/// is necessary to spill the vector being inserted into to memory, perform
348
/// the insert there, and then read the result back.
349
SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
350
                                                             SDValue Val,
351
                                                             SDValue Idx,
352
16
                                                             const SDLoc &dl) {
353
16
  SDValue Tmp1 = Vec;
354
16
  SDValue Tmp2 = Val;
355
16
  SDValue Tmp3 = Idx;
356
16
357
16
  // If the target doesn't support this, we have to spill the input vector
358
16
  // to a temporary stack slot, update the element, then reload it.  This is
359
16
  // badness.  We could also load the value into a vector register (either
360
16
  // with a "move to register" or "extload into register" instruction, then
361
16
  // permute it into place, if the idx is a constant and if the idx is
362
16
  // supported by the target.
363
16
  EVT VT    = Tmp1.getValueType();
364
16
  EVT EltVT = VT.getVectorElementType();
365
16
  SDValue StackPtr = DAG.CreateStackTemporary(VT);
366
16
367
16
  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
368
16
369
16
  // Store the vector.
370
16
  SDValue Ch = DAG.getStore(
371
16
      DAG.getEntryNode(), dl, Tmp1, StackPtr,
372
16
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
373
16
374
16
  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
375
16
376
16
  // Store the scalar value.
377
16
  Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
378
16
  // Load the updated vector.
379
16
  return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
380
16
                                               DAG.getMachineFunction(), SPFI));
381
16
}
382
383
SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
384
                                                      SDValue Idx,
385
404
                                                      const SDLoc &dl) {
386
404
  if (ConstantSDNode *
InsertPos404
= dyn_cast<ConstantSDNode>(Idx)) {
387
388
    // SCALAR_TO_VECTOR requires that the type of the value being inserted
388
388
    // match the element type of the vector being created, except for
389
388
    // integers in which case the inserted value can be over width.
390
388
    EVT EltVT = Vec.getValueType().getVectorElementType();
391
388
    if (Val.getValueType() == EltVT ||
392
388
        
(EltVT.isInteger() && 1
Val.getValueType().bitsGE(EltVT)1
)) {
393
388
      SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
394
388
                                  Vec.getValueType(), Val);
395
388
396
388
      unsigned NumElts = Vec.getValueType().getVectorNumElements();
397
388
      // We generate a shuffle of InVec and ScVec, so the shuffle mask
398
388
      // should be 0,1,2,3,4,5... with the appropriate element replaced with
399
388
      // elt 0 of the RHS.
400
388
      SmallVector<int, 8> ShufOps;
401
1.96k
      for (unsigned i = 0; 
i != NumElts1.96k
;
++i1.58k
)
402
1.58k
        
ShufOps.push_back(i != InsertPos->getZExtValue() ? 1.58k
i1.19k
:
NumElts388
);
403
388
404
388
      return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
405
388
    }
406
16
  }
407
16
  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
408
16
}
409
410
3.07M
SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
411
3.07M
  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
412
3.07M
  // FIXME: We shouldn't do this for TargetConstantFP's.
413
3.07M
  // FIXME: move this to the DAG Combiner!  Note that we can't regress due
414
3.07M
  // to phase ordering between legalized code and the dag combiner.  This
415
3.07M
  // probably means that we need to integrate dag combiner and legalizer
416
3.07M
  // together.
417
3.07M
  // We generally can't do this one for long doubles.
418
3.07M
  SDValue Chain = ST->getChain();
419
3.07M
  SDValue Ptr = ST->getBasePtr();
420
3.07M
  unsigned Alignment = ST->getAlignment();
421
3.07M
  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
422
3.07M
  AAMDNodes AAInfo = ST->getAAInfo();
423
3.07M
  SDLoc dl(ST);
424
3.07M
  if (ConstantFPSDNode *
CFP3.07M
= dyn_cast<ConstantFPSDNode>(ST->getValue())) {
425
170
    if (CFP->getValueType(0) == MVT::f32 &&
426
170
        
TLI.isTypeLegal(MVT::i32)1
) {
427
1
      SDValue Con = DAG.getConstant(CFP->getValueAPF().
428
1
                                      bitcastToAPInt().zextOrTrunc(32),
429
1
                                    SDLoc(CFP), MVT::i32);
430
1
      return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
431
1
                          MMOFlags, AAInfo);
432
1
    }
433
169
434
169
    
if (169
CFP->getValueType(0) == MVT::f64169
) {
435
8
      // If this target supports 64-bit registers, do a single 64-bit store.
436
8
      if (
TLI.isTypeLegal(MVT::i64)8
) {
437
0
        SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
438
0
                                      zextOrTrunc(64), SDLoc(CFP), MVT::i64);
439
0
        return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
440
0
                            Alignment, MMOFlags, AAInfo);
441
0
      }
442
8
443
8
      
if (8
TLI.isTypeLegal(MVT::i32) && 8
!ST->isVolatile()8
) {
444
0
        // Otherwise, if the target supports 32-bit registers, use 2 32-bit
445
0
        // stores.  If the target supports neither 32- nor 64-bits, this
446
0
        // xform is certainly not worth it.
447
0
        const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
448
0
        SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
449
0
        SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
450
0
        if (DAG.getDataLayout().isBigEndian())
451
0
          std::swap(Lo, Hi);
452
0
453
0
        Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
454
0
                          MMOFlags, AAInfo);
455
0
        Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
456
0
                          DAG.getConstant(4, dl, Ptr.getValueType()));
457
0
        Hi = DAG.getStore(Chain, dl, Hi, Ptr,
458
0
                          ST->getPointerInfo().getWithOffset(4),
459
0
                          MinAlign(Alignment, 4U), MMOFlags, AAInfo);
460
0
461
0
        return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
462
0
      }
463
3.07M
    }
464
170
  }
465
3.07M
  return SDValue(nullptr, 0);
466
3.07M
}
467
468
3.81M
void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
469
3.81M
    StoreSDNode *ST = cast<StoreSDNode>(Node);
470
3.81M
    SDValue Chain = ST->getChain();
471
3.81M
    SDValue Ptr = ST->getBasePtr();
472
3.81M
    SDLoc dl(Node);
473
3.81M
474
3.81M
    unsigned Alignment = ST->getAlignment();
475
3.81M
    MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
476
3.81M
    AAMDNodes AAInfo = ST->getAAInfo();
477
3.81M
478
3.81M
    if (
!ST->isTruncatingStore()3.81M
) {
479
3.07M
      if (SDNode *
OptStore3.07M
= OptimizeFloatStore(ST).getNode()) {
480
1
        ReplaceNode(ST, OptStore);
481
1
        return;
482
1
      }
483
3.07M
484
3.07M
      {
485
3.07M
        SDValue Value = ST->getValue();
486
3.07M
        MVT VT = Value.getSimpleValueType();
487
3.07M
        switch (TLI.getOperationAction(ISD::STORE, VT)) {
488
0
        
default: 0
llvm_unreachable0
("This action is not supported yet!");
489
2.83M
        case TargetLowering::Legal: {
490
2.83M
          // If this is an unaligned store and the target doesn't support it,
491
2.83M
          // expand it.
492
2.83M
          EVT MemVT = ST->getMemoryVT();
493
2.83M
          unsigned AS = ST->getAddressSpace();
494
2.83M
          unsigned Align = ST->getAlignment();
495
2.83M
          const DataLayout &DL = DAG.getDataLayout();
496
2.83M
          if (
!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)2.83M
) {
497
218
            SDValue Result = TLI.expandUnalignedStore(ST, DAG);
498
218
            ReplaceNode(SDValue(ST, 0), Result);
499
218
          }
500
2.83M
          break;
501
3.07M
        }
502
100k
        case TargetLowering::Custom: {
503
100k
          SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
504
100k
          if (
Res && 100k
Res != SDValue(Node, 0)33.5k
)
505
12.7k
            ReplaceNode(SDValue(Node, 0), Res);
506
100k
          return;
507
3.07M
        }
508
143k
        case TargetLowering::Promote: {
509
143k
          MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
510
143k
          assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
511
143k
                 "Can only promote stores to same size type");
512
143k
          Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
513
143k
          SDValue Result =
514
143k
              DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
515
143k
                           Alignment, MMOFlags, AAInfo);
516
143k
          ReplaceNode(SDValue(Node, 0), Result);
517
143k
          break;
518
2.97M
        }
519
2.97M
        }
520
2.97M
        return;
521
2.97M
      }
522
0
    } else {
523
739k
      SDValue Value = ST->getValue();
524
739k
525
739k
      EVT StVT = ST->getMemoryVT();
526
739k
      unsigned StWidth = StVT.getSizeInBits();
527
739k
      auto &DL = DAG.getDataLayout();
528
739k
529
739k
      if (
StWidth != StVT.getStoreSizeInBits()739k
) {
530
28.2k
        // Promote to a byte-sized store with upper bits zero if not
531
28.2k
        // storing an integral number of bytes.  For example, promote
532
28.2k
        // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
533
28.2k
        EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
534
28.2k
                                    StVT.getStoreSizeInBits());
535
28.2k
        Value = DAG.getZeroExtendInReg(Value, dl, StVT);
536
28.2k
        SDValue Result =
537
28.2k
            DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
538
28.2k
                              Alignment, MMOFlags, AAInfo);
539
28.2k
        ReplaceNode(SDValue(Node, 0), Result);
540
739k
      } else 
if (710k
StWidth & (StWidth - 1)710k
) {
541
907
        // If not storing a power-of-2 number of bits, expand as two stores.
542
907
        assert(!StVT.isVector() && "Unsupported truncstore!");
543
907
        unsigned RoundWidth = 1 << Log2_32(StWidth);
544
907
        assert(RoundWidth < StWidth);
545
907
        unsigned ExtraWidth = StWidth - RoundWidth;
546
907
        assert(ExtraWidth < RoundWidth);
547
907
        assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
548
907
               "Store size not an integral number of bytes!");
549
907
        EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
550
907
        EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
551
907
        SDValue Lo, Hi;
552
907
        unsigned IncrementSize;
553
907
554
907
        if (
DL.isLittleEndian()907
) {
555
897
          // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
556
897
          // Store the bottom RoundWidth bits.
557
897
          Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
558
897
                                 RoundVT, Alignment, MMOFlags, AAInfo);
559
897
560
897
          // Store the remaining ExtraWidth bits.
561
897
          IncrementSize = RoundWidth / 8;
562
897
          Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
563
897
                            DAG.getConstant(IncrementSize, dl,
564
897
                                            Ptr.getValueType()));
565
897
          Hi = DAG.getNode(
566
897
              ISD::SRL, dl, Value.getValueType(), Value,
567
897
              DAG.getConstant(RoundWidth, dl,
568
897
                              TLI.getShiftAmountTy(Value.getValueType(), DL)));
569
897
          Hi = DAG.getTruncStore(
570
897
              Chain, dl, Hi, Ptr,
571
897
              ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
572
897
              MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
573
907
        } else {
574
10
          // Big endian - avoid unaligned stores.
575
10
          // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
576
10
          // Store the top RoundWidth bits.
577
10
          Hi = DAG.getNode(
578
10
              ISD::SRL, dl, Value.getValueType(), Value,
579
10
              DAG.getConstant(ExtraWidth, dl,
580
10
                              TLI.getShiftAmountTy(Value.getValueType(), DL)));
581
10
          Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
582
10
                                 RoundVT, Alignment, MMOFlags, AAInfo);
583
10
584
10
          // Store the remaining ExtraWidth bits.
585
10
          IncrementSize = RoundWidth / 8;
586
10
          Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
587
10
                            DAG.getConstant(IncrementSize, dl,
588
10
                                            Ptr.getValueType()));
589
10
          Lo = DAG.getTruncStore(
590
10
              Chain, dl, Value, Ptr,
591
10
              ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
592
10
              MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
593
10
        }
594
907
595
907
        // The order of the stores doesn't matter.
596
907
        SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
597
907
        ReplaceNode(SDValue(Node, 0), Result);
598
710k
      } else {
599
710k
        switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
600
0
        
default: 0
llvm_unreachable0
("This action is not supported yet!");
601
706k
        case TargetLowering::Legal: {
602
706k
          EVT MemVT = ST->getMemoryVT();
603
706k
          unsigned AS = ST->getAddressSpace();
604
706k
          unsigned Align = ST->getAlignment();
605
706k
          // If this is an unaligned store and the target doesn't support it,
606
706k
          // expand it.
607
706k
          if (
!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)706k
) {
608
415
            SDValue Result = TLI.expandUnalignedStore(ST, DAG);
609
415
            ReplaceNode(SDValue(ST, 0), Result);
610
415
          }
611
706k
          break;
612
710k
        }
613
1.63k
        case TargetLowering::Custom: {
614
1.63k
          SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
615
1.63k
          if (
Res && 1.63k
Res != SDValue(Node, 0)1.49k
)
616
1.49k
            ReplaceNode(SDValue(Node, 0), Res);
617
1.63k
          return;
618
710k
        }
619
1.66k
        case TargetLowering::Expand:
620
1.66k
          assert(!StVT.isVector() &&
621
1.66k
                 "Vector Stores are handled in LegalizeVectorOps");
622
1.66k
623
1.66k
          // TRUNCSTORE:i16 i32 -> STORE i16
624
1.66k
          assert(TLI.isTypeLegal(StVT) &&
625
1.66k
                 "Do not know how to expand this store!");
626
1.66k
          Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
627
1.66k
          SDValue Result =
628
1.66k
              DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
629
1.66k
                           Alignment, MMOFlags, AAInfo);
630
1.66k
          ReplaceNode(SDValue(Node, 0), Result);
631
1.66k
          break;
632
737k
        }
633
737k
      }
634
739k
    }
635
3.81M
}
636
637
4.54M
void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
638
4.54M
  LoadSDNode *LD = cast<LoadSDNode>(Node);
639
4.54M
  SDValue Chain = LD->getChain();  // The chain.
640
4.54M
  SDValue Ptr = LD->getBasePtr();  // The base pointer.
641
4.54M
  SDValue Value;                   // The value returned by the load op.
642
4.54M
  SDLoc dl(Node);
643
4.54M
644
4.54M
  ISD::LoadExtType ExtType = LD->getExtensionType();
645
4.54M
  if (
ExtType == ISD::NON_EXTLOAD4.54M
) {
646
3.57M
    MVT VT = Node->getSimpleValueType(0);
647
3.57M
    SDValue RVal = SDValue(Node, 0);
648
3.57M
    SDValue RChain = SDValue(Node, 1);
649
3.57M
650
3.57M
    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
651
0
    
default: 0
llvm_unreachable0
("This action is not supported yet!");
652
3.38M
    case TargetLowering::Legal: {
653
3.38M
      EVT MemVT = LD->getMemoryVT();
654
3.38M
      unsigned AS = LD->getAddressSpace();
655
3.38M
      unsigned Align = LD->getAlignment();
656
3.38M
      const DataLayout &DL = DAG.getDataLayout();
657
3.38M
      // If this is an unaligned load and the target doesn't support it,
658
3.38M
      // expand it.
659
3.38M
      if (
!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)3.38M
) {
660
224
        std::tie(RVal, RChain) =  TLI.expandUnalignedLoad(LD, DAG);
661
224
      }
662
3.38M
      break;
663
3.57M
    }
664
124k
    case TargetLowering::Custom:
665
124k
      if (SDValue 
Res124k
= TLI.LowerOperation(RVal, DAG)) {
666
15.8k
        RVal = Res;
667
15.8k
        RChain = Res.getValue(1);
668
15.8k
      }
669
124k
      break;
670
3.57M
671
62.0k
    case TargetLowering::Promote: {
672
62.0k
      MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
673
62.0k
      assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
674
62.0k
             "Can only promote loads to same size type");
675
62.0k
676
62.0k
      SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
677
62.0k
      RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
678
62.0k
      RChain = Res.getValue(1);
679
62.0k
      break;
680
3.57M
    }
681
3.57M
    }
682
3.57M
    
if (3.57M
RChain.getNode() != Node3.57M
) {
683
75.6k
      assert(RVal.getNode() != Node && "Load must be completely replaced");
684
75.6k
      DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
685
75.6k
      DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
686
75.6k
      if (
UpdatedNodes75.6k
) {
687
1.75k
        UpdatedNodes->insert(RVal.getNode());
688
1.75k
        UpdatedNodes->insert(RChain.getNode());
689
1.75k
      }
690
75.6k
      ReplacedNode(Node);
691
75.6k
    }
692
3.57M
    return;
693
3.57M
  }
694
973k
695
973k
  EVT SrcVT = LD->getMemoryVT();
696
973k
  unsigned SrcWidth = SrcVT.getSizeInBits();
697
973k
  unsigned Alignment = LD->getAlignment();
698
973k
  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
699
973k
  AAMDNodes AAInfo = LD->getAAInfo();
700
973k
701
973k
  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
702
973k
      // Some targets pretend to have an i1 loading operation, and actually
703
973k
      // load an i8.  This trick is correct for ZEXTLOAD because the top 7
704
973k
      // bits are guaranteed to be zero; it helps the optimizers understand
705
973k
      // that these bits are zero.  It is also useful for EXTLOAD, since it
706
973k
      // tells the optimizers that those bits are undefined.  It would be
707
973k
      // nice to have an effective generic way of getting these benefits...
708
973k
      // Until such a way is found, don't insist on promoting i1 here.
709
8.04k
      (SrcVT != MVT::i1 ||
710
7.69k
       TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
711
973k
         TargetLowering::Promote)) {
712
611
    // Promote to a byte-sized load if not loading an integral number of
713
611
    // bytes.  For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
714
611
    unsigned NewWidth = SrcVT.getStoreSizeInBits();
715
611
    EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
716
611
    SDValue Ch;
717
611
718
611
    // The extra bits are guaranteed to be zero, since we stored them that
719
611
    // way.  A zext load from NVT thus automatically gives zext from SrcVT.
720
611
721
611
    ISD::LoadExtType NewExtType =
722
611
      ExtType == ISD::ZEXTLOAD ? 
ISD::ZEXTLOAD262
:
ISD::EXTLOAD349
;
723
611
724
611
    SDValue Result =
725
611
        DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
726
611
                       LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
727
611
728
611
    Ch = Result.getValue(1); // The chain.
729
611
730
611
    if (ExtType == ISD::SEXTLOAD)
731
611
      // Having the top bits zero doesn't help when sign extending.
732
81
      Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
733
81
                           Result.getValueType(),
734
81
                           Result, DAG.getValueType(SrcVT));
735
530
    else 
if (530
ExtType == ISD::ZEXTLOAD || 530
NVT == Result.getValueType()268
)
736
530
      // All the top bits are guaranteed to be zero - inform the optimizers.
737
335
      Result = DAG.getNode(ISD::AssertZext, dl,
738
335
                           Result.getValueType(), Result,
739
335
                           DAG.getValueType(SrcVT));
740
611
741
611
    Value = Result;
742
611
    Chain = Ch;
743
973k
  } else 
if (973k
SrcWidth & (SrcWidth - 1)973k
) {
744
647
    // If not loading a power-of-2 number of bits, expand as two loads.
745
647
    assert(!SrcVT.isVector() && "Unsupported extload!");
746
647
    unsigned RoundWidth = 1 << Log2_32(SrcWidth);
747
647
    assert(RoundWidth < SrcWidth);
748
647
    unsigned ExtraWidth = SrcWidth - RoundWidth;
749
647
    assert(ExtraWidth < RoundWidth);
750
647
    assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
751
647
           "Load size not an integral number of bytes!");
752
647
    EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
753
647
    EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
754
647
    SDValue Lo, Hi, Ch;
755
647
    unsigned IncrementSize;
756
647
    auto &DL = DAG.getDataLayout();
757
647
758
647
    if (
DL.isLittleEndian()647
) {
759
622
      // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
760
622
      // Load the bottom RoundWidth bits.
761
622
      Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
762
622
                          LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
763
622
                          AAInfo);
764
622
765
622
      // Load the remaining ExtraWidth bits.
766
622
      IncrementSize = RoundWidth / 8;
767
622
      Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
768
622
                         DAG.getConstant(IncrementSize, dl,
769
622
                                         Ptr.getValueType()));
770
622
      Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
771
622
                          LD->getPointerInfo().getWithOffset(IncrementSize),
772
622
                          ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
773
622
                          AAInfo);
774
622
775
622
      // Build a factor node to remember that this load is independent of
776
622
      // the other one.
777
622
      Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
778
622
                       Hi.getValue(1));
779
622
780
622
      // Move the top bits to the right place.
781
622
      Hi = DAG.getNode(
782
622
          ISD::SHL, dl, Hi.getValueType(), Hi,
783
622
          DAG.getConstant(RoundWidth, dl,
784
622
                          TLI.getShiftAmountTy(Hi.getValueType(), DL)));
785
622
786
622
      // Join the hi and lo parts.
787
622
      Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
788
647
    } else {
789
25
      // Big endian - avoid unaligned loads.
790
25
      // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
791
25
      // Load the top RoundWidth bits.
792
25
      Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
793
25
                          LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
794
25
                          AAInfo);
795
25
796
25
      // Load the remaining ExtraWidth bits.
797
25
      IncrementSize = RoundWidth / 8;
798
25
      Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
799
25
                         DAG.getConstant(IncrementSize, dl,
800
25
                                         Ptr.getValueType()));
801
25
      Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
802
25
                          LD->getPointerInfo().getWithOffset(IncrementSize),
803
25
                          ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
804
25
                          AAInfo);
805
25
806
25
      // Build a factor node to remember that this load is independent of
807
25
      // the other one.
808
25
      Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
809
25
                       Hi.getValue(1));
810
25
811
25
      // Move the top bits to the right place.
812
25
      Hi = DAG.getNode(
813
25
          ISD::SHL, dl, Hi.getValueType(), Hi,
814
25
          DAG.getConstant(ExtraWidth, dl,
815
25
                          TLI.getShiftAmountTy(Hi.getValueType(), DL)));
816
25
817
25
      // Join the hi and lo parts.
818
25
      Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
819
25
    }
820
647
821
647
    Chain = Ch;
822
973k
  } else {
823
972k
    bool isCustom = false;
824
972k
    switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
825
972k
                                 SrcVT.getSimpleVT())) {
826
0
    
default: 0
llvm_unreachable0
("This action is not supported yet!");
827
5.92k
    case TargetLowering::Custom:
828
5.92k
      isCustom = true;
829
5.92k
      LLVM_FALLTHROUGH;
830
972k
    case TargetLowering::Legal:
831
972k
      Value = SDValue(Node, 0);
832
972k
      Chain = SDValue(Node, 1);
833
972k
834
972k
      if (
isCustom972k
) {
835
5.92k
        if (SDValue 
Res5.92k
= TLI.LowerOperation(SDValue(Node, 0), DAG)) {
836
4.34k
          Value = Res;
837
4.34k
          Chain = Res.getValue(1);
838
4.34k
        }
839
972k
      } else {
840
966k
        // If this is an unaligned load and the target doesn't support it,
841
966k
        // expand it.
842
966k
        EVT MemVT = LD->getMemoryVT();
843
966k
        unsigned AS = LD->getAddressSpace();
844
966k
        unsigned Align = LD->getAlignment();
845
966k
        const DataLayout &DL = DAG.getDataLayout();
846
966k
        if (
!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)966k
) {
847
448
          std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
848
448
        }
849
966k
      }
850
972k
      break;
851
5.92k
852
323
    case TargetLowering::Expand: {
853
323
      EVT DestVT = Node->getValueType(0);
854
323
      if (
!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)323
) {
855
308
        // If the source type is not legal, see if there is a legal extload to
856
308
        // an intermediate type that we can then extend further.
857
308
        EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
858
308
        if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
859
308
            
TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)56
) {
860
308
          // If we are loading a legal type, this is a non-extload followed by a
861
308
          // full extend.
862
308
          ISD::LoadExtType MidExtType =
863
308
              (LoadVT == SrcVT) ? 
ISD::NON_EXTLOAD252
:
ExtType56
;
864
308
865
308
          SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
866
308
                                        SrcVT, LD->getMemOperand());
867
308
          unsigned ExtendOp =
868
308
              ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
869
308
          Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
870
308
          Chain = Load.getValue(1);
871
308
          break;
872
308
        }
873
0
874
0
        // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
875
0
        // normal undefined upper bits behavior to allow using an in-reg extend
876
0
        // with the illegal FP type, so load as an integer and do the
877
0
        // from-integer conversion.
878
0
        
if (0
SrcVT.getScalarType() == MVT::f160
) {
879
0
          EVT ISrcVT = SrcVT.changeTypeToInteger();
880
0
          EVT IDestVT = DestVT.changeTypeToInteger();
881
0
          EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
882
0
883
0
          SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
884
0
                                          Chain, Ptr, ISrcVT,
885
0
                                          LD->getMemOperand());
886
0
          Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
887
0
          Chain = Result.getValue(1);
888
0
          break;
889
0
        }
890
15
      }
891
15
892
0
      assert(!SrcVT.isVector() &&
893
15
             "Vector Loads are handled in LegalizeVectorOps");
894
15
895
15
      // FIXME: This does not work for vectors on most targets.  Sign-
896
15
      // and zero-extend operations are currently folded into extending
897
15
      // loads, whether they are legal or not, and then we end up here
898
15
      // without any support for legalizing them.
899
15
      assert(ExtType != ISD::EXTLOAD &&
900
15
             "EXTLOAD should always be supported!");
901
15
      // Turn the unsupported load into an EXTLOAD followed by an
902
15
      // explicit zero/sign extend inreg.
903
15
      SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
904
15
                                      Node->getValueType(0),
905
15
                                      Chain, Ptr, SrcVT,
906
15
                                      LD->getMemOperand());
907
15
      SDValue ValRes;
908
15
      if (ExtType == ISD::SEXTLOAD)
909
14
        ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
910
14
                             Result.getValueType(),
911
14
                             Result, DAG.getValueType(SrcVT));
912
15
      else
913
1
        ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
914
5.92k
      Value = ValRes;
915
5.92k
      Chain = Result.getValue(1);
916
5.92k
      break;
917
5.92k
    }
918
973k
    }
919
973k
  }
920
973k
921
973k
  // Since loads produce two values, make sure to remember that we legalized
922
973k
  // both of them.
923
973k
  
if (973k
Chain.getNode() != Node973k
) {
924
6.36k
    assert(Value.getNode() != Node && "Load must be completely replaced");
925
6.36k
    DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
926
6.36k
    DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
927
6.36k
    if (
UpdatedNodes6.36k
) {
928
1
      UpdatedNodes->insert(Value.getNode());
929
1
      UpdatedNodes->insert(Chain.getNode());
930
1
    }
931
6.36k
    ReplacedNode(Node);
932
6.36k
  }
933
4.54M
}
934
935
static TargetLowering::LegalizeAction
936
34
getStrictFPOpcodeAction(const TargetLowering &TLI, unsigned Opcode, EVT VT) {
937
34
  unsigned EqOpc;
938
34
  switch (Opcode) {
939
0
    
default: 0
llvm_unreachable0
("Unexpected FP pseudo-opcode");
940
4
    case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
941
2
    case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
942
2
    case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
943
6
    case ISD::STRICT_FMA: EqOpc = ISD::FMA; break;
944
2
    case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
945
2
    case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
946
2
    case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
947
2
    case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
948
2
    case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
949
2
    case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
950
2
    case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
951
3
    case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
952
3
    case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
953
34
  }
954
34
955
34
  auto Action = TLI.getOperationAction(EqOpc, VT);
956
34
957
34
  // We don't currently handle Custom or Promote for strict FP pseudo-ops.
958
34
  // For now, we just expand for those cases.
959
34
  if (Action != TargetLowering::Legal)
960
22
    Action = TargetLowering::Expand;
961
34
962
34
  return Action;
963
34
}
964
965
/// Return a legal replacement for the given operation, with all legal operands.
966
148M
void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
967
148M
  DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
968
148M
969
148M
  if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
970
2.74M
    return;
971
145M
972
#ifndef NDEBUG
973
  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
974
    assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
975
              TargetLowering::TypeLegal ||
976
            TLI.isTypeLegal(Node->getValueType(i))) &&
977
           "Unexpected illegal type!");
978
979
  for (const SDValue &Op : Node->op_values())
980
    assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
981
              TargetLowering::TypeLegal ||
982
            TLI.isTypeLegal(Op.getValueType()) ||
983
            Op.getOpcode() == ISD::TargetConstant) &&
984
            "Unexpected illegal type!");
985
#endif
986
987
145M
  // Figure out the correct action; the way to query this varies by opcode
988
145M
  TargetLowering::LegalizeAction Action = TargetLowering::Legal;
989
145M
  bool SimpleFinishLegalizing = true;
990
145M
  switch (Node->getOpcode()) {
991
324k
  case ISD::INTRINSIC_W_CHAIN:
992
324k
  case ISD::INTRINSIC_WO_CHAIN:
993
324k
  case ISD::INTRINSIC_VOID:
994
324k
  case ISD::STACKSAVE:
995
324k
    Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
996
324k
    break;
997
4
  case ISD::GET_DYNAMIC_AREA_OFFSET:
998
4
    Action = TLI.getOperationAction(Node->getOpcode(),
999
4
                                    Node->getValueType(0));
1000
4
    break;
1001
368
  case ISD::VAARG:
1002
368
    Action = TLI.getOperationAction(Node->getOpcode(),
1003
368
                                    Node->getValueType(0));
1004
368
    if (Action != TargetLowering::Promote)
1005
367
      Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1006
368
    break;
1007
499k
  case ISD::FP_TO_FP16:
1008
499k
  case ISD::SINT_TO_FP:
1009
499k
  case ISD::UINT_TO_FP:
1010
499k
  case ISD::EXTRACT_VECTOR_ELT:
1011
499k
    Action = TLI.getOperationAction(Node->getOpcode(),
1012
499k
                                    Node->getOperand(0).getValueType());
1013
499k
    break;
1014
65.2k
  case ISD::FP_ROUND_INREG:
1015
65.2k
  case ISD::SIGN_EXTEND_INREG: {
1016
65.2k
    EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1017
65.2k
    Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1018
65.2k
    break;
1019
65.2k
  }
1020
24.1k
  case ISD::ATOMIC_STORE:
1021
24.1k
    Action = TLI.getOperationAction(Node->getOpcode(),
1022
24.1k
                                    Node->getOperand(2).getValueType());
1023
24.1k
    break;
1024
1.93M
  case ISD::SELECT_CC:
1025
1.93M
  case ISD::SETCC:
1026
1.93M
  case ISD::BR_CC: {
1027
124k
    unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1028
1.81M
                         
Node->getOpcode() == ISD::SETCC ? 1.81M
272.6k
:
1029
1.73M
                         
Node->getOpcode() == ISD::SETCCE ? 1.73M
30
:
11.73M
;
1030
1.93M
    unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 
21.73M
:
0196k
;
1031
1.93M
    MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1032
1.93M
    ISD::CondCode CCCode =
1033
1.93M
        cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1034
1.93M
    Action = TLI.getCondCodeAction(CCCode, OpVT);
1035
1.93M
    if (
Action == TargetLowering::Legal1.93M
) {
1036
1.93M
      if (Node->getOpcode() == ISD::SELECT_CC)
1037
123k
        Action = TLI.getOperationAction(Node->getOpcode(),
1038
123k
                                        Node->getValueType(0));
1039
1.93M
      else
1040
1.81M
        Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1041
1.93M
    }
1042
1.93M
    break;
1043
1.93M
  }
1044
8.35M
  case ISD::LOAD:
1045
8.35M
  case ISD::STORE:
1046
8.35M
    // FIXME: Model these properly.  LOAD and STORE are complicated, and
1047
8.35M
    // STORE expects the unlegalized operand in some cases.
1048
8.35M
    SimpleFinishLegalizing = false;
1049
8.35M
    break;
1050
6.27M
  case ISD::CALLSEQ_START:
1051
6.27M
  case ISD::CALLSEQ_END:
1052
6.27M
    // FIXME: This shouldn't be necessary.  These nodes have special properties
1053
6.27M
    // dealing with the recursive nature of legalization.  Removing this
1054
6.27M
    // special case should be done as part of making LegalizeDAG non-recursive.
1055
6.27M
    SimpleFinishLegalizing = false;
1056
6.27M
    break;
1057
26.4k
  case ISD::EXTRACT_ELEMENT:
1058
26.4k
  case ISD::FLT_ROUNDS_:
1059
26.4k
  case ISD::MERGE_VALUES:
1060
26.4k
  case ISD::EH_RETURN:
1061
26.4k
  case ISD::FRAME_TO_ARGS_OFFSET:
1062
26.4k
  case ISD::EH_DWARF_CFA:
1063
26.4k
  case ISD::EH_SJLJ_SETJMP:
1064
26.4k
  case ISD::EH_SJLJ_LONGJMP:
1065
26.4k
  case ISD::EH_SJLJ_SETUP_DISPATCH:
1066
26.4k
    // These operations lie about being legal: when they claim to be legal,
1067
26.4k
    // they should actually be expanded.
1068
26.4k
    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1069
26.4k
    if (Action == TargetLowering::Legal)
1070
24.7k
      Action = TargetLowering::Expand;
1071
26.4k
    break;
1072
207
  case ISD::INIT_TRAMPOLINE:
1073
207
  case ISD::ADJUST_TRAMPOLINE:
1074
207
  case ISD::FRAMEADDR:
1075
207
  case ISD::RETURNADDR:
1076
207
  case ISD::ADDROFRETURNADDR:
1077
207
    // These operations lie about being legal: when they claim to be legal,
1078
207
    // they should actually be custom-lowered.
1079
207
    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1080
207
    if (Action == TargetLowering::Legal)
1081
203
      Action = TargetLowering::Custom;
1082
207
    break;
1083
25
  case ISD::READCYCLECOUNTER:
1084
25
    // READCYCLECOUNTER returns an i64, even if type legalization might have
1085
25
    // expanded that to several smaller types.
1086
25
    Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1087
25
    break;
1088
752
  case ISD::READ_REGISTER:
1089
752
  case ISD::WRITE_REGISTER:
1090
752
    // Named register is legal in the DAG, but blocked by register name
1091
752
    // selection if not implemented by target (to chose the correct register)
1092
752
    // They'll be converted to Copy(To/From)Reg.
1093
752
    Action = TargetLowering::Legal;
1094
752
    break;
1095
26
  case ISD::DEBUGTRAP:
1096
26
    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1097
26
    if (
Action == TargetLowering::Expand26
) {
1098
11
      // replace ISD::DEBUGTRAP with ISD::TRAP
1099
11
      SDValue NewVal;
1100
11
      NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1101
11
                           Node->getOperand(0));
1102
11
      ReplaceNode(Node, NewVal.getNode());
1103
11
      LegalizeOp(NewVal.getNode());
1104
11
      return;
1105
11
    }
1106
15
    break;
1107
34
  case ISD::STRICT_FSQRT:
1108
34
  case ISD::STRICT_FMA:
1109
34
  case ISD::STRICT_FPOW:
1110
34
  case ISD::STRICT_FPOWI:
1111
34
  case ISD::STRICT_FSIN:
1112
34
  case ISD::STRICT_FCOS:
1113
34
  case ISD::STRICT_FEXP:
1114
34
  case ISD::STRICT_FEXP2:
1115
34
  case ISD::STRICT_FLOG:
1116
34
  case ISD::STRICT_FLOG10:
1117
34
  case ISD::STRICT_FLOG2:
1118
34
  case ISD::STRICT_FRINT:
1119
34
  case ISD::STRICT_FNEARBYINT:
1120
34
    // These pseudo-ops get legalized as if they were their non-strict
1121
34
    // equivalent.  For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1122
34
    // is also legal, but if ISD::FSQRT requires expansion then so does
1123
34
    // ISD::STRICT_FSQRT.
1124
34
    Action = getStrictFPOpcodeAction(TLI, Node->getOpcode(),
1125
34
                                     Node->getValueType(0));
1126
34
    break;
1127
128M
  default:
1128
128M
    if (
Node->getOpcode() >= ISD::BUILTIN_OP_END128M
) {
1129
15.2M
      Action = TargetLowering::Legal;
1130
128M
    } else {
1131
113M
      Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1132
113M
    }
1133
324k
    break;
1134
145M
  }
1135
145M
1136
145M
  
if (145M
SimpleFinishLegalizing145M
) {
1137
131M
    SDNode *NewNode = Node;
1138
131M
    switch (Node->getOpcode()) {
1139
130M
    default: break;
1140
945k
    case ISD::SHL:
1141
945k
    case ISD::SRL:
1142
945k
    case ISD::SRA:
1143
945k
    case ISD::ROTL:
1144
945k
    case ISD::ROTR: {
1145
945k
      // Legalizing shifts/rotates requires adjusting the shift amount
1146
945k
      // to the appropriate width.
1147
945k
      SDValue Op0 = Node->getOperand(0);
1148
945k
      SDValue Op1 = Node->getOperand(1);
1149
945k
      if (
!Op1.getValueType().isVector()945k
) {
1150
940k
        SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1151
940k
        // The getShiftAmountOperand() may create a new operand node or
1152
940k
        // return the existing one. If new operand is created we need
1153
940k
        // to update the parent node.
1154
940k
        // Do not try to legalize SAO here! It will be automatically legalized
1155
940k
        // in the next round.
1156
940k
        if (SAO != Op1)
1157
15.0k
          NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1158
940k
      }
1159
945k
    }
1160
945k
    break;
1161
1.28k
    case ISD::SRL_PARTS:
1162
1.28k
    case ISD::SRA_PARTS:
1163
1.28k
    case ISD::SHL_PARTS: {
1164
1.28k
      // Legalizing shifts/rotates requires adjusting the shift amount
1165
1.28k
      // to the appropriate width.
1166
1.28k
      SDValue Op0 = Node->getOperand(0);
1167
1.28k
      SDValue Op1 = Node->getOperand(1);
1168
1.28k
      SDValue Op2 = Node->getOperand(2);
1169
1.28k
      if (
!Op2.getValueType().isVector()1.28k
) {
1170
1.28k
        SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1171
1.28k
        // The getShiftAmountOperand() may create a new operand node or
1172
1.28k
        // return the existing one. If new operand is created we need
1173
1.28k
        // to update the parent node.
1174
1.28k
        if (SAO != Op2)
1175
0
          NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1176
1.28k
      }
1177
1.28k
      break;
1178
131M
    }
1179
131M
    }
1180
131M
1181
131M
    
if (131M
NewNode != Node131M
) {
1182
145
      ReplaceNode(Node, NewNode);
1183
145
      Node = NewNode;
1184
145
    }
1185
131M
    switch (Action) {
1186
126M
    case TargetLowering::Legal:
1187
126M
      return;
1188
4.30M
    case TargetLowering::Custom:
1189
4.30M
      // FIXME: The handling for custom lowering with multiple results is
1190
4.30M
      // a complete mess.
1191
4.30M
      if (SDValue 
Res4.30M
= TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1192
4.20M
        if (
!(Res.getNode() != Node || 4.20M
Res.getResNo() != 0683k
))
1193
683k
          return;
1194
3.52M
1195
3.52M
        
if (3.52M
Node->getNumValues() == 13.52M
) {
1196
3.50M
          // We can just directly replace this node with the lowered value.
1197
3.50M
          ReplaceNode(SDValue(Node, 0), Res);
1198
3.50M
          return;
1199
3.50M
        }
1200
16.6k
1201
16.6k
        SmallVector<SDValue, 8> ResultVals;
1202
49.8k
        for (unsigned i = 0, e = Node->getNumValues(); 
i != e49.8k
;
++i33.2k
)
1203
33.2k
          ResultVals.push_back(Res.getValue(i));
1204
4.20M
        ReplaceNode(Node, ResultVals.data());
1205
4.20M
        return;
1206
4.20M
      }
1207
98.2k
      
LLVM_FALLTHROUGH98.2k
;
1208
334k
    case TargetLowering::Expand:
1209
334k
      if (ExpandNode(Node))
1210
186k
        return;
1211
147k
      
LLVM_FALLTHROUGH147k
;
1212
147k
    case TargetLowering::LibCall:
1213
147k
      ConvertNodeToLibcall(Node);
1214
147k
      return;
1215
6.94k
    case TargetLowering::Promote:
1216
6.94k
      PromoteNode(Node);
1217
6.94k
      return;
1218
14.6M
    }
1219
14.6M
  }
1220
14.6M
1221
14.6M
  switch (Node->getOpcode()) {
1222
0
  default:
1223
#ifndef NDEBUG
1224
    dbgs() << "NODE: ";
1225
    Node->dump( &DAG);
1226
    dbgs() << "\n";
1227
#endif
1228
0
    llvm_unreachable("Do not know how to legalize this operator!");
1229
14.6M
1230
6.27M
  case ISD::CALLSEQ_START:
1231
6.27M
  case ISD::CALLSEQ_END:
1232
6.27M
    break;
1233
4.54M
  case ISD::LOAD:
1234
4.54M
    return LegalizeLoadOps(Node);
1235
3.81M
  case ISD::STORE:
1236
3.81M
    return LegalizeStoreOps(Node);
1237
6.27M
  }
1238
6.27M
}
1239
1240
2.06k
SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1241
2.06k
  SDValue Vec = Op.getOperand(0);
1242
2.06k
  SDValue Idx = Op.getOperand(1);
1243
2.06k
  SDLoc dl(Op);
1244
2.06k
1245
2.06k
  // Before we generate a new store to a temporary stack slot, see if there is
1246
2.06k
  // already one that we can use. There often is because when we scalarize
1247
2.06k
  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1248
2.06k
  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1249
2.06k
  // the vector. If all are expanded here, we don't want one store per vector
1250
2.06k
  // element.
1251
2.06k
1252
2.06k
  // Caches for hasPredecessorHelper
1253
2.06k
  SmallPtrSet<const SDNode *, 32> Visited;
1254
2.06k
  SmallVector<const SDNode *, 16> Worklist;
1255
2.06k
  Worklist.push_back(Idx.getNode());
1256
2.06k
  SDValue StackPtr, Ch;
1257
2.06k
  for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1258
4.15k
       UE = Vec.getNode()->use_end(); 
UI != UE4.15k
;
++UI2.09k
) {
1259
3.73k
    SDNode *User = *UI;
1260
3.73k
    if (StoreSDNode *
ST3.73k
= dyn_cast<StoreSDNode>(User)) {
1261
1.64k
      if (
ST->isIndexed() || 1.64k
ST->isTruncatingStore()1.64k
||
1262
1.64k
          ST->getValue() != Vec)
1263
0
        continue;
1264
1.64k
1265
1.64k
      // Make sure that nothing else could have stored into the destination of
1266
1.64k
      // this store.
1267
1.64k
      
if (1.64k
!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())1.64k
)
1268
0
        continue;
1269
1.64k
1270
1.64k
      // If the index is dependent on the store we will introduce a cycle when
1271
1.64k
      // creating the load (the load uses the index, and by replacing the chain
1272
1.64k
      // we will make the index dependent on the load). Also, the store might be
1273
1.64k
      // dependent on the extractelement and introduce a cycle when creating
1274
1.64k
      // the load.
1275
1.64k
      
if (1.64k
SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1276
1.64k
          ST->hasPredecessor(Op.getNode()))
1277
2
        continue;
1278
1.64k
1279
1.64k
      StackPtr = ST->getBasePtr();
1280
1.64k
      Ch = SDValue(ST, 0);
1281
1.64k
      break;
1282
1.64k
    }
1283
3.73k
  }
1284
2.06k
1285
2.06k
  EVT VecVT = Vec.getValueType();
1286
2.06k
1287
2.06k
  if (
!Ch.getNode()2.06k
) {
1288
420
    // Store the value to a temporary stack slot, then LOAD the returned part.
1289
420
    StackPtr = DAG.CreateStackTemporary(VecVT);
1290
420
    Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1291
420
                      MachinePointerInfo());
1292
420
  }
1293
2.06k
1294
2.06k
  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1295
2.06k
1296
2.06k
  SDValue NewLoad;
1297
2.06k
1298
2.06k
  if (Op.getValueType().isVector())
1299
0
    NewLoad =
1300
0
        DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1301
2.06k
  else
1302
2.06k
    NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1303
2.06k
                             MachinePointerInfo(),
1304
2.06k
                             VecVT.getVectorElementType());
1305
2.06k
1306
2.06k
  // Replace the chain going out of the store, by the one out of the load.
1307
2.06k
  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1308
2.06k
1309
2.06k
  // We introduced a cycle though, so update the loads operands, making sure
1310
2.06k
  // to use the original store's chain as an incoming chain.
1311
2.06k
  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1312
2.06k
                                          NewLoad->op_end());
1313
2.06k
  NewLoadOperands[0] = Ch;
1314
2.06k
  NewLoad =
1315
2.06k
      SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1316
2.06k
  return NewLoad;
1317
2.06k
}
1318
1319
0
SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1320
0
  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1321
0
1322
0
  SDValue Vec  = Op.getOperand(0);
1323
0
  SDValue Part = Op.getOperand(1);
1324
0
  SDValue Idx  = Op.getOperand(2);
1325
0
  SDLoc dl(Op);
1326
0
1327
0
  // Store the value to a temporary stack slot, then LOAD the returned part.
1328
0
  EVT VecVT = Vec.getValueType();
1329
0
  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1330
0
  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1331
0
  MachinePointerInfo PtrInfo =
1332
0
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1333
0
1334
0
  // First store the whole vector.
1335
0
  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1336
0
1337
0
  // Then store the inserted part.
1338
0
  SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1339
0
1340
0
  // Store the subvector.
1341
0
  Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1342
0
1343
0
  // Finally, load the updated vector.
1344
0
  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1345
0
}
1346
1347
144
SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1348
144
  // We can't handle this case efficiently.  Allocate a sufficiently
1349
144
  // aligned object on the stack, store each element into it, then load
1350
144
  // the result as a vector.
1351
144
  // Create the stack frame object.
1352
144
  EVT VT = Node->getValueType(0);
1353
144
  EVT EltVT = VT.getVectorElementType();
1354
144
  SDLoc dl(Node);
1355
144
  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1356
144
  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1357
144
  MachinePointerInfo PtrInfo =
1358
144
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1359
144
1360
144
  // Emit a store of each element to the stack slot.
1361
144
  SmallVector<SDValue, 8> Stores;
1362
144
  unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1363
144
  // Store (in the right endianness) the elements to memory.
1364
1.16k
  for (unsigned i = 0, e = Node->getNumOperands(); 
i != e1.16k
;
++i1.01k
) {
1365
1.01k
    // Ignore undef elements.
1366
1.01k
    if (
Node->getOperand(i).isUndef()1.01k
)
continue125
;
1367
891
1368
891
    unsigned Offset = TypeByteSize*i;
1369
891
1370
891
    SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1371
891
    Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1372
891
1373
891
    // If the destination vector element type is narrower than the source
1374
891
    // element type, only store the bits necessary.
1375
891
    if (
EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())891
) {
1376
606
      Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1377
606
                                         Node->getOperand(i), Idx,
1378
606
                                         PtrInfo.getWithOffset(Offset), EltVT));
1379
606
    } else
1380
285
      Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1381
285
                                    Idx, PtrInfo.getWithOffset(Offset)));
1382
1.01k
  }
1383
144
1384
144
  SDValue StoreChain;
1385
144
  if (!Stores.empty())    // Not all undef elements?
1386
144
    StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1387
144
  else
1388
0
    StoreChain = DAG.getEntryNode();
1389
144
1390
144
  // Result is a load from the stack slot.
1391
144
  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1392
144
}
1393
1394
/// Bitcast a floating-point value to an integer value. Only bitcast the part
1395
/// containing the sign bit if the target has no integer value capable of
1396
/// holding all bits of the floating-point value.
1397
void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1398
                                             const SDLoc &DL,
1399
333
                                             SDValue Value) const {
1400
333
  EVT FloatVT = Value.getValueType();
1401
333
  unsigned NumBits = FloatVT.getSizeInBits();
1402
333
  State.FloatVT = FloatVT;
1403
333
  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1404
333
  // Convert to an integer of the same size.
1405
333
  if (
TLI.isTypeLegal(IVT)333
) {
1406
109
    State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1407
109
    State.SignMask = APInt::getSignMask(NumBits);
1408
109
    State.SignBit = NumBits - 1;
1409
109
    return;
1410
109
  }
1411
224
1412
224
  auto &DataLayout = DAG.getDataLayout();
1413
224
  // Store the float to memory, then load the sign part out as an integer.
1414
224
  MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1415
224
  // First create a temporary that is aligned for both the load and store.
1416
224
  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1417
224
  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1418
224
  // Then store the float to it.
1419
224
  State.FloatPtr = StackPtr;
1420
224
  MachineFunction &MF = DAG.getMachineFunction();
1421
224
  State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1422
224
  State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1423
224
                             State.FloatPointerInfo);
1424
224
1425
224
  SDValue IntPtr;
1426
224
  if (
DataLayout.isBigEndian()224
) {
1427
1
    assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1428
1
    // Load out a legal integer with the same sign bit as the float.
1429
1
    IntPtr = StackPtr;
1430
1
    State.IntPointerInfo = State.FloatPointerInfo;
1431
224
  } else {
1432
223
    // Advance the pointer so that the loaded byte will contain the sign bit.
1433
223
    unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1434
223
    IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1435
223
                      DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1436
223
    State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1437
223
                                                             ByteOffset);
1438
223
  }
1439
333
1440
333
  State.IntPtr = IntPtr;
1441
333
  State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1442
333
                                  State.IntPointerInfo, MVT::i8);
1443
333
  State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1444
333
  State.SignBit = 7;
1445
333
}
1446
1447
/// Replace the integer value produced by getSignAsIntValue() with a new value
1448
/// and cast the result back to a floating-point type.
1449
SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1450
                                              const SDLoc &DL,
1451
37
                                              SDValue NewIntValue) const {
1452
37
  if (!State.Chain)
1453
36
    return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1454
1
1455
1
  // Override the part containing the sign bit in the value stored on the stack.
1456
1
  SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1457
1
                                    State.IntPointerInfo, MVT::i8);
1458
1
  return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1459
1
                     State.FloatPointerInfo);
1460
1
}
1461
1462
296
SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1463
296
  SDLoc DL(Node);
1464
296
  SDValue Mag = Node->getOperand(0);
1465
296
  SDValue Sign = Node->getOperand(1);
1466
296
1467
296
  // Get sign bit into an integer value.
1468
296
  FloatSignAsInt SignAsInt;
1469
296
  getSignAsIntValue(SignAsInt, DL, Sign);
1470
296
1471
296
  EVT IntVT = SignAsInt.IntValue.getValueType();
1472
296
  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1473
296
  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1474
296
                                SignMask);
1475
296
1476
296
  // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1477
296
  EVT FloatVT = Mag.getValueType();
1478
296
  if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1479
296
      
TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)259
) {
1480
259
    SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1481
259
    SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1482
259
    SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1483
259
                                DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1484
259
    return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1485
259
  }
1486
37
1487
37
  // Transform Mag value to integer, and clear the sign bit.
1488
37
  FloatSignAsInt MagAsInt;
1489
37
  getSignAsIntValue(MagAsInt, DL, Mag);
1490
37
  EVT MagVT = MagAsInt.IntValue.getValueType();
1491
37
  SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1492
37
  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1493
37
                                    ClearSignMask);
1494
37
1495
37
  // Get the signbit at the right position for MagAsInt.
1496
37
  int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1497
37
  if (
SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()37
) {
1498
19
    if (
ShiftAmount > 019
) {
1499
19
      SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, IntVT);
1500
19
      SignBit = DAG.getNode(ISD::SRL, DL, IntVT, SignBit, ShiftCnst);
1501
19
    } else 
if (0
ShiftAmount < 00
) {
1502
0
      SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, IntVT);
1503
0
      SignBit = DAG.getNode(ISD::SHL, DL, IntVT, SignBit, ShiftCnst);
1504
0
    }
1505
19
    SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1506
37
  } else 
if (18
SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()18
) {
1507
0
    SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1508
0
    if (
ShiftAmount > 00
) {
1509
0
      SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, MagVT);
1510
0
      SignBit = DAG.getNode(ISD::SRL, DL, MagVT, SignBit, ShiftCnst);
1511
0
    } else 
if (0
ShiftAmount < 00
) {
1512
0
      SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, MagVT);
1513
0
      SignBit = DAG.getNode(ISD::SHL, DL, MagVT, SignBit, ShiftCnst);
1514
0
    }
1515
18
  }
1516
296
1517
296
  // Store the part with the modified sign and convert back to float.
1518
296
  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1519
296
  return modifySignAsInt(MagAsInt, DL, CopiedSign);
1520
296
}
1521
1522
32
SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1523
32
  SDLoc DL(Node);
1524
32
  SDValue Value = Node->getOperand(0);
1525
32
1526
32
  // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1527
32
  EVT FloatVT = Value.getValueType();
1528
32
  if (
TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)32
) {
1529
32
    SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1530
32
    return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1531
32
  }
1532
0
1533
0
  // Transform value to integer, clear the sign bit and transform back.
1534
0
  FloatSignAsInt ValueAsInt;
1535
0
  getSignAsIntValue(ValueAsInt, DL, Value);
1536
0
  EVT IntVT = ValueAsInt.IntValue.getValueType();
1537
0
  SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1538
0
  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1539
0
                                    ClearSignMask);
1540
0
  return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1541
0
}
1542
1543
void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1544
315
                                           SmallVectorImpl<SDValue> &Results) {
1545
315
  unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1546
315
  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1547
315
          " not tell us which reg is the stack pointer!");
1548
315
  SDLoc dl(Node);
1549
315
  EVT VT = Node->getValueType(0);
1550
315
  SDValue Tmp1 = SDValue(Node, 0);
1551
315
  SDValue Tmp2 = SDValue(Node, 1);
1552
315
  SDValue Tmp3 = Node->getOperand(2);
1553
315
  SDValue Chain = Tmp1.getOperand(0);
1554
315
1555
315
  // Chain the dynamic stack allocation so that it doesn't modify the stack
1556
315
  // pointer when other instructions are using the stack.
1557
315
  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1558
315
1559
315
  SDValue Size  = Tmp2.getOperand(1);
1560
315
  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1561
315
  Chain = SP.getValue(1);
1562
315
  unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1563
315
  unsigned StackAlign =
1564
315
      DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1565
315
  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size);       // Value
1566
315
  if (Align > StackAlign)
1567
51
    Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1568
51
                       DAG.getConstant(-(uint64_t)Align, dl, VT));
1569
315
  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
1570
315
1571
315
  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1572
315
                            DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1573
315
1574
315
  Results.push_back(Tmp1);
1575
315
  Results.push_back(Tmp2);
1576
315
}
1577
1578
/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1579
/// target.
1580
///
1581
/// If the SETCC has been legalized using AND / OR, then the legalized node
1582
/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1583
/// will be set to false.
1584
///
1585
/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1586
/// then the values of LHS and RHS will be swapped, CC will be set to the
1587
/// new condition, and NeedInvert will be set to false.
1588
///
1589
/// If the SETCC has been legalized using the inverse condcode, then LHS and
1590
/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1591
/// will be set to true. The caller must invert the result of the SETCC with
1592
/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1593
/// of a true/false result.
1594
///
1595
/// \returns true if the SetCC has been legalized, false if it hasn't.
1596
bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1597
                                                 SDValue &RHS, SDValue &CC,
1598
                                                 bool &NeedInvert,
1599
3.27k
                                                 const SDLoc &dl) {
1600
3.27k
  MVT OpVT = LHS.getSimpleValueType();
1601
3.27k
  ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1602
3.27k
  NeedInvert = false;
1603
3.27k
  switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1604
0
  
default: 0
llvm_unreachable0
("Unknown condition code action!");
1605
2.92k
  case TargetLowering::Legal:
1606
2.92k
    // Nothing to do.
1607
2.92k
    break;
1608
355
  case TargetLowering::Expand: {
1609
355
    ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1610
355
    if (
TLI.isCondCodeLegal(InvCC, OpVT)355
) {
1611
231
      std::swap(LHS, RHS);
1612
231
      CC = DAG.getCondCode(InvCC);
1613
231
      return true;
1614
231
    }
1615
124
    ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1616
124
    unsigned Opc = 0;
1617
124
    switch (CCCode) {
1618
0
    
default: 0
llvm_unreachable0
("Don't know how to expand this condition!");
1619
6
    case ISD::SETO:
1620
6
        assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)
1621
6
            == TargetLowering::Legal
1622
6
            && "If SETO is expanded, SETOEQ must be legal!");
1623
6
        CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1624
15
    case ISD::SETUO:
1625
15
        assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)
1626
15
            == TargetLowering::Legal
1627
15
            && "If SETUO is expanded, SETUNE must be legal!");
1628
15
        CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;  break;
1629
91
    case ISD::SETOEQ:
1630
91
    case ISD::SETOGT:
1631
91
    case ISD::SETOGE:
1632
91
    case ISD::SETOLT:
1633
91
    case ISD::SETOLE:
1634
91
    case ISD::SETONE:
1635
91
    case ISD::SETUEQ:
1636
91
    case ISD::SETUNE:
1637
91
    case ISD::SETUGT:
1638
91
    case ISD::SETUGE:
1639
91
    case ISD::SETULT:
1640
91
    case ISD::SETULE:
1641
91
        // If we are floating point, assign and break, otherwise fall through.
1642
91
        if (
!OpVT.isInteger()91
) {
1643
91
          // We can use the 4th bit to tell if we are the unordered
1644
91
          // or ordered version of the opcode.
1645
91
          CC2 = ((unsigned)CCCode & 0x8U) ? 
ISD::SETUO39
:
ISD::SETO52
;
1646
91
          Opc = ((unsigned)CCCode & 0x8U) ? 
ISD::OR39
:
ISD::AND52
;
1647
91
          CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1648
91
          break;
1649
91
        }
1650
0
        // Fallthrough if we are unsigned integer.
1651
0
        
LLVM_FALLTHROUGH0
;
1652
0
    case ISD::SETLE:
1653
0
    case ISD::SETGT:
1654
0
    case ISD::SETGE:
1655
0
    case ISD::SETLT:
1656
0
      // We only support using the inverted operation, which is computed above
1657
0
      // and not a different manner of supporting expanding these cases.
1658
0
      llvm_unreachable("Don't know how to expand this condition!");
1659
12
    case ISD::SETNE:
1660
12
    case ISD::SETEQ:
1661
12
      // Try inverting the result of the inverse condition.
1662
12
      InvCC = CCCode == ISD::SETEQ ? 
ISD::SETNE0
:
ISD::SETEQ12
;
1663
12
      if (
TLI.isCondCodeLegal(InvCC, OpVT)12
) {
1664
12
        CC = DAG.getCondCode(InvCC);
1665
12
        NeedInvert = true;
1666
12
        return true;
1667
12
      }
1668
0
      // If inverting the condition didn't work then we have no means to expand
1669
0
      // the condition.
1670
0
      
llvm_unreachable0
("Don't know how to expand this condition!");
1671
112
    }
1672
112
1673
112
    SDValue SetCC1, SetCC2;
1674
112
    if (
CCCode != ISD::SETO && 112
CCCode != ISD::SETUO106
) {
1675
91
      // If we aren't the ordered or unorder operation,
1676
91
      // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1677
91
      SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1678
91
      SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1679
112
    } else {
1680
21
      // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1681
21
      SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1682
21
      SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1683
21
    }
1684
355
    LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1685
355
    RHS = SDValue();
1686
355
    CC  = SDValue();
1687
355
    return true;
1688
355
  }
1689
2.92k
  }
1690
2.92k
  return false;
1691
2.92k
}
1692
1693
/// Emit a store/load combination to the stack.  This stores
1694
/// SrcOp to a stack slot of type SlotVT, truncating it if needed.  It then does
1695
/// a load from the stack slot to DestVT, extending it if needed.
1696
/// The resultant code need not be legal.
1697
SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1698
520
                                               EVT DestVT, const SDLoc &dl) {
1699
520
  // Create the stack frame object.
1700
520
  unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1701
520
      SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1702
520
  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1703
520
1704
520
  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1705
520
  int SPFI = StackPtrFI->getIndex();
1706
520
  MachinePointerInfo PtrInfo =
1707
520
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1708
520
1709
520
  unsigned SrcSize = SrcOp.getValueSizeInBits();
1710
520
  unsigned SlotSize = SlotVT.getSizeInBits();
1711
520
  unsigned DestSize = DestVT.getSizeInBits();
1712
520
  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1713
520
  unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1714
520
1715
520
  // Emit a store to the stack slot.  Use a truncstore if the input value is
1716
520
  // later than DestVT.
1717
520
  SDValue Store;
1718
520
1719
520
  if (SrcSize > SlotSize)
1720
0
    Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1721
0
                              SlotVT, SrcAlign);
1722
520
  else {
1723
520
    assert(SrcSize == SlotSize && "Invalid store");
1724
520
    Store =
1725
520
        DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1726
520
  }
1727
520
1728
520
  // Result is a load from the stack slot.
1729
520
  if (SlotSize == DestSize)
1730
520
    return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1731
0
1732
520
  assert(SlotSize < DestSize && "Unknown extension!");
1733
0
  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1734
0
                        DestAlign);
1735
0
}
1736
1737
18
SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1738
18
  SDLoc dl(Node);
1739
18
  // Create a vector sized/aligned stack slot, store the value to element #0,
1740
18
  // then load the whole vector back out.
1741
18
  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1742
18
1743
18
  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1744
18
  int SPFI = StackPtrFI->getIndex();
1745
18
1746
18
  SDValue Ch = DAG.getTruncStore(
1747
18
      DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1748
18
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1749
18
      Node->getValueType(0).getVectorElementType());
1750
18
  return DAG.getLoad(
1751
18
      Node->getValueType(0), dl, Ch, StackPtr,
1752
18
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1753
18
}
1754
1755
static bool
1756
ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1757
27
                     const TargetLowering &TLI, SDValue &Res) {
1758
27
  unsigned NumElems = Node->getNumOperands();
1759
27
  SDLoc dl(Node);
1760
27
  EVT VT = Node->getValueType(0);
1761
27
1762
27
  // Try to group the scalars into pairs, shuffle the pairs together, then
1763
27
  // shuffle the pairs of pairs together, etc. until the vector has
1764
27
  // been built. This will work only if all of the necessary shuffle masks
1765
27
  // are legal.
1766
27
1767
27
  // We do this in two phases; first to check the legality of the shuffles,
1768
27
  // and next, assuming that all shuffles are legal, to create the new nodes.
1769
81
  for (int Phase = 0; 
Phase < 281
;
++Phase54
) {
1770
54
    SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1771
54
                                                              NewIntermedVals;
1772
398
    for (unsigned i = 0; 
i < NumElems398
;
++i344
) {
1773
344
      SDValue V = Node->getOperand(i);
1774
344
      if (V.isUndef())
1775
16
        continue;
1776
328
1777
328
      SDValue Vec;
1778
328
      if (Phase)
1779
164
        Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1780
344
      IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1781
344
    }
1782
54
1783
130
    while (
IntermedVals.size() > 2130
) {
1784
76
      NewIntermedVals.clear();
1785
296
      for (unsigned i = 0, e = (IntermedVals.size() & ~1u); 
i < e296
;
i += 2220
) {
1786
220
        // This vector and the next vector are shuffled together (simply to
1787
220
        // append the one to the other).
1788
220
        SmallVector<int, 16> ShuffleVec(NumElems, -1);
1789
220
1790
220
        SmallVector<int, 16> FinalIndices;
1791
220
        FinalIndices.reserve(IntermedVals[i].second.size() +
1792
220
                             IntermedVals[i+1].second.size());
1793
220
1794
220
        int k = 0;
1795
520
        for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1796
300
             
++j, ++k300
) {
1797
300
          ShuffleVec[k] = j;
1798
300
          FinalIndices.push_back(IntermedVals[i].second[j]);
1799
300
        }
1800
520
        for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1801
300
             
++j, ++k300
) {
1802
300
          ShuffleVec[k] = NumElems + j;
1803
300
          FinalIndices.push_back(IntermedVals[i+1].second[j]);
1804
300
        }
1805
220
1806
220
        SDValue Shuffle;
1807
220
        if (Phase)
1808
110
          Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1809
110
                                         IntermedVals[i+1].first,
1810
110
                                         ShuffleVec);
1811
110
        else 
if (110
!TLI.isShuffleMaskLegal(ShuffleVec, VT)110
)
1812
0
          return false;
1813
220
        NewIntermedVals.push_back(
1814
220
            std::make_pair(Shuffle, std::move(FinalIndices)));
1815
220
      }
1816
76
1817
76
      // If we had an odd number of defined values, then append the last
1818
76
      // element to the array of new vectors.
1819
76
      
if (76
(IntermedVals.size() & 1) != 076
)
1820
0
        NewIntermedVals.push_back(IntermedVals.back());
1821
76
1822
76
      IntermedVals.swap(NewIntermedVals);
1823
76
    }
1824
54
1825
54
    assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1826
54
           "Invalid number of intermediate vectors");
1827
54
    SDValue Vec1 = IntermedVals[0].first;
1828
54
    SDValue Vec2;
1829
54
    if (IntermedVals.size() > 1)
1830
54
      Vec2 = IntermedVals[1].first;
1831
0
    else 
if (0
Phase0
)
1832
0
      Vec2 = DAG.getUNDEF(VT);
1833
54
1834
54
    SmallVector<int, 16> ShuffleVec(NumElems, -1);
1835
218
    for (unsigned i = 0, e = IntermedVals[0].second.size(); 
i != e218
;
++i164
)
1836
164
      ShuffleVec[IntermedVals[0].second[i]] = i;
1837
218
    for (unsigned i = 0, e = IntermedVals[1].second.size(); 
i != e218
;
++i164
)
1838
164
      ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1839
54
1840
54
    if (Phase)
1841
27
      Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1842
27
    else 
if (27
!TLI.isShuffleMaskLegal(ShuffleVec, VT)27
)
1843
0
      return false;
1844
54
  }
1845
27
1846
27
  return true;
1847
27
}
1848
1849
/// Expand a BUILD_VECTOR node on targets that don't
1850
/// support the operation, but do support the resultant vector type.
1851
63.0k
SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1852
63.0k
  unsigned NumElems = Node->getNumOperands();
1853
63.0k
  SDValue Value1, Value2;
1854
63.0k
  SDLoc dl(Node);
1855
63.0k
  EVT VT = Node->getValueType(0);
1856
63.0k
  EVT OpVT = Node->getOperand(0).getValueType();
1857
63.0k
  EVT EltVT = VT.getVectorElementType();
1858
63.0k
1859
63.0k
  // If the only non-undef value is the low element, turn this into a
1860
63.0k
  // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
1861
63.0k
  bool isOnlyLowElement = true;
1862
63.0k
  bool MoreThanTwoValues = false;
1863
63.0k
  bool isConstant = true;
1864
413k
  for (unsigned i = 0; 
i < NumElems413k
;
++i350k
) {
1865
350k
    SDValue V = Node->getOperand(i);
1866
350k
    if (V.isUndef())
1867
11.3k
      continue;
1868
338k
    
if (338k
i > 0338k
)
1869
276k
      isOnlyLowElement = false;
1870
338k
    if (
!isa<ConstantFPSDNode>(V) && 338k
!isa<ConstantSDNode>(V)314k
)
1871
9.06k
      isConstant = false;
1872
338k
1873
338k
    if (
!Value1.getNode()338k
) {
1874
63.0k
      Value1 = V;
1875
338k
    } else 
if (275k
!Value2.getNode()275k
) {
1876
113k
      if (V != Value1)
1877
58.8k
        Value2 = V;
1878
275k
    } else 
if (161k
V != Value1 && 161k
V != Value2141k
) {
1879
123k
      MoreThanTwoValues = true;
1880
123k
    }
1881
350k
  }
1882
63.0k
1883
63.0k
  if (!Value1.getNode())
1884
0
    return DAG.getUNDEF(VT);
1885
63.0k
1886
63.0k
  
if (63.0k
isOnlyLowElement63.0k
)
1887
15
    return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1888
63.0k
1889
63.0k
  // If all elements are constants, create a load from the constant pool.
1890
63.0k
  
if (63.0k
isConstant63.0k
) {
1891
61.1k
    SmallVector<Constant*, 16> CV;
1892
401k
    for (unsigned i = 0, e = NumElems; 
i != e401k
;
++i340k
) {
1893
340k
      if (ConstantFPSDNode *V =
1894
24.2k
          dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1895
24.2k
        CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1896
340k
      } else 
if (ConstantSDNode *316k
V316k
=
1897
305k
                 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1898
305k
        if (OpVT==EltVT)
1899
248k
          CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1900
56.5k
        else {
1901
56.5k
          // If OpVT and EltVT don't match, EltVT is not legal and the
1902
56.5k
          // element values have been promoted/truncated earlier.  Undo this;
1903
56.5k
          // we don't want a v16i8 to become a v16i32 for example.
1904
56.5k
          const ConstantInt *CI = V->getConstantIntValue();
1905
56.5k
          CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1906
56.5k
                                        CI->getZExtValue()));
1907
56.5k
        }
1908
316k
      } else {
1909
11.0k
        assert(Node->getOperand(i).isUndef());
1910
11.0k
        Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1911
11.0k
        CV.push_back(UndefValue::get(OpNTy));
1912
11.0k
      }
1913
340k
    }
1914
61.1k
    Constant *CP = ConstantVector::get(CV);
1915
61.1k
    SDValue CPIdx =
1916
61.1k
        DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1917
61.1k
    unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1918
61.1k
    return DAG.getLoad(
1919
61.1k
        VT, dl, DAG.getEntryNode(), CPIdx,
1920
61.1k
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1921
61.1k
        Alignment);
1922
61.1k
  }
1923
1.89k
1924
1.89k
  SmallSet<SDValue, 16> DefinedValues;
1925
11.2k
  for (unsigned i = 0; 
i < NumElems11.2k
;
++i9.35k
) {
1926
9.35k
    if (Node->getOperand(i).isUndef())
1927
261
      continue;
1928
9.09k
    DefinedValues.insert(Node->getOperand(i));
1929
9.09k
  }
1930
1.89k
1931
1.89k
  if (
TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())1.89k
) {
1932
1.82k
    if (
!MoreThanTwoValues1.82k
) {
1933
1.79k
      SmallVector<int, 8> ShuffleVec(NumElems, -1);
1934
10.2k
      for (unsigned i = 0; 
i < NumElems10.2k
;
++i8.44k
) {
1935
8.44k
        SDValue V = Node->getOperand(i);
1936
8.44k
        if (V.isUndef())
1937
228
          continue;
1938
8.21k
        
ShuffleVec[i] = V == Value1 ? 8.21k
07.01k
:
NumElems1.20k
;
1939
8.44k
      }
1940
1.79k
      if (
TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))1.79k
) {
1941
1.72k
        // Get the splatted value into the low element of a vector register.
1942
1.72k
        SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1943
1.72k
        SDValue Vec2;
1944
1.72k
        if (Value2.getNode())
1945
1.12k
          Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1946
1.72k
        else
1947
600
          Vec2 = DAG.getUNDEF(VT);
1948
1.72k
1949
1.72k
        // Return shuffle(LowValVec, undef, <0,0,0,0>)
1950
1.72k
        return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1951
1.72k
      }
1952
27
    } else {
1953
27
      SDValue Res;
1954
27
      if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1955
27
        return Res;
1956
144
    }
1957
1.82k
  }
1958
144
1959
144
  // Otherwise, we can't handle this case efficiently.
1960
144
  return ExpandVectorBuildThroughStack(Node);
1961
144
}
1962
1963
// Expand a node into a call to a libcall.  If the result value
1964
// does not fit into a register, return the lo part and set the hi part to the
1965
// by-reg argument.  If it does fit into a single register, return the result
1966
// and leave the Hi part unset.
1967
SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1968
2.52k
                                            bool isSigned) {
1969
2.52k
  TargetLowering::ArgListTy Args;
1970
2.52k
  TargetLowering::ArgListEntry Entry;
1971
3.67k
  for (const SDValue &Op : Node->op_values()) {
1972
3.67k
    EVT ArgVT = Op.getValueType();
1973
3.67k
    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1974
3.67k
    Entry.Node = Op;
1975
3.67k
    Entry.Ty = ArgTy;
1976
3.67k
    Entry.IsSExt = isSigned;
1977
3.67k
    Entry.IsZExt = !isSigned;
1978
3.67k
    Args.push_back(Entry);
1979
3.67k
  }
1980
2.52k
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1981
2.52k
                                         TLI.getPointerTy(DAG.getDataLayout()));
1982
2.52k
1983
2.52k
  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
1984
2.52k
1985
2.52k
  // By default, the input chain to this libcall is the entry node of the
1986
2.52k
  // function. If the libcall is going to be emitted as a tail call then
1987
2.52k
  // TLI.isUsedByReturnOnly will change it to the right chain if the return
1988
2.52k
  // node which is being folded has a non-entry input chain.
1989
2.52k
  SDValue InChain = DAG.getEntryNode();
1990
2.52k
1991
2.52k
  // isTailCall may be true since the callee does not reference caller stack
1992
2.52k
  // frame. Check if it's in the right position and that the return types match.
1993
2.52k
  SDValue TCChain = InChain;
1994
2.52k
  const Function *F = DAG.getMachineFunction().getFunction();
1995
2.52k
  bool isTailCall =
1996
2.52k
      TLI.isInTailCallPosition(DAG, Node, TCChain) &&
1997
285
      
(RetTy == F->getReturnType() || 285
F->getReturnType()->isVoidTy()6
);
1998
2.52k
  if (isTailCall)
1999
279
    InChain = TCChain;
2000
2.52k
2001
2.52k
  TargetLowering::CallLoweringInfo CLI(DAG);
2002
2.52k
  CLI.setDebugLoc(SDLoc(Node))
2003
2.52k
      .setChain(InChain)
2004
2.52k
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2005
2.52k
                    std::move(Args))
2006
2.52k
      .setTailCall(isTailCall)
2007
2.52k
      .setSExtResult(isSigned)
2008
2.52k
      .setZExtResult(!isSigned)
2009
2.52k
      .setIsPostTypeLegalization(true);
2010
2.52k
2011
2.52k
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2012
2.52k
2013
2.52k
  if (!CallInfo.second.getNode())
2014
2.52k
    // It's a tailcall, return the chain (which is the DAG root).
2015
233
    return DAG.getRoot();
2016
2.29k
2017
2.29k
  return CallInfo.first;
2018
2.29k
}
2019
2020
/// Generate a libcall taking the given operands as arguments
2021
/// and returning a result of type RetVT.
2022
SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT,
2023
                                            const SDValue *Ops, unsigned NumOps,
2024
5
                                            bool isSigned, const SDLoc &dl) {
2025
5
  TargetLowering::ArgListTy Args;
2026
5
  Args.reserve(NumOps);
2027
5
2028
5
  TargetLowering::ArgListEntry Entry;
2029
25
  for (unsigned i = 0; 
i != NumOps25
;
++i20
) {
2030
20
    Entry.Node = Ops[i];
2031
20
    Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
2032
20
    Entry.IsSExt = isSigned;
2033
20
    Entry.IsZExt = !isSigned;
2034
20
    Args.push_back(Entry);
2035
20
  }
2036
5
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2037
5
                                         TLI.getPointerTy(DAG.getDataLayout()));
2038
5
2039
5
  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2040
5
2041
5
  TargetLowering::CallLoweringInfo CLI(DAG);
2042
5
  CLI.setDebugLoc(dl)
2043
5
      .setChain(DAG.getEntryNode())
2044
5
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2045
5
                    std::move(Args))
2046
5
      .setSExtResult(isSigned)
2047
5
      .setZExtResult(!isSigned)
2048
5
      .setIsPostTypeLegalization(true);
2049
5
2050
5
  std::pair<SDValue,SDValue> CallInfo = TLI.LowerCallTo(CLI);
2051
5
2052
5
  return CallInfo.first;
2053
5
}
2054
2055
// Expand a node into a call to a libcall. Similar to
2056
// ExpandLibCall except that the first operand is the in-chain.
2057
std::pair<SDValue, SDValue>
2058
SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2059
                                         SDNode *Node,
2060
328
                                         bool isSigned) {
2061
328
  SDValue InChain = Node->getOperand(0);
2062
328
2063
328
  TargetLowering::ArgListTy Args;
2064
328
  TargetLowering::ArgListEntry Entry;
2065
1.00k
  for (unsigned i = 1, e = Node->getNumOperands(); 
i != e1.00k
;
++i673
) {
2066
673
    EVT ArgVT = Node->getOperand(i).getValueType();
2067
673
    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2068
673
    Entry.Node = Node->getOperand(i);
2069
673
    Entry.Ty = ArgTy;
2070
673
    Entry.IsSExt = isSigned;
2071
673
    Entry.IsZExt = !isSigned;
2072
673
    Args.push_back(Entry);
2073
673
  }
2074
328
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2075
328
                                         TLI.getPointerTy(DAG.getDataLayout()));
2076
328
2077
328
  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2078
328
2079
328
  TargetLowering::CallLoweringInfo CLI(DAG);
2080
328
  CLI.setDebugLoc(SDLoc(Node))
2081
328
      .setChain(InChain)
2082
328
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2083
328
                    std::move(Args))
2084
328
      .setSExtResult(isSigned)
2085
328
      .setZExtResult(!isSigned);
2086
328
2087
328
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2088
328
2089
328
  return CallInfo;
2090
328
}
2091
2092
SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2093
                                              RTLIB::Libcall Call_F32,
2094
                                              RTLIB::Libcall Call_F64,
2095
                                              RTLIB::Libcall Call_F80,
2096
                                              RTLIB::Libcall Call_F128,
2097
1.86k
                                              RTLIB::Libcall Call_PPCF128) {
2098
1.86k
  if (Node->isStrictFPOpcode())
2099
22
    Node = DAG.mutateStrictFPToFP(Node);
2100
1.86k
2101
1.86k
  RTLIB::Libcall LC;
2102
1.86k
  switch (Node->getSimpleValueType(0).SimpleTy) {
2103
0
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2104
783
  case MVT::f32: LC = Call_F32; break;
2105
1.00k
  case MVT::f64: LC = Call_F64; break;
2106
43
  case MVT::f80: LC = Call_F80; break;
2107
35
  case MVT::f128: LC = Call_F128; break;
2108
0
  case MVT::ppcf128: LC = Call_PPCF128; break;
2109
1.86k
  }
2110
1.86k
  return ExpandLibCall(LC, Node, false);
2111
1.86k
}
2112
2113
SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2114
                                               RTLIB::Libcall Call_I8,
2115
                                               RTLIB::Libcall Call_I16,
2116
                                               RTLIB::Libcall Call_I32,
2117
                                               RTLIB::Libcall Call_I64,
2118
173
                                               RTLIB::Libcall Call_I128) {
2119
173
  RTLIB::Libcall LC;
2120
173
  switch (Node->getSimpleValueType(0).SimpleTy) {
2121
0
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2122
0
  case MVT::i8:   LC = Call_I8; break;
2123
14
  case MVT::i16:  LC = Call_I16; break;
2124
159
  case MVT::i32:  LC = Call_I32; break;
2125
0
  case MVT::i64:  LC = Call_I64; break;
2126
0
  case MVT::i128: LC = Call_I128; break;
2127
173
  }
2128
173
  return ExpandLibCall(LC, Node, isSigned);
2129
173
}
2130
2131
/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2132
void
2133
SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2134
14
                                          SmallVectorImpl<SDValue> &Results) {
2135
14
  unsigned Opcode = Node->getOpcode();
2136
14
  bool isSigned = Opcode == ISD::SDIVREM;
2137
14
2138
14
  RTLIB::Libcall LC;
2139
14
  switch (Node->getSimpleValueType(0).SimpleTy) {
2140
0
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2141
0
  
case MVT::i8: LC= isSigned ? 0
RTLIB::SDIVREM_I80
:
RTLIB::UDIVREM_I80
; break;
2142
0
  
case MVT::i16: LC= isSigned ? 0
RTLIB::SDIVREM_I160
:
RTLIB::UDIVREM_I160
; break;
2143
14
  
case MVT::i32: LC= isSigned ? 14
RTLIB::SDIVREM_I329
:
RTLIB::UDIVREM_I325
; break;
2144
0
  
case MVT::i64: LC= isSigned ? 0
RTLIB::SDIVREM_I640
:
RTLIB::UDIVREM_I640
; break;
2145
0
  
case MVT::i128: LC= isSigned ? 0
RTLIB::SDIVREM_I1280
:
RTLIB::UDIVREM_I1280
; break;
2146
14
  }
2147
14
2148
14
  // The input chain to this libcall is the entry node of the function.
2149
14
  // Legalizing the call will automatically add the previous call to the
2150
14
  // dependence.
2151
14
  SDValue InChain = DAG.getEntryNode();
2152
14
2153
14
  EVT RetVT = Node->getValueType(0);
2154
14
  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2155
14
2156
14
  TargetLowering::ArgListTy Args;
2157
14
  TargetLowering::ArgListEntry Entry;
2158
28
  for (const SDValue &Op : Node->op_values()) {
2159
28
    EVT ArgVT = Op.getValueType();
2160
28
    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2161
28
    Entry.Node = Op;
2162
28
    Entry.Ty = ArgTy;
2163
28
    Entry.IsSExt = isSigned;
2164
28
    Entry.IsZExt = !isSigned;
2165
28
    Args.push_back(Entry);
2166
28
  }
2167
14
2168
14
  // Also pass the return address of the remainder.
2169
14
  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2170
14
  Entry.Node = FIPtr;
2171
14
  Entry.Ty = RetTy->getPointerTo();
2172
14
  Entry.IsSExt = isSigned;
2173
14
  Entry.IsZExt = !isSigned;
2174
14
  Args.push_back(Entry);
2175
14
2176
14
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2177
14
                                         TLI.getPointerTy(DAG.getDataLayout()));
2178
14
2179
14
  SDLoc dl(Node);
2180
14
  TargetLowering::CallLoweringInfo CLI(DAG);
2181
14
  CLI.setDebugLoc(dl)
2182
14
      .setChain(InChain)
2183
14
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2184
14
                    std::move(Args))
2185
14
      .setSExtResult(isSigned)
2186
14
      .setZExtResult(!isSigned);
2187
14
2188
14
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2189
14
2190
14
  // Remainder is loaded back from the stack frame.
2191
14
  SDValue Rem =
2192
14
      DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2193
14
  Results.push_back(CallInfo.first);
2194
14
  Results.push_back(Rem);
2195
14
}
2196
2197
/// Return true if sincos libcall is available.
2198
253
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2199
253
  RTLIB::Libcall LC;
2200
253
  switch (Node->getSimpleValueType(0).SimpleTy) {
2201
0
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2202
136
  case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
2203
86
  case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
2204
19
  case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
2205
12
  case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
2206
0
  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2207
253
  }
2208
253
  return TLI.getLibcallName(LC) != nullptr;
2209
253
}
2210
2211
/// Only issue sincos libcall if both sin and cos are needed.
2212
704
static bool useSinCos(SDNode *Node) {
2213
704
  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2214
704
    ? 
ISD::FCOS302
:
ISD::FSIN402
;
2215
704
2216
704
  SDValue Op0 = Node->getOperand(0);
2217
704
  for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2218
1.06k
       UE = Op0.getNode()->use_end(); 
UI != UE1.06k
;
++UI360
) {
2219
770
    SDNode *User = *UI;
2220
770
    if (User == Node)
2221
294
      continue;
2222
476
    // The other user might have been turned into sincos already.
2223
476
    
if (476
User->getOpcode() == OtherOpcode || 476
User->getOpcode() == ISD::FSINCOS271
)
2224
410
      return true;
2225
770
  }
2226
294
  return false;
2227
704
}
2228
2229
/// Issue libcalls to sincos to compute sin / cos pairs.
2230
void
2231
SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2232
30
                                          SmallVectorImpl<SDValue> &Results) {
2233
30
  RTLIB::Libcall LC;
2234
30
  switch (Node->getSimpleValueType(0).SimpleTy) {
2235
0
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2236
11
  case MVT::f32:     LC = RTLIB::SINCOS_F32; break;
2237
11
  case MVT::f64:     LC = RTLIB::SINCOS_F64; break;
2238
5
  case MVT::f80:     LC = RTLIB::SINCOS_F80; break;
2239
3
  case MVT::f128:    LC = RTLIB::SINCOS_F128; break;
2240
0
  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2241
30
  }
2242
30
2243
30
  // The input chain to this libcall is the entry node of the function.
2244
30
  // Legalizing the call will automatically add the previous call to the
2245
30
  // dependence.
2246
30
  SDValue InChain = DAG.getEntryNode();
2247
30
2248
30
  EVT RetVT = Node->getValueType(0);
2249
30
  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2250
30
2251
30
  TargetLowering::ArgListTy Args;
2252
30
  TargetLowering::ArgListEntry Entry;
2253
30
2254
30
  // Pass the argument.
2255
30
  Entry.Node = Node->getOperand(0);
2256
30
  Entry.Ty = RetTy;
2257
30
  Entry.IsSExt = false;
2258
30
  Entry.IsZExt = false;
2259
30
  Args.push_back(Entry);
2260
30
2261
30
  // Pass the return address of sin.
2262
30
  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2263
30
  Entry.Node = SinPtr;
2264
30
  Entry.Ty = RetTy->getPointerTo();
2265
30
  Entry.IsSExt = false;
2266
30
  Entry.IsZExt = false;
2267
30
  Args.push_back(Entry);
2268
30
2269
30
  // Also pass the return address of the cos.
2270
30
  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2271
30
  Entry.Node = CosPtr;
2272
30
  Entry.Ty = RetTy->getPointerTo();
2273
30
  Entry.IsSExt = false;
2274
30
  Entry.IsZExt = false;
2275
30
  Args.push_back(Entry);
2276
30
2277
30
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2278
30
                                         TLI.getPointerTy(DAG.getDataLayout()));
2279
30
2280
30
  SDLoc dl(Node);
2281
30
  TargetLowering::CallLoweringInfo CLI(DAG);
2282
30
  CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2283
30
      TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2284
30
      std::move(Args));
2285
30
2286
30
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2287
30
2288
30
  Results.push_back(
2289
30
      DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2290
30
  Results.push_back(
2291
30
      DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2292
30
}
2293
2294
/// This function is responsible for legalizing a
2295
/// INT_TO_FP operation of the specified operand when the target requests that
2296
/// we expand it.  At this point, we know that the result and operand types are
2297
/// legal for the target.
2298
SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2299
                                                   EVT DestVT,
2300
138
                                                   const SDLoc &dl) {
2301
138
  // TODO: Should any fast-math-flags be set for the created nodes?
2302
138
2303
138
  if (
Op0.getValueType() == MVT::i32 && 138
TLI.isTypeLegal(MVT::f64)51
) {
2304
50
    // simple 32-bit [signed|unsigned] integer to float/double expansion
2305
50
2306
50
    // Get the stack frame index of a 8 byte buffer.
2307
50
    SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2308
50
2309
50
    // word offset constant for Hi/Lo address computation
2310
50
    SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2311
50
                                      StackSlot.getValueType());
2312
50
    // set up Hi and Lo (into buffer) address based on endian
2313
50
    SDValue Hi = StackSlot;
2314
50
    SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2315
50
                             StackSlot, WordOff);
2316
50
    if (DAG.getDataLayout().isLittleEndian())
2317
20
      std::swap(Hi, Lo);
2318
50
2319
50
    // if signed map to unsigned space
2320
50
    SDValue Op0Mapped;
2321
50
    if (
isSigned50
) {
2322
4
      // constant used to invert sign bit (signed to unsigned mapping)
2323
4
      SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2324
4
      Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2325
50
    } else {
2326
46
      Op0Mapped = Op0;
2327
46
    }
2328
50
    // store the lo of the constructed double - based on integer input
2329
50
    SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2330
50
                                  MachinePointerInfo());
2331
50
    // initial hi portion of constructed double
2332
50
    SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2333
50
    // store the hi of the constructed double - biased exponent
2334
50
    SDValue Store2 =
2335
50
        DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2336
50
    // load the constructed double
2337
50
    SDValue Load =
2338
50
        DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2339
50
    // FP constant to bias correct the final result
2340
50
    SDValue Bias = DAG.getConstantFP(isSigned ?
2341
4
                                     BitsToDouble(0x4330000080000000ULL) :
2342
46
                                     BitsToDouble(0x4330000000000000ULL),
2343
50
                                     dl, MVT::f64);
2344
50
    // subtract the bias
2345
50
    SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2346
50
    // final result
2347
50
    SDValue Result;
2348
50
    // handle final rounding
2349
50
    if (
DestVT == MVT::f6450
) {
2350
10
      // do nothing
2351
10
      Result = Sub;
2352
50
    } else 
if (40
DestVT.bitsLT(MVT::f64)40
) {
2353
36
      Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
2354
36
                           DAG.getIntPtrConstant(0, dl));
2355
40
    } else 
if (4
DestVT.bitsGT(MVT::f64)4
) {
2356
4
      Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
2357
4
    }
2358
50
    return Result;
2359
50
  }
2360
138
  assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
2361
88
  // Code below here assumes !isSigned without checking again.
2362
88
2363
88
  // Implementation of unsigned i64 to f64 following the algorithm in
2364
88
  // __floatundidf in compiler_rt. This implementation has the advantage
2365
88
  // of performing rounding correctly, both in the default rounding mode
2366
88
  // and in all alternate rounding modes.
2367
88
  // TODO: Generalize this for use with other types.
2368
88
  if (
Op0.getValueType() == MVT::i64 && 88
DestVT == MVT::f6487
) {
2369
4
    SDValue TwoP52 =
2370
4
      DAG.getConstant(UINT64_C(0x4330000000000000), dl, MVT::i64);
2371
4
    SDValue TwoP84PlusTwoP52 =
2372
4
      DAG.getConstantFP(BitsToDouble(UINT64_C(0x4530000000100000)), dl,
2373
4
                        MVT::f64);
2374
4
    SDValue TwoP84 =
2375
4
      DAG.getConstant(UINT64_C(0x4530000000000000), dl, MVT::i64);
2376
4
2377
4
    SDValue Lo = DAG.getZeroExtendInReg(Op0, dl, MVT::i32);
2378
4
    SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0,
2379
4
                             DAG.getConstant(32, dl, MVT::i64));
2380
4
    SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52);
2381
4
    SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84);
2382
4
    SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr);
2383
4
    SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr);
2384
4
    SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt,
2385
4
                                TwoP84PlusTwoP52);
2386
4
    return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub);
2387
4
  }
2388
84
2389
84
  // Implementation of unsigned i64 to f32.
2390
84
  // TODO: Generalize this for use with other types.
2391
84
  
if (84
Op0.getValueType() == MVT::i64 && 84
DestVT == MVT::f3283
) {
2392
82
    // For unsigned conversions, convert them to signed conversions using the
2393
82
    // algorithm from the x86_64 __floatundidf in compiler_rt.
2394
82
    if (
!isSigned82
) {
2395
82
      SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0);
2396
82
2397
82
      SDValue ShiftConst = DAG.getConstant(
2398
82
          1, dl, TLI.getShiftAmountTy(Op0.getValueType(), DAG.getDataLayout()));
2399
82
      SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst);
2400
82
      SDValue AndConst = DAG.getConstant(1, dl, MVT::i64);
2401
82
      SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst);
2402
82
      SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr);
2403
82
2404
82
      SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or);
2405
82
      SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt);
2406
82
2407
82
      // TODO: This really should be implemented using a branch rather than a
2408
82
      // select.  We happen to get lucky and machinesink does the right
2409
82
      // thing most of the time.  This would be a good candidate for a
2410
82
      //pseudo-op, or, even better, for whole-function isel.
2411
82
      SDValue SignBitTest = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
2412
82
        Op0, DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
2413
82
      return DAG.getSelect(dl, MVT::f32, SignBitTest, Slow, Fast);
2414
82
    }
2415
0
2416
0
    // Otherwise, implement the fully general conversion.
2417
0
2418
0
    SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2419
0
         DAG.getConstant(UINT64_C(0xfffffffffffff800), dl, MVT::i64));
2420
0
    SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And,
2421
0
         DAG.getConstant(UINT64_C(0x800), dl, MVT::i64));
2422
0
    SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0,
2423
0
         DAG.getConstant(UINT64_C(0x7ff), dl, MVT::i64));
2424
0
    SDValue Ne = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), And2,
2425
0
                              DAG.getConstant(UINT64_C(0), dl, MVT::i64),
2426
0
                              ISD::SETNE);
2427
0
    SDValue Sel = DAG.getSelect(dl, MVT::i64, Ne, Or, Op0);
2428
0
    SDValue Ge = DAG.getSetCC(dl, getSetCCResultType(MVT::i64), Op0,
2429
0
                              DAG.getConstant(UINT64_C(0x0020000000000000), dl,
2430
0
                                              MVT::i64),
2431
0
                              ISD::SETUGE);
2432
0
    SDValue Sel2 = DAG.getSelect(dl, MVT::i64, Ge, Sel, Op0);
2433
0
    EVT SHVT = TLI.getShiftAmountTy(Sel2.getValueType(), DAG.getDataLayout());
2434
0
2435
0
    SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2,
2436
0
                             DAG.getConstant(32, dl, SHVT));
2437
0
    SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh);
2438
0
    SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc);
2439
0
    SDValue TwoP32 =
2440
0
      DAG.getConstantFP(BitsToDouble(UINT64_C(0x41f0000000000000)), dl,
2441
0
                        MVT::f64);
2442
0
    SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt);
2443
0
    SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2);
2444
0
    SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo);
2445
0
    SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2);
2446
0
    return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd,
2447
0
                       DAG.getIntPtrConstant(0, dl));
2448
0
  }
2449
2
2450
2
  SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2451
2
2452
2
  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(Op0.getValueType()),
2453
2
                                 Op0,
2454
2
                                 DAG.getConstant(0, dl, Op0.getValueType()),
2455
2
                                 ISD::SETLT);
2456
2
  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2457
2
          Four = DAG.getIntPtrConstant(4, dl);
2458
2
  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2459
2
                                    SignSet, Four, Zero);
2460
2
2461
2
  // If the sign bit of the integer is set, the large number will be treated
2462
2
  // as a negative number.  To counteract this, the dynamic code adds an
2463
2
  // offset depending on the data type.
2464
2
  uint64_t FF;
2465
2
  switch (Op0.getSimpleValueType().SimpleTy) {
2466
0
  
default: 0
llvm_unreachable0
("Unsupported integer type!");
2467
0
  case MVT::i8 : FF = 0x43800000ULL; break;  // 2^8  (as a float)
2468
0
  case MVT::i16: FF = 0x47800000ULL; break;  // 2^16 (as a float)
2469
1
  case MVT::i32: FF = 0x4F800000ULL; break;  // 2^32 (as a float)
2470
1
  case MVT::i64: FF = 0x5F800000ULL; break;  // 2^64 (as a float)
2471
2
  }
2472
2
  
if (2
DAG.getDataLayout().isLittleEndian()2
)
2473
0
    FF <<= 32;
2474
2
  Constant *FudgeFactor = ConstantInt::get(
2475
2
                                       Type::getInt64Ty(*DAG.getContext()), FF);
2476
2
2477
2
  SDValue CPIdx =
2478
2
      DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2479
2
  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2480
2
  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2481
2
  Alignment = std::min(Alignment, 4u);
2482
2
  SDValue FudgeInReg;
2483
2
  if (DestVT == MVT::f32)
2484
1
    FudgeInReg = DAG.getLoad(
2485
1
        MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2486
1
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2487
1
        Alignment);
2488
1
  else {
2489
1
    SDValue Load = DAG.getExtLoad(
2490
1
        ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2491
1
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2492
1
        Alignment);
2493
1
    HandleSDNode Handle(Load);
2494
1
    LegalizeOp(Load.getNode());
2495
1
    FudgeInReg = Handle.getValue();
2496
1
  }
2497
138
2498
138
  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2499
138
}
2500
2501
/// This function is responsible for legalizing a
2502
/// *INT_TO_FP operation of the specified operand when the target requests that
2503
/// we promote it.  At this point, we know that the result and operand types are
2504
/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2505
/// operation that takes a larger input.
2506
SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2507
                                                    bool isSigned,
2508
88
                                                    const SDLoc &dl) {
2509
88
  // First step, figure out the appropriate *INT_TO_FP operation to use.
2510
88
  EVT NewInTy = LegalOp.getValueType();
2511
88
2512
88
  unsigned OpToUse = 0;
2513
88
2514
88
  // Scan for the appropriate larger type to use.
2515
103
  while (
true103
) {
2516
103
    NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2517
103
    assert(NewInTy.isInteger() && "Ran out of possibilities!");
2518
103
2519
103
    // If the target supports SINT_TO_FP of this type, use it.
2520
103
    if (
TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)103
) {
2521
88
      OpToUse = ISD::SINT_TO_FP;
2522
88
      break;
2523
88
    }
2524
15
    
if (15
isSigned15
)
continue14
;
2525
1
2526
1
    // If the target supports UINT_TO_FP of this type, use it.
2527
1
    
if (1
TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)1
) {
2528
0
      OpToUse = ISD::UINT_TO_FP;
2529
0
      break;
2530
0
    }
2531
103
2532
103
    // Otherwise, try a larger type.
2533
103
  }
2534
88
2535
88
  // Okay, we found the operation and type to use.  Zero extend our input to the
2536
88
  // desired type then run the operation on it.
2537
88
  return DAG.getNode(OpToUse, dl, DestVT,
2538
88
                     DAG.getNode(isSigned ? 
ISD::SIGN_EXTEND26
:
ISD::ZERO_EXTEND62
,
2539
88
                                 dl, NewInTy, LegalOp));
2540
88
}
2541
2542
/// This function is responsible for legalizing a
2543
/// FP_TO_*INT operation of the specified operand when the target requests that
2544
/// we promote it.  At this point, we know that the result and operand types are
2545
/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2546
/// operation that returns a larger result.
2547
SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2548
                                                    bool isSigned,
2549
136
                                                    const SDLoc &dl) {
2550
136
  // First step, figure out the appropriate FP_TO*INT operation to use.
2551
136
  EVT NewOutTy = DestVT;
2552
136
2553
136
  unsigned OpToUse = 0;
2554
136
2555
136
  // Scan for the appropriate larger type to use.
2556
156
  while (
true156
) {
2557
156
    NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2558
156
    assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2559
156
2560
156
    // A larger signed type can hold all unsigned values of the requested type,
2561
156
    // so using FP_TO_SINT is valid
2562
156
    if (
TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)156
) {
2563
136
      OpToUse = ISD::FP_TO_SINT;
2564
136
      break;
2565
136
    }
2566
20
2567
20
    // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2568
20
    
if (20
!isSigned && 20
TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)4
) {
2569
0
      OpToUse = ISD::FP_TO_UINT;
2570
0
      break;
2571
0
    }
2572
156
2573
156
    // Otherwise, try a larger type.
2574
156
  }
2575
136
2576
136
  // Okay, we found the operation and type to use.
2577
136
  SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2578
136
2579
136
  // Truncate the result of the extended FP_TO_*INT operation to the desired
2580
136
  // size.
2581
136
  return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2582
136
}
2583
2584
/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2585
92
SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2586
92
  EVT VT = Op.getValueType();
2587
92
  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2588
92
  unsigned Sz = VT.getScalarSizeInBits();
2589
92
2590
92
  SDValue Tmp, Tmp2, Tmp3;
2591
92
2592
92
  // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2593
92
  // and finally the i1 pairs.
2594
92
  // TODO: We can easily support i4/i2 legal types if any target ever does.
2595
92
  if (
Sz >= 8 && 92
isPowerOf2_32(Sz)92
) {
2596
92
    // Create the masks - repeating the pattern every byte.
2597
92
    APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0);
2598
92
    APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0);
2599
424
    for (unsigned J = 0; 
J != Sz424
;
J += 8332
) {
2600
332
      MaskHi4 = MaskHi4 | (0xF0ull << J);
2601
332
      MaskLo4 = MaskLo4 | (0x0Full << J);
2602
332
      MaskHi2 = MaskHi2 | (0xCCull << J);
2603
332
      MaskLo2 = MaskLo2 | (0x33ull << J);
2604
332
      MaskHi1 = MaskHi1 | (0xAAull << J);
2605
332
      MaskLo1 = MaskLo1 | (0x55ull << J);
2606
332
    }
2607
92
2608
92
    // BSWAP if the type is wider than a single byte.
2609
92
    Tmp = (Sz > 8 ? 
DAG.getNode(ISD::BSWAP, dl, VT, Op)52
:
Op40
);
2610
92
2611
92
    // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2612
92
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2613
92
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2614
92
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, VT));
2615
92
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, VT));
2616
92
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2617
92
2618
92
    // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2619
92
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2620
92
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2621
92
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, VT));
2622
92
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, VT));
2623
92
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2624
92
2625
92
    // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2626
92
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2627
92
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2628
92
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, VT));
2629
92
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, VT));
2630
92
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2631
92
    return Tmp;
2632
92
  }
2633
0
2634
0
  Tmp = DAG.getConstant(0, dl, VT);
2635
0
  for (unsigned I = 0, J = Sz-1; 
I < Sz0
;
++I, --J0
) {
2636
0
    if (I < J)
2637
0
      Tmp2 =
2638
0
          DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2639
0
    else
2640
0
      Tmp2 =
2641
0
          DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2642
0
2643
0
    APInt Shift(Sz, 1);
2644
0
    Shift <<= J;
2645
0
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2646
0
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2647
0
  }
2648
92
2649
92
  return Tmp;
2650
92
}
2651
2652
/// Open code the operations for BSWAP of the specified operation.
2653
118
SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2654
118
  EVT VT = Op.getValueType();
2655
118
  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2656
118
  SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2657
118
  switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2658
0
  
default: 0
llvm_unreachable0
("Unhandled Expand type in BSWAP!");
2659
55
  case MVT::i16:
2660
55
    Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2661
55
    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2662
55
    return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
2663
25
  case MVT::i32:
2664
25
    Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2665
25
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2666
25
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2667
25
    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2668
25
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2669
25
                       DAG.getConstant(0xFF0000, dl, VT));
2670
25
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2671
25
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2672
25
    Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2673
25
    return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2674
38
  case MVT::i64:
2675
38
    Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2676
38
    Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2677
38
    Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2678
38
    Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2679
38
    Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2680
38
    Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2681
38
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2682
38
    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2683
38
    Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2684
38
                       DAG.getConstant(255ULL<<48, dl, VT));
2685
38
    Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2686
38
                       DAG.getConstant(255ULL<<40, dl, VT));
2687
38
    Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2688
38
                       DAG.getConstant(255ULL<<32, dl, VT));
2689
38
    Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2690
38
                       DAG.getConstant(255ULL<<24, dl, VT));
2691
38
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2692
38
                       DAG.getConstant(255ULL<<16, dl, VT));
2693
38
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2694
38
                       DAG.getConstant(255ULL<<8 , dl, VT));
2695
38
    Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2696
38
    Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2697
38
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2698
38
    Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2699
38
    Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2700
38
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2701
38
    return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2702
0
  }
2703
0
}
2704
2705
/// Expand the specified bitcount instruction into operations.
2706
SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op,
2707
3.50k
                                             const SDLoc &dl) {
2708
3.50k
  switch (Opc) {
2709
0
  
default: 0
llvm_unreachable0
("Cannot expand this yet!");
2710
134
  case ISD::CTPOP: {
2711
134
    EVT VT = Op.getValueType();
2712
134
    EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2713
134
    unsigned Len = VT.getSizeInBits();
2714
134
2715
134
    assert(VT.isInteger() && Len <= 128 && Len % 8 == 0 &&
2716
134
           "CTPOP not implemented for this type.");
2717
134
2718
134
    // This is the "best" algorithm from
2719
134
    // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
2720
134
2721
134
    SDValue Mask55 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)),
2722
134
                                     dl, VT);
2723
134
    SDValue Mask33 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)),
2724
134
                                     dl, VT);
2725
134
    SDValue Mask0F = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)),
2726
134
                                     dl, VT);
2727
134
    SDValue Mask01 = DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)),
2728
134
                                     dl, VT);
2729
134
2730
134
    // v = v - ((v >> 1) & 0x55555555...)
2731
134
    Op = DAG.getNode(ISD::SUB, dl, VT, Op,
2732
134
                     DAG.getNode(ISD::AND, dl, VT,
2733
134
                                 DAG.getNode(ISD::SRL, dl, VT, Op,
2734
134
                                             DAG.getConstant(1, dl, ShVT)),
2735
134
                                 Mask55));
2736
134
    // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
2737
134
    Op = DAG.getNode(ISD::ADD, dl, VT,
2738
134
                     DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
2739
134
                     DAG.getNode(ISD::AND, dl, VT,
2740
134
                                 DAG.getNode(ISD::SRL, dl, VT, Op,
2741
134
                                             DAG.getConstant(2, dl, ShVT)),
2742
134
                                 Mask33));
2743
134
    // v = (v + (v >> 4)) & 0x0F0F0F0F...
2744
134
    Op = DAG.getNode(ISD::AND, dl, VT,
2745
134
                     DAG.getNode(ISD::ADD, dl, VT, Op,
2746
134
                                 DAG.getNode(ISD::SRL, dl, VT, Op,
2747
134
                                             DAG.getConstant(4, dl, ShVT))),
2748
134
                     Mask0F);
2749
134
    // v = (v * 0x01010101...) >> (Len - 8)
2750
134
    Op = DAG.getNode(ISD::SRL, dl, VT,
2751
134
                     DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
2752
134
                     DAG.getConstant(Len - 8, dl, ShVT));
2753
134
2754
134
    return Op;
2755
3.50k
  }
2756
3.08k
  case ISD::CTLZ_ZERO_UNDEF:
2757
3.08k
    // This trivially expands to CTLZ.
2758
3.08k
    return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op);
2759
119
  case ISD::CTLZ: {
2760
119
    EVT VT = Op.getValueType();
2761
119
    unsigned len = VT.getSizeInBits();
2762
119
2763
119
    if (
TLI.isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)119
) {
2764
40
      EVT SetCCVT = getSetCCResultType(VT);
2765
40
      SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
2766
40
      SDValue Zero = DAG.getConstant(0, dl, VT);
2767
40
      SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
2768
40
      return DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
2769
40
                         DAG.getConstant(len, dl, VT), CTLZ);
2770
40
    }
2771
79
2772
79
    // for now, we do this:
2773
79
    // x = x | (x >> 1);
2774
79
    // x = x | (x >> 2);
2775
79
    // ...
2776
79
    // x = x | (x >>16);
2777
79
    // x = x | (x >>32); // for 64-bit input
2778
79
    // return popcount(~x);
2779
79
    //
2780
79
    // Ref: "Hacker's Delight" by Henry Warren
2781
79
    EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2782
480
    for (unsigned i = 0; 
(1U << i) <= (len / 2)480
;
++i401
) {
2783
401
      SDValue Tmp3 = DAG.getConstant(1ULL << i, dl, ShVT);
2784
401
      Op = DAG.getNode(ISD::OR, dl, VT, Op,
2785
401
                       DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3));
2786
401
    }
2787
79
    Op = DAG.getNOT(dl, Op, VT);
2788
79
    return DAG.getNode(ISD::CTPOP, dl, VT, Op);
2789
79
  }
2790
140
  case ISD::CTTZ_ZERO_UNDEF:
2791
140
    // This trivially expands to CTTZ.
2792
140
    return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op);
2793
25
  case ISD::CTTZ: {
2794
25
    // for now, we use: { return popcount(~x & (x - 1)); }
2795
25
    // unless the target has ctlz but not ctpop, in which case we use:
2796
25
    // { return 32 - nlz(~x & (x-1)); }
2797
25
    // Ref: "Hacker's Delight" by Henry Warren
2798
25
    EVT VT = Op.getValueType();
2799
25
    SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT,
2800
25
                               DAG.getNOT(dl, Op, VT),
2801
25
                               DAG.getNode(ISD::SUB, dl, VT, Op,
2802
25
                                           DAG.getConstant(1, dl, VT)));
2803
25
    // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
2804
25
    if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) &&
2805
17
        TLI.isOperationLegalOrCustom(ISD::CTLZ, VT))
2806
5
      return DAG.getNode(ISD::SUB, dl, VT,
2807
5
                         DAG.getConstant(VT.getSizeInBits(), dl, VT),
2808
5
                         DAG.getNode(ISD::CTLZ, dl, VT, Tmp3));
2809
20
    return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3);
2810
20
  }
2811
3.50k
  }
2812
3.50k
}
2813
2814
334k
bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2815
334k
  SmallVector<SDValue, 8> Results;
2816
334k
  SDLoc dl(Node);
2817
334k
  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2818
334k
  bool NeedInvert;
2819
334k
  switch (Node->getOpcode()) {
2820
3.50k
  case ISD::CTPOP:
2821
3.50k
  case ISD::CTLZ:
2822
3.50k
  case ISD::CTLZ_ZERO_UNDEF:
2823
3.50k
  case ISD::CTTZ:
2824
3.50k
  case ISD::CTTZ_ZERO_UNDEF:
2825
3.50k
    Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl);
2826
3.50k
    Results.push_back(Tmp1);
2827
3.50k
    break;
2828
92
  case ISD::BITREVERSE:
2829
92
    Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2830
92
    break;
2831
118
  case ISD::BSWAP:
2832
118
    Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2833
118
    break;
2834
0
  case ISD::FRAMEADDR:
2835
0
  case ISD::RETURNADDR:
2836
0
  case ISD::FRAME_TO_ARGS_OFFSET:
2837
0
    Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2838
0
    break;
2839
6
  case ISD::EH_DWARF_CFA: {
2840
6
    SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2841
6
                                        TLI.getPointerTy(DAG.getDataLayout()));
2842
6
    SDValue Offset = DAG.getNode(ISD::ADD, dl,
2843
6
                                 CfaArg.getValueType(),
2844
6
                                 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2845
6
                                             CfaArg.getValueType()),
2846
6
                                 CfaArg);
2847
6
    SDValue FA = DAG.getNode(
2848
6
        ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2849
6
        DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2850
6
    Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2851
6
                                  FA, Offset));
2852
6
    break;
2853
0
  }
2854
1
  case ISD::FLT_ROUNDS_:
2855
1
    Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2856
1
    break;
2857
470
  case ISD::EH_RETURN:
2858
470
  case ISD::EH_LABEL:
2859
470
  case ISD::PREFETCH:
2860
470
  case ISD::VAEND:
2861
470
  case ISD::EH_SJLJ_LONGJMP:
2862
470
    // If the target didn't expand these, there's nothing to do, so just
2863
470
    // preserve the chain and be done.
2864
470
    Results.push_back(Node->getOperand(0));
2865
470
    break;
2866
3
  case ISD::READCYCLECOUNTER:
2867
3
    // If the target didn't expand this, just return 'zero' and preserve the
2868
3
    // chain.
2869
3
    Results.append(Node->getNumValues() - 1,
2870
3
                   DAG.getConstant(0, dl, Node->getValueType(0)));
2871
3
    Results.push_back(Node->getOperand(0));
2872
3
    break;
2873
0
  case ISD::EH_SJLJ_SETJMP:
2874
0
    // If the target didn't expand this, just return 'zero' and preserve the
2875
0
    // chain.
2876
0
    Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2877
0
    Results.push_back(Node->getOperand(0));
2878
0
    break;
2879
6
  case ISD::ATOMIC_LOAD: {
2880
6
    // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2881
6
    SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2882
6
    SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2883
6
    SDValue Swap = DAG.getAtomicCmpSwap(
2884
6
        ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2885
6
        Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2886
6
        cast<AtomicSDNode>(Node)->getMemOperand());
2887
6
    Results.push_back(Swap.getValue(0));
2888
6
    Results.push_back(Swap.getValue(1));
2889
6
    break;
2890
470
  }
2891
14
  case ISD::ATOMIC_STORE: {
2892
14
    // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2893
14
    SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2894
14
                                 cast<AtomicSDNode>(Node)->getMemoryVT(),
2895
14
                                 Node->getOperand(0),
2896
14
                                 Node->getOperand(1), Node->getOperand(2),
2897
14
                                 cast<AtomicSDNode>(Node)->getMemOperand());
2898
14
    Results.push_back(Swap.getValue(1));
2899
14
    break;
2900
470
  }
2901
1.99k
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2902
1.99k
    // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2903
1.99k
    // splits out the success value as a comparison. Expanding the resulting
2904
1.99k
    // ATOMIC_CMP_SWAP will produce a libcall.
2905
1.99k
    SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2906
1.99k
    SDValue Res = DAG.getAtomicCmpSwap(
2907
1.99k
        ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2908
1.99k
        Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2909
1.99k
        Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2910
1.99k
2911
1.99k
    SDValue ExtRes = Res;
2912
1.99k
    SDValue LHS = Res;
2913
1.99k
    SDValue RHS = Node->getOperand(1);
2914
1.99k
2915
1.99k
    EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2916
1.99k
    EVT OuterType = Node->getValueType(0);
2917
1.99k
    switch (TLI.getExtendForAtomicOps()) {
2918
55
    case ISD::SIGN_EXTEND:
2919
55
      LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2920
55
                        DAG.getValueType(AtomicType));
2921
55
      RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2922
55
                        Node->getOperand(2), DAG.getValueType(AtomicType));
2923
55
      ExtRes = LHS;
2924
55
      break;
2925
1.90k
    case ISD::ZERO_EXTEND:
2926
1.90k
      LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2927
1.90k
                        DAG.getValueType(AtomicType));
2928
1.90k
      RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2929
1.90k
      ExtRes = LHS;
2930
1.90k
      break;
2931
39
    case ISD::ANY_EXTEND:
2932
39
      LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2933
39
      RHS = DAG.getNode(ISD::ZERO_EXTEND, dl, OuterType, Node->getOperand(2));
2934
39
      break;
2935
0
    default:
2936
0
      llvm_unreachable("Invalid atomic op extension");
2937
1.99k
    }
2938
1.99k
2939
1.99k
    SDValue Success =
2940
1.99k
        DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2941
1.99k
2942
1.99k
    Results.push_back(ExtRes.getValue(0));
2943
1.99k
    Results.push_back(Success);
2944
1.99k
    Results.push_back(Res.getValue(1));
2945
1.99k
    break;
2946
1.99k
  }
2947
315
  case ISD::DYNAMIC_STACKALLOC:
2948
315
    ExpandDYNAMIC_STACKALLOC(Node, Results);
2949
315
    break;
2950
26.2k
  case ISD::MERGE_VALUES:
2951
78.6k
    for (unsigned i = 0; 
i < Node->getNumValues()78.6k
;
i++52.4k
)
2952
52.4k
      Results.push_back(Node->getOperand(i));
2953
26.2k
    break;
2954
448
  case ISD::UNDEF: {
2955
448
    EVT VT = Node->getValueType(0);
2956
448
    if (VT.isInteger())
2957
330
      Results.push_back(DAG.getConstant(0, dl, VT));
2958
118
    else {
2959
118
      assert(VT.isFloatingPoint() && "Unknown value type!");
2960
118
      Results.push_back(DAG.getConstantFP(0, dl, VT));
2961
118
    }
2962
448
    break;
2963
1.99k
  }
2964
520
  case ISD::FP_ROUND:
2965
520
  case ISD::BITCAST:
2966
520
    Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2967
520
                            Node->getValueType(0), dl);
2968
520
    Results.push_back(Tmp1);
2969
520
    break;
2970
0
  case ISD::FP_EXTEND:
2971
0
    Tmp1 = EmitStackConvert(Node->getOperand(0),
2972
0
                            Node->getOperand(0).getValueType(),
2973
0
                            Node->getValueType(0), dl);
2974
0
    Results.push_back(Tmp1);
2975
0
    break;
2976
843
  case ISD::SIGN_EXTEND_INREG: {
2977
843
    EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2978
843
    EVT VT = Node->getValueType(0);
2979
843
2980
843
    // An in-register sign-extend of a boolean is a negation:
2981
843
    // 'true' (1) sign-extended is -1.
2982
843
    // 'false' (0) sign-extended is 0.
2983
843
    // However, we must mask the high bits of the source operand because the
2984
843
    // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2985
843
2986
843
    // TODO: Do this for vectors too?
2987
843
    if (
ExtraVT.getSizeInBits() == 1843
) {
2988
232
      SDValue One = DAG.getConstant(1, dl, VT);
2989
232
      SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2990
232
      SDValue Zero = DAG.getConstant(0, dl, VT);
2991
232
      SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2992
232
      Results.push_back(Neg);
2993
232
      break;
2994
232
    }
2995
611
2996
611
    // NOTE: we could fall back on load/store here too for targets without
2997
611
    // SRA.  However, it is doubtful that any exist.
2998
611
    EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2999
611
    unsigned BitsDiff = VT.getScalarSizeInBits() -
3000
611
                        ExtraVT.getScalarSizeInBits();
3001
611
    SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3002
611
    Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3003
611
                       Node->getOperand(0), ShiftCst);
3004
611
    Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3005
611
    Results.push_back(Tmp1);
3006
611
    break;
3007
611
  }
3008
0
  case ISD::FP_ROUND_INREG: {
3009
0
    // The only way we can lower this is to turn it into a TRUNCSTORE,
3010
0
    // EXTLOAD pair, targeting a temporary location (a stack slot).
3011
0
3012
0
    // NOTE: there is a choice here between constantly creating new stack
3013
0
    // slots and always reusing the same one.  We currently always create
3014
0
    // new ones, as reuse may inhibit scheduling.
3015
0
    EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3016
0
    Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
3017
0
                            Node->getValueType(0), dl);
3018
0
    Results.push_back(Tmp1);
3019
0
    break;
3020
611
  }
3021
138
  case ISD::SINT_TO_FP:
3022
138
  case ISD::UINT_TO_FP:
3023
138
    Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
3024
138
                                Node->getOperand(0), Node->getValueType(0), dl);
3025
138
    Results.push_back(Tmp1);
3026
138
    break;
3027
60
  case ISD::FP_TO_SINT:
3028
60
    if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3029
60
      Results.push_back(Tmp1);
3030
60
    break;
3031
190
  case ISD::FP_TO_UINT: {
3032
190
    SDValue True, False;
3033
190
    EVT VT =  Node->getOperand(0).getValueType();
3034
190
    EVT NVT = Node->getValueType(0);
3035
190
    APFloat apf(DAG.EVTToAPFloatSemantics(VT),
3036
190
                APInt::getNullValue(VT.getSizeInBits()));
3037
190
    APInt x = APInt::getSignMask(NVT.getSizeInBits());
3038
190
    (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3039
190
    Tmp1 = DAG.getConstantFP(apf, dl, VT);
3040
190
    Tmp2 = DAG.getSetCC(dl, getSetCCResultType(VT),
3041
190
                        Node->getOperand(0),
3042
190
                        Tmp1, ISD::SETLT);
3043
190
    True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0));
3044
190
    // TODO: Should any fast-math-flags be set for the FSUB?
3045
190
    False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT,
3046
190
                        DAG.getNode(ISD::FSUB, dl, VT,
3047
190
                                    Node->getOperand(0), Tmp1));
3048
190
    False = DAG.getNode(ISD::XOR, dl, NVT, False,
3049
190
                        DAG.getConstant(x, dl, NVT));
3050
190
    Tmp1 = DAG.getSelect(dl, NVT, Tmp2, True, False);
3051
190
    Results.push_back(Tmp1);
3052
190
    break;
3053
138
  }
3054
24
  case ISD::VAARG:
3055
24
    Results.push_back(DAG.expandVAArg(Node));
3056
24
    Results.push_back(Results[0].getValue(1));
3057
24
    break;
3058
31
  case ISD::VACOPY:
3059
31
    Results.push_back(DAG.expandVACopy(Node));
3060
31
    break;
3061
2.20k
  case ISD::EXTRACT_VECTOR_ELT:
3062
2.20k
    if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3063
2.20k
      // This must be an access of the only element.  Return it.
3064
136
      Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3065
136
                         Node->getOperand(0));
3066
2.20k
    else
3067
2.06k
      Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3068
2.20k
    Results.push_back(Tmp1);
3069
2.20k
    break;
3070
0
  case ISD::EXTRACT_SUBVECTOR:
3071
0
    Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3072
0
    break;
3073
0
  case ISD::INSERT_SUBVECTOR:
3074
0
    Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3075
0
    break;
3076
0
  case ISD::CONCAT_VECTORS:
3077
0
    Results.push_back(ExpandVectorBuildThroughStack(Node));
3078
0
    break;
3079
18
  case ISD::SCALAR_TO_VECTOR:
3080
18
    Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3081
18
    break;
3082
404
  case ISD::INSERT_VECTOR_ELT:
3083
404
    Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3084
404
                                              Node->getOperand(1),
3085
404
                                              Node->getOperand(2), dl));
3086
404
    break;
3087
183
  case ISD::VECTOR_SHUFFLE: {
3088
183
    SmallVector<int, 32> NewMask;
3089
183
    ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3090
183
3091
183
    EVT VT = Node->getValueType(0);
3092
183
    EVT EltVT = VT.getVectorElementType();
3093
183
    SDValue Op0 = Node->getOperand(0);
3094
183
    SDValue Op1 = Node->getOperand(1);
3095
183
    if (
!TLI.isTypeLegal(EltVT)183
) {
3096
11
      EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3097
11
3098
11
      // BUILD_VECTOR operands are allowed to be wider than the element type.
3099
11
      // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3100
11
      // it.
3101
11
      if (
NewEltVT.bitsLT(EltVT)11
) {
3102
0
        // Convert shuffle node.
3103
0
        // If original node was v4i64 and the new EltVT is i32,
3104
0
        // cast operands to v8i32 and re-build the mask.
3105
0
3106
0
        // Calculate new VT, the size of the new VT should be equal to original.
3107
0
        EVT NewVT =
3108
0
            EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3109
0
                             VT.getSizeInBits() / NewEltVT.getSizeInBits());
3110
0
        assert(NewVT.bitsEq(VT));
3111
0
3112
0
        // cast operands to new VT
3113
0
        Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3114
0
        Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3115
0
3116
0
        // Convert the shuffle mask
3117
0
        unsigned int factor =
3118
0
                         NewVT.getVectorNumElements()/VT.getVectorNumElements();
3119
0
3120
0
        // EltVT gets smaller
3121
0
        assert(factor > 0);
3122
0
3123
0
        for (unsigned i = 0; 
i < VT.getVectorNumElements()0
;
++i0
) {
3124
0
          if (
Mask[i] < 00
) {
3125
0
            for (unsigned fi = 0; 
fi < factor0
;
++fi0
)
3126
0
              NewMask.push_back(Mask[i]);
3127
0
          }
3128
0
          else {
3129
0
            for (unsigned fi = 0; 
fi < factor0
;
++fi0
)
3130
0
              NewMask.push_back(Mask[i]*factor+fi);
3131
0
          }
3132
0
        }
3133
0
        Mask = NewMask;
3134
0
        VT = NewVT;
3135
0
      }
3136
11
      EltVT = NewEltVT;
3137
11
    }
3138
183
    unsigned NumElems = VT.getVectorNumElements();
3139
183
    SmallVector<SDValue, 16> Ops;
3140
1.31k
    for (unsigned i = 0; 
i != NumElems1.31k
;
++i1.12k
) {
3141
1.12k
      if (
Mask[i] < 01.12k
) {
3142
23
        Ops.push_back(DAG.getUNDEF(EltVT));
3143
23
        continue;
3144
23
      }
3145
1.10k
      unsigned Idx = Mask[i];
3146
1.10k
      if (Idx < NumElems)
3147
597
        Ops.push_back(DAG.getNode(
3148
597
            ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3149
597
            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3150
1.10k
      else
3151
508
        Ops.push_back(DAG.getNode(
3152
508
            ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3153
508
            DAG.getConstant(Idx - NumElems, dl,
3154
508
                            TLI.getVectorIdxTy(DAG.getDataLayout()))));
3155
1.12k
    }
3156
183
3157
183
    Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3158
183
    // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3159
183
    Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3160
183
    Results.push_back(Tmp1);
3161
183
    break;
3162
138
  }
3163
144
  case ISD::EXTRACT_ELEMENT: {
3164
144
    EVT OpTy = Node->getOperand(0).getValueType();
3165
144
    if (
cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()144
) {
3166
56
      // 1 -> Hi
3167
56
      Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3168
56
                         DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3169
56
                                         TLI.getShiftAmountTy(
3170
56
                                             Node->getOperand(0).getValueType(),
3171
56
                                             DAG.getDataLayout())));
3172
56
      Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3173
144
    } else {
3174
88
      // 0 -> Lo
3175
88
      Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3176
88
                         Node->getOperand(0));
3177
88
    }
3178
144
    Results.push_back(Tmp1);
3179
144
    break;
3180
138
  }
3181
111
  case ISD::STACKSAVE:
3182
111
    // Expand to CopyFromReg if the target set
3183
111
    // StackPointerRegisterToSaveRestore.
3184
111
    if (unsigned 
SP111
= TLI.getStackPointerRegisterToSaveRestore()) {
3185
111
      Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3186
111
                                           Node->getValueType(0)));
3187
111
      Results.push_back(Results[0].getValue(1));
3188
111
    } else {
3189
0
      Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3190
0
      Results.push_back(Node->getOperand(0));
3191
0
    }
3192
111
    break;
3193
27
  case ISD::STACKRESTORE:
3194
27
    // Expand to CopyToReg if the target set
3195
27
    // StackPointerRegisterToSaveRestore.
3196
27
    if (unsigned 
SP27
= TLI.getStackPointerRegisterToSaveRestore()) {
3197
27
      Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3198
27
                                         Node->getOperand(1)));
3199
27
    } else {
3200
0
      Results.push_back(Node->getOperand(0));
3201
0
    }
3202
27
    break;
3203
0
  case ISD::GET_DYNAMIC_AREA_OFFSET:
3204
0
    Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3205
0
    Results.push_back(Results[0].getValue(0));
3206
0
    break;
3207
296
  case ISD::FCOPYSIGN:
3208
296
    Results.push_back(ExpandFCOPYSIGN(Node));
3209
296
    break;
3210
3
  case ISD::FNEG:
3211
3
    // Expand Y = FNEG(X) ->  Y = SUB -0.0, X
3212
3
    Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3213
3
    // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3214
3
    Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3215
3
                       Node->getOperand(0));
3216
3
    Results.push_back(Tmp1);
3217
3
    break;
3218
32
  case ISD::FABS:
3219
32
    Results.push_back(ExpandFABS(Node));
3220
32
    break;
3221
0
  case ISD::SMIN:
3222
0
  case ISD::SMAX:
3223
0
  case ISD::UMIN:
3224
0
  case ISD::UMAX: {
3225
0
    // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3226
0
    ISD::CondCode Pred;
3227
0
    switch (Node->getOpcode()) {
3228
0
    
default: 0
llvm_unreachable0
("How did we get here?");
3229
0
    case ISD::SMAX: Pred = ISD::SETGT; break;
3230
0
    case ISD::SMIN: Pred = ISD::SETLT; break;
3231
0
    case ISD::UMAX: Pred = ISD::SETUGT; break;
3232
0
    case ISD::UMIN: Pred = ISD::SETULT; break;
3233
0
    }
3234
0
    Tmp1 = Node->getOperand(0);
3235
0
    Tmp2 = Node->getOperand(1);
3236
0
    Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3237
0
    Results.push_back(Tmp1);
3238
0
    break;
3239
0
  }
3240
0
3241
860
  case ISD::FSIN:
3242
860
  case ISD::FCOS: {
3243
860
    EVT VT = Node->getValueType(0);
3244
860
    // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3245
860
    // fcos which share the same operand and both are used.
3246
860
    if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3247
253
         isSinCosLibcallAvailable(Node, TLI))
3248
860
        && 
useSinCos(Node)704
) {
3249
410
      SDVTList VTs = DAG.getVTList(VT, VT);
3250
410
      Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3251
410
      if (Node->getOpcode() == ISD::FCOS)
3252
205
        Tmp1 = Tmp1.getValue(1);
3253
410
      Results.push_back(Tmp1);
3254
410
    }
3255
860
    break;
3256
860
  }
3257
0
  case ISD::FMAD:
3258
0
    llvm_unreachable("Illegal fmad should never be formed");
3259
860
3260
157
  case ISD::FP16_TO_FP:
3261
157
    if (
Node->getValueType(0) != MVT::f32157
) {
3262
0
      // We can extend to types bigger than f32 in two steps without changing
3263
0
      // the result. Since "f16 -> f32" is much more commonly available, give
3264
0
      // CodeGen the option of emitting that before resorting to a libcall.
3265
0
      SDValue Res =
3266
0
          DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3267
0
      Results.push_back(
3268
0
          DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3269
0
    }
3270
157
    break;
3271
345
  case ISD::FP_TO_FP16:
3272
345
    if (
!TLI.useSoftFloat() && 345
TM.Options.UnsafeFPMath345
) {
3273
15
      SDValue Op = Node->getOperand(0);
3274
15
      MVT SVT = Op.getSimpleValueType();
3275
15
      if (
(SVT == MVT::f64 || 15
SVT == MVT::f802
) &&
3276
15
          
TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)15
) {
3277
13
        // Under fastmath, we can expand this node into a fround followed by
3278
13
        // a float-half conversion.
3279
13
        SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3280
13
                                       DAG.getIntPtrConstant(0, dl));
3281
13
        Results.push_back(
3282
13
            DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3283
13
      }
3284
15
    }
3285
345
    break;
3286
122k
  case ISD::ConstantFP: {
3287
122k
    ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3288
122k
    // Check to see if this FP immediate is already legal.
3289
122k
    // If this is a legal constant, turn it into a TargetConstantFP node.
3290
122k
    if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0)))
3291
34.1k
      Results.push_back(ExpandConstantFP(CFP, true));
3292
122k
    break;
3293
860
  }
3294
0
  case ISD::Constant: {
3295
0
    ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3296
0
    Results.push_back(ExpandConstant(CP));
3297
0
    break;
3298
860
  }
3299
180
  case ISD::FSUB: {
3300
180
    EVT VT = Node->getValueType(0);
3301
180
    if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3302
180
        
TLI.isOperationLegalOrCustom(ISD::FNEG, VT)164
) {
3303
164
      const SDNodeFlags Flags = Node->getFlags();
3304
164
      Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3305
164
      Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3306
164
      Results.push_back(Tmp1);
3307
164
    }
3308
180
    break;
3309
860
  }
3310
0
  case ISD::SUB: {
3311
0
    EVT VT = Node->getValueType(0);
3312
0
    assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3313
0
           TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3314
0
           "Don't know how to expand this subtraction!");
3315
0
    Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3316
0
               DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3317
0
                               VT));
3318
0
    Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3319
0
    Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3320
0
    break;
3321
860
  }
3322
4.95k
  case ISD::UREM:
3323
4.95k
  case ISD::SREM: {
3324
4.95k
    EVT VT = Node->getValueType(0);
3325
4.95k
    bool isSigned = Node->getOpcode() == ISD::SREM;
3326
4.95k
    unsigned DivOpc = isSigned ? 
ISD::SDIV3.74k
:
ISD::UDIV1.21k
;
3327
4.95k
    unsigned DivRemOpc = isSigned ? 
ISD::SDIVREM3.74k
:
ISD::UDIVREM1.21k
;
3328
4.95k
    Tmp2 = Node->getOperand(0);
3329
4.95k
    Tmp3 = Node->getOperand(1);
3330
4.95k
    if (
TLI.isOperationLegalOrCustom(DivRemOpc, VT)4.95k
) {
3331
720
      SDVTList VTs = DAG.getVTList(VT, VT);
3332
720
      Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3333
720
      Results.push_back(Tmp1);
3334
4.95k
    } else 
if (4.23k
TLI.isOperationLegalOrCustom(DivOpc, VT)4.23k
) {
3335
4.19k
      // X % Y -> X-X/Y*Y
3336
4.19k
      Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3337
4.19k
      Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3338
4.19k
      Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3339
4.19k
      Results.push_back(Tmp1);
3340
4.19k
    }
3341
4.95k
    break;
3342
4.95k
  }
3343
751
  case ISD::UDIV:
3344
751
  case ISD::SDIV: {
3345
751
    bool isSigned = Node->getOpcode() == ISD::SDIV;
3346
751
    unsigned DivRemOpc = isSigned ? 
ISD::SDIVREM331
:
ISD::UDIVREM420
;
3347
751
    EVT VT = Node->getValueType(0);
3348
751
    if (
TLI.isOperationLegalOrCustom(DivRemOpc, VT)751
) {
3349
743
      SDVTList VTs = DAG.getVTList(VT, VT);
3350
743
      Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3351
743
                         Node->getOperand(1));
3352
743
      Results.push_back(Tmp1);
3353
743
    }
3354
751
    break;
3355
751
  }
3356
384
  case ISD::MULHU:
3357
384
  case ISD::MULHS: {
3358
384
    unsigned ExpandOpcode =
3359
384
        Node->getOpcode() == ISD::MULHU ? 
ISD::UMUL_LOHI314
:
ISD::SMUL_LOHI70
;
3360
384
    EVT VT = Node->getValueType(0);
3361
384
    SDVTList VTs = DAG.getVTList(VT, VT);
3362
384
3363
384
    Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3364
384
                       Node->getOperand(1));
3365
384
    Results.push_back(Tmp1.getValue(1));
3366
384
    break;
3367
384
  }
3368
0
  case ISD::UMUL_LOHI:
3369
0
  case ISD::SMUL_LOHI: {
3370
0
    SDValue LHS = Node->getOperand(0);
3371
0
    SDValue RHS = Node->getOperand(1);
3372
0
    MVT VT = LHS.getSimpleValueType();
3373
0
    unsigned MULHOpcode =
3374
0
        Node->getOpcode() == ISD::UMUL_LOHI ? 
ISD::MULHU0
:
ISD::MULHS0
;
3375
0
3376
0
    if (
TLI.isOperationLegalOrCustom(MULHOpcode, VT)0
) {
3377
0
      Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3378
0
      Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3379
0
      break;
3380
0
    }
3381
0
3382
0
    SmallVector<SDValue, 4> Halves;
3383
0
    EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3384
0
    assert(TLI.isTypeLegal(HalfType));
3385
0
    if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3386
0
                           HalfType, DAG,
3387
0
                           TargetLowering::MulExpansionKind::Always)) {
3388
0
      for (unsigned i = 0; 
i < 20
;
++i0
) {
3389
0
        SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3390
0
        SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3391
0
        SDValue Shift = DAG.getConstant(
3392
0
            HalfType.getScalarSizeInBits(), dl,
3393
0
            TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3394
0
        Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3395
0
        Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3396
0
      }
3397
0
      break;
3398
0
    }
3399
0
    break;
3400
0
  }
3401
50
  case ISD::MUL: {
3402
50
    EVT VT = Node->getValueType(0);
3403
50
    SDVTList VTs = DAG.getVTList(VT, VT);
3404
50
    // See if multiply or divide can be lowered using two-result operations.
3405
50
    // We just need the low half of the multiply; try both the signed
3406
50
    // and unsigned forms. If the target supports both SMUL_LOHI and
3407
50
    // UMUL_LOHI, form a preference by checking which forms of plain
3408
50
    // MULH it supports.
3409
50
    bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3410
50
    bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3411
50
    bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3412
50
    bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3413
50
    unsigned OpToUse = 0;
3414
50
    if (
HasSMUL_LOHI && 50
!HasMULHS7
) {
3415
7
      OpToUse = ISD::SMUL_LOHI;
3416
50
    } else 
if (43
HasUMUL_LOHI && 43
!HasMULHU0
) {
3417
0
      OpToUse = ISD::UMUL_LOHI;
3418
43
    } else 
if (43
HasSMUL_LOHI43
) {
3419
0
      OpToUse = ISD::SMUL_LOHI;
3420
43
    } else 
if (43
HasUMUL_LOHI43
) {
3421
0
      OpToUse = ISD::UMUL_LOHI;
3422
0
    }
3423
50
    if (
OpToUse50
) {
3424
7
      Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3425
7
                                    Node->getOperand(1)));
3426
7
      break;
3427
7
    }
3428
43
3429
43
    SDValue Lo, Hi;
3430
43
    EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3431
43
    if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3432
43
        TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3433
43
        TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3434
43
        TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3435
43
        TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3436
43
                      TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3437
37
      Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3438
37
      Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3439
37
      SDValue Shift =
3440
37
          DAG.getConstant(HalfType.getSizeInBits(), dl,
3441
37
                          TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3442
37
      Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3443
37
      Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3444
37
    }
3445
43
    break;
3446
43
  }
3447
25
  case ISD::SADDO:
3448
25
  case ISD::SSUBO: {
3449
25
    SDValue LHS = Node->getOperand(0);
3450
25
    SDValue RHS = Node->getOperand(1);
3451
25
    SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3452
25
                              
ISD::ADD12
:
ISD::SUB13
, dl, LHS.getValueType(),
3453
25
                              LHS, RHS);
3454
25
    Results.push_back(Sum);
3455
25
    EVT ResultType = Node->getValueType(1);
3456
25
    EVT OType = getSetCCResultType(Node->getValueType(0));
3457
25
3458
25
    SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3459
25
3460
25
    //   LHSSign -> LHS >= 0
3461
25
    //   RHSSign -> RHS >= 0
3462
25
    //   SumSign -> Sum >= 0
3463
25
    //
3464
25
    //   Add:
3465
25
    //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3466
25
    //   Sub:
3467
25
    //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3468
25
    SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3469
25
    SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3470
25
    SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3471
25
                                      Node->getOpcode() == ISD::SADDO ?
3472
25
                                      
ISD::SETEQ12
:
ISD::SETNE13
);
3473
25
3474
25
    SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3475
25
    SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3476
25
3477
25
    SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3478
25
    Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3479
25
    break;
3480
25
  }
3481
14
  case ISD::UADDO:
3482
14
  case ISD::USUBO: {
3483
14
    SDValue LHS = Node->getOperand(0);
3484
14
    SDValue RHS = Node->getOperand(1);
3485
14
    SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ?
3486
14
                              
ISD::ADD7
:
ISD::SUB7
, dl, LHS.getValueType(),
3487
14
                              LHS, RHS);
3488
14
    Results.push_back(Sum);
3489
14
3490
14
    EVT ResultType = Node->getValueType(1);
3491
14
    EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3492
14
    ISD::CondCode CC
3493
14
      = Node->getOpcode() == ISD::UADDO ? 
ISD::SETULT7
:
ISD::SETUGT7
;
3494
14
    SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3495
14
3496
14
    Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3497
14
    break;
3498
14
  }
3499
10
  case ISD::UMULO:
3500
10
  case ISD::SMULO: {
3501
10
    EVT VT = Node->getValueType(0);
3502
10
    EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2);
3503
10
    SDValue LHS = Node->getOperand(0);
3504
10
    SDValue RHS = Node->getOperand(1);
3505
10
    SDValue BottomHalf;
3506
10
    SDValue TopHalf;
3507
10
    static const unsigned Ops[2][3] =
3508
10
        { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
3509
10
          { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
3510
10
    bool isSigned = Node->getOpcode() == ISD::SMULO;
3511
10
    if (
TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)10
) {
3512
5
      BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3513
5
      TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
3514
10
    } else 
if (5
TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)5
) {
3515
0
      BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
3516
0
                               RHS);
3517
0
      TopHalf = BottomHalf.getValue(1);
3518
5
    } else 
if (5
TLI.isTypeLegal(WideVT)5
) {
3519
0
      LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
3520
0
      RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
3521
0
      Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
3522
0
      BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3523
0
                               DAG.getIntPtrConstant(0, dl));
3524
0
      TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1,
3525
0
                            DAG.getIntPtrConstant(1, dl));
3526
5
    } else {
3527
5
      // We can fall back to a libcall with an illegal type for the MUL if we
3528
5
      // have a libcall big enough.
3529
5
      // Also, we can fall back to a division in some cases, but that's a big
3530
5
      // performance hit in the general case.
3531
5
      RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3532
5
      if (WideVT == MVT::i16)
3533
0
        LC = RTLIB::MUL_I16;
3534
5
      else 
if (5
WideVT == MVT::i325
)
3535
1
        LC = RTLIB::MUL_I32;
3536
4
      else 
if (4
WideVT == MVT::i644
)
3537
4
        LC = RTLIB::MUL_I64;
3538
0
      else 
if (0
WideVT == MVT::i1280
)
3539
0
        LC = RTLIB::MUL_I128;
3540
5
      assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
3541
5
3542
5
      SDValue HiLHS;
3543
5
      SDValue HiRHS;
3544
5
      if (
isSigned5
) {
3545
1
        // The high part is obtained by SRA'ing all but one of the bits of low
3546
1
        // part.
3547
1
        unsigned LoSize = VT.getSizeInBits();
3548
1
        HiLHS =
3549
1
            DAG.getNode(ISD::SRA, dl, VT, LHS,
3550
1
                        DAG.getConstant(LoSize - 1, dl,
3551
1
                                        TLI.getPointerTy(DAG.getDataLayout())));
3552
1
        HiRHS =
3553
1
            DAG.getNode(ISD::SRA, dl, VT, RHS,
3554
1
                        DAG.getConstant(LoSize - 1, dl,
3555
1
                                        TLI.getPointerTy(DAG.getDataLayout())));
3556
5
      } else {
3557
4
          HiLHS = DAG.getConstant(0, dl, VT);
3558
4
          HiRHS = DAG.getConstant(0, dl, VT);
3559
4
      }
3560
5
3561
5
      // Here we're passing the 2 arguments explicitly as 4 arguments that are
3562
5
      // pre-lowered to the correct types. This all depends upon WideVT not
3563
5
      // being a legal type for the architecture and thus has to be split to
3564
5
      // two arguments.
3565
5
      SDValue Ret;
3566
5
      if(
DAG.getDataLayout().isLittleEndian()5
) {
3567
5
        // Halves of WideVT are packed into registers in different order
3568
5
        // depending on platform endianness. This is usually handled by
3569
5
        // the C calling convention, but we can't defer to it in
3570
5
        // the legalizer.
3571
5
        SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
3572
5
        Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3573
5
      } else {
3574
0
        SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
3575
0
        Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl);
3576
0
      }
3577
5
      assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
3578
5
             "Ret value is a collection of constituent nodes holding result.");
3579
5
      BottomHalf = Ret.getOperand(0);
3580
5
      TopHalf = Ret.getOperand(1);
3581
5
    }
3582
10
3583
10
    if (
isSigned10
) {
3584
5
      Tmp1 = DAG.getConstant(
3585
5
          VT.getSizeInBits() - 1, dl,
3586
5
          TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
3587
5
      Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1);
3588
5
      TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1,
3589
5
                             ISD::SETNE);
3590
10
    } else {
3591
5
      TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf,
3592
5
                             DAG.getConstant(0, dl, VT), ISD::SETNE);
3593
5
    }
3594
10
3595
10
    // Truncate the result if SetCC returns a larger type than needed.
3596
10
    EVT RType = Node->getValueType(1);
3597
10
    if (RType.getSizeInBits() < TopHalf.getValueSizeInBits())
3598
1
      TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf);
3599
10
3600
10
    assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() &&
3601
10
           "Unexpected result type for S/UMULO legalization");
3602
10
3603
10
    Results.push_back(BottomHalf);
3604
10
    Results.push_back(TopHalf);
3605
10
    break;
3606
10
  }
3607
10
  case ISD::BUILD_PAIR: {
3608
10
    EVT PairTy = Node->getValueType(0);
3609
10
    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3610
10
    Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3611
10
    Tmp2 = DAG.getNode(
3612
10
        ISD::SHL, dl, PairTy, Tmp2,
3613
10
        DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3614
10
                        TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3615
10
    Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3616
10
    break;
3617
10
  }
3618
188
  case ISD::SELECT:
3619
188
    Tmp1 = Node->getOperand(0);
3620
188
    Tmp2 = Node->getOperand(1);
3621
188
    Tmp3 = Node->getOperand(2);
3622
188
    if (
Tmp1.getOpcode() == ISD::SETCC188
) {
3623
139
      Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3624
139
                             Tmp2, Tmp3,
3625
139
                             cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3626
188
    } else {
3627
49
      Tmp1 = DAG.getSelectCC(dl, Tmp1,
3628
49
                             DAG.getConstant(0, dl, Tmp1.getValueType()),
3629
49
                             Tmp2, Tmp3, ISD::SETNE);
3630
49
    }
3631
188
    Results.push_back(Tmp1);
3632
188
    break;
3633
4.80k
  case ISD::BR_JT: {
3634
4.80k
    SDValue Chain = Node->getOperand(0);
3635
4.80k
    SDValue Table = Node->getOperand(1);
3636
4.80k
    SDValue Index = Node->getOperand(2);
3637
4.80k
3638
4.80k
    const DataLayout &TD = DAG.getDataLayout();
3639
4.80k
    EVT PTy = TLI.getPointerTy(TD);
3640
4.80k
3641
4.80k
    unsigned EntrySize =
3642
4.80k
      DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3643
4.80k
3644
4.80k
    Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3645
4.80k
                        DAG.getConstant(EntrySize, dl, Index.getValueType()));
3646
4.80k
    SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3647
4.80k
                               Index, Table);
3648
4.80k
3649
4.80k
    EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3650
4.80k
    SDValue LD = DAG.getExtLoad(
3651
4.80k
        ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3652
4.80k
        MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3653
4.80k
    Addr = LD;
3654
4.80k
    if (
TLI.isJumpTableRelative()4.80k
) {
3655
4.66k
      // For PIC, the sequence is:
3656
4.66k
      // BRIND(load(Jumptable + index) + RelocBase)
3657
4.66k
      // RelocBase can be JumpTable, GOT or some sort of global base.
3658
4.66k
      Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3659
4.66k
                          TLI.getPICJumpTableRelocBase(Table, DAG));
3660
4.66k
    }
3661
4.80k
    Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr);
3662
4.80k
    Results.push_back(Tmp1);
3663
4.80k
    break;
3664
10
  }
3665
32.8k
  case ISD::BRCOND:
3666
32.8k
    // Expand brcond's setcc into its constituent parts and create a BR_CC
3667
32.8k
    // Node.
3668
32.8k
    Tmp1 = Node->getOperand(0);
3669
32.8k
    Tmp2 = Node->getOperand(1);
3670
32.8k
    if (
Tmp2.getOpcode() == ISD::SETCC32.8k
) {
3671
294
      Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3672
294
                         Tmp1, Tmp2.getOperand(2),
3673
294
                         Tmp2.getOperand(0), Tmp2.getOperand(1),
3674
294
                         Node->getOperand(2));
3675
32.8k
    } else {
3676
32.5k
      // We test only the i1 bit.  Skip the AND if UNDEF or another AND.
3677
32.5k
      if (Tmp2.isUndef() ||
3678
32.5k
          (Tmp2.getOpcode() == ISD::AND &&
3679
7.40k
           isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3680
6.75k
           dyn_cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3681
6.75k
        Tmp3 = Tmp2;
3682
32.5k
      else
3683
25.7k
        Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3684
25.7k
                           DAG.getConstant(1, dl, Tmp2.getValueType()));
3685
32.5k
      Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3686
32.5k
                         DAG.getCondCode(ISD::SETNE), Tmp3,
3687
32.5k
                         DAG.getConstant(0, dl, Tmp3.getValueType()),
3688
32.5k
                         Node->getOperand(2));
3689
32.5k
    }
3690
32.8k
    Results.push_back(Tmp1);
3691
32.8k
    break;
3692
3.23k
  case ISD::SETCC: {
3693
3.23k
    Tmp1 = Node->getOperand(0);
3694
3.23k
    Tmp2 = Node->getOperand(1);
3695
3.23k
    Tmp3 = Node->getOperand(2);
3696
3.23k
    bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3697
3.23k
                                           Tmp3, NeedInvert, dl);
3698
3.23k
3699
3.23k
    if (
Legalized3.23k
) {
3700
313
      // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3701
313
      // condition code, create a new SETCC node.
3702
313
      if (Tmp3.getNode())
3703
230
        Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3704
230
                           Tmp1, Tmp2, Tmp3);
3705
313
3706
313
      // If we expanded the SETCC by inverting the condition code, then wrap
3707
313
      // the existing SETCC in a NOT to restore the intended condition.
3708
313
      if (NeedInvert)
3709
12
        Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3710
313
3711
313
      Results.push_back(Tmp1);
3712
313
      break;
3713
313
    }
3714
2.92k
3715
2.92k
    // Otherwise, SETCC for the given comparison type must be completely
3716
2.92k
    // illegal; expand it into a SELECT_CC.
3717
2.92k
    EVT VT = Node->getValueType(0);
3718
2.92k
    int TrueValue;
3719
2.92k
    switch (TLI.getBooleanContents(Tmp1->getValueType(0))) {
3720
1.55k
    case TargetLowering::ZeroOrOneBooleanContent:
3721
1.55k
    case TargetLowering::UndefinedBooleanContent:
3722
1.55k
      TrueValue = 1;
3723
1.55k
      break;
3724
1.36k
    case TargetLowering::ZeroOrNegativeOneBooleanContent:
3725
1.36k
      TrueValue = -1;
3726
1.36k
      break;
3727
2.92k
    }
3728
2.92k
    Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3729
2.92k
                       DAG.getConstant(TrueValue, dl, VT),
3730
2.92k
                       DAG.getConstant(0, dl, VT),
3731
2.92k
                       Tmp3);
3732
2.92k
    Results.push_back(Tmp1);
3733
2.92k
    break;
3734
2.92k
  }
3735
3.51k
  case ISD::SELECT_CC: {
3736
3.51k
    Tmp1 = Node->getOperand(0);   // LHS
3737
3.51k
    Tmp2 = Node->getOperand(1);   // RHS
3738
3.51k
    Tmp3 = Node->getOperand(2);   // True
3739
3.51k
    Tmp4 = Node->getOperand(3);   // False
3740
3.51k
    EVT VT = Node->getValueType(0);
3741
3.51k
    SDValue CC = Node->getOperand(4);
3742
3.51k
    ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3743
3.51k
3744
3.51k
    if (
TLI.isCondCodeLegal(CCOp, Tmp1.getSimpleValueType())3.51k
) {
3745
3.21k
      // If the condition code is legal, then we need to expand this
3746
3.21k
      // node using SETCC and SELECT.
3747
3.21k
      EVT CmpVT = Tmp1.getValueType();
3748
3.21k
      assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3749
3.21k
             "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3750
3.21k
             "expanded.");
3751
3.21k
      EVT CCVT =
3752
3.21k
          TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3753
3.21k
      SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3754
3.21k
      Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3755
3.21k
      break;
3756
3.21k
    }
3757
301
3758
301
    // SELECT_CC is legal, so the condition code must not be.
3759
301
    bool Legalized = false;
3760
301
    // Try to legalize by inverting the condition.  This is for targets that
3761
301
    // might support an ordered version of a condition, but not the unordered
3762
301
    // version (or vice versa).
3763
301
    ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3764
301
                                               Tmp1.getValueType().isInteger());
3765
301
    if (
TLI.isCondCodeLegal(InvCC, Tmp1.getSimpleValueType())301
) {
3766
256
      // Use the new condition code and swap true and false
3767
256
      Legalized = true;
3768
256
      Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3769
301
    } else {
3770
45
      // If The inverse is not legal, then try to swap the arguments using
3771
45
      // the inverse condition code.
3772
45
      ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3773
45
      if (
TLI.isCondCodeLegal(SwapInvCC, Tmp1.getSimpleValueType())45
) {
3774
12
        // The swapped inverse condition is legal, so swap true and false,
3775
12
        // lhs and rhs.
3776
12
        Legalized = true;
3777
12
        Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3778
12
      }
3779
45
    }
3780
301
3781
301
    if (
!Legalized301
) {
3782
33
      Legalized = LegalizeSetCCCondCode(
3783
33
          getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3784
33
          dl);
3785
33
3786
33
      assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3787
33
3788
33
      // If we expanded the SETCC by inverting the condition code, then swap
3789
33
      // the True/False operands to match.
3790
33
      if (NeedInvert)
3791
0
        std::swap(Tmp3, Tmp4);
3792
33
3793
33
      // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3794
33
      // condition code, create a new SELECT_CC node.
3795
33
      if (
CC.getNode()33
) {
3796
13
        Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3797
13
                           Tmp1, Tmp2, Tmp3, Tmp4, CC);
3798
33
      } else {
3799
20
        Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3800
20
        CC = DAG.getCondCode(ISD::SETNE);
3801
20
        Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3802
20
                           Tmp2, Tmp3, Tmp4, CC);
3803
20
      }
3804
33
    }
3805
301
    Results.push_back(Tmp1);
3806
301
    break;
3807
301
  }
3808
9
  case ISD::BR_CC: {
3809
9
    Tmp1 = Node->getOperand(0);              // Chain
3810
9
    Tmp2 = Node->getOperand(2);              // LHS
3811
9
    Tmp3 = Node->getOperand(3);              // RHS
3812
9
    Tmp4 = Node->getOperand(1);              // CC
3813
9
3814
9
    bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3815
9
        Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3816
9
    (void)Legalized;
3817
9
    assert(Legalized && "Can't legalize BR_CC with legal condition!");
3818
9
3819
9
    // If we expanded the SETCC by inverting the condition code, then wrap
3820
9
    // the existing SETCC in a NOT to restore the intended condition.
3821
9
    if (NeedInvert)
3822
0
      Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
3823
9
3824
9
    // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3825
9
    // node.
3826
9
    if (
Tmp4.getNode()9
) {
3827
0
      Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3828
0
                         Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3829
9
    } else {
3830
9
      Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3831
9
      Tmp4 = DAG.getCondCode(ISD::SETNE);
3832
9
      Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3833
9
                         Tmp2, Tmp3, Node->getOperand(4));
3834
9
    }
3835
9
    Results.push_back(Tmp1);
3836
9
    break;
3837
301
  }
3838
63.0k
  case ISD::BUILD_VECTOR:
3839
63.0k
    Results.push_back(ExpandBUILD_VECTOR(Node));
3840
63.0k
    break;
3841
10
  case ISD::SRA:
3842
10
  case ISD::SRL:
3843
10
  case ISD::SHL: {
3844
10
    // Scalarize vector SRA/SRL/SHL.
3845
10
    EVT VT = Node->getValueType(0);
3846
10
    assert(VT.isVector() && "Unable to legalize non-vector shift");
3847
10
    assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3848
10
    unsigned NumElem = VT.getVectorNumElements();
3849
10
3850
10
    SmallVector<SDValue, 8> Scalars;
3851
30
    for (unsigned Idx = 0; 
Idx < NumElem30
;
Idx++20
) {
3852
20
      SDValue Ex = DAG.getNode(
3853
20
          ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3854
20
          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3855
20
      SDValue Sh = DAG.getNode(
3856
20
          ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3857
20
          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3858
20
      Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3859
20
                                    VT.getScalarType(), Ex, Sh));
3860
20
    }
3861
10
3862
10
    SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3863
10
    ReplaceNode(SDValue(Node, 0), Result);
3864
10
    break;
3865
10
  }
3866
30.2k
  case ISD::GLOBAL_OFFSET_TABLE:
3867
30.2k
  case ISD::GlobalAddress:
3868
30.2k
  case ISD::GlobalTLSAddress:
3869
30.2k
  case ISD::ExternalSymbol:
3870
30.2k
  case ISD::ConstantPool:
3871
30.2k
  case ISD::JumpTable:
3872
30.2k
  case ISD::INTRINSIC_W_CHAIN:
3873
30.2k
  case ISD::INTRINSIC_WO_CHAIN:
3874
30.2k
  case ISD::INTRINSIC_VOID:
3875
30.2k
    // FIXME: Custom lowering for these operations shouldn't return null!
3876
30.2k
    break;
3877
334k
  }
3878
334k
3879
334k
  // Replace the original node with the legalized result.
3880
334k
  
if (334k
Results.empty()334k
)
3881
147k
    return false;
3882
186k
3883
186k
  ReplaceNode(Node, Results.data());
3884
186k
  return true;
3885
186k
}
3886
3887
147k
void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3888
147k
  SmallVector<SDValue, 8> Results;
3889
147k
  SDLoc dl(Node);
3890
147k
  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3891
147k
  unsigned Opc = Node->getOpcode();
3892
147k
  switch (Opc) {
3893
13
  case ISD::ATOMIC_FENCE: {
3894
13
    // If the target didn't lower this, lower it to '__sync_synchronize()' call
3895
13
    // FIXME: handle "fence singlethread" more efficiently.
3896
13
    TargetLowering::ArgListTy Args;
3897
13
3898
13
    TargetLowering::CallLoweringInfo CLI(DAG);
3899
13
    CLI.setDebugLoc(dl)
3900
13
        .setChain(Node->getOperand(0))
3901
13
        .setLibCallee(
3902
13
            CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3903
13
            DAG.getExternalSymbol("__sync_synchronize",
3904
13
                                  TLI.getPointerTy(DAG.getDataLayout())),
3905
13
            std::move(Args));
3906
13
3907
13
    std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3908
13
3909
13
    Results.push_back(CallResult.second);
3910
13
    break;
3911
147k
  }
3912
147k
  // By default, atomic intrinsics are marked Legal and lowered. Targets
3913
147k
  // which don't support them directly, however, may want libcalls, in which
3914
147k
  // case they mark them Expand, and we get here.
3915
328
  case ISD::ATOMIC_SWAP:
3916
328
  case ISD::ATOMIC_LOAD_ADD:
3917
328
  case ISD::ATOMIC_LOAD_SUB:
3918
328
  case ISD::ATOMIC_LOAD_AND:
3919
328
  case ISD::ATOMIC_LOAD_OR:
3920
328
  case ISD::ATOMIC_LOAD_XOR:
3921
328
  case ISD::ATOMIC_LOAD_NAND:
3922
328
  case ISD::ATOMIC_LOAD_MIN:
3923
328
  case ISD::ATOMIC_LOAD_MAX:
3924
328
  case ISD::ATOMIC_LOAD_UMIN:
3925
328
  case ISD::ATOMIC_LOAD_UMAX:
3926
328
  case ISD::ATOMIC_CMP_SWAP: {
3927
328
    MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3928
328
    RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3929
328
    assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
3930
328
3931
328
    std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3932
328
    Results.push_back(Tmp.first);
3933
328
    Results.push_back(Tmp.second);
3934
328
    break;
3935
328
  }
3936
2
  case ISD::TRAP: {
3937
2
    // If this operation is not supported, lower it to 'abort()' call
3938
2
    TargetLowering::ArgListTy Args;
3939
2
    TargetLowering::CallLoweringInfo CLI(DAG);
3940
2
    CLI.setDebugLoc(dl)
3941
2
        .setChain(Node->getOperand(0))
3942
2
        .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3943
2
                      DAG.getExternalSymbol(
3944
2
                          "abort", TLI.getPointerTy(DAG.getDataLayout())),
3945
2
                      std::move(Args));
3946
2
    std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3947
2
3948
2
    Results.push_back(CallResult.second);
3949
2
    break;
3950
328
  }
3951
45
  case ISD::FMINNUM:
3952
45
    Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3953
45
                                      RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3954
45
                                      RTLIB::FMIN_PPCF128));
3955
45
    break;
3956
101
  case ISD::FMAXNUM:
3957
101
    Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3958
101
                                      RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3959
101
                                      RTLIB::FMAX_PPCF128));
3960
101
    break;
3961
4
  case ISD::FSQRT:
3962
4
  case ISD::STRICT_FSQRT:
3963
4
    Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3964
4
                                      RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3965
4
                                      RTLIB::SQRT_PPCF128));
3966
4
    break;
3967
186
  case ISD::FSIN:
3968
186
  case ISD::STRICT_FSIN:
3969
186
    Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3970
186
                                      RTLIB::SIN_F80, RTLIB::SIN_F128,
3971
186
                                      RTLIB::SIN_PPCF128));
3972
186
    break;
3973
268
  case ISD::FCOS:
3974
268
  case ISD::STRICT_FCOS:
3975
268
    Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3976
268
                                      RTLIB::COS_F80, RTLIB::COS_F128,
3977
268
                                      RTLIB::COS_PPCF128));
3978
268
    break;
3979
30
  case ISD::FSINCOS:
3980
30
    // Expand into sincos libcall.
3981
30
    ExpandSinCosLibCall(Node, Results);
3982
30
    break;
3983
56
  case ISD::FLOG:
3984
56
  case ISD::STRICT_FLOG:
3985
56
    Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3986
56
                                      RTLIB::LOG_F80, RTLIB::LOG_F128,
3987
56
                                      RTLIB::LOG_PPCF128));
3988
56
    break;
3989
59
  case ISD::FLOG2:
3990
59
  case ISD::STRICT_FLOG2:
3991
59
    Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3992
59
                                      RTLIB::LOG2_F80, RTLIB::LOG2_F128,
3993
59
                                      RTLIB::LOG2_PPCF128));
3994
59
    break;
3995
60
  case ISD::FLOG10:
3996
60
  case ISD::STRICT_FLOG10:
3997
60
    Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3998
60
                                      RTLIB::LOG10_F80, RTLIB::LOG10_F128,
3999
60
                                      RTLIB::LOG10_PPCF128));
4000
60
    break;
4001
160
  case ISD::FEXP:
4002
160
  case ISD::STRICT_FEXP:
4003
160
    Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
4004
160
                                      RTLIB::EXP_F80, RTLIB::EXP_F128,
4005
160
                                      RTLIB::EXP_PPCF128));
4006
160
    break;
4007
83
  case ISD::FEXP2:
4008
83
  case ISD::STRICT_FEXP2:
4009
83
    Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
4010
83
                                      RTLIB::EXP2_F80, RTLIB::EXP2_F128,
4011
83
                                      RTLIB::EXP2_PPCF128));
4012
83
    break;
4013
30
  case ISD::FTRUNC:
4014
30
    Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4015
30
                                      RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4016
30
                                      RTLIB::TRUNC_PPCF128));
4017
30
    break;
4018
38
  case ISD::FFLOOR:
4019
38
    Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4020
38
                                      RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4021
38
                                      RTLIB::FLOOR_PPCF128));
4022
38
    break;
4023
20
  case ISD::FCEIL:
4024
20
    Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4025
20
                                      RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4026
20
                                      RTLIB::CEIL_PPCF128));
4027
20
    break;
4028
25
  case ISD::FRINT:
4029
25
  case ISD::STRICT_FRINT:
4030
25
    Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4031
25
                                      RTLIB::RINT_F80, RTLIB::RINT_F128,
4032
25
                                      RTLIB::RINT_PPCF128));
4033
25
    break;
4034
48
  case ISD::FNEARBYINT:
4035
48
  case ISD::STRICT_FNEARBYINT:
4036
48
    Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4037
48
                                      RTLIB::NEARBYINT_F64,
4038
48
                                      RTLIB::NEARBYINT_F80,
4039
48
                                      RTLIB::NEARBYINT_F128,
4040
48
                                      RTLIB::NEARBYINT_PPCF128));
4041
48
    break;
4042
17
  case ISD::FROUND:
4043
17
    Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4044
17
                                      RTLIB::ROUND_F64,
4045
17
                                      RTLIB::ROUND_F80,
4046
17
                                      RTLIB::ROUND_F128,
4047
17
                                      RTLIB::ROUND_PPCF128));
4048
17
    break;
4049
99
  case ISD::FPOWI:
4050
99
  case ISD::STRICT_FPOWI:
4051
99
    Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4052
99
                                      RTLIB::POWI_F80, RTLIB::POWI_F128,
4053
99
                                      RTLIB::POWI_PPCF128));
4054
99
    break;
4055
177
  case ISD::FPOW:
4056
177
  case ISD::STRICT_FPOW:
4057
177
    Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
4058
177
                                      RTLIB::POW_F80, RTLIB::POW_F128,
4059
177
                                      RTLIB::POW_PPCF128));
4060
177
    break;
4061
13
  case ISD::FDIV:
4062
13
    Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4063
13
                                      RTLIB::DIV_F80, RTLIB::DIV_F128,
4064
13
                                      RTLIB::DIV_PPCF128));
4065
13
    break;
4066
85
  case ISD::FREM:
4067
85
    Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4068
85
                                      RTLIB::REM_F80, RTLIB::REM_F128,
4069
85
                                      RTLIB::REM_PPCF128));
4070
85
    break;
4071
160
  case ISD::FMA:
4072
160
  case ISD::STRICT_FMA:
4073
160
    Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4074
160
                                      RTLIB::FMA_F80, RTLIB::FMA_F128,
4075
160
                                      RTLIB::FMA_PPCF128));
4076
160
    break;
4077
36
  case ISD::FADD:
4078
36
    Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4079
36
                                      RTLIB::ADD_F80, RTLIB::ADD_F128,
4080
36
                                      RTLIB::ADD_PPCF128));
4081
36
    break;
4082
81
  case ISD::FMUL:
4083
81
    Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4084
81
                                      RTLIB::MUL_F80, RTLIB::MUL_F128,
4085
81
                                      RTLIB::MUL_PPCF128));
4086
81
    break;
4087
157
  case ISD::FP16_TO_FP:
4088
157
    if (
Node->getValueType(0) == MVT::f32157
) {
4089
157
      Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4090
157
    }
4091
157
    break;
4092
332
  case ISD::FP_TO_FP16: {
4093
332
    RTLIB::Libcall LC =
4094
332
        RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4095
332
    assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4096
332
    Results.push_back(ExpandLibCall(LC, Node, false));
4097
332
    break;
4098
160
  }
4099
16
  case ISD::FSUB:
4100
16
    Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4101
16
                                      RTLIB::SUB_F80, RTLIB::SUB_F128,
4102
16
                                      RTLIB::SUB_PPCF128));
4103
16
    break;
4104
19
  case ISD::SREM:
4105
19
    Results.push_back(ExpandIntLibCall(Node, true,
4106
19
                                       RTLIB::SREM_I8,
4107
19
                                       RTLIB::SREM_I16, RTLIB::SREM_I32,
4108
19
                                       RTLIB::SREM_I64, RTLIB::SREM_I128));
4109
19
    break;
4110
30
  case ISD::UREM:
4111
30
    Results.push_back(ExpandIntLibCall(Node, false,
4112
30
                                       RTLIB::UREM_I8,
4113
30
                                       RTLIB::UREM_I16, RTLIB::UREM_I32,
4114
30
                                       RTLIB::UREM_I64, RTLIB::UREM_I128));
4115
30
    break;
4116
56
  case ISD::SDIV:
4117
56
    Results.push_back(ExpandIntLibCall(Node, true,
4118
56
                                       RTLIB::SDIV_I8,
4119
56
                                       RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4120
56
                                       RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4121
56
    break;
4122
52
  case ISD::UDIV:
4123
52
    Results.push_back(ExpandIntLibCall(Node, false,
4124
52
                                       RTLIB::UDIV_I8,
4125
52
                                       RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4126
52
                                       RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4127
52
    break;
4128
14
  case ISD::SDIVREM:
4129
14
  case ISD::UDIVREM:
4130
14
    // Expand into divrem libcall
4131
14
    ExpandDivRemLibCall(Node, Results);
4132
14
    break;
4133
16
  case ISD::MUL:
4134
16
    Results.push_back(ExpandIntLibCall(Node, false,
4135
16
                                       RTLIB::MUL_I8,
4136
16
                                       RTLIB::MUL_I16, RTLIB::MUL_I32,
4137
16
                                       RTLIB::MUL_I64, RTLIB::MUL_I128));
4138
16
    break;
4139
147k
  }
4140
147k
4141
147k
  // Replace the original node with the legalized result.
4142
147k
  
if (147k
!Results.empty()147k
)
4143
2.91k
    ReplaceNode(Node, Results.data());
4144
147k
}
4145
4146
// Determine the vector type to use in place of an original scalar element when
4147
// promoting equally sized vectors.
4148
static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4149
2.80k
                                        MVT EltVT, MVT NewEltVT) {
4150
2.80k
  unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4151
2.80k
  MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4152
2.80k
  assert(TLI.isTypeLegal(MidVT) && "unexpected");
4153
2.80k
  return MidVT;
4154
2.80k
}
4155
4156
6.94k
void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4157
6.94k
  SmallVector<SDValue, 8> Results;
4158
6.94k
  MVT OVT = Node->getSimpleValueType(0);
4159
6.94k
  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4160
6.88k
      Node->getOpcode() == ISD::SINT_TO_FP ||
4161
6.85k
      Node->getOpcode() == ISD::SETCC ||
4162
6.67k
      Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4163
6.94k
      
Node->getOpcode() == ISD::INSERT_VECTOR_ELT4.26k
) {
4164
2.67k
    OVT = Node->getOperand(0).getSimpleValueType();
4165
2.67k
  }
4166
6.94k
  if (Node->getOpcode() == ISD::BR_CC)
4167
1
    OVT = Node->getOperand(2).getSimpleValueType();
4168
6.94k
  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4169
6.94k
  SDLoc dl(Node);
4170
6.94k
  SDValue Tmp1, Tmp2, Tmp3;
4171
6.94k
  switch (Node->getOpcode()) {
4172
33
  case ISD::CTTZ:
4173
33
  case ISD::CTTZ_ZERO_UNDEF:
4174
33
  case ISD::CTLZ:
4175
33
  case ISD::CTLZ_ZERO_UNDEF:
4176
33
  case ISD::CTPOP:
4177
33
    // Zero extend the argument.
4178
33
    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4179
33
    if (
Node->getOpcode() == ISD::CTTZ33
) {
4180
4
      // The count is the same in the promoted type except if the original
4181
4
      // value was zero.  This can be handled by setting the bit just off
4182
4
      // the top of the original type.
4183
4
      auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4184
4
                                        OVT.getSizeInBits());
4185
4
      Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4186
4
                         DAG.getConstant(TopBit, dl, NVT));
4187
4
    }
4188
33
    // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4189
33
    // already the correct result.
4190
33
    Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4191
33
    if (Node->getOpcode() == ISD::CTLZ ||
4192
33
        
Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF28
) {
4193
11
      // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4194
11
      Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4195
11
                          DAG.getConstant(NVT.getSizeInBits() -
4196
11
                                          OVT.getSizeInBits(), dl, NVT));
4197
11
    }
4198
33
    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4199
33
    break;
4200
0
  case ISD::BITREVERSE:
4201
0
  case ISD::BSWAP: {
4202
0
    unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4203
0
    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4204
0
    Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4205
0
    Tmp1 = DAG.getNode(
4206
0
        ISD::SRL, dl, NVT, Tmp1,
4207
0
        DAG.getConstant(DiffBits, dl,
4208
0
                        TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4209
0
    Results.push_back(Tmp1);
4210
0
    break;
4211
0
  }
4212
136
  case ISD::FP_TO_UINT:
4213
136
  case ISD::FP_TO_SINT:
4214
136
    Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4215
136
                                 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4216
136
    Results.push_back(Tmp1);
4217
136
    break;
4218
88
  case ISD::UINT_TO_FP:
4219
88
  case ISD::SINT_TO_FP:
4220
88
    Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4221
88
                                 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4222
88
    Results.push_back(Tmp1);
4223
88
    break;
4224
1
  case ISD::VAARG: {
4225
1
    SDValue Chain = Node->getOperand(0); // Get the chain.
4226
1
    SDValue Ptr = Node->getOperand(1); // Get the pointer.
4227
1
4228
1
    unsigned TruncOp;
4229
1
    if (
OVT.isVector()1
) {
4230
0
      TruncOp = ISD::BITCAST;
4231
1
    } else {
4232
1
      assert(OVT.isInteger()
4233
1
        && "VAARG promotion is supported only for vectors or integer types");
4234
1
      TruncOp = ISD::TRUNCATE;
4235
1
    }
4236
1
4237
1
    // Perform the larger operation, then convert back
4238
1
    Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4239
1
             Node->getConstantOperandVal(3));
4240
1
    Chain = Tmp1.getValue(1);
4241
1
4242
1
    Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4243
1
4244
1
    // Modified the chain result - switch anything that used the old chain to
4245
1
    // use the new one.
4246
1
    DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4247
1
    DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4248
1
    if (
UpdatedNodes1
) {
4249
0
      UpdatedNodes->insert(Tmp2.getNode());
4250
0
      UpdatedNodes->insert(Chain.getNode());
4251
0
    }
4252
1
    ReplacedNode(Node);
4253
1
    break;
4254
88
  }
4255
2.68k
  case ISD::MUL:
4256
2.68k
  case ISD::SDIV:
4257
2.68k
  case ISD::SREM:
4258
2.68k
  case ISD::UDIV:
4259
2.68k
  case ISD::UREM:
4260
2.68k
  case ISD::AND:
4261
2.68k
  case ISD::OR:
4262
2.68k
  case ISD::XOR: {
4263
2.68k
    unsigned ExtOp, TruncOp;
4264
2.68k
    if (
OVT.isVector()2.68k
) {
4265
2.67k
      ExtOp   = ISD::BITCAST;
4266
2.67k
      TruncOp = ISD::BITCAST;
4267
2.68k
    } else {
4268
10
      assert(OVT.isInteger() && "Cannot promote logic operation");
4269
10
4270
10
      switch (Node->getOpcode()) {
4271
1
      default:
4272
1
        ExtOp = ISD::ANY_EXTEND;
4273
1
        break;
4274
3
      case ISD::SDIV:
4275
3
      case ISD::SREM:
4276
3
        ExtOp = ISD::SIGN_EXTEND;
4277
3
        break;
4278
6
      case ISD::UDIV:
4279
6
      case ISD::UREM:
4280
6
        ExtOp = ISD::ZERO_EXTEND;
4281
6
        break;
4282
10
      }
4283
10
      TruncOp = ISD::TRUNCATE;
4284
10
    }
4285
2.68k
    // Promote each of the values to the new type.
4286
2.68k
    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4287
2.68k
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4288
2.68k
    // Perform the larger operation, then convert back
4289
2.68k
    Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4290
2.68k
    Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4291
2.68k
    break;
4292
2.68k
  }
4293
0
  case ISD::UMUL_LOHI:
4294
0
  case ISD::SMUL_LOHI: {
4295
0
    // Promote to a multiply in a wider integer type.
4296
0
    unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4297
0
                                                         : ISD::SIGN_EXTEND;
4298
0
    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4299
0
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4300
0
    Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4301
0
4302
0
    auto &DL = DAG.getDataLayout();
4303
0
    unsigned OriginalSize = OVT.getScalarSizeInBits();
4304
0
    Tmp2 = DAG.getNode(
4305
0
        ISD::SRL, dl, NVT, Tmp1,
4306
0
        DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4307
0
    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4308
0
    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4309
0
    break;
4310
0
  }
4311
160
  case ISD::SELECT: {
4312
160
    unsigned ExtOp, TruncOp;
4313
160
    if (Node->getValueType(0).isVector() ||
4314
160
        
Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()160
) {
4315
152
      ExtOp   = ISD::BITCAST;
4316
152
      TruncOp = ISD::BITCAST;
4317
160
    } else 
if (8
Node->getValueType(0).isInteger()8
) {
4318
4
      ExtOp   = ISD::ANY_EXTEND;
4319
4
      TruncOp = ISD::TRUNCATE;
4320
8
    } else {
4321
4
      ExtOp   = ISD::FP_EXTEND;
4322
4
      TruncOp = ISD::FP_ROUND;
4323
4
    }
4324
160
    Tmp1 = Node->getOperand(0);
4325
160
    // Promote each of the values to the new type.
4326
160
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4327
160
    Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4328
160
    // Perform the larger operation, then round down.
4329
160
    Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4330
160
    if (TruncOp != ISD::FP_ROUND)
4331
156
      Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4332
160
    else
4333
4
      Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4334
4
                         DAG.getIntPtrConstant(0, dl));
4335
160
    Results.push_back(Tmp1);
4336
160
    break;
4337
0
  }
4338
402
  case ISD::VECTOR_SHUFFLE: {
4339
402
    ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4340
402
4341
402
    // Cast the two input vectors.
4342
402
    Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4343
402
    Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4344
402
4345
402
    // Convert the shuffle mask to the right # elements.
4346
402
    Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4347
402
    Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4348
402
    Results.push_back(Tmp1);
4349
402
    break;
4350
0
  }
4351
177
  case ISD::SETCC: {
4352
177
    unsigned ExtOp = ISD::FP_EXTEND;
4353
177
    if (
NVT.isInteger()177
) {
4354
60
      ISD::CondCode CCCode =
4355
60
        cast<CondCodeSDNode>(Node->getOperand(2))->get();
4356
60
      ExtOp = isSignedIntSetCC(CCCode) ? 
ISD::SIGN_EXTEND0
:
ISD::ZERO_EXTEND60
;
4357
60
    }
4358
177
    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4359
177
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4360
177
    Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4361
177
                                  Tmp1, Tmp2, Node->getOperand(2)));
4362
177
    break;
4363
0
  }
4364
1
  case ISD::BR_CC: {
4365
1
    unsigned ExtOp = ISD::FP_EXTEND;
4366
1
    if (
NVT.isInteger()1
) {
4367
0
      ISD::CondCode CCCode =
4368
0
        cast<CondCodeSDNode>(Node->getOperand(1))->get();
4369
0
      ExtOp = isSignedIntSetCC(CCCode) ? 
ISD::SIGN_EXTEND0
:
ISD::ZERO_EXTEND0
;
4370
0
    }
4371
1
    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4372
1
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4373
1
    Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4374
1
                                  Node->getOperand(0), Node->getOperand(1),
4375
1
                                  Tmp1, Tmp2, Node->getOperand(4)));
4376
1
    break;
4377
0
  }
4378
151
  case ISD::FADD:
4379
151
  case ISD::FSUB:
4380
151
  case ISD::FMUL:
4381
151
  case ISD::FDIV:
4382
151
  case ISD::FREM:
4383
151
  case ISD::FMINNUM:
4384
151
  case ISD::FMAXNUM:
4385
151
  case ISD::FPOW:
4386
151
    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4387
151
    Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4388
151
    Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4389
151
                       Node->getFlags());
4390
151
    Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4391
151
                                  Tmp3, DAG.getIntPtrConstant(0, dl)));
4392
151
    break;
4393
25
  case ISD::FMA:
4394
25
    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4395
25
    Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4396
25
    Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4397
25
    Results.push_back(
4398
25
        DAG.getNode(ISD::FP_ROUND, dl, OVT,
4399
25
                    DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4400
25
                    DAG.getIntPtrConstant(0, dl)));
4401
25
    break;
4402
12
  case ISD::FCOPYSIGN:
4403
12
  case ISD::FPOWI: {
4404
12
    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4405
12
    Tmp2 = Node->getOperand(1);
4406
12
    Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4407
12
4408
12
    // fcopysign doesn't change anything but the sign bit, so
4409
12
    //   (fp_round (fcopysign (fpext a), b))
4410
12
    // is as precise as
4411
12
    //   (fp_round (fpext a))
4412
12
    // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4413
12
    const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4414
12
    Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4415
12
                                  Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4416
12
    break;
4417
12
  }
4418
163
  case ISD::FFLOOR:
4419
163
  case ISD::FCEIL:
4420
163
  case ISD::FRINT:
4421
163
  case ISD::FNEARBYINT:
4422
163
  case ISD::FROUND:
4423
163
  case ISD::FTRUNC:
4424
163
  case ISD::FNEG:
4425
163
  case ISD::FSQRT:
4426
163
  case ISD::FSIN:
4427
163
  case ISD::FCOS:
4428
163
  case ISD::FLOG:
4429
163
  case ISD::FLOG2:
4430
163
  case ISD::FLOG10:
4431
163
  case ISD::FABS:
4432
163
  case ISD::FEXP:
4433
163
  case ISD::FEXP2:
4434
163
    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4435
163
    Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4436
163
    Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4437
163
                                  Tmp2, DAG.getIntPtrConstant(0, dl)));
4438
163
    break;
4439
363
  case ISD::BUILD_VECTOR: {
4440
363
    MVT EltVT = OVT.getVectorElementType();
4441
363
    MVT NewEltVT = NVT.getVectorElementType();
4442
363
4443
363
    // Handle bitcasts to a different vector type with the same total bit size
4444
363
    //
4445
363
    // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4446
363
    //  =>
4447
363
    //  v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4448
363
4449
363
    assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4450
363
           "Invalid promote type for build_vector");
4451
363
    assert(NewEltVT.bitsLT(EltVT) && "not handled");
4452
363
4453
363
    MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4454
363
4455
363
    SmallVector<SDValue, 8> NewOps;
4456
1.08k
    for (unsigned I = 0, E = Node->getNumOperands(); 
I != E1.08k
;
++I726
) {
4457
726
      SDValue Op = Node->getOperand(I);
4458
726
      NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4459
726
    }
4460
363
4461
363
    SDLoc SL(Node);
4462
363
    SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4463
363
    SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4464
363
    Results.push_back(CvtVec);
4465
363
    break;
4466
163
  }
4467
2.41k
  case ISD::EXTRACT_VECTOR_ELT: {
4468
2.41k
    MVT EltVT = OVT.getVectorElementType();
4469
2.41k
    MVT NewEltVT = NVT.getVectorElementType();
4470
2.41k
4471
2.41k
    // Handle bitcasts to a different vector type with the same total bit size.
4472
2.41k
    //
4473
2.41k
    // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4474
2.41k
    //  =>
4475
2.41k
    //  v4i32:castx = bitcast x:v2i64
4476
2.41k
    //
4477
2.41k
    // i64 = bitcast
4478
2.41k
    //   (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4479
2.41k
    //                       (i32 (extract_vector_elt castx, (2 * y + 1)))
4480
2.41k
    //
4481
2.41k
4482
2.41k
    assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4483
2.41k
           "Invalid promote type for extract_vector_elt");
4484
2.41k
    assert(NewEltVT.bitsLT(EltVT) && "not handled");
4485
2.41k
4486
2.41k
    MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4487
2.41k
    unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4488
2.41k
4489
2.41k
    SDValue Idx = Node->getOperand(1);
4490
2.41k
    EVT IdxVT = Idx.getValueType();
4491
2.41k
    SDLoc SL(Node);
4492
2.41k
    SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4493
2.41k
    SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4494
2.41k
4495
2.41k
    SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4496
2.41k
4497
2.41k
    SmallVector<SDValue, 8> NewOps;
4498
7.23k
    for (unsigned I = 0; 
I < NewEltsPerOldElt7.23k
;
++I4.82k
) {
4499
4.82k
      SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4500
4.82k
      SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4501
4.82k
4502
4.82k
      SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4503
4.82k
                                CastVec, TmpIdx);
4504
4.82k
      NewOps.push_back(Elt);
4505
4.82k
    }
4506
2.41k
4507
2.41k
    SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4508
2.41k
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4509
2.41k
    break;
4510
163
  }
4511
4
  case ISD::INSERT_VECTOR_ELT: {
4512
4
    MVT EltVT = OVT.getVectorElementType();
4513
4
    MVT NewEltVT = NVT.getVectorElementType();
4514
4
4515
4
    // Handle bitcasts to a different vector type with the same total bit size
4516
4
    //
4517
4
    // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4518
4
    //  =>
4519
4
    //  v4i32:castx = bitcast x:v2i64
4520
4
    //  v2i32:casty = bitcast y:i64
4521
4
    //
4522
4
    // v2i64 = bitcast
4523
4
    //   (v4i32 insert_vector_elt
4524
4
    //       (v4i32 insert_vector_elt v4i32:castx,
4525
4
    //                                (extract_vector_elt casty, 0), 2 * z),
4526
4
    //        (extract_vector_elt casty, 1), (2 * z + 1))
4527
4
4528
4
    assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4529
4
           "Invalid promote type for insert_vector_elt");
4530
4
    assert(NewEltVT.bitsLT(EltVT) && "not handled");
4531
4
4532
4
    MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4533
4
    unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4534
4
4535
4
    SDValue Val = Node->getOperand(1);
4536
4
    SDValue Idx = Node->getOperand(2);
4537
4
    EVT IdxVT = Idx.getValueType();
4538
4
    SDLoc SL(Node);
4539
4
4540
4
    SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4541
4
    SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4542
4
4543
4
    SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4544
4
    SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4545
4
4546
4
    SDValue NewVec = CastVec;
4547
12
    for (unsigned I = 0; 
I < NewEltsPerOldElt12
;
++I8
) {
4548
8
      SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4549
8
      SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4550
8
4551
8
      SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4552
8
                                CastVal, IdxOffset);
4553
8
4554
8
      NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4555
8
                           NewVec, Elt, InEltIdx);
4556
8
    }
4557
4
4558
4
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4559
4
    break;
4560
163
  }
4561
28
  case ISD::SCALAR_TO_VECTOR: {
4562
28
    MVT EltVT = OVT.getVectorElementType();
4563
28
    MVT NewEltVT = NVT.getVectorElementType();
4564
28
4565
28
    // Handle bitcasts to different vector type with the same total bit size.
4566
28
    //
4567
28
    // e.g. v2i64 = scalar_to_vector x:i64
4568
28
    //   =>
4569
28
    //  concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4570
28
    //
4571
28
4572
28
    MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4573
28
    SDValue Val = Node->getOperand(0);
4574
28
    SDLoc SL(Node);
4575
28
4576
28
    SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4577
28
    SDValue Undef = DAG.getUNDEF(MidVT);
4578
28
4579
28
    SmallVector<SDValue, 8> NewElts;
4580
28
    NewElts.push_back(CastVal);
4581
56
    for (unsigned I = 1, NElts = OVT.getVectorNumElements(); 
I != NElts56
;
++I28
)
4582
28
      NewElts.push_back(Undef);
4583
28
4584
28
    SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4585
28
    SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4586
28
    Results.push_back(CvtVec);
4587
28
    break;
4588
6.94k
  }
4589
6.94k
  }
4590
6.94k
4591
6.94k
  // Replace the original node with the legalized result.
4592
6.94k
  
if (6.94k
!Results.empty()6.94k
)
4593
6.83k
    ReplaceNode(Node, Results.data());
4594
6.94k
}
4595
4596
/// This is the entry point for the file.
4597
3.42M
void SelectionDAG::Legalize() {
4598
3.42M
  AssignTopologicalOrder();
4599
3.42M
4600
3.42M
  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4601
3.42M
  // Use a delete listener to remove nodes which were deleted during
4602
3.42M
  // legalization from LegalizeNodes. This is needed to handle the situation
4603
3.42M
  // where a new node is allocated by the object pool to the same address of a
4604
3.42M
  // previously deleted node.
4605
3.42M
  DAGNodeDeletedListener DeleteListener(
4606
3.42M
      *this,
4607
244
      [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4608
3.42M
4609
3.42M
  SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4610
3.42M
4611
3.42M
  // Visit all the nodes. We start in topological order, so that we see
4612
3.42M
  // nodes with their original operands intact. Legalization can produce
4613
3.42M
  // new nodes which may themselves need to be legalized. Iterate until all
4614
3.42M
  // nodes have been legalized.
4615
9.45M
  while (
true9.45M
) {
4616
9.45M
    bool AnyLegalized = false;
4617
215M
    for (auto NI = allnodes_end(); 
NI != allnodes_begin()215M
;) {
4618
205M
      --NI;
4619
205M
4620
205M
      SDNode *N = &*NI;
4621
205M
      if (
N->use_empty() && 205M
N != getRoot().getNode()12.7M
) {
4622
3.30M
        ++NI;
4623
3.30M
        DeleteNode(N);
4624
3.30M
        continue;
4625
3.30M
      }
4626
202M
4627
202M
      
if (202M
LegalizedNodes.insert(N).second202M
) {
4628
75.7M
        AnyLegalized = true;
4629
75.7M
        Legalizer.LegalizeOp(N);
4630
75.7M
4631
75.7M
        if (
N->use_empty() && 75.7M
N != getRoot().getNode()7.41M
) {
4632
3.98M
          ++NI;
4633
3.98M
          DeleteNode(N);
4634
3.98M
        }
4635
75.7M
      }
4636
205M
    }
4637
9.45M
    if (!AnyLegalized)
4638
3.42M
      break;
4639
9.45M
4640
9.45M
  }
4641
3.42M
4642
3.42M
  // Remove dead nodes now.
4643
3.42M
  RemoveDeadNodes();
4644
3.42M
}
4645
4646
bool SelectionDAG::LegalizeOp(SDNode *N,
4647
72.8M
                              SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4648
72.8M
  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4649
72.8M
  SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4650
72.8M
4651
72.8M
  // Directly insert the node in question, and legalize it. This will recurse
4652
72.8M
  // as needed through operands.
4653
72.8M
  LegalizedNodes.insert(N);
4654
72.8M
  Legalizer.LegalizeOp(N);
4655
72.8M
4656
72.8M
  return LegalizedNodes.count(N);
4657
72.8M
}