Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements the SelectionDAG::Legalize method.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "llvm/ADT/APFloat.h"
14
#include "llvm/ADT/APInt.h"
15
#include "llvm/ADT/ArrayRef.h"
16
#include "llvm/ADT/SetVector.h"
17
#include "llvm/ADT/SmallPtrSet.h"
18
#include "llvm/ADT/SmallSet.h"
19
#include "llvm/ADT/SmallVector.h"
20
#include "llvm/CodeGen/ISDOpcodes.h"
21
#include "llvm/CodeGen/MachineFunction.h"
22
#include "llvm/CodeGen/MachineJumpTableInfo.h"
23
#include "llvm/CodeGen/MachineMemOperand.h"
24
#include "llvm/CodeGen/RuntimeLibcalls.h"
25
#include "llvm/CodeGen/SelectionDAG.h"
26
#include "llvm/CodeGen/SelectionDAGNodes.h"
27
#include "llvm/CodeGen/TargetFrameLowering.h"
28
#include "llvm/CodeGen/TargetLowering.h"
29
#include "llvm/CodeGen/TargetSubtargetInfo.h"
30
#include "llvm/CodeGen/ValueTypes.h"
31
#include "llvm/IR/CallingConv.h"
32
#include "llvm/IR/Constants.h"
33
#include "llvm/IR/DataLayout.h"
34
#include "llvm/IR/DerivedTypes.h"
35
#include "llvm/IR/Function.h"
36
#include "llvm/IR/Metadata.h"
37
#include "llvm/IR/Type.h"
38
#include "llvm/Support/Casting.h"
39
#include "llvm/Support/Compiler.h"
40
#include "llvm/Support/Debug.h"
41
#include "llvm/Support/ErrorHandling.h"
42
#include "llvm/Support/MachineValueType.h"
43
#include "llvm/Support/MathExtras.h"
44
#include "llvm/Support/raw_ostream.h"
45
#include "llvm/Target/TargetMachine.h"
46
#include "llvm/Target/TargetOptions.h"
47
#include <algorithm>
48
#include <cassert>
49
#include <cstdint>
50
#include <tuple>
51
#include <utility>
52
53
using namespace llvm;
54
55
#define DEBUG_TYPE "legalizedag"
56
57
namespace {
58
59
/// Keeps track of state when getting the sign of a floating-point value as an
60
/// integer.
61
struct FloatSignAsInt {
62
  EVT FloatVT;
63
  SDValue Chain;
64
  SDValue FloatPtr;
65
  SDValue IntPtr;
66
  MachinePointerInfo IntPointerInfo;
67
  MachinePointerInfo FloatPointerInfo;
68
  SDValue IntValue;
69
  APInt SignMask;
70
  uint8_t SignBit;
71
};
72
73
//===----------------------------------------------------------------------===//
74
/// This takes an arbitrary SelectionDAG as input and
75
/// hacks on it until the target machine can handle it.  This involves
76
/// eliminating value sizes the machine cannot handle (promoting small sizes to
77
/// large sizes or splitting up large values into small values) as well as
78
/// eliminating operations the machine cannot handle.
79
///
80
/// This code also does a small amount of optimization and recognition of idioms
81
/// as part of its processing.  For example, if a target does not support a
82
/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
83
/// will attempt merge setcc and brc instructions into brcc's.
84
class SelectionDAGLegalize {
85
  const TargetMachine &TM;
86
  const TargetLowering &TLI;
87
  SelectionDAG &DAG;
88
89
  /// The set of nodes which have already been legalized. We hold a
90
  /// reference to it in order to update as necessary on node deletion.
91
  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
92
93
  /// A set of all the nodes updated during legalization.
94
  SmallSetVector<SDNode *, 16> *UpdatedNodes;
95
96
2.15k
  EVT getSetCCResultType(EVT VT) const {
97
2.15k
    return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
98
2.15k
  }
99
100
  // Libcall insertion helpers.
101
102
public:
103
  SelectionDAGLegalize(SelectionDAG &DAG,
104
                       SmallPtrSetImpl<SDNode *> &LegalizedNodes,
105
                       SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
106
      : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
107
28.7M
        LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
108
109
  /// Legalizes the given operation.
110
  void LegalizeOp(SDNode *Node);
111
112
private:
113
  SDValue OptimizeFloatStore(StoreSDNode *ST);
114
115
  void LegalizeLoadOps(SDNode *Node);
116
  void LegalizeStoreOps(SDNode *Node);
117
118
  /// Some targets cannot handle a variable
119
  /// insertion index for the INSERT_VECTOR_ELT instruction.  In this case, it
120
  /// is necessary to spill the vector being inserted into to memory, perform
121
  /// the insert there, and then read the result back.
122
  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
123
                                         const SDLoc &dl);
124
  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
125
                                  const SDLoc &dl);
126
127
  /// Return a vector shuffle operation which
128
  /// performs the same shuffe in terms of order or result bytes, but on a type
129
  /// whose vector element type is narrower than the original shuffle type.
130
  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
131
  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
132
                                     SDValue N1, SDValue N2,
133
                                     ArrayRef<int> Mask) const;
134
135
  bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
136
                             bool &NeedInvert, const SDLoc &dl);
137
138
  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
139
140
  std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
141
                                                 SDNode *Node, bool isSigned);
142
  SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
143
                          RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
144
                          RTLIB::Libcall Call_F128,
145
                          RTLIB::Libcall Call_PPCF128);
146
  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
147
                           RTLIB::Libcall Call_I8,
148
                           RTLIB::Libcall Call_I16,
149
                           RTLIB::Libcall Call_I32,
150
                           RTLIB::Libcall Call_I64,
151
                           RTLIB::Libcall Call_I128);
152
  SDValue ExpandArgFPLibCall(SDNode *Node,
153
                             RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
154
                             RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
155
                             RTLIB::Libcall Call_PPCF128);
156
  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157
  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
158
159
  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160
                           const SDLoc &dl);
161
  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
162
                           const SDLoc &dl, SDValue ChainIn);
163
  SDValue ExpandBUILD_VECTOR(SDNode *Node);
164
  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
165
  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
166
                                SmallVectorImpl<SDValue> &Results);
167
  void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
168
                         SDValue Value) const;
169
  SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
170
                          SDValue NewIntValue) const;
171
  SDValue ExpandFCOPYSIGN(SDNode *Node) const;
172
  SDValue ExpandFABS(SDNode *Node) const;
173
  SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
174
                               const SDLoc &dl);
175
  SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
176
                                const SDLoc &dl);
177
  SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
178
                                const SDLoc &dl);
179
180
  SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
181
  SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
182
183
  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
184
  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
185
  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
186
187
  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
188
  SDValue ExpandConstant(ConstantSDNode *CP);
189
190
  // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
191
  bool ExpandNode(SDNode *Node);
192
  void ConvertNodeToLibcall(SDNode *Node);
193
  void PromoteNode(SDNode *Node);
194
195
public:
196
  // Node replacement helpers
197
198
1.87M
  void ReplacedNode(SDNode *N) {
199
1.87M
    LegalizedNodes.erase(N);
200
1.87M
    if (UpdatedNodes)
201
11.9k
      UpdatedNodes->insert(N);
202
1.87M
  }
203
204
336
  void ReplaceNode(SDNode *Old, SDNode *New) {
205
336
    LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
206
336
               dbgs() << "     with:      "; New->dump(&DAG));
207
336
208
336
    assert(Old->getNumValues() == New->getNumValues() &&
209
336
           "Replacing one node with another that produces a different number "
210
336
           "of values!");
211
336
    DAG.ReplaceAllUsesWith(Old, New);
212
336
    if (UpdatedNodes)
213
5
      UpdatedNodes->insert(New);
214
336
    ReplacedNode(Old);
215
336
  }
216
217
1.62M
  void ReplaceNode(SDValue Old, SDValue New) {
218
1.62M
    LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
219
1.62M
               dbgs() << "     with:      "; New->dump(&DAG));
220
1.62M
221
1.62M
    DAG.ReplaceAllUsesWith(Old, New);
222
1.62M
    if (UpdatedNodes)
223
6.28k
      UpdatedNodes->insert(New.getNode());
224
1.62M
    ReplacedNode(Old.getNode());
225
1.62M
  }
226
227
196k
  void ReplaceNode(SDNode *Old, const SDValue *New) {
228
196k
    LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
229
196k
230
196k
    DAG.ReplaceAllUsesWith(Old, New);
231
461k
    for (unsigned i = 0, e = Old->getNumValues(); i != e; 
++i265k
) {
232
265k
      LLVM_DEBUG(dbgs() << (i == 0 ? "     with:      " : "      and:      ");
233
265k
                 New[i]->dump(&DAG));
234
265k
      if (UpdatedNodes)
235
7.12k
        UpdatedNodes->insert(New[i].getNode());
236
265k
    }
237
196k
    ReplacedNode(Old);
238
196k
  }
239
};
240
241
} // end anonymous namespace
242
243
/// Return a vector shuffle operation which
244
/// performs the same shuffle in terms of order or result bytes, but on a type
245
/// whose vector element type is narrower than the original shuffle type.
246
/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
247
SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
248
    EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
249
1.19k
    ArrayRef<int> Mask) const {
250
1.19k
  unsigned NumMaskElts = VT.getVectorNumElements();
251
1.19k
  unsigned NumDestElts = NVT.getVectorNumElements();
252
1.19k
  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
253
1.19k
254
1.19k
  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
255
1.19k
256
1.19k
  if (NumEltsGrowth == 1)
257
0
    return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
258
1.19k
259
1.19k
  SmallVector<int, 8> NewMask;
260
9.49k
  for (unsigned i = 0; i != NumMaskElts; 
++i8.29k
) {
261
8.29k
    int Idx = Mask[i];
262
29.6k
    for (unsigned j = 0; j != NumEltsGrowth; 
++j21.3k
) {
263
21.3k
      if (Idx < 0)
264
6.25k
        NewMask.push_back(-1);
265
15.1k
      else
266
15.1k
        NewMask.push_back(Idx * NumEltsGrowth + j);
267
21.3k
    }
268
8.29k
  }
269
1.19k
  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
270
1.19k
  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
271
1.19k
  return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
272
1.19k
}
273
274
/// Expands the ConstantFP node to an integer constant or
275
/// a load from the constant pool.
276
SDValue
277
12.4k
SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
278
12.4k
  bool Extend = false;
279
12.4k
  SDLoc dl(CFP);
280
12.4k
281
12.4k
  // If a FP immediate is precise when represented as a float and if the
282
12.4k
  // target can do an extending load from float to double, we put it into
283
12.4k
  // the constant pool as a float, even if it's is statically typed as a
284
12.4k
  // double.  This shrinks FP constants and canonicalizes them for targets where
285
12.4k
  // an FP extending load is the same cost as a normal load (such as on the x87
286
12.4k
  // fp stack or PPC FP unit).
287
12.4k
  EVT VT = CFP->getValueType(0);
288
12.4k
  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
289
12.4k
  if (!UseCP) {
290
0
    assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
291
0
    return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
292
0
                           (VT == MVT::f64) ? MVT::i64 : MVT::i32);
293
0
  }
294
12.4k
295
12.4k
  APFloat APF = CFP->getValueAPF();
296
12.4k
  EVT OrigVT = VT;
297
12.4k
  EVT SVT = VT;
298
12.4k
299
12.4k
  // We don't want to shrink SNaNs. Converting the SNaN back to its real type
300
12.4k
  // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
301
12.4k
  if (!APF.isSignaling()) {
302
21.8k
    while (SVT != MVT::f32 && 
SVT != MVT::f169.69k
) {
303
9.38k
      SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
304
9.38k
      if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
305
9.38k
          // Only do this if the target has a native EXTLOAD instruction from
306
9.38k
          // smaller type.
307
9.38k
          
TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT)3.19k
&&
308
9.38k
          
TLI.ShouldShrinkFPConstant(OrigVT)1.39k
) {
309
1.32k
        Type *SType = SVT.getTypeForEVT(*DAG.getContext());
310
1.32k
        LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
311
1.32k
        VT = SVT;
312
1.32k
        Extend = true;
313
1.32k
      }
314
9.38k
    }
315
12.4k
  }
316
12.4k
317
12.4k
  SDValue CPIdx =
318
12.4k
      DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
319
12.4k
  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
320
12.4k
  if (Extend) {
321
992
    SDValue Result = DAG.getExtLoad(
322
992
        ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
323
992
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
324
992
        Alignment);
325
992
    return Result;
326
992
  }
327
11.4k
  SDValue Result = DAG.getLoad(
328
11.4k
      OrigVT, dl, DAG.getEntryNode(), CPIdx,
329
11.4k
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
330
11.4k
  return Result;
331
11.4k
}
332
333
/// Expands the Constant node to a load from the constant pool.
334
0
SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
335
0
  SDLoc dl(CP);
336
0
  EVT VT = CP->getValueType(0);
337
0
  SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
338
0
                                      TLI.getPointerTy(DAG.getDataLayout()));
339
0
  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
340
0
  SDValue Result = DAG.getLoad(
341
0
      VT, dl, DAG.getEntryNode(), CPIdx,
342
0
      MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
343
0
  return Result;
344
0
}
345
346
/// Some target cannot handle a variable insertion index for the
347
/// INSERT_VECTOR_ELT instruction.  In this case, it
348
/// is necessary to spill the vector being inserted into to memory, perform
349
/// the insert there, and then read the result back.
350
SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
351
                                                             SDValue Val,
352
                                                             SDValue Idx,
353
32
                                                             const SDLoc &dl) {
354
32
  SDValue Tmp1 = Vec;
355
32
  SDValue Tmp2 = Val;
356
32
  SDValue Tmp3 = Idx;
357
32
358
32
  // If the target doesn't support this, we have to spill the input vector
359
32
  // to a temporary stack slot, update the element, then reload it.  This is
360
32
  // badness.  We could also load the value into a vector register (either
361
32
  // with a "move to register" or "extload into register" instruction, then
362
32
  // permute it into place, if the idx is a constant and if the idx is
363
32
  // supported by the target.
364
32
  EVT VT    = Tmp1.getValueType();
365
32
  EVT EltVT = VT.getVectorElementType();
366
32
  SDValue StackPtr = DAG.CreateStackTemporary(VT);
367
32
368
32
  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
369
32
370
32
  // Store the vector.
371
32
  SDValue Ch = DAG.getStore(
372
32
      DAG.getEntryNode(), dl, Tmp1, StackPtr,
373
32
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
374
32
375
32
  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
376
32
377
32
  // Store the scalar value.
378
32
  Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
379
32
  // Load the updated vector.
380
32
  return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
381
32
                                               DAG.getMachineFunction(), SPFI));
382
32
}
383
384
SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
385
                                                      SDValue Idx,
386
1.88k
                                                      const SDLoc &dl) {
387
1.88k
  if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
388
1.85k
    // SCALAR_TO_VECTOR requires that the type of the value being inserted
389
1.85k
    // match the element type of the vector being created, except for
390
1.85k
    // integers in which case the inserted value can be over width.
391
1.85k
    EVT EltVT = Vec.getValueType().getVectorElementType();
392
1.85k
    if (Val.getValueType() == EltVT ||
393
1.85k
        
(1
EltVT.isInteger()1
&&
Val.getValueType().bitsGE(EltVT)1
)) {
394
1.85k
      SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
395
1.85k
                                  Vec.getValueType(), Val);
396
1.85k
397
1.85k
      unsigned NumElts = Vec.getValueType().getVectorNumElements();
398
1.85k
      // We generate a shuffle of InVec and ScVec, so the shuffle mask
399
1.85k
      // should be 0,1,2,3,4,5... with the appropriate element replaced with
400
1.85k
      // elt 0 of the RHS.
401
1.85k
      SmallVector<int, 8> ShufOps;
402
7.96k
      for (unsigned i = 0; i != NumElts; 
++i6.10k
)
403
6.10k
        ShufOps.push_back(i != InsertPos->getZExtValue() ? 
i4.24k
:
NumElts1.85k
);
404
1.85k
405
1.85k
      return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
406
1.85k
    }
407
32
  }
408
32
  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
409
32
}
410
411
1.77M
SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
412
1.77M
  LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
413
1.77M
  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
414
1.77M
  // FIXME: We shouldn't do this for TargetConstantFP's.
415
1.77M
  // FIXME: move this to the DAG Combiner!  Note that we can't regress due
416
1.77M
  // to phase ordering between legalized code and the dag combiner.  This
417
1.77M
  // probably means that we need to integrate dag combiner and legalizer
418
1.77M
  // together.
419
1.77M
  // We generally can't do this one for long doubles.
420
1.77M
  SDValue Chain = ST->getChain();
421
1.77M
  SDValue Ptr = ST->getBasePtr();
422
1.77M
  unsigned Alignment = ST->getAlignment();
423
1.77M
  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
424
1.77M
  AAMDNodes AAInfo = ST->getAAInfo();
425
1.77M
  SDLoc dl(ST);
426
1.77M
  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
427
226
    if (CFP->getValueType(0) == MVT::f32 &&
428
226
        
TLI.isTypeLegal(MVT::i32)0
) {
429
0
      SDValue Con = DAG.getConstant(CFP->getValueAPF().
430
0
                                      bitcastToAPInt().zextOrTrunc(32),
431
0
                                    SDLoc(CFP), MVT::i32);
432
0
      return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
433
0
                          MMOFlags, AAInfo);
434
0
    }
435
226
436
226
    if (CFP->getValueType(0) == MVT::f64) {
437
13
      // If this target supports 64-bit registers, do a single 64-bit store.
438
13
      if (TLI.isTypeLegal(MVT::i64)) {
439
5
        SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
440
5
                                      zextOrTrunc(64), SDLoc(CFP), MVT::i64);
441
5
        return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
442
5
                            Alignment, MMOFlags, AAInfo);
443
5
      }
444
8
445
8
      if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
446
0
        // Otherwise, if the target supports 32-bit registers, use 2 32-bit
447
0
        // stores.  If the target supports neither 32- nor 64-bits, this
448
0
        // xform is certainly not worth it.
449
0
        const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
450
0
        SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
451
0
        SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
452
0
        if (DAG.getDataLayout().isBigEndian())
453
0
          std::swap(Lo, Hi);
454
0
455
0
        Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
456
0
                          MMOFlags, AAInfo);
457
0
        Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
458
0
                          DAG.getConstant(4, dl, Ptr.getValueType()));
459
0
        Hi = DAG.getStore(Chain, dl, Hi, Ptr,
460
0
                          ST->getPointerInfo().getWithOffset(4),
461
0
                          MinAlign(Alignment, 4U), MMOFlags, AAInfo);
462
0
463
0
        return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
464
0
      }
465
1.77M
    }
466
226
  }
467
1.77M
  return SDValue(nullptr, 0);
468
1.77M
}
469
470
2.00M
void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
471
2.00M
  StoreSDNode *ST = cast<StoreSDNode>(Node);
472
2.00M
  SDValue Chain = ST->getChain();
473
2.00M
  SDValue Ptr = ST->getBasePtr();
474
2.00M
  SDLoc dl(Node);
475
2.00M
476
2.00M
  unsigned Alignment = ST->getAlignment();
477
2.00M
  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
478
2.00M
  AAMDNodes AAInfo = ST->getAAInfo();
479
2.00M
480
2.00M
  if (!ST->isTruncatingStore()) {
481
1.77M
    LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
482
1.77M
    if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
483
5
      ReplaceNode(ST, OptStore);
484
5
      return;
485
5
    }
486
1.77M
487
1.77M
    SDValue Value = ST->getValue();
488
1.77M
    MVT VT = Value.getSimpleValueType();
489
1.77M
    switch (TLI.getOperationAction(ISD::STORE, VT)) {
490
1.77M
    
default: 0
llvm_unreachable0
("This action is not supported yet!");
491
1.77M
    case TargetLowering::Legal: {
492
1.48M
      // If this is an unaligned store and the target doesn't support it,
493
1.48M
      // expand it.
494
1.48M
      EVT MemVT = ST->getMemoryVT();
495
1.48M
      const DataLayout &DL = DAG.getDataLayout();
496
1.48M
      if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
497
1.48M
                                  *ST->getMemOperand())) {
498
226
        LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
499
226
        SDValue Result = TLI.expandUnalignedStore(ST, DAG);
500
226
        ReplaceNode(SDValue(ST, 0), Result);
501
226
      } else
502
1.48M
        LLVM_DEBUG(dbgs() << "Legal store\n");
503
1.48M
      break;
504
1.77M
    }
505
1.77M
    case TargetLowering::Custom: {
506
148k
      LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
507
148k
      SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
508
148k
      if (Res && 
Res != SDValue(Node, 0)20.1k
)
509
17.6k
        ReplaceNode(SDValue(Node, 0), Res);
510
148k
      return;
511
1.77M
    }
512
1.77M
    case TargetLowering::Promote: {
513
148k
      MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
514
148k
      assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
515
148k
             "Can only promote stores to same size type");
516
148k
      Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
517
148k
      SDValue Result =
518
148k
          DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
519
148k
                       Alignment, MMOFlags, AAInfo);
520
148k
      ReplaceNode(SDValue(Node, 0), Result);
521
148k
      break;
522
1.63M
    }
523
1.63M
    }
524
1.63M
    return;
525
1.63M
  }
526
223k
527
223k
  LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
528
223k
  SDValue Value = ST->getValue();
529
223k
  EVT StVT = ST->getMemoryVT();
530
223k
  unsigned StWidth = StVT.getSizeInBits();
531
223k
  auto &DL = DAG.getDataLayout();
532
223k
533
223k
  if (StWidth != StVT.getStoreSizeInBits()) {
534
10.6k
    // Promote to a byte-sized store with upper bits zero if not
535
10.6k
    // storing an integral number of bytes.  For example, promote
536
10.6k
    // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
537
10.6k
    EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
538
10.6k
                                StVT.getStoreSizeInBits());
539
10.6k
    Value = DAG.getZeroExtendInReg(Value, dl, StVT);
540
10.6k
    SDValue Result =
541
10.6k
        DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
542
10.6k
                          Alignment, MMOFlags, AAInfo);
543
10.6k
    ReplaceNode(SDValue(Node, 0), Result);
544
212k
  } else if (StWidth & (StWidth - 1)) {
545
1.01k
    // If not storing a power-of-2 number of bits, expand as two stores.
546
1.01k
    assert(!StVT.isVector() && "Unsupported truncstore!");
547
1.01k
    unsigned LogStWidth = Log2_32(StWidth);
548
1.01k
    assert(LogStWidth < 32);
549
1.01k
    unsigned RoundWidth = 1 << LogStWidth;
550
1.01k
    assert(RoundWidth < StWidth);
551
1.01k
    unsigned ExtraWidth = StWidth - RoundWidth;
552
1.01k
    assert(ExtraWidth < RoundWidth);
553
1.01k
    assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
554
1.01k
           "Store size not an integral number of bytes!");
555
1.01k
    EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
556
1.01k
    EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
557
1.01k
    SDValue Lo, Hi;
558
1.01k
    unsigned IncrementSize;
559
1.01k
560
1.01k
    if (DL.isLittleEndian()) {
561
1.00k
      // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
562
1.00k
      // Store the bottom RoundWidth bits.
563
1.00k
      Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
564
1.00k
                             RoundVT, Alignment, MMOFlags, AAInfo);
565
1.00k
566
1.00k
      // Store the remaining ExtraWidth bits.
567
1.00k
      IncrementSize = RoundWidth / 8;
568
1.00k
      Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
569
1.00k
                        DAG.getConstant(IncrementSize, dl,
570
1.00k
                                        Ptr.getValueType()));
571
1.00k
      Hi = DAG.getNode(
572
1.00k
          ISD::SRL, dl, Value.getValueType(), Value,
573
1.00k
          DAG.getConstant(RoundWidth, dl,
574
1.00k
                          TLI.getShiftAmountTy(Value.getValueType(), DL)));
575
1.00k
      Hi = DAG.getTruncStore(
576
1.00k
          Chain, dl, Hi, Ptr,
577
1.00k
          ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
578
1.00k
          MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
579
1.00k
    } else {
580
12
      // Big endian - avoid unaligned stores.
581
12
      // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
582
12
      // Store the top RoundWidth bits.
583
12
      Hi = DAG.getNode(
584
12
          ISD::SRL, dl, Value.getValueType(), Value,
585
12
          DAG.getConstant(ExtraWidth, dl,
586
12
                          TLI.getShiftAmountTy(Value.getValueType(), DL)));
587
12
      Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
588
12
                             RoundVT, Alignment, MMOFlags, AAInfo);
589
12
590
12
      // Store the remaining ExtraWidth bits.
591
12
      IncrementSize = RoundWidth / 8;
592
12
      Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
593
12
                        DAG.getConstant(IncrementSize, dl,
594
12
                                        Ptr.getValueType()));
595
12
      Lo = DAG.getTruncStore(
596
12
          Chain, dl, Value, Ptr,
597
12
          ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
598
12
          MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
599
12
    }
600
1.01k
601
1.01k
    // The order of the stores doesn't matter.
602
1.01k
    SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
603
1.01k
    ReplaceNode(SDValue(Node, 0), Result);
604
211k
  } else {
605
211k
    switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
606
211k
    
default: 0
llvm_unreachable0
("This action is not supported yet!");
607
211k
    case TargetLowering::Legal: {
608
207k
      EVT MemVT = ST->getMemoryVT();
609
207k
      // If this is an unaligned store and the target doesn't support it,
610
207k
      // expand it.
611
207k
      if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
612
207k
                                  *ST->getMemOperand())) {
613
440
        SDValue Result = TLI.expandUnalignedStore(ST, DAG);
614
440
        ReplaceNode(SDValue(ST, 0), Result);
615
440
      }
616
207k
      break;
617
211k
    }
618
211k
    case TargetLowering::Custom: {
619
1.72k
      SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
620
1.72k
      if (Res && 
Res != SDValue(Node, 0)1.55k
)
621
1.55k
        ReplaceNode(SDValue(Node, 0), Res);
622
1.72k
      return;
623
211k
    }
624
211k
    case TargetLowering::Expand:
625
1.90k
      assert(!StVT.isVector() &&
626
1.90k
             "Vector Stores are handled in LegalizeVectorOps");
627
1.90k
628
1.90k
      SDValue Result;
629
1.90k
630
1.90k
      // TRUNCSTORE:i16 i32 -> STORE i16
631
1.90k
      if (TLI.isTypeLegal(StVT)) {
632
1.87k
        Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
633
1.87k
        Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
634
1.87k
                              Alignment, MMOFlags, AAInfo);
635
1.87k
      } else {
636
28
        // The in-memory type isn't legal. Truncate to the type it would promote
637
28
        // to, and then do a truncstore.
638
28
        Value = DAG.getNode(ISD::TRUNCATE, dl,
639
28
                            TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
640
28
                            Value);
641
28
        Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
642
28
                                   StVT, Alignment, MMOFlags, AAInfo);
643
28
      }
644
1.90k
645
1.90k
      ReplaceNode(SDValue(Node, 0), Result);
646
1.90k
      break;
647
211k
    }
648
211k
  }
649
223k
}
650
651
1.83M
void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
652
1.83M
  LoadSDNode *LD = cast<LoadSDNode>(Node);
653
1.83M
  SDValue Chain = LD->getChain();  // The chain.
654
1.83M
  SDValue Ptr = LD->getBasePtr();  // The base pointer.
655
1.83M
  SDValue Value;                   // The value returned by the load op.
656
1.83M
  SDLoc dl(Node);
657
1.83M
658
1.83M
  ISD::LoadExtType ExtType = LD->getExtensionType();
659
1.83M
  if (ExtType == ISD::NON_EXTLOAD) {
660
1.46M
    LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
661
1.46M
    MVT VT = Node->getSimpleValueType(0);
662
1.46M
    SDValue RVal = SDValue(Node, 0);
663
1.46M
    SDValue RChain = SDValue(Node, 1);
664
1.46M
665
1.46M
    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
666
1.46M
    
default: 0
llvm_unreachable0
("This action is not supported yet!");
667
1.46M
    case TargetLowering::Legal: {
668
1.27M
      EVT MemVT = LD->getMemoryVT();
669
1.27M
      const DataLayout &DL = DAG.getDataLayout();
670
1.27M
      // If this is an unaligned load and the target doesn't support it,
671
1.27M
      // expand it.
672
1.27M
      if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
673
1.27M
                                  *LD->getMemOperand())) {
674
291
        std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
675
291
      }
676
1.27M
      break;
677
1.46M
    }
678
1.46M
    case TargetLowering::Custom:
679
148k
      if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
680
19.1k
        RVal = Res;
681
19.1k
        RChain = Res.getValue(1);
682
19.1k
      }
683
148k
      break;
684
1.46M
685
1.46M
    case TargetLowering::Promote: {
686
38.2k
      MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
687
38.2k
      assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
688
38.2k
             "Can only promote loads to same size type");
689
38.2k
690
38.2k
      SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
691
38.2k
      RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
692
38.2k
      RChain = Res.getValue(1);
693
38.2k
      break;
694
1.46M
    }
695
1.46M
    }
696
1.46M
    if (RChain.getNode() != Node) {
697
50.7k
      assert(RVal.getNode() != Node && "Load must be completely replaced");
698
50.7k
      DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
699
50.7k
      DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
700
50.7k
      if (UpdatedNodes) {
701
40
        UpdatedNodes->insert(RVal.getNode());
702
40
        UpdatedNodes->insert(RChain.getNode());
703
40
      }
704
50.7k
      ReplacedNode(Node);
705
50.7k
    }
706
1.46M
    return;
707
1.46M
  }
708
369k
709
369k
  LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
710
369k
  EVT SrcVT = LD->getMemoryVT();
711
369k
  unsigned SrcWidth = SrcVT.getSizeInBits();
712
369k
  unsigned Alignment = LD->getAlignment();
713
369k
  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
714
369k
  AAMDNodes AAInfo = LD->getAAInfo();
715
369k
716
369k
  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
717
369k
      // Some targets pretend to have an i1 loading operation, and actually
718
369k
      // load an i8.  This trick is correct for ZEXTLOAD because the top 7
719
369k
      // bits are guaranteed to be zero; it helps the optimizers understand
720
369k
      // that these bits are zero.  It is also useful for EXTLOAD, since it
721
369k
      // tells the optimizers that those bits are undefined.  It would be
722
369k
      // nice to have an effective generic way of getting these benefits...
723
369k
      // Until such a way is found, don't insist on promoting i1 here.
724
369k
      
(2.15k
SrcVT != MVT::i12.15k
||
725
2.15k
       TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
726
1.77k
         TargetLowering::Promote)) {
727
550
    // Promote to a byte-sized load if not loading an integral number of
728
550
    // bytes.  For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
729
550
    unsigned NewWidth = SrcVT.getStoreSizeInBits();
730
550
    EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
731
550
    SDValue Ch;
732
550
733
550
    // The extra bits are guaranteed to be zero, since we stored them that
734
550
    // way.  A zext load from NVT thus automatically gives zext from SrcVT.
735
550
736
550
    ISD::LoadExtType NewExtType =
737
550
      ExtType == ISD::ZEXTLOAD ? 
ISD::ZEXTLOAD149
:
ISD::EXTLOAD401
;
738
550
739
550
    SDValue Result =
740
550
        DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
741
550
                       LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
742
550
743
550
    Ch = Result.getValue(1); // The chain.
744
550
745
550
    if (ExtType == ISD::SEXTLOAD)
746
103
      // Having the top bits zero doesn't help when sign extending.
747
103
      Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
748
103
                           Result.getValueType(),
749
103
                           Result, DAG.getValueType(SrcVT));
750
447
    else if (ExtType == ISD::ZEXTLOAD || 
NVT == Result.getValueType()298
)
751
206
      // All the top bits are guaranteed to be zero - inform the optimizers.
752
206
      Result = DAG.getNode(ISD::AssertZext, dl,
753
206
                           Result.getValueType(), Result,
754
206
                           DAG.getValueType(SrcVT));
755
550
756
550
    Value = Result;
757
550
    Chain = Ch;
758
368k
  } else if (SrcWidth & (SrcWidth - 1)) {
759
531
    // If not loading a power-of-2 number of bits, expand as two loads.
760
531
    assert(!SrcVT.isVector() && "Unsupported extload!");
761
531
    unsigned LogSrcWidth = Log2_32(SrcWidth);
762
531
    assert(LogSrcWidth < 32);
763
531
    unsigned RoundWidth = 1 << LogSrcWidth;
764
531
    assert(RoundWidth < SrcWidth);
765
531
    unsigned ExtraWidth = SrcWidth - RoundWidth;
766
531
    assert(ExtraWidth < RoundWidth);
767
531
    assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
768
531
           "Load size not an integral number of bytes!");
769
531
    EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
770
531
    EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
771
531
    SDValue Lo, Hi, Ch;
772
531
    unsigned IncrementSize;
773
531
    auto &DL = DAG.getDataLayout();
774
531
775
531
    if (DL.isLittleEndian()) {
776
505
      // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
777
505
      // Load the bottom RoundWidth bits.
778
505
      Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
779
505
                          LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
780
505
                          AAInfo);
781
505
782
505
      // Load the remaining ExtraWidth bits.
783
505
      IncrementSize = RoundWidth / 8;
784
505
      Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
785
505
                         DAG.getConstant(IncrementSize, dl,
786
505
                                         Ptr.getValueType()));
787
505
      Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
788
505
                          LD->getPointerInfo().getWithOffset(IncrementSize),
789
505
                          ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
790
505
                          AAInfo);
791
505
792
505
      // Build a factor node to remember that this load is independent of
793
505
      // the other one.
794
505
      Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
795
505
                       Hi.getValue(1));
796
505
797
505
      // Move the top bits to the right place.
798
505
      Hi = DAG.getNode(
799
505
          ISD::SHL, dl, Hi.getValueType(), Hi,
800
505
          DAG.getConstant(RoundWidth, dl,
801
505
                          TLI.getShiftAmountTy(Hi.getValueType(), DL)));
802
505
803
505
      // Join the hi and lo parts.
804
505
      Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
805
505
    } else {
806
26
      // Big endian - avoid unaligned loads.
807
26
      // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
808
26
      // Load the top RoundWidth bits.
809
26
      Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
810
26
                          LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
811
26
                          AAInfo);
812
26
813
26
      // Load the remaining ExtraWidth bits.
814
26
      IncrementSize = RoundWidth / 8;
815
26
      Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
816
26
                         DAG.getConstant(IncrementSize, dl,
817
26
                                         Ptr.getValueType()));
818
26
      Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
819
26
                          LD->getPointerInfo().getWithOffset(IncrementSize),
820
26
                          ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
821
26
                          AAInfo);
822
26
823
26
      // Build a factor node to remember that this load is independent of
824
26
      // the other one.
825
26
      Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
826
26
                       Hi.getValue(1));
827
26
828
26
      // Move the top bits to the right place.
829
26
      Hi = DAG.getNode(
830
26
          ISD::SHL, dl, Hi.getValueType(), Hi,
831
26
          DAG.getConstant(ExtraWidth, dl,
832
26
                          TLI.getShiftAmountTy(Hi.getValueType(), DL)));
833
26
834
26
      // Join the hi and lo parts.
835
26
      Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
836
26
    }
837
531
838
531
    Chain = Ch;
839
368k
  } else {
840
368k
    bool isCustom = false;
841
368k
    switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
842
368k
                                 SrcVT.getSimpleVT())) {
843
368k
    
default: 0
llvm_unreachable0
("This action is not supported yet!");
844
368k
    case TargetLowering::Custom:
845
6.51k
      isCustom = true;
846
6.51k
      LLVM_FALLTHROUGH;
847
367k
    case TargetLowering::Legal:
848
367k
      Value = SDValue(Node, 0);
849
367k
      Chain = SDValue(Node, 1);
850
367k
851
367k
      if (isCustom) {
852
6.51k
        if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
853
4.47k
          Value = Res;
854
4.47k
          Chain = Res.getValue(1);
855
4.47k
        }
856
361k
      } else {
857
361k
        // If this is an unaligned load and the target doesn't support it,
858
361k
        // expand it.
859
361k
        EVT MemVT = LD->getMemoryVT();
860
361k
        const DataLayout &DL = DAG.getDataLayout();
861
361k
        if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
862
361k
                                    *LD->getMemOperand())) {
863
523
          std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
864
523
        }
865
361k
      }
866
367k
      break;
867
6.51k
868
6.51k
    case TargetLowering::Expand: {
869
481
      EVT DestVT = Node->getValueType(0);
870
481
      if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
871
455
        // If the source type is not legal, see if there is a legal extload to
872
455
        // an intermediate type that we can then extend further.
873
455
        EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
874
455
        if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
875
455
            
TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)93
) {
876
455
          // If we are loading a legal type, this is a non-extload followed by a
877
455
          // full extend.
878
455
          ISD::LoadExtType MidExtType =
879
455
              (LoadVT == SrcVT) ? 
ISD::NON_EXTLOAD362
:
ExtType93
;
880
455
881
455
          SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
882
455
                                        SrcVT, LD->getMemOperand());
883
455
          unsigned ExtendOp =
884
455
              ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
885
455
          Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
886
455
          Chain = Load.getValue(1);
887
455
          break;
888
455
        }
889
0
890
0
        // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
891
0
        // normal undefined upper bits behavior to allow using an in-reg extend
892
0
        // with the illegal FP type, so load as an integer and do the
893
0
        // from-integer conversion.
894
0
        if (SrcVT.getScalarType() == MVT::f16) {
895
0
          EVT ISrcVT = SrcVT.changeTypeToInteger();
896
0
          EVT IDestVT = DestVT.changeTypeToInteger();
897
0
          EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
898
0
899
0
          SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
900
0
                                          Chain, Ptr, ISrcVT,
901
0
                                          LD->getMemOperand());
902
0
          Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
903
0
          Chain = Result.getValue(1);
904
0
          break;
905
0
        }
906
26
      }
907
26
908
26
      assert(!SrcVT.isVector() &&
909
26
             "Vector Loads are handled in LegalizeVectorOps");
910
26
911
26
      // FIXME: This does not work for vectors on most targets.  Sign-
912
26
      // and zero-extend operations are currently folded into extending
913
26
      // loads, whether they are legal or not, and then we end up here
914
26
      // without any support for legalizing them.
915
26
      assert(ExtType != ISD::EXTLOAD &&
916
26
             "EXTLOAD should always be supported!");
917
26
      // Turn the unsupported load into an EXTLOAD followed by an
918
26
      // explicit zero/sign extend inreg.
919
26
      SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
920
26
                                      Node->getValueType(0),
921
26
                                      Chain, Ptr, SrcVT,
922
26
                                      LD->getMemOperand());
923
26
      SDValue ValRes;
924
26
      if (ExtType == ISD::SEXTLOAD)
925
25
        ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
926
25
                             Result.getValueType(),
927
25
                             Result, DAG.getValueType(SrcVT));
928
1
      else
929
1
        ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
930
26
      Value = ValRes;
931
26
      Chain = Result.getValue(1);
932
26
      break;
933
26
    }
934
368k
    }
935
368k
  }
936
369k
937
369k
  // Since loads produce two values, make sure to remember that we legalized
938
369k
  // both of them.
939
369k
  if (Chain.getNode() != Node) {
940
6.54k
    assert(Value.getNode() != Node && "Load must be completely replaced");
941
6.54k
    DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
942
6.54k
    DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
943
6.54k
    if (UpdatedNodes) {
944
1
      UpdatedNodes->insert(Value.getNode());
945
1
      UpdatedNodes->insert(Chain.getNode());
946
1
    }
947
6.54k
    ReplacedNode(Node);
948
6.54k
  }
949
369k
}
950
951
/// Return a legal replacement for the given operation, with all legal operands.
952
55.6M
void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
953
55.6M
  LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
954
55.6M
955
55.6M
  // Allow illegal target nodes and illegal registers.
956
55.6M
  if (Node->getOpcode() == ISD::TargetConstant ||
957
55.6M
      
Node->getOpcode() == ISD::Register54.4M
)
958
9.04M
    return;
959
46.6M
960
#ifndef NDEBUG
961
  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
962
    assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
963
              TargetLowering::TypeLegal ||
964
            TLI.isTypeLegal(Node->getValueType(i))) &&
965
           "Unexpected illegal type!");
966
967
  for (const SDValue &Op : Node->op_values())
968
    assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
969
              TargetLowering::TypeLegal ||
970
            TLI.isTypeLegal(Op.getValueType()) ||
971
            Op.getOpcode() == ISD::TargetConstant ||
972
            Op.getOpcode() == ISD::Register) &&
973
            "Unexpected illegal type!");
974
#endif
975
976
46.6M
  // Figure out the correct action; the way to query this varies by opcode
977
46.6M
  TargetLowering::LegalizeAction Action = TargetLowering::Legal;
978
46.6M
  bool SimpleFinishLegalizing = true;
979
46.6M
  switch (Node->getOpcode()) {
980
46.6M
  case ISD::INTRINSIC_W_CHAIN:
981
226k
  case ISD::INTRINSIC_WO_CHAIN:
982
226k
  case ISD::INTRINSIC_VOID:
983
226k
  case ISD::STACKSAVE:
984
226k
    Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
985
226k
    break;
986
226k
  case ISD::GET_DYNAMIC_AREA_OFFSET:
987
4
    Action = TLI.getOperationAction(Node->getOpcode(),
988
4
                                    Node->getValueType(0));
989
4
    break;
990
226k
  case ISD::VAARG:
991
361
    Action = TLI.getOperationAction(Node->getOpcode(),
992
361
                                    Node->getValueType(0));
993
361
    if (Action != TargetLowering::Promote)
994
360
      Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
995
361
    break;
996
628k
  case ISD::FP_TO_FP16:
997
628k
  case ISD::SINT_TO_FP:
998
628k
  case ISD::UINT_TO_FP:
999
628k
  case ISD::EXTRACT_VECTOR_ELT:
1000
628k
  case ISD::LROUND:
1001
628k
  case ISD::LLROUND:
1002
628k
  case ISD::LRINT:
1003
628k
  case ISD::LLRINT:
1004
628k
    Action = TLI.getOperationAction(Node->getOpcode(),
1005
628k
                                    Node->getOperand(0).getValueType());
1006
628k
    break;
1007
628k
  case ISD::FP_ROUND_INREG:
1008
36.8k
  case ISD::SIGN_EXTEND_INREG: {
1009
36.8k
    EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1010
36.8k
    Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1011
36.8k
    break;
1012
36.8k
  }
1013
36.8k
  case ISD::ATOMIC_STORE:
1014
13.4k
    Action = TLI.getOperationAction(Node->getOpcode(),
1015
13.4k
                                    Node->getOperand(2).getValueType());
1016
13.4k
    break;
1017
529k
  case ISD::SELECT_CC:
1018
529k
  case ISD::SETCC:
1019
529k
  case ISD::BR_CC: {
1020
529k
    unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 
442.4k
:
1021
529k
                         
Node->getOpcode() == ISD::SETCC 487k
?
269.5k
:
1417k
;
1022
529k
    unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 
2417k
:
0111k
;
1023
529k
    MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1024
529k
    ISD::CondCode CCCode =
1025
529k
        cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1026
529k
    Action = TLI.getCondCodeAction(CCCode, OpVT);
1027
529k
    if (Action == TargetLowering::Legal) {
1028
528k
      if (Node->getOpcode() == ISD::SELECT_CC)
1029
42.1k
        Action = TLI.getOperationAction(Node->getOpcode(),
1030
42.1k
                                        Node->getValueType(0));
1031
486k
      else
1032
486k
        Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1033
528k
    }
1034
529k
    break;
1035
529k
  }
1036
3.83M
  case ISD::LOAD:
1037
3.83M
  case ISD::STORE:
1038
3.83M
    // FIXME: Model these properly.  LOAD and STORE are complicated, and
1039
3.83M
    // STORE expects the unlegalized operand in some cases.
1040
3.83M
    SimpleFinishLegalizing = false;
1041
3.83M
    break;
1042
3.83M
  case ISD::CALLSEQ_START:
1043
1.58M
  case ISD::CALLSEQ_END:
1044
1.58M
    // FIXME: This shouldn't be necessary.  These nodes have special properties
1045
1.58M
    // dealing with the recursive nature of legalization.  Removing this
1046
1.58M
    // special case should be done as part of making LegalizeDAG non-recursive.
1047
1.58M
    SimpleFinishLegalizing = false;
1048
1.58M
    break;
1049
1.58M
  case ISD::EXTRACT_ELEMENT:
1050
36.5k
  case ISD::FLT_ROUNDS_:
1051
36.5k
  case ISD::MERGE_VALUES:
1052
36.5k
  case ISD::EH_RETURN:
1053
36.5k
  case ISD::FRAME_TO_ARGS_OFFSET:
1054
36.5k
  case ISD::EH_DWARF_CFA:
1055
36.5k
  case ISD::EH_SJLJ_SETJMP:
1056
36.5k
  case ISD::EH_SJLJ_LONGJMP:
1057
36.5k
  case ISD::EH_SJLJ_SETUP_DISPATCH:
1058
36.5k
    // These operations lie about being legal: when they claim to be legal,
1059
36.5k
    // they should actually be expanded.
1060
36.5k
    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1061
36.5k
    if (Action == TargetLowering::Legal)
1062
33.8k
      Action = TargetLowering::Expand;
1063
36.5k
    break;
1064
36.5k
  case ISD::INIT_TRAMPOLINE:
1065
16.9k
  case ISD::ADJUST_TRAMPOLINE:
1066
16.9k
  case ISD::FRAMEADDR:
1067
16.9k
  case ISD::RETURNADDR:
1068
16.9k
  case ISD::ADDROFRETURNADDR:
1069
16.9k
  case ISD::SPONENTRY:
1070
16.9k
    // These operations lie about being legal: when they claim to be legal,
1071
16.9k
    // they should actually be custom-lowered.
1072
16.9k
    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1073
16.9k
    if (Action == TargetLowering::Legal)
1074
16.9k
      Action = TargetLowering::Custom;
1075
16.9k
    break;
1076
16.9k
  case ISD::READCYCLECOUNTER:
1077
33
    // READCYCLECOUNTER returns an i64, even if type legalization might have
1078
33
    // expanded that to several smaller types.
1079
33
    Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1080
33
    break;
1081
16.9k
  case ISD::READ_REGISTER:
1082
780
  case ISD::WRITE_REGISTER:
1083
780
    // Named register is legal in the DAG, but blocked by register name
1084
780
    // selection if not implemented by target (to chose the correct register)
1085
780
    // They'll be converted to Copy(To/From)Reg.
1086
780
    Action = TargetLowering::Legal;
1087
780
    break;
1088
780
  case ISD::DEBUGTRAP:
1089
58
    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1090
58
    if (Action == TargetLowering::Expand) {
1091
2
      // replace ISD::DEBUGTRAP with ISD::TRAP
1092
2
      SDValue NewVal;
1093
2
      NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1094
2
                           Node->getOperand(0));
1095
2
      ReplaceNode(Node, NewVal.getNode());
1096
2
      LegalizeOp(NewVal.getNode());
1097
2
      return;
1098
2
    }
1099
56
    break;
1100
3.50k
  case ISD::STRICT_FADD:
1101
3.50k
  case ISD::STRICT_FSUB:
1102
3.50k
  case ISD::STRICT_FMUL:
1103
3.50k
  case ISD::STRICT_FDIV:
1104
3.50k
  case ISD::STRICT_FREM:
1105
3.50k
  case ISD::STRICT_FSQRT:
1106
3.50k
  case ISD::STRICT_FMA:
1107
3.50k
  case ISD::STRICT_FPOW:
1108
3.50k
  case ISD::STRICT_FPOWI:
1109
3.50k
  case ISD::STRICT_FSIN:
1110
3.50k
  case ISD::STRICT_FCOS:
1111
3.50k
  case ISD::STRICT_FEXP:
1112
3.50k
  case ISD::STRICT_FEXP2:
1113
3.50k
  case ISD::STRICT_FLOG:
1114
3.50k
  case ISD::STRICT_FLOG10:
1115
3.50k
  case ISD::STRICT_FLOG2:
1116
3.50k
  case ISD::STRICT_FRINT:
1117
3.50k
  case ISD::STRICT_FNEARBYINT:
1118
3.50k
  case ISD::STRICT_FMAXNUM:
1119
3.50k
  case ISD::STRICT_FMINNUM:
1120
3.50k
  case ISD::STRICT_FCEIL:
1121
3.50k
  case ISD::STRICT_FFLOOR:
1122
3.50k
  case ISD::STRICT_FROUND:
1123
3.50k
  case ISD::STRICT_FTRUNC:
1124
3.50k
  case ISD::STRICT_FP_ROUND:
1125
3.50k
  case ISD::STRICT_FP_EXTEND:
1126
3.50k
    // These pseudo-ops get legalized as if they were their non-strict
1127
3.50k
    // equivalent.  For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1128
3.50k
    // is also legal, but if ISD::FSQRT requires expansion then so does
1129
3.50k
    // ISD::STRICT_FSQRT.
1130
3.50k
    Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
1131
3.50k
                                            Node->getValueType(0));
1132
3.50k
    break;
1133
4.61k
  case ISD::SADDSAT:
1134
4.61k
  case ISD::UADDSAT:
1135
4.61k
  case ISD::SSUBSAT:
1136
4.61k
  case ISD::USUBSAT: {
1137
4.61k
    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1138
4.61k
    break;
1139
4.61k
  }
1140
4.61k
  case ISD::SMULFIX:
1141
74
  case ISD::SMULFIXSAT:
1142
74
  case ISD::UMULFIX: {
1143
74
    unsigned Scale = Node->getConstantOperandVal(2);
1144
74
    Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1145
74
                                              Node->getValueType(0), Scale);
1146
74
    break;
1147
74
  }
1148
113
  case ISD::MSCATTER:
1149
113
    Action = TLI.getOperationAction(Node->getOpcode(),
1150
113
                    cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1151
113
    break;
1152
1.25k
  case ISD::MSTORE:
1153
1.25k
    Action = TLI.getOperationAction(Node->getOpcode(),
1154
1.25k
                    cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1155
1.25k
    break;
1156
74
  case ISD::VECREDUCE_FADD:
1157
0
  case ISD::VECREDUCE_FMUL:
1158
0
  case ISD::VECREDUCE_ADD:
1159
0
  case ISD::VECREDUCE_MUL:
1160
0
  case ISD::VECREDUCE_AND:
1161
0
  case ISD::VECREDUCE_OR:
1162
0
  case ISD::VECREDUCE_XOR:
1163
0
  case ISD::VECREDUCE_SMAX:
1164
0
  case ISD::VECREDUCE_SMIN:
1165
0
  case ISD::VECREDUCE_UMAX:
1166
0
  case ISD::VECREDUCE_UMIN:
1167
0
  case ISD::VECREDUCE_FMAX:
1168
0
  case ISD::VECREDUCE_FMIN:
1169
0
    Action = TLI.getOperationAction(
1170
0
        Node->getOpcode(), Node->getOperand(0).getValueType());
1171
0
    break;
1172
39.7M
  default:
1173
39.7M
    if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1174
5.73M
      Action = TargetLowering::Legal;
1175
33.9M
    } else {
1176
33.9M
      Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1177
33.9M
    }
1178
39.7M
    break;
1179
46.6M
  }
1180
46.6M
1181
46.6M
  if (SimpleFinishLegalizing) {
1182
41.2M
    SDNode *NewNode = Node;
1183
41.2M
    switch (Node->getOpcode()) {
1184
41.2M
    
default: break40.6M
;
1185
41.2M
    case ISD::SHL:
1186
538k
    case ISD::SRL:
1187
538k
    case ISD::SRA:
1188
538k
    case ISD::ROTL:
1189
538k
    case ISD::ROTR: {
1190
538k
      // Legalizing shifts/rotates requires adjusting the shift amount
1191
538k
      // to the appropriate width.
1192
538k
      SDValue Op0 = Node->getOperand(0);
1193
538k
      SDValue Op1 = Node->getOperand(1);
1194
538k
      if (!Op1.getValueType().isVector()) {
1195
530k
        SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1196
530k
        // The getShiftAmountOperand() may create a new operand node or
1197
530k
        // return the existing one. If new operand is created we need
1198
530k
        // to update the parent node.
1199
530k
        // Do not try to legalize SAO here! It will be automatically legalized
1200
530k
        // in the next round.
1201
530k
        if (SAO != Op1)
1202
28.3k
          NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1203
530k
      }
1204
538k
    }
1205
538k
    break;
1206
538k
    case ISD::FSHL:
1207
3.49k
    case ISD::FSHR:
1208
3.49k
    case ISD::SRL_PARTS:
1209
3.49k
    case ISD::SRA_PARTS:
1210
3.49k
    case ISD::SHL_PARTS: {
1211
3.49k
      // Legalizing shifts/rotates requires adjusting the shift amount
1212
3.49k
      // to the appropriate width.
1213
3.49k
      SDValue Op0 = Node->getOperand(0);
1214
3.49k
      SDValue Op1 = Node->getOperand(1);
1215
3.49k
      SDValue Op2 = Node->getOperand(2);
1216
3.49k
      if (!Op2.getValueType().isVector()) {
1217
3.49k
        SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1218
3.49k
        // The getShiftAmountOperand() may create a new operand node or
1219
3.49k
        // return the existing one. If new operand is created we need
1220
3.49k
        // to update the parent node.
1221
3.49k
        if (SAO != Op2)
1222
108
          NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1223
3.49k
      }
1224
3.49k
      break;
1225
41.2M
    }
1226
41.2M
    }
1227
41.2M
1228
41.2M
    if (NewNode != Node) {
1229
329
      ReplaceNode(Node, NewNode);
1230
329
      Node = NewNode;
1231
329
    }
1232
41.2M
    switch (Action) {
1233
41.2M
    case TargetLowering::Legal:
1234
39.0M
      LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1235
39.0M
      return;
1236
41.2M
    case TargetLowering::Custom:
1237
2.09M
      LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1238
2.09M
      // FIXME: The handling for custom lowering with multiple results is
1239
2.09M
      // a complete mess.
1240
2.09M
      if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1241
1.99M
        if (!(Res.getNode() != Node || 
Res.getResNo() != 0525k
))
1242
525k
          return;
1243
1.47M
1244
1.47M
        if (Node->getNumValues() == 1) {
1245
1.44M
          LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1246
1.44M
          // We can just directly replace this node with the lowered value.
1247
1.44M
          ReplaceNode(SDValue(Node, 0), Res);
1248
1.44M
          return;
1249
1.44M
        }
1250
27.4k
1251
27.4k
        SmallVector<SDValue, 8> ResultVals;
1252
83.5k
        for (unsigned i = 0, e = Node->getNumValues(); i != e; 
++i56.0k
)
1253
56.0k
          ResultVals.push_back(Res.getValue(i));
1254
27.4k
        LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1255
27.4k
        ReplaceNode(Node, ResultVals.data());
1256
27.4k
        return;
1257
27.4k
      }
1258
96.9k
      LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1259
96.9k
      LLVM_FALLTHROUGH;
1260
210k
    case TargetLowering::Expand:
1261
210k
      if (ExpandNode(Node))
1262
145k
        return;
1263
64.7k
      LLVM_FALLTHROUGH;
1264
64.8k
    case TargetLowering::LibCall:
1265
64.8k
      ConvertNodeToLibcall(Node);
1266
64.8k
      return;
1267
64.7k
    case TargetLowering::Promote:
1268
17.7k
      PromoteNode(Node);
1269
17.7k
      return;
1270
5.42M
    }
1271
5.42M
  }
1272
5.42M
1273
5.42M
  switch (Node->getOpcode()) {
1274
5.42M
  default:
1275
#ifndef NDEBUG
1276
    dbgs() << "NODE: ";
1277
    Node->dump( &DAG);
1278
    dbgs() << "\n";
1279
#endif
1280
0
    llvm_unreachable("Do not know how to legalize this operator!");
1281
5.42M
1282
5.42M
  case ISD::CALLSEQ_START:
1283
1.58M
  case ISD::CALLSEQ_END:
1284
1.58M
    break;
1285
1.83M
  case ISD::LOAD:
1286
1.83M
    return LegalizeLoadOps(Node);
1287
2.00M
  case ISD::STORE:
1288
2.00M
    return LegalizeStoreOps(Node);
1289
5.42M
  }
1290
5.42M
}
1291
1292
1.51k
SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1293
1.51k
  SDValue Vec = Op.getOperand(0);
1294
1.51k
  SDValue Idx = Op.getOperand(1);
1295
1.51k
  SDLoc dl(Op);
1296
1.51k
1297
1.51k
  // Before we generate a new store to a temporary stack slot, see if there is
1298
1.51k
  // already one that we can use. There often is because when we scalarize
1299
1.51k
  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1300
1.51k
  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1301
1.51k
  // the vector. If all are expanded here, we don't want one store per vector
1302
1.51k
  // element.
1303
1.51k
1304
1.51k
  // Caches for hasPredecessorHelper
1305
1.51k
  SmallPtrSet<const SDNode *, 32> Visited;
1306
1.51k
  SmallVector<const SDNode *, 16> Worklist;
1307
1.51k
  Visited.insert(Op.getNode());
1308
1.51k
  Worklist.push_back(Idx.getNode());
1309
1.51k
  SDValue StackPtr, Ch;
1310
1.51k
  for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1311
3.06k
       UE = Vec.getNode()->use_end(); UI != UE; 
++UI1.55k
) {
1312
2.60k
    SDNode *User = *UI;
1313
2.60k
    if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1314
1.05k
      if (ST->isIndexed() || ST->isTruncatingStore() ||
1315
1.05k
          ST->getValue() != Vec)
1316
0
        continue;
1317
1.05k
1318
1.05k
      // Make sure that nothing else could have stored into the destination of
1319
1.05k
      // this store.
1320
1.05k
      if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1321
0
        continue;
1322
1.05k
1323
1.05k
      // If the index is dependent on the store we will introduce a cycle when
1324
1.05k
      // creating the load (the load uses the index, and by replacing the chain
1325
1.05k
      // we will make the index dependent on the load). Also, the store might be
1326
1.05k
      // dependent on the extractelement and introduce a cycle when creating
1327
1.05k
      // the load.
1328
1.05k
      if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1329
1.05k
          
ST->hasPredecessor(Op.getNode())1.05k
)
1330
2
        continue;
1331
1.04k
1332
1.04k
      StackPtr = ST->getBasePtr();
1333
1.04k
      Ch = SDValue(ST, 0);
1334
1.04k
      break;
1335
1.04k
    }
1336
2.60k
  }
1337
1.51k
1338
1.51k
  EVT VecVT = Vec.getValueType();
1339
1.51k
1340
1.51k
  if (!Ch.getNode()) {
1341
468
    // Store the value to a temporary stack slot, then LOAD the returned part.
1342
468
    StackPtr = DAG.CreateStackTemporary(VecVT);
1343
468
    Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1344
468
                      MachinePointerInfo());
1345
468
  }
1346
1.51k
1347
1.51k
  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1348
1.51k
1349
1.51k
  SDValue NewLoad;
1350
1.51k
1351
1.51k
  if (Op.getValueType().isVector())
1352
0
    NewLoad =
1353
0
        DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1354
1.51k
  else
1355
1.51k
    NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1356
1.51k
                             MachinePointerInfo(),
1357
1.51k
                             VecVT.getVectorElementType());
1358
1.51k
1359
1.51k
  // Replace the chain going out of the store, by the one out of the load.
1360
1.51k
  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1361
1.51k
1362
1.51k
  // We introduced a cycle though, so update the loads operands, making sure
1363
1.51k
  // to use the original store's chain as an incoming chain.
1364
1.51k
  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1365
1.51k
                                          NewLoad->op_end());
1366
1.51k
  NewLoadOperands[0] = Ch;
1367
1.51k
  NewLoad =
1368
1.51k
      SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1369
1.51k
  return NewLoad;
1370
1.51k
}
1371
1372
0
SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1373
0
  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1374
0
1375
0
  SDValue Vec  = Op.getOperand(0);
1376
0
  SDValue Part = Op.getOperand(1);
1377
0
  SDValue Idx  = Op.getOperand(2);
1378
0
  SDLoc dl(Op);
1379
0
1380
0
  // Store the value to a temporary stack slot, then LOAD the returned part.
1381
0
  EVT VecVT = Vec.getValueType();
1382
0
  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1383
0
  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1384
0
  MachinePointerInfo PtrInfo =
1385
0
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1386
0
1387
0
  // First store the whole vector.
1388
0
  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1389
0
1390
0
  // Then store the inserted part.
1391
0
  SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1392
0
1393
0
  // Store the subvector.
1394
0
  Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1395
0
1396
0
  // Finally, load the updated vector.
1397
0
  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1398
0
}
1399
1400
177
SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1401
177
  // We can't handle this case efficiently.  Allocate a sufficiently
1402
177
  // aligned object on the stack, store each element into it, then load
1403
177
  // the result as a vector.
1404
177
  // Create the stack frame object.
1405
177
  EVT VT = Node->getValueType(0);
1406
177
  EVT EltVT = VT.getVectorElementType();
1407
177
  SDLoc dl(Node);
1408
177
  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1409
177
  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1410
177
  MachinePointerInfo PtrInfo =
1411
177
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1412
177
1413
177
  // Emit a store of each element to the stack slot.
1414
177
  SmallVector<SDValue, 8> Stores;
1415
177
  unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1416
177
  assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1417
177
  // Store (in the right endianness) the elements to memory.
1418
873
  for (unsigned i = 0, e = Node->getNumOperands(); i != e; 
++i696
) {
1419
696
    // Ignore undef elements.
1420
696
    if (Node->getOperand(i).isUndef()) 
continue257
;
1421
439
1422
439
    unsigned Offset = TypeByteSize*i;
1423
439
1424
439
    SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1425
439
    Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1426
439
1427
439
    // If the destination vector element type is narrower than the source
1428
439
    // element type, only store the bits necessary.
1429
439
    if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1430
132
      Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1431
132
                                         Node->getOperand(i), Idx,
1432
132
                                         PtrInfo.getWithOffset(Offset), EltVT));
1433
132
    } else
1434
307
      Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1435
307
                                    Idx, PtrInfo.getWithOffset(Offset)));
1436
439
  }
1437
177
1438
177
  SDValue StoreChain;
1439
177
  if (!Stores.empty())    // Not all undef elements?
1440
177
    StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1441
0
  else
1442
0
    StoreChain = DAG.getEntryNode();
1443
177
1444
177
  // Result is a load from the stack slot.
1445
177
  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1446
177
}
1447
1448
/// Bitcast a floating-point value to an integer value. Only bitcast the part
1449
/// containing the sign bit if the target has no integer value capable of
1450
/// holding all bits of the floating-point value.
1451
void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1452
                                             const SDLoc &DL,
1453
368
                                             SDValue Value) const {
1454
368
  EVT FloatVT = Value.getValueType();
1455
368
  unsigned NumBits = FloatVT.getSizeInBits();
1456
368
  State.FloatVT = FloatVT;
1457
368
  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1458
368
  // Convert to an integer of the same size.
1459
368
  if (TLI.isTypeLegal(IVT)) {
1460
119
    State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1461
119
    State.SignMask = APInt::getSignMask(NumBits);
1462
119
    State.SignBit = NumBits - 1;
1463
119
    return;
1464
119
  }
1465
249
1466
249
  auto &DataLayout = DAG.getDataLayout();
1467
249
  // Store the float to memory, then load the sign part out as an integer.
1468
249
  MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1469
249
  // First create a temporary that is aligned for both the load and store.
1470
249
  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1471
249
  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1472
249
  // Then store the float to it.
1473
249
  State.FloatPtr = StackPtr;
1474
249
  MachineFunction &MF = DAG.getMachineFunction();
1475
249
  State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1476
249
  State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1477
249
                             State.FloatPointerInfo);
1478
249
1479
249
  SDValue IntPtr;
1480
249
  if (DataLayout.isBigEndian()) {
1481
1
    assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1482
1
    // Load out a legal integer with the same sign bit as the float.
1483
1
    IntPtr = StackPtr;
1484
1
    State.IntPointerInfo = State.FloatPointerInfo;
1485
248
  } else {
1486
248
    // Advance the pointer so that the loaded byte will contain the sign bit.
1487
248
    unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1488
248
    IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1489
248
                      DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1490
248
    State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1491
248
                                                             ByteOffset);
1492
248
  }
1493
249
1494
249
  State.IntPtr = IntPtr;
1495
249
  State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1496
249
                                  State.IntPointerInfo, MVT::i8);
1497
249
  State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1498
249
  State.SignBit = 7;
1499
249
}
1500
1501
/// Replace the integer value produced by getSignAsIntValue() with a new value
1502
/// and cast the result back to a floating-point type.
1503
SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1504
                                              const SDLoc &DL,
1505
43
                                              SDValue NewIntValue) const {
1506
43
  if (!State.Chain)
1507
40
    return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1508
3
1509
3
  // Override the part containing the sign bit in the value stored on the stack.
1510
3
  SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1511
3
                                    State.IntPointerInfo, MVT::i8);
1512
3
  return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1513
3
                     State.FloatPointerInfo);
1514
3
}
1515
1516
325
SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1517
325
  SDLoc DL(Node);
1518
325
  SDValue Mag = Node->getOperand(0);
1519
325
  SDValue Sign = Node->getOperand(1);
1520
325
1521
325
  // Get sign bit into an integer value.
1522
325
  FloatSignAsInt SignAsInt;
1523
325
  getSignAsIntValue(SignAsInt, DL, Sign);
1524
325
1525
325
  EVT IntVT = SignAsInt.IntValue.getValueType();
1526
325
  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1527
325
  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1528
325
                                SignMask);
1529
325
1530
325
  // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1531
325
  EVT FloatVT = Mag.getValueType();
1532
325
  if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1533
325
      
TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)283
) {
1534
283
    SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1535
283
    SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1536
283
    SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1537
283
                                DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1538
283
    return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1539
283
  }
1540
42
1541
42
  // Transform Mag value to integer, and clear the sign bit.
1542
42
  FloatSignAsInt MagAsInt;
1543
42
  getSignAsIntValue(MagAsInt, DL, Mag);
1544
42
  EVT MagVT = MagAsInt.IntValue.getValueType();
1545
42
  SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1546
42
  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1547
42
                                    ClearSignMask);
1548
42
1549
42
  // Get the signbit at the right position for MagAsInt.
1550
42
  int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1551
42
  EVT ShiftVT = IntVT;
1552
42
  if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1553
0
    SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1554
0
    ShiftVT = MagVT;
1555
0
  }
1556
42
  if (ShiftAmount > 0) {
1557
22
    SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1558
22
    SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1559
22
  } else 
if (20
ShiftAmount < 020
) {
1560
0
    SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1561
0
    SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1562
0
  }
1563
42
  if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1564
21
    SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1565
21
  }
1566
42
1567
42
  // Store the part with the modified sign and convert back to float.
1568
42
  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1569
42
  return modifySignAsInt(MagAsInt, DL, CopiedSign);
1570
42
}
1571
1572
37
SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1573
37
  SDLoc DL(Node);
1574
37
  SDValue Value = Node->getOperand(0);
1575
37
1576
37
  // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1577
37
  EVT FloatVT = Value.getValueType();
1578
37
  if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1579
36
    SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1580
36
    return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1581
36
  }
1582
1
1583
1
  // Transform value to integer, clear the sign bit and transform back.
1584
1
  FloatSignAsInt ValueAsInt;
1585
1
  getSignAsIntValue(ValueAsInt, DL, Value);
1586
1
  EVT IntVT = ValueAsInt.IntValue.getValueType();
1587
1
  SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1588
1
  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1589
1
                                    ClearSignMask);
1590
1
  return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1591
1
}
1592
1593
void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1594
350
                                           SmallVectorImpl<SDValue> &Results) {
1595
350
  unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1596
350
  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1597
350
          " not tell us which reg is the stack pointer!");
1598
350
  SDLoc dl(Node);
1599
350
  EVT VT = Node->getValueType(0);
1600
350
  SDValue Tmp1 = SDValue(Node, 0);
1601
350
  SDValue Tmp2 = SDValue(Node, 1);
1602
350
  SDValue Tmp3 = Node->getOperand(2);
1603
350
  SDValue Chain = Tmp1.getOperand(0);
1604
350
1605
350
  // Chain the dynamic stack allocation so that it doesn't modify the stack
1606
350
  // pointer when other instructions are using the stack.
1607
350
  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1608
350
1609
350
  SDValue Size  = Tmp2.getOperand(1);
1610
350
  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1611
350
  Chain = SP.getValue(1);
1612
350
  unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1613
350
  unsigned StackAlign =
1614
350
      DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1615
350
  Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size);       // Value
1616
350
  if (Align > StackAlign)
1617
63
    Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1618
63
                       DAG.getConstant(-(uint64_t)Align, dl, VT));
1619
350
  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);     // Output chain
1620
350
1621
350
  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1622
350
                            DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1623
350
1624
350
  Results.push_back(Tmp1);
1625
350
  Results.push_back(Tmp2);
1626
350
}
1627
1628
/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1629
/// target.
1630
///
1631
/// If the SETCC has been legalized using AND / OR, then the legalized node
1632
/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1633
/// will be set to false.
1634
///
1635
/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1636
/// then the values of LHS and RHS will be swapped, CC will be set to the
1637
/// new condition, and NeedInvert will be set to false.
1638
///
1639
/// If the SETCC has been legalized using the inverse condcode, then LHS and
1640
/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1641
/// will be set to true. The caller must invert the result of the SETCC with
1642
/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1643
/// of a true/false result.
1644
///
1645
/// \returns true if the SetCC has been legalized, false if it hasn't.
1646
bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1647
                                                 SDValue &RHS, SDValue &CC,
1648
                                                 bool &NeedInvert,
1649
4.57k
                                                 const SDLoc &dl) {
1650
4.57k
  MVT OpVT = LHS.getSimpleValueType();
1651
4.57k
  ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1652
4.57k
  NeedInvert = false;
1653
4.57k
  bool NeedSwap = false;
1654
4.57k
  switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1655
4.57k
  
default: 0
llvm_unreachable0
("Unknown condition code action!");
1656
4.57k
  case TargetLowering::Legal:
1657
3.77k
    // Nothing to do.
1658
3.77k
    break;
1659
4.57k
  case TargetLowering::Expand: {
1660
797
    ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1661
797
    if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1662
200
      std::swap(LHS, RHS);
1663
200
      CC = DAG.getCondCode(InvCC);
1664
200
      return true;
1665
200
    }
1666
597
    // Swapping operands didn't work. Try inverting the condition.
1667
597
    InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
1668
597
    if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1669
414
      // If inverting the condition is not enough, try swapping operands
1670
414
      // on top of it.
1671
414
      InvCC = ISD::getSetCCSwappedOperands(InvCC);
1672
414
      NeedSwap = true;
1673
414
    }
1674
597
    if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1675
222
      CC = DAG.getCondCode(InvCC);
1676
222
      NeedInvert = true;
1677
222
      if (NeedSwap)
1678
39
        std::swap(LHS, RHS);
1679
222
      return true;
1680
222
    }
1681
375
1682
375
    ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1683
375
    unsigned Opc = 0;
1684
375
    switch (CCCode) {
1685
375
    
default: 0
llvm_unreachable0
("Don't know how to expand this condition!");
1686
375
    case ISD::SETO:
1687
86
        assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)
1688
86
            && "If SETO is expanded, SETOEQ must be legal!");
1689
86
        CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1690
375
    case ISD::SETUO:
1691
88
        assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)
1692
88
            && "If SETUO is expanded, SETUNE must be legal!");
1693
88
        CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;  break;
1694
375
    case ISD::SETOEQ:
1695
201
    case ISD::SETOGT:
1696
201
    case ISD::SETOGE:
1697
201
    case ISD::SETOLT:
1698
201
    case ISD::SETOLE:
1699
201
    case ISD::SETONE:
1700
201
    case ISD::SETUEQ:
1701
201
    case ISD::SETUNE:
1702
201
    case ISD::SETUGT:
1703
201
    case ISD::SETUGE:
1704
201
    case ISD::SETULT:
1705
201
    case ISD::SETULE:
1706
201
        // If we are floating point, assign and break, otherwise fall through.
1707
201
        if (!OpVT.isInteger()) {
1708
201
          // We can use the 4th bit to tell if we are the unordered
1709
201
          // or ordered version of the opcode.
1710
201
          CC2 = ((unsigned)CCCode & 0x8U) ? 
ISD::SETUO90
:
ISD::SETO111
;
1711
201
          Opc = ((unsigned)CCCode & 0x8U) ? 
ISD::OR90
:
ISD::AND111
;
1712
201
          CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1713
201
          break;
1714
201
        }
1715
0
        // Fallthrough if we are unsigned integer.
1716
0
        LLVM_FALLTHROUGH;
1717
0
    case ISD::SETLE:
1718
0
    case ISD::SETGT:
1719
0
    case ISD::SETGE:
1720
0
    case ISD::SETLT:
1721
0
    case ISD::SETNE:
1722
0
    case ISD::SETEQ:
1723
0
      // If all combinations of inverting the condition and swapping operands
1724
0
      // didn't work then we have no means to expand the condition.
1725
0
      llvm_unreachable("Don't know how to expand this condition!");
1726
375
    }
1727
375
1728
375
    SDValue SetCC1, SetCC2;
1729
375
    if (CCCode != ISD::SETO && 
CCCode != ISD::SETUO289
) {
1730
201
      // If we aren't the ordered or unorder operation,
1731
201
      // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1732
201
      SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1733
201
      SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1734
201
    } else {
1735
174
      // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1736
174
      SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1737
174
      SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1738
174
    }
1739
375
    LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1740
375
    RHS = SDValue();
1741
375
    CC  = SDValue();
1742
375
    return true;
1743
375
  }
1744
3.77k
  }
1745
3.77k
  return false;
1746
3.77k
}
1747
1748
/// Emit a store/load combination to the stack.  This stores
1749
/// SrcOp to a stack slot of type SlotVT, truncating it if needed.  It then does
1750
/// a load from the stack slot to DestVT, extending it if needed.
1751
/// The resultant code need not be legal.
1752
SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1753
841
                                               EVT DestVT, const SDLoc &dl) {
1754
841
  return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1755
841
}
1756
1757
SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1758
                                               EVT DestVT, const SDLoc &dl,
1759
841
                                               SDValue Chain) {
1760
841
  // Create the stack frame object.
1761
841
  unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1762
841
      SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1763
841
  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1764
841
1765
841
  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1766
841
  int SPFI = StackPtrFI->getIndex();
1767
841
  MachinePointerInfo PtrInfo =
1768
841
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1769
841
1770
841
  unsigned SrcSize = SrcOp.getValueSizeInBits();
1771
841
  unsigned SlotSize = SlotVT.getSizeInBits();
1772
841
  unsigned DestSize = DestVT.getSizeInBits();
1773
841
  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1774
841
  unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1775
841
1776
841
  // Emit a store to the stack slot.  Use a truncstore if the input value is
1777
841
  // later than DestVT.
1778
841
  SDValue Store;
1779
841
1780
841
  if (SrcSize > SlotSize) 
1781
0
    Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1782
0
                              SlotVT, SrcAlign);
1783
841
  else {
1784
841
    assert(SrcSize == SlotSize && "Invalid store");
1785
841
    Store =
1786
841
        DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1787
841
  }
1788
841
1789
841
  // Result is a load from the stack slot.
1790
841
  if (SlotSize == DestSize)
1791
841
    return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1792
0
    
1793
0
  assert(SlotSize < DestSize && "Unknown extension!");
1794
0
  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1795
0
                        DestAlign);
1796
0
}
1797
1798
28
SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1799
28
  SDLoc dl(Node);
1800
28
  // Create a vector sized/aligned stack slot, store the value to element #0,
1801
28
  // then load the whole vector back out.
1802
28
  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1803
28
1804
28
  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1805
28
  int SPFI = StackPtrFI->getIndex();
1806
28
1807
28
  SDValue Ch = DAG.getTruncStore(
1808
28
      DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1809
28
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1810
28
      Node->getValueType(0).getVectorElementType());
1811
28
  return DAG.getLoad(
1812
28
      Node->getValueType(0), dl, Ch, StackPtr,
1813
28
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1814
28
}
1815
1816
static bool
1817
ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1818
158
                     const TargetLowering &TLI, SDValue &Res) {
1819
158
  unsigned NumElems = Node->getNumOperands();
1820
158
  SDLoc dl(Node);
1821
158
  EVT VT = Node->getValueType(0);
1822
158
1823
158
  // Try to group the scalars into pairs, shuffle the pairs together, then
1824
158
  // shuffle the pairs of pairs together, etc. until the vector has
1825
158
  // been built. This will work only if all of the necessary shuffle masks
1826
158
  // are legal.
1827
158
1828
158
  // We do this in two phases; first to check the legality of the shuffles,
1829
158
  // and next, assuming that all shuffles are legal, to create the new nodes.
1830
474
  for (int Phase = 0; Phase < 2; 
++Phase316
) {
1831
316
    SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1832
316
                                                              NewIntermedVals;
1833
3.00k
    for (unsigned i = 0; i < NumElems; 
++i2.68k
) {
1834
2.68k
      SDValue V = Node->getOperand(i);
1835
2.68k
      if (V.isUndef())
1836
688
        continue;
1837
2.00k
1838
2.00k
      SDValue Vec;
1839
2.00k
      if (Phase)
1840
1.00k
        Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1841
2.00k
      IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1842
2.00k
    }
1843
316
1844
808
    while (IntermedVals.size() > 2) {
1845
492
      NewIntermedVals.clear();
1846
1.86k
      for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; 
i += 21.36k
) {
1847
1.36k
        // This vector and the next vector are shuffled together (simply to
1848
1.36k
        // append the one to the other).
1849
1.36k
        SmallVector<int, 16> ShuffleVec(NumElems, -1);
1850
1.36k
1851
1.36k
        SmallVector<int, 16> FinalIndices;
1852
1.36k
        FinalIndices.reserve(IntermedVals[i].second.size() +
1853
1.36k
                             IntermedVals[i+1].second.size());
1854
1.36k
1855
1.36k
        int k = 0;
1856
3.29k
        for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1857
1.92k
             ++j, ++k) {
1858
1.92k
          ShuffleVec[k] = j;
1859
1.92k
          FinalIndices.push_back(IntermedVals[i].second[j]);
1860
1.92k
        }
1861
3.28k
        for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1862
1.92k
             ++j, ++k) {
1863
1.92k
          ShuffleVec[k] = NumElems + j;
1864
1.92k
          FinalIndices.push_back(IntermedVals[i+1].second[j]);
1865
1.92k
        }
1866
1.36k
1867
1.36k
        SDValue Shuffle;
1868
1.36k
        if (Phase)
1869
684
          Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1870
684
                                         IntermedVals[i+1].first,
1871
684
                                         ShuffleVec);
1872
684
        else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1873
0
          return false;
1874
1.36k
        NewIntermedVals.push_back(
1875
1.36k
            std::make_pair(Shuffle, std::move(FinalIndices)));
1876
1.36k
      }
1877
492
1878
492
      // If we had an odd number of defined values, then append the last
1879
492
      // element to the array of new vectors.
1880
492
      if ((IntermedVals.size() & 1) != 0)
1881
104
        NewIntermedVals.push_back(IntermedVals.back());
1882
492
1883
492
      IntermedVals.swap(NewIntermedVals);
1884
492
    }
1885
316
1886
316
    assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1887
316
           "Invalid number of intermediate vectors");
1888
316
    SDValue Vec1 = IntermedVals[0].first;
1889
316
    SDValue Vec2;
1890
316
    if (IntermedVals.size() > 1)
1891
316
      Vec2 = IntermedVals[1].first;
1892
0
    else if (Phase)
1893
0
      Vec2 = DAG.getUNDEF(VT);
1894
316
1895
316
    SmallVector<int, 16> ShuffleVec(NumElems, -1);
1896
1.37k
    for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; 
++i1.05k
)
1897
1.05k
      ShuffleVec[IntermedVals[0].second[i]] = i;
1898
1.26k
    for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; 
++i944
)
1899
944
      ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1900
316
1901
316
    if (Phase)
1902
158
      Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1903
158
    else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1904
0
      return false;
1905
316
  }
1906
158
1907
158
  return true;
1908
158
}
1909
1910
/// Expand a BUILD_VECTOR node on targets that don't
1911
/// support the operation, but do support the resultant vector type.
1912
55.7k
SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1913
55.7k
  unsigned NumElems = Node->getNumOperands();
1914
55.7k
  SDValue Value1, Value2;
1915
55.7k
  SDLoc dl(Node);
1916
55.7k
  EVT VT = Node->getValueType(0);
1917
55.7k
  EVT OpVT = Node->getOperand(0).getValueType();
1918
55.7k
  EVT EltVT = VT.getVectorElementType();
1919
55.7k
1920
55.7k
  // If the only non-undef value is the low element, turn this into a
1921
55.7k
  // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
1922
55.7k
  bool isOnlyLowElement = true;
1923
55.7k
  bool MoreThanTwoValues = false;
1924
55.7k
  bool isConstant = true;
1925
520k
  for (unsigned i = 0; i < NumElems; 
++i464k
) {
1926
464k
    SDValue V = Node->getOperand(i);
1927
464k
    if (V.isUndef())
1928
21.0k
      continue;
1929
443k
    if (i > 0)
1930
388k
      isOnlyLowElement = false;
1931
443k
    if (!isa<ConstantFPSDNode>(V) && 
!isa<ConstantSDNode>(V)432k
)
1932
12.2k
      isConstant = false;
1933
443k
1934
443k
    if (!Value1.getNode()) {
1935
55.7k
      Value1 = V;
1936
387k
    } else if (!Value2.getNode()) {
1937
175k
      if (V != Value1)
1938
43.7k
        Value2 = V;
1939
212k
    } else if (V != Value1 && 
V != Value2185k
) {
1940
148k
      MoreThanTwoValues = true;
1941
148k
    }
1942
443k
  }
1943
55.7k
1944
55.7k
  if (!Value1.getNode())
1945
0
    return DAG.getUNDEF(VT);
1946
55.7k
1947
55.7k
  if (isOnlyLowElement)
1948
61
    return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1949
55.6k
1950
55.6k
  // If all elements are constants, create a load from the constant pool.
1951
55.6k
  if (isConstant) {
1952
52.8k
    SmallVector<Constant*, 16> CV;
1953
503k
    for (unsigned i = 0, e = NumElems; i != e; 
++i450k
) {
1954
450k
      if (ConstantFPSDNode *V =
1955
10.6k
          dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1956
10.6k
        CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1957
440k
      } else if (ConstantSDNode *V =
1958
420k
                 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1959
420k
        if (OpVT==EltVT)
1960
385k
          CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1961
35.5k
        else {
1962
35.5k
          // If OpVT and EltVT don't match, EltVT is not legal and the
1963
35.5k
          // element values have been promoted/truncated earlier.  Undo this;
1964
35.5k
          // we don't want a v16i8 to become a v16i32 for example.
1965
35.5k
          const ConstantInt *CI = V->getConstantIntValue();
1966
35.5k
          CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1967
35.5k
                                        CI->getZExtValue()));
1968
35.5k
        }
1969
420k
      } else {
1970
19.7k
        assert(Node->getOperand(i).isUndef());
1971
19.7k
        Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1972
19.7k
        CV.push_back(UndefValue::get(OpNTy));
1973
19.7k
      }
1974
450k
    }
1975
52.8k
    Constant *CP = ConstantVector::get(CV);
1976
52.8k
    SDValue CPIdx =
1977
52.8k
        DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1978
52.8k
    unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1979
52.8k
    return DAG.getLoad(
1980
52.8k
        VT, dl, DAG.getEntryNode(), CPIdx,
1981
52.8k
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1982
52.8k
        Alignment);
1983
52.8k
  }
1984
2.86k
1985
2.86k
  SmallSet<SDValue, 16> DefinedValues;
1986
16.3k
  for (unsigned i = 0; i < NumElems; 
++i13.4k
) {
1987
13.4k
    if (Node->getOperand(i).isUndef())
1988
1.15k
      continue;
1989
12.2k
    DefinedValues.insert(Node->getOperand(i));
1990
12.2k
  }
1991
2.86k
1992
2.86k
  if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1993
2.79k
    if (!MoreThanTwoValues) {
1994
2.63k
      SmallVector<int, 8> ShuffleVec(NumElems, -1);
1995
14.5k
      for (unsigned i = 0; i < NumElems; 
++i11.8k
) {
1996
11.8k
        SDValue V = Node->getOperand(i);
1997
11.8k
        if (V.isUndef())
1998
809
          continue;
1999
11.0k
        ShuffleVec[i] = V == Value1 ? 
09.46k
:
NumElems1.62k
;
2000
11.0k
      }
2001
2.63k
      if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2002
2.53k
        // Get the splatted value into the low element of a vector register.
2003
2.53k
        SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2004
2.53k
        SDValue Vec2;
2005
2.53k
        if (Value2.getNode())
2006
1.51k
          Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2007
1.01k
        else
2008
1.01k
          Vec2 = DAG.getUNDEF(VT);
2009
2.53k
2010
2.53k
        // Return shuffle(LowValVec, undef, <0,0,0,0>)
2011
2.53k
        return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2012
2.53k
      }
2013
158
    } else {
2014
158
      SDValue Res;
2015
158
      if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2016
158
        return Res;
2017
177
    }
2018
2.79k
  }
2019
177
2020
177
  // Otherwise, we can't handle this case efficiently.
2021
177
  return ExpandVectorBuildThroughStack(Node);
2022
177
}
2023
2024
// Expand a node into a call to a libcall.  If the result value
2025
// does not fit into a register, return the lo part and set the hi part to the
2026
// by-reg argument.  If it does fit into a single register, return the result
2027
// and leave the Hi part unset.
2028
SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2029
5.64k
                                            bool isSigned) {
2030
5.64k
  TargetLowering::ArgListTy Args;
2031
5.64k
  TargetLowering::ArgListEntry Entry;
2032
7.94k
  for (const SDValue &Op : Node->op_values()) {
2033
7.94k
    EVT ArgVT = Op.getValueType();
2034
7.94k
    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2035
7.94k
    Entry.Node = Op;
2036
7.94k
    Entry.Ty = ArgTy;
2037
7.94k
    Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2038
7.94k
    Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2039
7.94k
    Args.push_back(Entry);
2040
7.94k
  }
2041
5.64k
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2042
5.64k
                                         TLI.getPointerTy(DAG.getDataLayout()));
2043
5.64k
2044
5.64k
  EVT RetVT = Node->getValueType(0);
2045
5.64k
  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2046
5.64k
2047
5.64k
  // By default, the input chain to this libcall is the entry node of the
2048
5.64k
  // function. If the libcall is going to be emitted as a tail call then
2049
5.64k
  // TLI.isUsedByReturnOnly will change it to the right chain if the return
2050
5.64k
  // node which is being folded has a non-entry input chain.
2051
5.64k
  SDValue InChain = DAG.getEntryNode();
2052
5.64k
2053
5.64k
  // isTailCall may be true since the callee does not reference caller stack
2054
5.64k
  // frame. Check if it's in the right position and that the return types match.
2055
5.64k
  SDValue TCChain = InChain;
2056
5.64k
  const Function &F = DAG.getMachineFunction().getFunction();
2057
5.64k
  bool isTailCall =
2058
5.64k
      TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2059
5.64k
      
(495
RetTy == F.getReturnType()495
||
F.getReturnType()->isVoidTy()42
);
2060
5.64k
  if (isTailCall)
2061
453
    InChain = TCChain;
2062
5.64k
2063
5.64k
  TargetLowering::CallLoweringInfo CLI(DAG);
2064
5.64k
  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2065
5.64k
  CLI.setDebugLoc(SDLoc(Node))
2066
5.64k
      .setChain(InChain)
2067
5.64k
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2068
5.64k
                    std::move(Args))
2069
5.64k
      .setTailCall(isTailCall)
2070
5.64k
      .setSExtResult(signExtend)
2071
5.64k
      .setZExtResult(!signExtend)
2072
5.64k
      .setIsPostTypeLegalization(true);
2073
5.64k
2074
5.64k
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2075
5.64k
2076
5.64k
  if (!CallInfo.second.getNode()) {
2077
404
    LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump());
2078
404
    // It's a tailcall, return the chain (which is the DAG root).
2079
404
    return DAG.getRoot();
2080
404
  }
2081
5.24k
2082
5.24k
  LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump());
2083
5.24k
  return CallInfo.first;
2084
5.24k
}
2085
2086
// Expand a node into a call to a libcall. Similar to
2087
// ExpandLibCall except that the first operand is the in-chain.
2088
std::pair<SDValue, SDValue>
2089
SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2090
                                         SDNode *Node,
2091
86
                                         bool isSigned) {
2092
86
  SDValue InChain = Node->getOperand(0);
2093
86
2094
86
  TargetLowering::ArgListTy Args;
2095
86
  TargetLowering::ArgListEntry Entry;
2096
275
  for (unsigned i = 1, e = Node->getNumOperands(); i != e; 
++i189
) {
2097
189
    EVT ArgVT = Node->getOperand(i).getValueType();
2098
189
    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2099
189
    Entry.Node = Node->getOperand(i);
2100
189
    Entry.Ty = ArgTy;
2101
189
    Entry.IsSExt = isSigned;
2102
189
    Entry.IsZExt = !isSigned;
2103
189
    Args.push_back(Entry);
2104
189
  }
2105
86
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2106
86
                                         TLI.getPointerTy(DAG.getDataLayout()));
2107
86
2108
86
  Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2109
86
2110
86
  TargetLowering::CallLoweringInfo CLI(DAG);
2111
86
  CLI.setDebugLoc(SDLoc(Node))
2112
86
      .setChain(InChain)
2113
86
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2114
86
                    std::move(Args))
2115
86
      .setSExtResult(isSigned)
2116
86
      .setZExtResult(!isSigned);
2117
86
2118
86
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2119
86
2120
86
  return CallInfo;
2121
86
}
2122
2123
SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2124
                                              RTLIB::Libcall Call_F32,
2125
                                              RTLIB::Libcall Call_F64,
2126
                                              RTLIB::Libcall Call_F80,
2127
                                              RTLIB::Libcall Call_F128,
2128
4.14k
                                              RTLIB::Libcall Call_PPCF128) {
2129
4.14k
  if (Node->isStrictFPOpcode())
2130
1.09k
    Node = DAG.mutateStrictFPToFP(Node);
2131
4.14k
2132
4.14k
  RTLIB::Libcall LC;
2133
4.14k
  switch (Node->getSimpleValueType(0).SimpleTy) {
2134
4.14k
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2135
4.14k
  
case MVT::f32: LC = Call_F32; break2.04k
;
2136
4.14k
  
case MVT::f64: LC = Call_F64; break1.94k
;
2137
4.14k
  
case MVT::f80: LC = Call_F80; break84
;
2138
4.14k
  
case MVT::f128: LC = Call_F128; break69
;
2139
4.14k
  
case MVT::ppcf128: LC = Call_PPCF128; break0
;
2140
4.14k
  }
2141
4.14k
  return ExpandLibCall(LC, Node, false);
2142
4.14k
}
2143
2144
SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2145
                                               RTLIB::Libcall Call_I8,
2146
                                               RTLIB::Libcall Call_I16,
2147
                                               RTLIB::Libcall Call_I32,
2148
                                               RTLIB::Libcall Call_I64,
2149
268
                                               RTLIB::Libcall Call_I128) {
2150
268
  RTLIB::Libcall LC;
2151
268
  switch (Node->getSimpleValueType(0).SimpleTy) {
2152
268
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2153
268
  
case MVT::i8: LC = Call_I8; break0
;
2154
268
  
case MVT::i16: LC = Call_I16; break13
;
2155
268
  
case MVT::i32: LC = Call_I32; break237
;
2156
268
  
case MVT::i64: LC = Call_I64; break18
;
2157
268
  
case MVT::i128: LC = Call_I128; break0
;
2158
268
  }
2159
268
  return ExpandLibCall(LC, Node, isSigned);
2160
268
}
2161
2162
/// Expand the node to a libcall based on first argument type (for instance
2163
/// lround and its variant).
2164
SDValue SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2165
                                                 RTLIB::Libcall Call_F32,
2166
                                                 RTLIB::Libcall Call_F64,
2167
                                                 RTLIB::Libcall Call_F80,
2168
                                                 RTLIB::Libcall Call_F128,
2169
59
                                                 RTLIB::Libcall Call_PPCF128) {
2170
59
  RTLIB::Libcall LC;
2171
59
  switch (Node->getOperand(0).getValueType().getSimpleVT().SimpleTy) {
2172
59
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2173
59
  
case MVT::f32: LC = Call_F32; break21
;
2174
59
  
case MVT::f64: LC = Call_F64; break22
;
2175
59
  
case MVT::f80: LC = Call_F80; break8
;
2176
59
  
case MVT::f128: LC = Call_F128; break8
;
2177
59
  
case MVT::ppcf128: LC = Call_PPCF128; break0
;
2178
59
  }
2179
59
2180
59
  return ExpandLibCall(LC, Node, false);
2181
59
}
2182
2183
/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2184
void
2185
SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2186
14
                                          SmallVectorImpl<SDValue> &Results) {
2187
14
  unsigned Opcode = Node->getOpcode();
2188
14
  bool isSigned = Opcode == ISD::SDIVREM;
2189
14
2190
14
  RTLIB::Libcall LC;
2191
14
  switch (Node->getSimpleValueType(0).SimpleTy) {
2192
14
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2193
14
  
case MVT::i8: LC= isSigned 0
?
RTLIB::SDIVREM_I80
:
RTLIB::UDIVREM_I80
; break;
2194
14
  
case MVT::i16: LC= isSigned 0
?
RTLIB::SDIVREM_I160
:
RTLIB::UDIVREM_I160
; break;
2195
14
  case MVT::i32:  LC= isSigned ? 
RTLIB::SDIVREM_I329
:
RTLIB::UDIVREM_I325
; break;
2196
14
  
case MVT::i64: LC= isSigned 0
?
RTLIB::SDIVREM_I640
:
RTLIB::UDIVREM_I640
; break;
2197
14
  
case MVT::i128: LC= isSigned 0
?
RTLIB::SDIVREM_I1280
:
RTLIB::UDIVREM_I1280
; break;
2198
14
  }
2199
14
2200
14
  // The input chain to this libcall is the entry node of the function.
2201
14
  // Legalizing the call will automatically add the previous call to the
2202
14
  // dependence.
2203
14
  SDValue InChain = DAG.getEntryNode();
2204
14
2205
14
  EVT RetVT = Node->getValueType(0);
2206
14
  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2207
14
2208
14
  TargetLowering::ArgListTy Args;
2209
14
  TargetLowering::ArgListEntry Entry;
2210
28
  for (const SDValue &Op : Node->op_values()) {
2211
28
    EVT ArgVT = Op.getValueType();
2212
28
    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2213
28
    Entry.Node = Op;
2214
28
    Entry.Ty = ArgTy;
2215
28
    Entry.IsSExt = isSigned;
2216
28
    Entry.IsZExt = !isSigned;
2217
28
    Args.push_back(Entry);
2218
28
  }
2219
14
2220
14
  // Also pass the return address of the remainder.
2221
14
  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2222
14
  Entry.Node = FIPtr;
2223
14
  Entry.Ty = RetTy->getPointerTo();
2224
14
  Entry.IsSExt = isSigned;
2225
14
  Entry.IsZExt = !isSigned;
2226
14
  Args.push_back(Entry);
2227
14
2228
14
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2229
14
                                         TLI.getPointerTy(DAG.getDataLayout()));
2230
14
2231
14
  SDLoc dl(Node);
2232
14
  TargetLowering::CallLoweringInfo CLI(DAG);
2233
14
  CLI.setDebugLoc(dl)
2234
14
      .setChain(InChain)
2235
14
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2236
14
                    std::move(Args))
2237
14
      .setSExtResult(isSigned)
2238
14
      .setZExtResult(!isSigned);
2239
14
2240
14
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2241
14
2242
14
  // Remainder is loaded back from the stack frame.
2243
14
  SDValue Rem =
2244
14
      DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2245
14
  Results.push_back(CallInfo.first);
2246
14
  Results.push_back(Rem);
2247
14
}
2248
2249
/// Return true if sincos libcall is available.
2250
476
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2251
476
  RTLIB::Libcall LC;
2252
476
  switch (Node->getSimpleValueType(0).SimpleTy) {
2253
476
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2254
476
  
case MVT::f32: LC = RTLIB::SINCOS_F32; break320
;
2255
476
  
case MVT::f64: LC = RTLIB::SINCOS_F64; break123
;
2256
476
  
case MVT::f80: LC = RTLIB::SINCOS_F80; break19
;
2257
476
  
case MVT::f128: LC = RTLIB::SINCOS_F128; break14
;
2258
476
  
case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break0
;
2259
476
  }
2260
476
  return TLI.getLibcallName(LC) != nullptr;
2261
476
}
2262
2263
/// Only issue sincos libcall if both sin and cos are needed.
2264
213
static bool useSinCos(SDNode *Node) {
2265
213
  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2266
213
    ? 
ISD::FCOS106
:
ISD::FSIN107
;
2267
213
2268
213
  SDValue Op0 = Node->getOperand(0);
2269
213
  for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2270
284
       UE = Op0.getNode()->use_end(); UI != UE; 
++UI71
) {
2271
213
    SDNode *User = *UI;
2272
213
    if (User == Node)
2273
71
      continue;
2274
142
    // The other user might have been turned into sincos already.
2275
142
    if (User->getOpcode() == OtherOpcode || 
User->getOpcode() == ISD::FSINCOS71
)
2276
142
      return true;
2277
142
  }
2278
213
  
return false71
;
2279
213
}
2280
2281
/// Issue libcalls to sincos to compute sin / cos pairs.
2282
void
2283
SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2284
34
                                          SmallVectorImpl<SDValue> &Results) {
2285
34
  RTLIB::Libcall LC;
2286
34
  switch (Node->getSimpleValueType(0).SimpleTy) {
2287
34
  
default: 0
llvm_unreachable0
("Unexpected request for libcall!");
2288
34
  
case MVT::f32: LC = RTLIB::SINCOS_F32; break13
;
2289
34
  
case MVT::f64: LC = RTLIB::SINCOS_F64; break13
;
2290
34
  
case MVT::f80: LC = RTLIB::SINCOS_F80; break5
;
2291
34
  
case MVT::f128: LC = RTLIB::SINCOS_F128; break3
;
2292
34
  
case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break0
;
2293
34
  }
2294
34
2295
34
  // The input chain to this libcall is the entry node of the function.
2296
34
  // Legalizing the call will automatically add the previous call to the
2297
34
  // dependence.
2298
34
  SDValue InChain = DAG.getEntryNode();
2299
34
2300
34
  EVT RetVT = Node->getValueType(0);
2301
34
  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2302
34
2303
34
  TargetLowering::ArgListTy Args;
2304
34
  TargetLowering::ArgListEntry Entry;
2305
34
2306
34
  // Pass the argument.
2307
34
  Entry.Node = Node->getOperand(0);
2308
34
  Entry.Ty = RetTy;
2309
34
  Entry.IsSExt = false;
2310
34
  Entry.IsZExt = false;
2311
34
  Args.push_back(Entry);
2312
34
2313
34
  // Pass the return address of sin.
2314
34
  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2315
34
  Entry.Node = SinPtr;
2316
34
  Entry.Ty = RetTy->getPointerTo();
2317
34
  Entry.IsSExt = false;
2318
34
  Entry.IsZExt = false;
2319
34
  Args.push_back(Entry);
2320
34
2321
34
  // Also pass the return address of the cos.
2322
34
  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2323
34
  Entry.Node = CosPtr;
2324
34
  Entry.Ty = RetTy->getPointerTo();
2325
34
  Entry.IsSExt = false;
2326
34
  Entry.IsZExt = false;
2327
34
  Args.push_back(Entry);
2328
34
2329
34
  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2330
34
                                         TLI.getPointerTy(DAG.getDataLayout()));
2331
34
2332
34
  SDLoc dl(Node);
2333
34
  TargetLowering::CallLoweringInfo CLI(DAG);
2334
34
  CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2335
34
      TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2336
34
      std::move(Args));
2337
34
2338
34
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2339
34
2340
34
  Results.push_back(
2341
34
      DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2342
34
  Results.push_back(
2343
34
      DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2344
34
}
2345
2346
/// This function is responsible for legalizing a
2347
/// INT_TO_FP operation of the specified operand when the target requests that
2348
/// we expand it.  At this point, we know that the result and operand types are
2349
/// legal for the target.
2350
SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2351
                                                   EVT DestVT,
2352
56
                                                   const SDLoc &dl) {
2353
56
  EVT SrcVT = Op0.getValueType();
2354
56
2355
56
  // TODO: Should any fast-math-flags be set for the created nodes?
2356
56
  LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2357
56
  if (SrcVT == MVT::i32 && 
TLI.isTypeLegal(MVT::f64)55
) {
2358
54
    LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2359
54
                         "expansion\n");
2360
54
2361
54
    // Get the stack frame index of a 8 byte buffer.
2362
54
    SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2363
54
2364
54
    // word offset constant for Hi/Lo address computation
2365
54
    SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2366
54
                                      StackSlot.getValueType());
2367
54
    // set up Hi and Lo (into buffer) address based on endian
2368
54
    SDValue Hi = StackSlot;
2369
54
    SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2370
54
                             StackSlot, WordOff);
2371
54
    if (DAG.getDataLayout().isLittleEndian())
2372
22
      std::swap(Hi, Lo);
2373
54
2374
54
    // if signed map to unsigned space
2375
54
    SDValue Op0Mapped;
2376
54
    if (isSigned) {
2377
4
      // constant used to invert sign bit (signed to unsigned mapping)
2378
4
      SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2379
4
      Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2380
50
    } else {
2381
50
      Op0Mapped = Op0;
2382
50
    }
2383
54
    // store the lo of the constructed double - based on integer input
2384
54
    SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2385
54
                                  MachinePointerInfo());
2386
54
    // initial hi portion of constructed double
2387
54
    SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2388
54
    // store the hi of the constructed double - biased exponent
2389
54
    SDValue Store2 =
2390
54
        DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2391
54
    // load the constructed double
2392
54
    SDValue Load =
2393
54
        DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2394
54
    // FP constant to bias correct the final result
2395
54
    SDValue Bias = DAG.getConstantFP(isSigned ?
2396
4
                                     BitsToDouble(0x4330000080000000ULL) :
2397
54
                                     
BitsToDouble(0x4330000000000000ULL)50
,
2398
54
                                     dl, MVT::f64);
2399
54
    // subtract the bias
2400
54
    SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2401
54
    // final result
2402
54
    SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2403
54
    return Result;
2404
54
  }
2405
2
  assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
2406
2
  // Code below here assumes !isSigned without checking again.
2407
2
2408
2
  SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2409
2
2410
2
  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2411
2
                                 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2412
2
  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2413
2
          Four = DAG.getIntPtrConstant(4, dl);
2414
2
  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2415
2
                                    SignSet, Four, Zero);
2416
2
2417
2
  // If the sign bit of the integer is set, the large number will be treated
2418
2
  // as a negative number.  To counteract this, the dynamic code adds an
2419
2
  // offset depending on the data type.
2420
2
  uint64_t FF;
2421
2
  switch (SrcVT.getSimpleVT().SimpleTy) {
2422
2
  
default: 0
llvm_unreachable0
("Unsupported integer type!");
2423
2
  
case MVT::i8 : FF = 0x43800000ULL; break0
; // 2^8 (as a float)
2424
2
  
case MVT::i16: FF = 0x47800000ULL; break0
; // 2^16 (as a float)
2425
2
  
case MVT::i32: FF = 0x4F800000ULL; break1
; // 2^32 (as a float)
2426
2
  
case MVT::i64: FF = 0x5F800000ULL; break1
; // 2^64 (as a float)
2427
2
  }
2428
2
  if (DAG.getDataLayout().isLittleEndian())
2429
0
    FF <<= 32;
2430
2
  Constant *FudgeFactor = ConstantInt::get(
2431
2
                                       Type::getInt64Ty(*DAG.getContext()), FF);
2432
2
2433
2
  SDValue CPIdx =
2434
2
      DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2435
2
  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2436
2
  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2437
2
  Alignment = std::min(Alignment, 4u);
2438
2
  SDValue FudgeInReg;
2439
2
  if (DestVT == MVT::f32)
2440
1
    FudgeInReg = DAG.getLoad(
2441
1
        MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2442
1
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2443
1
        Alignment);
2444
1
  else {
2445
1
    SDValue Load = DAG.getExtLoad(
2446
1
        ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2447
1
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2448
1
        Alignment);
2449
1
    HandleSDNode Handle(Load);
2450
1
    LegalizeOp(Load.getNode());
2451
1
    FudgeInReg = Handle.getValue();
2452
1
  }
2453
2
2454
2
  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2455
2
}
2456
2457
/// This function is responsible for legalizing a
2458
/// *INT_TO_FP operation of the specified operand when the target requests that
2459
/// we promote it.  At this point, we know that the result and operand types are
2460
/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2461
/// operation that takes a larger input.
2462
SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2463
                                                    bool isSigned,
2464
177
                                                    const SDLoc &dl) {
2465
177
  // First step, figure out the appropriate *INT_TO_FP operation to use.
2466
177
  EVT NewInTy = LegalOp.getValueType();
2467
177
2468
177
  unsigned OpToUse = 0;
2469
177
2470
177
  // Scan for the appropriate larger type to use.
2471
187
  while (true) {
2472
187
    NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2473
187
    assert(NewInTy.isInteger() && "Ran out of possibilities!");
2474
187
2475
187
    // If the target supports SINT_TO_FP of this type, use it.
2476
187
    if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2477
177
      OpToUse = ISD::SINT_TO_FP;
2478
177
      break;
2479
177
    }
2480
10
    if (isSigned) 
continue9
;
2481
1
2482
1
    // If the target supports UINT_TO_FP of this type, use it.
2483
1
    if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2484
0
      OpToUse = ISD::UINT_TO_FP;
2485
0
      break;
2486
0
    }
2487
1
2488
1
    // Otherwise, try a larger type.
2489
1
  }
2490
177
2491
177
  // Okay, we found the operation and type to use.  Zero extend our input to the
2492
177
  // desired type then run the operation on it.
2493
177
  return DAG.getNode(OpToUse, dl, DestVT,
2494
177
                     DAG.getNode(isSigned ? 
ISD::SIGN_EXTEND39
:
ISD::ZERO_EXTEND138
,
2495
177
                                 dl, NewInTy, LegalOp));
2496
177
}
2497
2498
/// This function is responsible for legalizing a
2499
/// FP_TO_*INT operation of the specified operand when the target requests that
2500
/// we promote it.  At this point, we know that the result and operand types are
2501
/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2502
/// operation that returns a larger result.
2503
SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2504
                                                    bool isSigned,
2505
286
                                                    const SDLoc &dl) {
2506
286
  // First step, figure out the appropriate FP_TO*INT operation to use.
2507
286
  EVT NewOutTy = DestVT;
2508
286
2509
286
  unsigned OpToUse = 0;
2510
286
2511
286
  // Scan for the appropriate larger type to use.
2512
286
  while (true) {
2513
286
    NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2514
286
    assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2515
286
2516
286
    // A larger signed type can hold all unsigned values of the requested type,
2517
286
    // so using FP_TO_SINT is valid
2518
286
    if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2519
286
      OpToUse = ISD::FP_TO_SINT;
2520
286
      break;
2521
286
    }
2522
0
2523
0
    // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2524
0
    if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2525
0
      OpToUse = ISD::FP_TO_UINT;
2526
0
      break;
2527
0
    }
2528
0
2529
0
    // Otherwise, try a larger type.
2530
0
  }
2531
286
2532
286
  // Okay, we found the operation and type to use.
2533
286
  SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2534
286
2535
286
  // Truncate the result of the extended FP_TO_*INT operation to the desired
2536
286
  // size.
2537
286
  return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2538
286
}
2539
2540
/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2541
119
SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2542
119
  EVT VT = Op.getValueType();
2543
119
  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2544
119
  unsigned Sz = VT.getScalarSizeInBits();
2545
119
2546
119
  SDValue Tmp, Tmp2, Tmp3;
2547
119
2548
119
  // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2549
119
  // and finally the i1 pairs.
2550
119
  // TODO: We can easily support i4/i2 legal types if any target ever does.
2551
119
  if (Sz >= 8 && isPowerOf2_32(Sz)) {
2552
119
    // Create the masks - repeating the pattern every byte.
2553
119
    APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0));
2554
119
    APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC));
2555
119
    APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA));
2556
119
    APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F));
2557
119
    APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33));
2558
119
    APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55));
2559
119
2560
119
    // BSWAP if the type is wider than a single byte.
2561
119
    Tmp = (Sz > 8 ? 
DAG.getNode(ISD::BSWAP, dl, VT, Op)78
:
Op41
);
2562
119
2563
119
    // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2564
119
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2565
119
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2566
119
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT));
2567
119
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
2568
119
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2569
119
2570
119
    // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2571
119
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2572
119
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2573
119
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT));
2574
119
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
2575
119
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2576
119
2577
119
    // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2578
119
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2579
119
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2580
119
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT));
2581
119
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
2582
119
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2583
119
    return Tmp;
2584
119
  }
2585
0
2586
0
  Tmp = DAG.getConstant(0, dl, VT);
2587
0
  for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2588
0
    if (I < J)
2589
0
      Tmp2 =
2590
0
          DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2591
0
    else
2592
0
      Tmp2 =
2593
0
          DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2594
0
2595
0
    APInt Shift(Sz, 1);
2596
0
    Shift <<= J;
2597
0
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2598
0
    Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2599
0
  }
2600
0
2601
0
  return Tmp;
2602
0
}
2603
2604
/// Open code the operations for BSWAP of the specified operation.
2605
176
SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2606
176
  EVT VT = Op.getValueType();
2607
176
  EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2608
176
  SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2609
176
  switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2610
176
  
default: 0
llvm_unreachable0
("Unhandled Expand type in BSWAP!");
2611
176
  case MVT::i16:
2612
105
    // Use a rotate by 8. This can be further expanded if necessary.
2613
105
    return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2614
176
  case MVT::i32:
2615
31
    Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2616
31
    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2617
31
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2618
31
    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2619
31
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2620
31
                       DAG.getConstant(0xFF0000, dl, VT));
2621
31
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2622
31
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2623
31
    Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2624
31
    return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2625
176
  case MVT::i64:
2626
40
    Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2627
40
    Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2628
40
    Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2629
40
    Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2630
40
    Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2631
40
    Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2632
40
    Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2633
40
    Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2634
40
    Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2635
40
                       DAG.getConstant(255ULL<<48, dl, VT));
2636
40
    Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2637
40
                       DAG.getConstant(255ULL<<40, dl, VT));
2638
40
    Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2639
40
                       DAG.getConstant(255ULL<<32, dl, VT));
2640
40
    Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2641
40
                       DAG.getConstant(255ULL<<24, dl, VT));
2642
40
    Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2643
40
                       DAG.getConstant(255ULL<<16, dl, VT));
2644
40
    Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2645
40
                       DAG.getConstant(255ULL<<8 , dl, VT));
2646
40
    Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2647
40
    Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2648
40
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2649
40
    Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2650
40
    Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2651
40
    Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2652
40
    return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2653
176
  }
2654
176
}
2655
2656
210k
bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2657
210k
  LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2658
210k
  SmallVector<SDValue, 8> Results;
2659
210k
  SDLoc dl(Node);
2660
210k
  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2661
210k
  bool NeedInvert;
2662
210k
  switch (Node->getOpcode()) {
2663
210k
  case ISD::ABS:
2664
413
    if (TLI.expandABS(Node, Tmp1, DAG))
2665
413
      Results.push_back(Tmp1);
2666
413
    break;
2667
210k
  case ISD::CTPOP:
2668
205
    if (TLI.expandCTPOP(Node, Tmp1, DAG))
2669
205
      Results.push_back(Tmp1);
2670
205
    break;
2671
210k
  case ISD::CTLZ:
2672
1.81k
  case ISD::CTLZ_ZERO_UNDEF:
2673
1.81k
    if (TLI.expandCTLZ(Node, Tmp1, DAG))
2674
1.81k
      Results.push_back(Tmp1);
2675
1.81k
    break;
2676
1.81k
  case ISD::CTTZ:
2677
273
  case ISD::CTTZ_ZERO_UNDEF:
2678
273
    if (TLI.expandCTTZ(Node, Tmp1, DAG))
2679
273
      Results.push_back(Tmp1);
2680
273
    break;
2681
273
  case ISD::BITREVERSE:
2682
119
    Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2683
119
    break;
2684
273
  case ISD::BSWAP:
2685
176
    Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2686
176
    break;
2687
273
  case ISD::FRAMEADDR:
2688
5
  case ISD::RETURNADDR:
2689
5
  case ISD::FRAME_TO_ARGS_OFFSET:
2690
5
    Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2691
5
    break;
2692
6
  case ISD::EH_DWARF_CFA: {
2693
6
    SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2694
6
                                        TLI.getPointerTy(DAG.getDataLayout()));
2695
6
    SDValue Offset = DAG.getNode(ISD::ADD, dl,
2696
6
                                 CfaArg.getValueType(),
2697
6
                                 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2698
6
                                             CfaArg.getValueType()),
2699
6
                                 CfaArg);
2700
6
    SDValue FA = DAG.getNode(
2701
6
        ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2702
6
        DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2703
6
    Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2704
6
                                  FA, Offset));
2705
6
    break;
2706
5
  }
2707
5
  case ISD::FLT_ROUNDS_:
2708
3
    Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2709
3
    break;
2710
830
  case ISD::EH_RETURN:
2711
830
  case ISD::EH_LABEL:
2712
830
  case ISD::PREFETCH:
2713
830
  case ISD::VAEND:
2714
830
  case ISD::EH_SJLJ_LONGJMP:
2715
830
    // If the target didn't expand these, there's nothing to do, so just
2716
830
    // preserve the chain and be done.
2717
830
    Results.push_back(Node->getOperand(0));
2718
830
    break;
2719
830
  case ISD::READCYCLECOUNTER:
2720
3
    // If the target didn't expand this, just return 'zero' and preserve the
2721
3
    // chain.
2722
3
    Results.append(Node->getNumValues() - 1,
2723
3
                   DAG.getConstant(0, dl, Node->getValueType(0)));
2724
3
    Results.push_back(Node->getOperand(0));
2725
3
    break;
2726
830
  case ISD::EH_SJLJ_SETJMP:
2727
0
    // If the target didn't expand this, just return 'zero' and preserve the
2728
0
    // chain.
2729
0
    Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2730
0
    Results.push_back(Node->getOperand(0));
2731
0
    break;
2732
830
  case ISD::ATOMIC_LOAD: {
2733
6
    // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2734
6
    SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2735
6
    SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2736
6
    SDValue Swap = DAG.getAtomicCmpSwap(
2737
6
        ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2738
6
        Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2739
6
        cast<AtomicSDNode>(Node)->getMemOperand());
2740
6
    Results.push_back(Swap.getValue(0));
2741
6
    Results.push_back(Swap.getValue(1));
2742
6
    break;
2743
830
  }
2744
830
  case ISD::ATOMIC_STORE: {
2745
14
    // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2746
14
    SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2747
14
                                 cast<AtomicSDNode>(Node)->getMemoryVT(),
2748
14
                                 Node->getOperand(0),
2749
14
                                 Node->getOperand(1), Node->getOperand(2),
2750
14
                                 cast<AtomicSDNode>(Node)->getMemOperand());
2751
14
    Results.push_back(Swap.getValue(1));
2752
14
    break;
2753
830
  }
2754
1.25k
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2755
1.25k
    // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2756
1.25k
    // splits out the success value as a comparison. Expanding the resulting
2757
1.25k
    // ATOMIC_CMP_SWAP will produce a libcall.
2758
1.25k
    SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2759
1.25k
    SDValue Res = DAG.getAtomicCmpSwap(
2760
1.25k
        ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2761
1.25k
        Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2762
1.25k
        Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2763
1.25k
2764
1.25k
    SDValue ExtRes = Res;
2765
1.25k
    SDValue LHS = Res;
2766
1.25k
    SDValue RHS = Node->getOperand(1);
2767
1.25k
2768
1.25k
    EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2769
1.25k
    EVT OuterType = Node->getValueType(0);
2770
1.25k
    switch (TLI.getExtendForAtomicOps()) {
2771
1.25k
    case ISD::SIGN_EXTEND:
2772
120
      LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2773
120
                        DAG.getValueType(AtomicType));
2774
120
      RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2775
120
                        Node->getOperand(2), DAG.getValueType(AtomicType));
2776
120
      ExtRes = LHS;
2777
120
      break;
2778
1.25k
    case ISD::ZERO_EXTEND:
2779
1.08k
      LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2780
1.08k
                        DAG.getValueType(AtomicType));
2781
1.08k
      RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2782
1.08k
      ExtRes = LHS;
2783
1.08k
      break;
2784
1.25k
    case ISD::ANY_EXTEND:
2785
51
      LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2786
51
      RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2787
51
      break;
2788
1.25k
    default:
2789
0
      llvm_unreachable("Invalid atomic op extension");
2790
1.25k
    }
2791
1.25k
2792
1.25k
    SDValue Success =
2793
1.25k
        DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2794
1.25k
2795
1.25k
    Results.push_back(ExtRes.getValue(0));
2796
1.25k
    Results.push_back(Success);
2797
1.25k
    Results.push_back(Res.getValue(1));
2798
1.25k
    break;
2799
1.25k
  }
2800
1.25k
  case ISD::DYNAMIC_STACKALLOC:
2801
350
    ExpandDYNAMIC_STACKALLOC(Node, Results);
2802
350
    break;
2803
35.6k
  case ISD::MERGE_VALUES:
2804
108k
    for (unsigned i = 0; i < Node->getNumValues(); 
i++72.5k
)
2805
72.5k
      Results.push_back(Node->getOperand(i));
2806
35.6k
    break;
2807
1.25k
  case ISD::UNDEF: {
2808
170
    EVT VT = Node->getValueType(0);
2809
170
    if (VT.isInteger())
2810
157
      Results.push_back(DAG.getConstant(0, dl, VT));
2811
13
    else {
2812
13
      assert(VT.isFloatingPoint() && "Unknown value type!");
2813
13
      Results.push_back(DAG.getConstantFP(0, dl, VT));
2814
13
    }
2815
170
    break;
2816
1.25k
  }
2817
1.25k
  case ISD::STRICT_FP_ROUND:
2818
0
    Tmp1 = EmitStackConvert(Node->getOperand(1), 
2819
0
                            Node->getValueType(0),
2820
0
                            Node->getValueType(0), dl, Node->getOperand(0));
2821
0
    ReplaceNode(Node, Tmp1.getNode());
2822
0
    LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
2823
0
    return true;
2824
1.25k
  case ISD::FP_ROUND:
2825
841
  case ISD::BITCAST:
2826
841
    Tmp1 = EmitStackConvert(Node->getOperand(0), 
2827
841
                            Node->getValueType(0),
2828
841
                            Node->getValueType(0), dl);
2829
841
    Results.push_back(Tmp1);
2830
841
    break;
2831
841
  case ISD::STRICT_FP_EXTEND:
2832
0
    Tmp1 = EmitStackConvert(Node->getOperand(1),
2833
0
                            Node->getOperand(1).getValueType(),
2834
0
                            Node->getValueType(0), dl, Node->getOperand(0));
2835
0
    ReplaceNode(Node, Tmp1.getNode());
2836
0
    LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
2837
0
    return true;
2838
841
  case ISD::FP_EXTEND:
2839
0
    Tmp1 = EmitStackConvert(Node->getOperand(0),
2840
0
                            Node->getOperand(0).getValueType(),
2841
0
                            Node->getValueType(0), dl);
2842
0
    Results.push_back(Tmp1);
2843
0
    break;
2844
2.64k
  case ISD::SIGN_EXTEND_INREG: {
2845
2.64k
    EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2846
2.64k
    EVT VT = Node->getValueType(0);
2847
2.64k
2848
2.64k
    // An in-register sign-extend of a boolean is a negation:
2849
2.64k
    // 'true' (1) sign-extended is -1.
2850
2.64k
    // 'false' (0) sign-extended is 0.
2851
2.64k
    // However, we must mask the high bits of the source operand because the
2852
2.64k
    // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2853
2.64k
2854
2.64k
    // TODO: Do this for vectors too?
2855
2.64k
    if (ExtraVT.getSizeInBits() == 1) {
2856
827
      SDValue One = DAG.getConstant(1, dl, VT);
2857
827
      SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2858
827
      SDValue Zero = DAG.getConstant(0, dl, VT);
2859
827
      SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2860
827
      Results.push_back(Neg);
2861
827
      break;
2862
827
    }
2863
1.82k
2864
1.82k
    // NOTE: we could fall back on load/store here too for targets without
2865
1.82k
    // SRA.  However, it is doubtful that any exist.
2866
1.82k
    EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2867
1.82k
    unsigned BitsDiff = VT.getScalarSizeInBits() -
2868
1.82k
                        ExtraVT.getScalarSizeInBits();
2869
1.82k
    SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2870
1.82k
    Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2871
1.82k
                       Node->getOperand(0), ShiftCst);
2872
1.82k
    Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2873
1.82k
    Results.push_back(Tmp1);
2874
1.82k
    break;
2875
1.82k
  }
2876
1.82k
  case ISD::FP_ROUND_INREG: {
2877
0
    // The only way we can lower this is to turn it into a TRUNCSTORE,
2878
0
    // EXTLOAD pair, targeting a temporary location (a stack slot).
2879
0
2880
0
    // NOTE: there is a choice here between constantly creating new stack
2881
0
    // slots and always reusing the same one.  We currently always create
2882
0
    // new ones, as reuse may inhibit scheduling.
2883
0
    EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2884
0
    Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
2885
0
                            Node->getValueType(0), dl);
2886
0
    Results.push_back(Tmp1);
2887
0
    break;
2888
1.82k
  }
2889
1.82k
  case ISD::UINT_TO_FP:
2890
301
    if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) {
2891
249
      Results.push_back(Tmp1);
2892
249
      break;
2893
249
    }
2894
52
    LLVM_FALLTHROUGH;
2895
56
  case ISD::SINT_TO_FP:
2896
56
    Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2897
56
                                Node->getOperand(0), Node->getValueType(0), dl);
2898
56
    Results.push_back(Tmp1);
2899
56
    break;
2900
61
  case ISD::FP_TO_SINT:
2901
61
    if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2902
61
      Results.push_back(Tmp1);
2903
61
    break;
2904
345
  case ISD::FP_TO_UINT:
2905
345
    if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG))
2906
345
      Results.push_back(Tmp1);
2907
345
    break;
2908
52
  case ISD::LROUND:
2909
17
    Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
2910
17
                                         RTLIB::LROUND_F64, RTLIB::LROUND_F80,
2911
17
                                         RTLIB::LROUND_F128,
2912
17
                                         RTLIB::LROUND_PPCF128));
2913
17
    break;
2914
52
  case ISD::LLROUND:
2915
14
    Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
2916
14
                                         RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
2917
14
                                         RTLIB::LLROUND_F128,
2918
14
                                         RTLIB::LLROUND_PPCF128));
2919
14
    break;
2920
52
  case ISD::LRINT:
2921
16
    Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
2922
16
                                         RTLIB::LRINT_F64, RTLIB::LRINT_F80,
2923
16
                                         RTLIB::LRINT_F128,
2924
16
                                         RTLIB::LRINT_PPCF128));
2925
16
    break;
2926
52
  case ISD::LLRINT:
2927
12
    Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
2928
12
                                         RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
2929
12
                                         RTLIB::LLRINT_F128,
2930
12
                                         RTLIB::LLRINT_PPCF128));
2931
12
    break;
2932
118
  case ISD::VAARG:
2933
118
    Results.push_back(DAG.expandVAArg(Node));
2934
118
    Results.push_back(Results[0].getValue(1));
2935
118
    break;
2936
100
  case ISD::VACOPY:
2937
100
    Results.push_back(DAG.expandVACopy(Node));
2938
100
    break;
2939
1.62k
  case ISD::EXTRACT_VECTOR_ELT:
2940
1.62k
    if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2941
112
      // This must be an access of the only element.  Return it.
2942
112
      Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2943
112
                         Node->getOperand(0));
2944
1.51k
    else
2945
1.51k
      Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2946
1.62k
    Results.push_back(Tmp1);
2947
1.62k
    break;
2948
52
  case ISD::EXTRACT_SUBVECTOR:
2949
0
    Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2950
0
    break;
2951
52
  case ISD::INSERT_SUBVECTOR:
2952
0
    Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2953
0
    break;
2954
52
  case ISD::CONCAT_VECTORS:
2955
0
    Results.push_back(ExpandVectorBuildThroughStack(Node));
2956
0
    break;
2957
52
  case ISD::SCALAR_TO_VECTOR:
2958
28
    Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2959
28
    break;
2960
1.88k
  case ISD::INSERT_VECTOR_ELT:
2961
1.88k
    Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2962
1.88k
                                              Node->getOperand(1),
2963
1.88k
                                              Node->getOperand(2), dl));
2964
1.88k
    break;
2965
453
  case ISD::VECTOR_SHUFFLE: {
2966
453
    SmallVector<int, 32> NewMask;
2967
453
    ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
2968
453
2969
453
    EVT VT = Node->getValueType(0);
2970
453
    EVT EltVT = VT.getVectorElementType();
2971
453
    SDValue Op0 = Node->getOperand(0);
2972
453
    SDValue Op1 = Node->getOperand(1);
2973
453
    if (!TLI.isTypeLegal(EltVT)) {
2974
12
      EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
2975
12
2976
12
      // BUILD_VECTOR operands are allowed to be wider than the element type.
2977
12
      // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
2978
12
      // it.
2979
12
      if (NewEltVT.bitsLT(EltVT)) {
2980
0
        // Convert shuffle node.
2981
0
        // If original node was v4i64 and the new EltVT is i32,
2982
0
        // cast operands to v8i32 and re-build the mask.
2983
0
2984
0
        // Calculate new VT, the size of the new VT should be equal to original.
2985
0
        EVT NewVT =
2986
0
            EVT::getVectorVT(*DAG.getContext(), NewEltVT,
2987
0
                             VT.getSizeInBits() / NewEltVT.getSizeInBits());
2988
0
        assert(NewVT.bitsEq(VT));
2989
0
2990
0
        // cast operands to new VT
2991
0
        Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
2992
0
        Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
2993
0
2994
0
        // Convert the shuffle mask
2995
0
        unsigned int factor =
2996
0
                         NewVT.getVectorNumElements()/VT.getVectorNumElements();
2997
0
2998
0
        // EltVT gets smaller
2999
0
        assert(factor > 0);
3000
0
3001
0
        for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3002
0
          if (Mask[i] < 0) {
3003
0
            for (unsigned fi = 0; fi < factor; ++fi)
3004
0
              NewMask.push_back(Mask[i]);
3005
0
          }
3006
0
          else {
3007
0
            for (unsigned fi = 0; fi < factor; ++fi)
3008
0
              NewMask.push_back(Mask[i]*factor+fi);
3009
0
          }
3010
0
        }
3011
0
        Mask = NewMask;
3012
0
        VT = NewVT;
3013
0
      }
3014
12
      EltVT = NewEltVT;
3015
12
    }
3016
453
    unsigned NumElems = VT.getVectorNumElements();
3017
453
    SmallVector<SDValue, 16> Ops;
3018
1.76k
    for (unsigned i = 0; i != NumElems; 
++i1.31k
) {
3019
1.31k
      if (Mask[i] < 0) {
3020
22
        Ops.push_back(DAG.getUNDEF(EltVT));
3021
22
        continue;
3022
22
      }
3023
1.29k
      unsigned Idx = Mask[i];
3024
1.29k
      if (Idx < NumElems)
3025
848
        Ops.push_back(DAG.getNode(
3026
848
            ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3027
848
            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
3028
442
      else
3029
442
        Ops.push_back(DAG.getNode(
3030
442
            ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3031
442
            DAG.getConstant(Idx - NumElems, dl,
3032
442
                            TLI.getVectorIdxTy(DAG.getDataLayout()))));
3033
1.29k
    }
3034
453
3035
453
    Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3036
453
    // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3037
453
    Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3038
453
    Results.push_back(Tmp1);
3039
453
    break;
3040
52
  }
3041
712
  case ISD::EXTRACT_ELEMENT: {
3042
712
    EVT OpTy = Node->getOperand(0).getValueType();
3043
712
    if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3044
336
      // 1 -> Hi
3045
336
      Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3046
336
                         DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3047
336
                                         TLI.getShiftAmountTy(
3048
336
                                             Node->getOperand(0).getValueType(),
3049
336
                                             DAG.getDataLayout())));
3050
336
      Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3051
376
    } else {
3052
376
      // 0 -> Lo
3053
376
      Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3054
376
                         Node->getOperand(0));
3055
376
    }
3056
712
    Results.push_back(Tmp1);
3057
712
    break;
3058
52
  }
3059
221
  case ISD::STACKSAVE:
3060
221
    // Expand to CopyFromReg if the target set
3061
221
    // StackPointerRegisterToSaveRestore.
3062
221
    if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3063
221
      Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3064
221
                                           Node->getValueType(0)));
3065
221
      Results.push_back(Results[0].getValue(1));
3066
221
    } else {
3067
0
      Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3068
0
      Results.push_back(Node->getOperand(0));
3069
0
    }
3070
221
    break;
3071
82
  case ISD::STACKRESTORE:
3072
82
    // Expand to CopyToReg if the target set
3073
82
    // StackPointerRegisterToSaveRestore.
3074
82
    if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
3075
82
      Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3076
82
                                         Node->getOperand(1)));
3077
82
    } else {
3078
0
      Results.push_back(Node->getOperand(0));
3079
0
    }
3080
82
    break;
3081
52
  case ISD::GET_DYNAMIC_AREA_OFFSET:
3082
0
    Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3083
0
    Results.push_back(Results[0].getValue(0));
3084
0
    break;
3085
325
  case ISD::FCOPYSIGN:
3086
325
    Results.push_back(ExpandFCOPYSIGN(Node));
3087
325
    break;
3088
52
  case ISD::FNEG:
3089
33
    // Expand Y = FNEG(X) ->  Y = SUB -0.0, X
3090
33
    Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3091
33
    // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3092
33
    Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3093
33
                       Node->getOperand(0));
3094
33
    Results.push_back(Tmp1);
3095
33
    break;
3096
52
  case ISD::FABS:
3097
37
    Results.push_back(ExpandFABS(Node));
3098
37
    break;
3099
52
  case ISD::SMIN:
3100
2
  case ISD::SMAX:
3101
2
  case ISD::UMIN:
3102
2
  case ISD::UMAX: {
3103
2
    // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3104
2
    ISD::CondCode Pred;
3105
2
    switch (Node->getOpcode()) {
3106
2
    
default: 0
llvm_unreachable0
("How did we get here?");
3107
2
    
case ISD::SMAX: Pred = ISD::SETGT; break0
;
3108
2
    
case ISD::SMIN: Pred = ISD::SETLT; break0
;
3109
2
    case ISD::UMAX: Pred = ISD::SETUGT; break;
3110
2
    
case ISD::UMIN: Pred = ISD::SETULT; break0
;
3111
2
    }
3112
2
    Tmp1 = Node->getOperand(0);
3113
2
    Tmp2 = Node->getOperand(1);
3114
2
    Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3115
2
    Results.push_back(Tmp1);
3116
2
    break;
3117
2
  }
3118
157
  case ISD::FMINNUM:
3119
157
  case ISD::FMAXNUM: {
3120
157
    if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3121
4
      Results.push_back(Expanded);
3122
157
    break;
3123
157
  }
3124
577
  case ISD::FSIN:
3125
577
  case ISD::FCOS: {
3126
577
    EVT VT = Node->getValueType(0);
3127
577
    // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3128
577
    // fcos which share the same operand and both are used.
3129
577
    if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3130
577
         
isSinCosLibcallAvailable(Node, TLI)476
)
3131
577
        && 
useSinCos(Node)213
) {
3132
142
      SDVTList VTs = DAG.getVTList(VT, VT);
3133
142
      Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3134
142
      if (Node->getOpcode() == ISD::FCOS)
3135
71
        Tmp1 = Tmp1.getValue(1);
3136
142
      Results.push_back(Tmp1);
3137
142
    }
3138
577
    break;
3139
577
  }
3140
577
  case ISD::FMAD:
3141
0
    llvm_unreachable("Illegal fmad should never be formed");
3142
577
3143
577
  case ISD::FP16_TO_FP:
3144
565
    if (Node->getValueType(0) != MVT::f32) {
3145
0
      // We can extend to types bigger than f32 in two steps without changing
3146
0
      // the result. Since "f16 -> f32" is much more commonly available, give
3147
0
      // CodeGen the option of emitting that before resorting to a libcall.
3148
0
      SDValue Res =
3149
0
          DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3150
0
      Results.push_back(
3151
0
          DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3152
0
    }
3153
565
    break;
3154
577
  case ISD::FP_TO_FP16:
3155
564
    LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3156
564
    if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3157
15
      SDValue Op = Node->getOperand(0);
3158
15
      MVT SVT = Op.getSimpleValueType();
3159
15
      if ((SVT == MVT::f64 || 
SVT == MVT::f802
) &&
3160
15
          TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3161
13
        // Under fastmath, we can expand this node into a fround followed by
3162
13
        // a float-half conversion.
3163
13
        SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3164
13
                                       DAG.getIntPtrConstant(0, dl));
3165
13
        Results.push_back(
3166
13
            DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3167
13
      }
3168
15
    }
3169
564
    break;
3170
28.7k
  case ISD::ConstantFP: {
3171
28.7k
    ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3172
28.7k
    // Check to see if this FP immediate is already legal.
3173
28.7k
    // If this is a legal constant, turn it into a TargetConstantFP node.
3174
28.7k
    if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3175
28.7k
                          DAG.getMachineFunction().getFunction().hasOptSize()))
3176
12.4k
      Results.push_back(ExpandConstantFP(CFP, true));
3177
28.7k
    break;
3178
577
  }
3179
577
  case ISD::Constant: {
3180
0
    ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3181
0
    Results.push_back(ExpandConstant(CP));
3182
0
    break;
3183
577
  }
3184
577
  case ISD::FSUB: {
3185
299
    EVT VT = Node->getValueType(0);
3186
299
    if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3187
299
        
TLI.isOperationLegalOrCustom(ISD::FNEG, VT)181
) {
3188
181
      const SDNodeFlags Flags = Node->getFlags();
3189
181
      Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3190
181
      Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3191
181
      Results.push_back(Tmp1);
3192
181
    }
3193
299
    break;
3194
577
  }
3195
577
  case ISD::SUB: {
3196
0
    EVT VT = Node->getValueType(0);
3197
0
    assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3198
0
           TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3199
0
           "Don't know how to expand this subtraction!");
3200
0
    Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3201
0
               DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3202
0
                               VT));
3203
0
    Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3204
0
    Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3205
0
    break;
3206
577
  }
3207
3.40k
  case ISD::UREM:
3208
3.40k
  case ISD::SREM: {
3209
3.40k
    EVT VT = Node->getValueType(0);
3210
3.40k
    bool isSigned = Node->getOpcode() == ISD::SREM;
3211
3.40k
    unsigned DivOpc = isSigned ? 
ISD::SDIV1.96k
:
ISD::UDIV1.43k
;
3212
3.40k
    unsigned DivRemOpc = isSigned ? 
ISD::SDIVREM1.96k
:
ISD::UDIVREM1.43k
;
3213
3.40k
    Tmp2 = Node->getOperand(0);
3214
3.40k
    Tmp3 = Node->getOperand(1);
3215
3.40k
    if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3216
1.44k
      SDVTList VTs = DAG.getVTList(VT, VT);
3217
1.44k
      Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3218
1.44k
      Results.push_back(Tmp1);
3219
1.95k
    } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3220
1.86k
      // X % Y -> X-X/Y*Y
3221
1.86k
      Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3222
1.86k
      Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3223
1.86k
      Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3224
1.86k
      Results.push_back(Tmp1);
3225
1.86k
    }
3226
3.40k
    break;
3227
3.40k
  }
3228
3.40k
  case ISD::UDIV:
3229
1.10k
  case ISD::SDIV: {
3230
1.10k
    bool isSigned = Node->getOpcode() == ISD::SDIV;
3231
1.10k
    unsigned DivRemOpc = isSigned ? 
ISD::SDIVREM310
:
ISD::UDIVREM792
;
3232
1.10k
    EVT VT = Node->getValueType(0);
3233
1.10k
    if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3234
1.08k
      SDVTList VTs = DAG.getVTList(VT, VT);
3235
1.08k
      Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3236
1.08k
                         Node->getOperand(1));
3237
1.08k
      Results.push_back(Tmp1);
3238
1.08k
    }
3239
1.10k
    break;
3240
1.10k
  }
3241
1.10k
  case ISD::MULHU:
3242
784
  case ISD::MULHS: {
3243
784
    unsigned ExpandOpcode =
3244
784
        Node->getOpcode() == ISD::MULHU ? 
ISD::UMUL_LOHI653
:
ISD::SMUL_LOHI131
;
3245
784
    EVT VT = Node->getValueType(0);
3246
784
    SDVTList VTs = DAG.getVTList(VT, VT);
3247
784
3248
784
    Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3249
784
                       Node->getOperand(1));
3250
784
    Results.push_back(Tmp1.getValue(1));
3251
784
    break;
3252
784
  }
3253
784
  case ISD::UMUL_LOHI:
3254
690
  case ISD::SMUL_LOHI: {
3255
690
    SDValue LHS = Node->getOperand(0);
3256
690
    SDValue RHS = Node->getOperand(1);
3257
690
    MVT VT = LHS.getSimpleValueType();
3258
690
    unsigned MULHOpcode =
3259
690
        Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : 
ISD::MULHS0
;
3260
690
3261
690
    if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3262
552
      Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3263
552
      Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3264
552
      break;
3265
552
    }
3266
138
3267
138
    SmallVector<SDValue, 4> Halves;
3268
138
    EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3269
138
    assert(TLI.isTypeLegal(HalfType));
3270
138
    if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3271
138
                           HalfType, DAG,
3272
138
                           TargetLowering::MulExpansionKind::Always)) {
3273
414
      for (unsigned i = 0; i < 2; 
++i276
) {
3274
276
        SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3275
276
        SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3276
276
        SDValue Shift = DAG.getConstant(
3277
276
            HalfType.getScalarSizeInBits(), dl,
3278
276
            TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3279
276
        Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3280
276
        Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3281
276
      }
3282
138
      break;
3283
138
    }
3284
0
    break;
3285
0
  }
3286
706
  case ISD::MUL: {
3287
706
    EVT VT = Node->getValueType(0);
3288
706
    SDVTList VTs = DAG.getVTList(VT, VT);
3289
706
    // See if multiply or divide can be lowered using two-result operations.
3290
706
    // We just need the low half of the multiply; try both the signed
3291
706
    // and unsigned forms. If the target supports both SMUL_LOHI and
3292
706
    // UMUL_LOHI, form a preference by checking which forms of plain
3293
706
    // MULH it supports.
3294
706
    bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3295
706
    bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3296
706
    bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3297
706
    bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3298
706
    unsigned OpToUse = 0;
3299
706
    if (HasSMUL_LOHI && 
!HasMULHS7
) {
3300
7
      OpToUse = ISD::SMUL_LOHI;
3301
699
    } else if (HasUMUL_LOHI && 
!HasMULHU0
) {
3302
0
      OpToUse = ISD::UMUL_LOHI;
3303
699
    } else if (HasSMUL_LOHI) {
3304
0
      OpToUse = ISD::SMUL_LOHI;
3305
699
    } else if (HasUMUL_LOHI) {
3306
0
      OpToUse = ISD::UMUL_LOHI;
3307
0
    }
3308
706
    if (OpToUse) {
3309
7
      Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3310
7
                                    Node->getOperand(1)));
3311
7
      break;
3312
7
    }
3313
699
3314
699
    SDValue Lo, Hi;
3315
699
    EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3316
699
    if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3317
699
        TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3318
699
        TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3319
699
        TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3320
699
        TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3321
699
                      TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3322
671
      Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3323
671
      Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3324
671
      SDValue Shift =
3325
671
          DAG.getConstant(HalfType.getSizeInBits(), dl,
3326
671
                          TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3327
671
      Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3328
671
      Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3329
671
    }
3330
699
    break;
3331
699
  }
3332
699
  case ISD::FSHL:
3333
30
  case ISD::FSHR:
3334
30
    if (TLI.expandFunnelShift(Node, Tmp1, DAG))
3335
30
      Results.push_back(Tmp1);
3336
30
    break;
3337
30
  case ISD::ROTL:
3338
0
  case ISD::ROTR:
3339
0
    if (TLI.expandROT(Node, Tmp1, DAG))
3340
0
      Results.push_back(Tmp1);
3341
0
    break;
3342
212
  case ISD::SADDSAT:
3343
212
  case ISD::UADDSAT:
3344
212
  case ISD::SSUBSAT:
3345
212
  case ISD::USUBSAT:
3346
212
    Results.push_back(TLI.expandAddSubSat(Node, DAG));
3347
212
    break;
3348
212
  case ISD::SMULFIX:
3349
74
  case ISD::SMULFIXSAT:
3350
74
  case ISD::UMULFIX:
3351
74
    Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3352
74
    break;
3353
74
  case ISD::ADDCARRY:
3354
4
  case ISD::SUBCARRY: {
3355
4
    SDValue LHS = Node->getOperand(0);
3356
4
    SDValue RHS = Node->getOperand(1);
3357
4
    SDValue Carry = Node->getOperand(2);
3358
4
3359
4
    bool IsAdd = Node->getOpcode() == ISD::ADDCARRY;
3360
4
3361
4
    // Initial add of the 2 operands.
3362
4
    unsigned Op = IsAdd ? ISD::ADD : 
ISD::SUB0
;
3363
4
    EVT VT = LHS.getValueType();
3364
4
    SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3365
4
3366
4
    // Initial check for overflow.
3367
4
    EVT CarryType = Node->getValueType(1);
3368
4
    EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3369
4
    ISD::CondCode CC = IsAdd ? ISD::SETULT : 
ISD::SETUGT0
;
3370
4
    SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3371
4
3372
4
    // Add of the sum and the carry.
3373
4
    SDValue CarryExt =
3374
4
        DAG.getZeroExtendInReg(DAG.getZExtOrTrunc(Carry, dl, VT), dl, MVT::i1);
3375
4
    SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3376
4
3377
4
    // Second check for overflow. If we are adding, we can only overflow if the
3378
4
    // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3379
4
    // If we are subtracting, we can only overflow if the initial sum is 0 and
3380
4
    // the carry is set, resulting in a new sum of all 1s.
3381
4
    SDValue Zero = DAG.getConstant(0, dl, VT);
3382
4
    SDValue Overflow2 =
3383
4
        IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3384
4
              : 
DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ)0
;
3385
4
    Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3386
4
                            DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3387
4
3388
4
    SDValue ResultCarry =
3389
4
        DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3390
4
3391
4
    Results.push_back(Sum2);
3392
4
    Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3393
4
    break;
3394
4
  }
3395
44
  case ISD::SADDO:
3396
44
  case ISD::SSUBO: {
3397
44
    SDValue Result, Overflow;
3398
44
    TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3399
44
    Results.push_back(Result);
3400
44
    Results.push_back(Overflow);
3401
44
    break;
3402
44
  }
3403
44
  case ISD::UADDO:
3404
39
  case ISD::USUBO: {
3405
39
    SDValue Result, Overflow;
3406
39
    TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3407
39
    Results.push_back(Result);
3408
39
    Results.push_back(Overflow);
3409
39
    break;
3410
39
  }
3411
60
  case ISD::UMULO:
3412
60
  case ISD::SMULO: {
3413
60
    SDValue Result, Overflow;
3414
60
    if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3415
60
      Results.push_back(Result);
3416
60
      Results.push_back(Overflow);
3417
60
    }
3418
60
    break;
3419
60
  }
3420
60
  case ISD::BUILD_PAIR: {
3421
24
    EVT PairTy = Node->getValueType(0);
3422
24
    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3423
24
    Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3424
24
    Tmp2 = DAG.getNode(
3425
24
        ISD::SHL, dl, PairTy, Tmp2,
3426
24
        DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3427
24
                        TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3428
24
    Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3429
24
    break;
3430
60
  }
3431
212
  case ISD::SELECT:
3432
212
    Tmp1 = Node->getOperand(0);
3433
212
    Tmp2 = Node->getOperand(1);
3434
212
    Tmp3 = Node->getOperand(2);
3435
212
    if (Tmp1.getOpcode() == ISD::SETCC) {
3436
155
      Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3437
155
                             Tmp2, Tmp3,
3438
155
                             cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3439
155
    } else {
3440
57
      Tmp1 = DAG.getSelectCC(dl, Tmp1,
3441
57
                             DAG.getConstant(0, dl, Tmp1.getValueType()),
3442
57
                             Tmp2, Tmp3, ISD::SETNE);
3443
57
    }
3444
212
    Tmp1->setFlags(Node->getFlags());
3445
212
    Results.push_back(Tmp1);
3446
212
    break;
3447
374
  case ISD::BR_JT: {
3448
374
    SDValue Chain = Node->getOperand(0);
3449
374
    SDValue Table = Node->getOperand(1);
3450
374
    SDValue Index = Node->getOperand(2);
3451
374
3452
374
    const DataLayout &TD = DAG.getDataLayout();
3453
374
    EVT PTy = TLI.getPointerTy(TD);
3454
374
3455
374
    unsigned EntrySize =
3456
374
      DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3457
374
3458
374
    // For power-of-two jumptable entry sizes convert multiplication to a shift.
3459
374
    // This transformation needs to be done here since otherwise the MIPS
3460
374
    // backend will end up emitting a three instruction multiply sequence
3461
374
    // instead of a single shift and MSP430 will call a runtime function.
3462
374
    if (llvm::isPowerOf2_32(EntrySize))
3463
374
      Index = DAG.getNode(
3464
374
          ISD::SHL, dl, Index.getValueType(), Index,
3465
374
          DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3466
0
    else
3467
0
      Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3468
0
                          DAG.getConstant(EntrySize, dl, Index.getValueType()));
3469
374
    SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3470
374
                               Index, Table);
3471
374
3472
374
    EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3473
374
    SDValue LD = DAG.getExtLoad(
3474
374
        ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3475
374
        MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3476
374
    Addr = LD;
3477
374
    if (TLI.isJumpTableRelative()) {
3478
280
      // For PIC, the sequence is:
3479
280
      // BRIND(load(Jumptable + index) + RelocBase)
3480
280
      // RelocBase can be JumpTable, GOT or some sort of global base.
3481
280
      Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3482
280
                          TLI.getPICJumpTableRelocBase(Table, DAG));
3483
280
    }
3484
374
3485
374
    Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3486
374
    Results.push_back(Tmp1);
3487
374
    break;
3488
60
  }
3489
11.4k
  case ISD::BRCOND:
3490
11.4k
    // Expand brcond's setcc into its constituent parts and create a BR_CC
3491
11.4k
    // Node.
3492
11.4k
    Tmp1 = Node->getOperand(0);
3493
11.4k
    Tmp2 = Node->getOperand(1);
3494
11.4k
    if (Tmp2.getOpcode() == ISD::SETCC) {
3495
1
      Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3496
1
                         Tmp1, Tmp2.getOperand(2),
3497
1
                         Tmp2.getOperand(0), Tmp2.getOperand(1),
3498
1
                         Node->getOperand(2));
3499
11.4k
    } else {
3500
11.4k
      // We test only the i1 bit.  Skip the AND if UNDEF or another AND.
3501
11.4k
      if (Tmp2.isUndef() ||
3502
11.4k
          (Tmp2.getOpcode() == ISD::AND &&
3503
11.4k
           
isa<ConstantSDNode>(Tmp2.getOperand(1))1.57k
&&
3504
11.4k
           
cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 11.35k
))
3505
1.35k
        Tmp3 = Tmp2;
3506
10.0k
      else
3507
10.0k
        Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3508
10.0k
                           DAG.getConstant(1, dl, Tmp2.getValueType()));
3509
11.4k
      Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3510
11.4k
                         DAG.getCondCode(ISD::SETNE), Tmp3,
3511
11.4k
                         DAG.getConstant(0, dl, Tmp3.getValueType()),
3512
11.4k
                         Node->getOperand(2));
3513
11.4k
    }
3514
11.4k
    Results.push_back(Tmp1);
3515
11.4k
    break;
3516
4.52k
  case ISD::SETCC: {
3517
4.52k
    Tmp1 = Node->getOperand(0);
3518
4.52k
    Tmp2 = Node->getOperand(1);
3519
4.52k
    Tmp3 = Node->getOperand(2);
3520
4.52k
    bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3521
4.52k
                                           Tmp3, NeedInvert, dl);
3522
4.52k
3523
4.52k
    if (Legalized) {
3524
748
      // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3525
748
      // condition code, create a new SETCC node.
3526
748
      if (Tmp3.getNode())
3527
409
        Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3528
409
                           Tmp1, Tmp2, Tmp3, Node->getFlags());
3529
748
3530
748
      // If we expanded the SETCC by inverting the condition code, then wrap
3531
748
      // the existing SETCC in a NOT to restore the intended condition.
3532
748
      if (NeedInvert)
3533
222
        Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3534
748
3535
748
      Results.push_back(Tmp1);
3536
748
      break;
3537
748
    }
3538
3.77k
3539
3.77k
    // Otherwise, SETCC for the given comparison type must be completely
3540
3.77k
    // illegal; expand it into a SELECT_CC.
3541
3.77k
    EVT VT = Node->getValueType(0);
3542
3.77k
    int TrueValue;
3543
3.77k
    switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3544
3.77k
    case TargetLowering::ZeroOrOneBooleanContent:
3545
2.41k
    case TargetLowering::UndefinedBooleanContent:
3546
2.41k
      TrueValue = 1;
3547
2.41k
      break;
3548
2.41k
    case TargetLowering::ZeroOrNegativeOneBooleanContent:
3549
1.36k
      TrueValue = -1;
3550
1.36k
      break;
3551
3.77k
    }
3552
3.77k
    Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3553
3.77k
                       DAG.getConstant(TrueValue, dl, VT),
3554
3.77k
                       DAG.getConstant(0, dl, VT),
3555
3.77k
                       Tmp3);
3556
3.77k
    Tmp1->setFlags(Node->getFlags());
3557
3.77k
    Results.push_back(Tmp1);
3558
3.77k
    break;
3559
3.77k
  }
3560
3.77k
  case ISD::SELECT_CC: {
3561
2.12k
    Tmp1 = Node->getOperand(0);   // LHS
3562
2.12k
    Tmp2 = Node->getOperand(1);   // RHS
3563
2.12k
    Tmp3 = Node->getOperand(2);   // True
3564
2.12k
    Tmp4 = Node->getOperand(3);   // False
3565
2.12k
    EVT VT = Node->getValueType(0);
3566
2.12k
    SDValue CC = Node->getOperand(4);
3567
2.12k
    ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3568
2.12k
3569
2.12k
    if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3570
1.81k
      // If the condition code is legal, then we need to expand this
3571
1.81k
      // node using SETCC and SELECT.
3572
1.81k
      EVT CmpVT = Tmp1.getValueType();
3573
1.81k
      assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3574
1.81k
             "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3575
1.81k
             "expanded.");
3576
1.81k
      EVT CCVT = getSetCCResultType(CmpVT);
3577
1.81k
      SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
3578
1.81k
      Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3579
1.81k
      break;
3580
1.81k
    }
3581
309
3582
309
    // SELECT_CC is legal, so the condition code must not be.
3583
309
    bool Legalized = false;
3584
309
    // Try to legalize by inverting the condition.  This is for targets that
3585
309
    // might support an ordered version of a condition, but not the unordered
3586
309
    // version (or vice versa).
3587
309
    ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3588
309
                                               Tmp1.getValueType().isInteger());
3589
309
    if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3590
257
      // Use the new condition code and swap true and false
3591
257
      Legalized = true;
3592
257
      Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3593
257
      Tmp1->setFlags(Node->getFlags());
3594
257
    } else {
3595
52
      // If The inverse is not legal, then try to swap the arguments using
3596
52
      // the inverse condition code.
3597
52
      ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3598
52
      if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3599
13
        // The swapped inverse condition is legal, so swap true and false,
3600
13
        // lhs and rhs.
3601
13
        Legalized = true;
3602
13
        Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3603
13
        Tmp1->setFlags(Node->getFlags());
3604
13
      }
3605
52
    }
3606
309
3607
309
    if (!Legalized) {
3608
39
      Legalized = LegalizeSetCCCondCode(
3609
39
          getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3610
39
          dl);
3611
39
3612
39
      assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3613
39
3614
39
      // If we expanded the SETCC by inverting the condition code, then swap
3615
39
      // the True/False operands to match.
3616
39
      if (NeedInvert)
3617
0
        std::swap(Tmp3, Tmp4);
3618
39
3619
39
      // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3620
39
      // condition code, create a new SELECT_CC node.
3621
39
      if (CC.getNode()) {
3622
13
        Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3623
13
                           Tmp1, Tmp2, Tmp3, Tmp4, CC);
3624
26
      } else {
3625
26
        Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3626
26
        CC = DAG.getCondCode(ISD::SETNE);
3627
26
        Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3628
26
                           Tmp2, Tmp3, Tmp4, CC);
3629
26
      }
3630
39
      Tmp1->setFlags(Node->getFlags());
3631
39
    }
3632
309
    Results.push_back(Tmp1);
3633
309
    break;
3634
309
  }
3635
309
  case ISD::BR_CC: {
3636
10
    Tmp1 = Node->getOperand(0);              // Chain
3637
10
    Tmp2 = Node->getOperand(2);              // LHS
3638
10
    Tmp3 = Node->getOperand(3);              // RHS
3639
10
    Tmp4 = Node->getOperand(1);              // CC
3640
10
3641
10
    bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3642
10
        Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3643
10
    (void)Legalized;
3644
10
    assert(Legalized && "Can't legalize BR_CC with legal condition!");
3645
10
3646
10
    assert(!NeedInvert && "Don't know how to invert BR_CC!");
3647
10
3648
10
    // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3649
10
    // node.
3650
10
    if (Tmp4.getNode()) {
3651
0
      Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3652
0
                         Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3653
10
    } else {
3654
10
      Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3655
10
      Tmp4 = DAG.getCondCode(ISD::SETNE);
3656
10
      Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3657
10
                         Tmp2, Tmp3, Node->getOperand(4));
3658
10
    }
3659
10
    Results.push_back(Tmp1);
3660
10
    break;
3661
309
  }
3662
55.7k
  case ISD::BUILD_VECTOR:
3663
55.7k
    Results.push_back(ExpandBUILD_VECTOR(Node));
3664
55.7k
    break;
3665
309
  case ISD::SRA:
3666
38
  case ISD::SRL:
3667
38
  case ISD::SHL: {
3668
38
    // Scalarize vector SRA/SRL/SHL.
3669
38
    EVT VT = Node->getValueType(0);
3670
38
    assert(VT.isVector() && "Unable to legalize non-vector shift");
3671
38
    assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3672
38
    unsigned NumElem = VT.getVectorNumElements();
3673
38
3674
38
    SmallVector<SDValue, 8> Scalars;
3675
138
    for (unsigned Idx = 0; Idx < NumElem; 
Idx++100
) {
3676
100
      SDValue Ex = DAG.getNode(
3677
100
          ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3678
100
          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3679
100
      SDValue Sh = DAG.getNode(
3680
100
          ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3681
100
          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3682
100
      Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3683
100
                                    VT.getScalarType(), Ex, Sh));
3684
100
    }
3685
38
3686
38
    SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3687
38
    ReplaceNode(SDValue(Node, 0), Result);
3688
38
    break;
3689
38
  }
3690
38
  case ISD::VECREDUCE_FADD:
3691
0
  case ISD::VECREDUCE_FMUL:
3692
0
  case ISD::VECREDUCE_ADD:
3693
0
  case ISD::VECREDUCE_MUL:
3694
0
  case ISD::VECREDUCE_AND:
3695
0
  case ISD::VECREDUCE_OR:
3696
0
  case ISD::VECREDUCE_XOR:
3697
0
  case ISD::VECREDUCE_SMAX:
3698
0
  case ISD::VECREDUCE_SMIN:
3699
0
  case ISD::VECREDUCE_UMAX:
3700
0
  case ISD::VECREDUCE_UMIN:
3701
0
  case ISD::VECREDUCE_FMAX:
3702
0
  case ISD::VECREDUCE_FMIN:
3703
0
    Results.push_back(TLI.expandVecReduce(Node, DAG));
3704
0
    break;
3705
25.3k
  case ISD::GLOBAL_OFFSET_TABLE:
3706
25.3k
  case ISD::GlobalAddress:
3707
25.3k
  case ISD::GlobalTLSAddress:
3708
25.3k
  case ISD::ExternalSymbol:
3709
25.3k
  case ISD::ConstantPool:
3710
25.3k
  case ISD::JumpTable:
3711
25.3k
  case ISD::INTRINSIC_W_CHAIN:
3712
25.3k
  case ISD::INTRINSIC_WO_CHAIN:
3713
25.3k
  case ISD::INTRINSIC_VOID:
3714
25.3k
    // FIXME: Custom lowering for these operations shouldn't return null!
3715
25.3k
    break;
3716
210k
  }
3717
210k
3718
210k
  // Replace the original node with the legalized result.
3719
210k
  if (Results.empty()) {
3720
64.6k
    LLVM_DEBUG(dbgs() << "Cannot expand node\n");
3721
64.6k
    return false;
3722
64.6k
  }
3723
145k
3724
145k
  LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
3725
145k
  ReplaceNode(Node, Results.data());
3726
145k
  return true;
3727
145k
}
3728
3729
64.8k
void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3730
64.8k
  LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
3731
64.8k
  SmallVector<SDValue, 8> Results;
3732
64.8k
  SDLoc dl(Node);
3733
64.8k
  // FIXME: Check flags on the node to see if we can use a finite call.
3734
64.8k
  bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && 
TM.Options.NoNaNsFPMath110
;
3735
64.8k
  unsigned Opc = Node->getOpcode();
3736
64.8k
  switch (Opc) {
3737
64.8k
  case ISD::ATOMIC_FENCE: {
3738
13
    // If the target didn't lower this, lower it to '__sync_synchronize()' call
3739
13
    // FIXME: handle "fence singlethread" more efficiently.
3740
13
    TargetLowering::ArgListTy Args;
3741
13
3742
13
    TargetLowering::CallLoweringInfo CLI(DAG);
3743
13
    CLI.setDebugLoc(dl)
3744
13
        .setChain(Node->getOperand(0))
3745
13
        .setLibCallee(
3746
13
            CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3747
13
            DAG.getExternalSymbol("__sync_synchronize",
3748
13
                                  TLI.getPointerTy(DAG.getDataLayout())),
3749
13
            std::move(Args));
3750
13
3751
13
    std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3752
13
3753
13
    Results.push_back(CallResult.second);
3754
13
    break;
3755
64.8k
  }
3756
64.8k
  // By default, atomic intrinsics are marked Legal and lowered. Targets
3757
64.8k
  // which don't support them directly, however, may want libcalls, in which
3758
64.8k
  // case they mark them Expand, and we get here.
3759
64.8k
  case ISD::ATOMIC_SWAP:
3760
86
  case ISD::ATOMIC_LOAD_ADD:
3761
86
  case ISD::ATOMIC_LOAD_SUB:
3762
86
  case ISD::ATOMIC_LOAD_AND:
3763
86
  case ISD::ATOMIC_LOAD_CLR:
3764
86
  case ISD::ATOMIC_LOAD_OR:
3765
86
  case ISD::ATOMIC_LOAD_XOR:
3766
86
  case ISD::ATOMIC_LOAD_NAND:
3767
86
  case ISD::ATOMIC_LOAD_MIN:
3768
86
  case ISD::ATOMIC_LOAD_MAX:
3769
86
  case ISD::ATOMIC_LOAD_UMIN:
3770
86
  case ISD::ATOMIC_LOAD_UMAX:
3771
86
  case ISD::ATOMIC_CMP_SWAP: {
3772
86
    MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3773
86
    RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3774
86
    assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
3775
86
3776
86
    std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3777
86
    Results.push_back(Tmp.first);
3778
86
    Results.push_back(Tmp.second);
3779
86
    break;
3780
86
  }
3781
86
  case ISD::TRAP: {
3782
0
    // If this operation is not supported, lower it to 'abort()' call
3783
0
    TargetLowering::ArgListTy Args;
3784
0
    TargetLowering::CallLoweringInfo CLI(DAG);
3785
0
    CLI.setDebugLoc(dl)
3786
0
        .setChain(Node->getOperand(0))
3787
0
        .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3788
0
                      DAG.getExternalSymbol(
3789
0
                          "abort", TLI.getPointerTy(DAG.getDataLayout())),
3790
0
                      std::move(Args));
3791
0
    std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3792
0
3793
0
    Results.push_back(CallResult.second);
3794
0
    break;
3795
86
  }
3796
117
  case ISD::FMINNUM:
3797
117
  case ISD::STRICT_FMINNUM:
3798
117
    Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3799
117
                                      RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3800
117
                                      RTLIB::FMIN_PPCF128));
3801
117
    break;
3802
160
  case ISD::FMAXNUM:
3803
160
  case ISD::STRICT_FMAXNUM:
3804
160
    Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3805
160
                                      RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3806
160
                                      RTLIB::FMAX_PPCF128));
3807
160
    break;
3808
160
  case ISD::FSQRT:
3809
7
  case ISD::STRICT_FSQRT:
3810
7
    Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3811
7
                                      RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3812
7
                                      RTLIB::SQRT_PPCF128));
3813
7
    break;
3814
7
  case ISD::FCBRT:
3815
2
    Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3816
2
                                      RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3817
2
                                      RTLIB::CBRT_PPCF128));
3818
2
    break;
3819
330
  case ISD::FSIN:
3820
330
  case ISD::STRICT_FSIN:
3821
330
    Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3822
330
                                      RTLIB::SIN_F80, RTLIB::SIN_F128,
3823
330
                                      RTLIB::SIN_PPCF128));
3824
330
    break;
3825
330
  case ISD::FCOS:
3826
265
  case ISD::STRICT_FCOS:
3827
265
    Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3828
265
                                      RTLIB::COS_F80, RTLIB::COS_F128,
3829
265
                                      RTLIB::COS_PPCF128));
3830
265
    break;
3831
265
  case ISD::FSINCOS:
3832
34
    // Expand into sincos libcall.
3833
34
    ExpandSinCosLibCall(Node, Results);
3834
34
    break;
3835
265
  case ISD::FLOG:
3836
229
  case ISD::STRICT_FLOG:
3837
229
    if (CanUseFiniteLibCall && 
DAG.getLibInfo().has(LibFunc_log_finite)11
)
3838
4
      Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
3839
4
                                        RTLIB::LOG_FINITE_F64,
3840
4
                                        RTLIB::LOG_FINITE_F80,
3841
4
                                        RTLIB::LOG_FINITE_F128,
3842
4
                                        RTLIB::LOG_FINITE_PPCF128));
3843
225
    else
3844
225
      Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3845
225
                                        RTLIB::LOG_F80, RTLIB::LOG_F128,
3846
225
                                        RTLIB::LOG_PPCF128));
3847
229
    break;
3848
229
  case ISD::FLOG2:
3849
223
  case ISD::STRICT_FLOG2:
3850
223
    if (CanUseFiniteLibCall && 
DAG.getLibInfo().has(LibFunc_log2_finite)11
)
3851
4
      Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
3852
4
                                        RTLIB::LOG2_FINITE_F64,
3853
4
                                        RTLIB::LOG2_FINITE_F80,
3854
4
                                        RTLIB::LOG2_FINITE_F128,
3855
4
                                        RTLIB::LOG2_FINITE_PPCF128));
3856
219
    else
3857
219
      Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3858
219
                                        RTLIB::LOG2_F80, RTLIB::LOG2_F128,
3859
219
                                        RTLIB::LOG2_PPCF128));
3860
223
    break;
3861
255
  case ISD::FLOG10:
3862
255
  case ISD::STRICT_FLOG10:
3863
255
    if (CanUseFiniteLibCall && 
DAG.getLibInfo().has(LibFunc_log10_finite)11
)
3864
4
      Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
3865
4
                                        RTLIB::LOG10_FINITE_F64,
3866
4
                                        RTLIB::LOG10_FINITE_F80,
3867
4
                                        RTLIB::LOG10_FINITE_F128,
3868
4
                                        RTLIB::LOG10_FINITE_PPCF128));
3869
251
    else
3870
251
      Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3871
251
                                        RTLIB::LOG10_F80, RTLIB::LOG10_F128,
3872
251
                                        RTLIB::LOG10_PPCF128));
3873
255
    break;
3874
336
  case ISD::FEXP:
3875
336
  case ISD::STRICT_FEXP:
3876
336
    if (CanUseFiniteLibCall && 
DAG.getLibInfo().has(LibFunc_exp_finite)11
)
3877
4
      Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
3878
4
                                        RTLIB::EXP_FINITE_F64,
3879
4
                                        RTLIB::EXP_FINITE_F80,
3880
4
                                        RTLIB::EXP_FINITE_F128,
3881
4
                                        RTLIB::EXP_FINITE_PPCF128));
3882
332
    else
3883
332
      Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3884
332
                                        RTLIB::EXP_F80, RTLIB::EXP_F128,
3885
332
                                        RTLIB::EXP_PPCF128));
3886
336
    break;
3887
336
  case ISD::FEXP2:
3888
233
  case ISD::STRICT_FEXP2:
3889
233
    if (CanUseFiniteLibCall && 
DAG.getLibInfo().has(LibFunc_exp2_finite)11
)
3890
4
      Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
3891
4
                                        RTLIB::EXP2_FINITE_F64,
3892
4
                                        RTLIB::EXP2_FINITE_F80,
3893
4
                                        RTLIB::EXP2_FINITE_F128,
3894
4
                                        RTLIB::EXP2_FINITE_PPCF128));
3895
229
    else
3896
229
      Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
3897
229
                                        RTLIB::EXP2_F80, RTLIB::EXP2_F128,
3898
229
                                        RTLIB::EXP2_PPCF128));
3899
233
    break;
3900
233
  case ISD::FTRUNC:
3901
68
  case ISD::STRICT_FTRUNC:
3902
68
    Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3903
68
                                      RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
3904
68
                                      RTLIB::TRUNC_PPCF128));
3905
68
    break;
3906
101
  case ISD::FFLOOR:
3907
101
  case ISD::STRICT_FFLOOR:
3908
101
    Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3909
101
                                      RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
3910
101
                                      RTLIB::FLOOR_PPCF128));
3911
101
    break;
3912
101
  case ISD::FCEIL:
3913
58
  case ISD::STRICT_FCEIL:
3914
58
    Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3915
58
                                      RTLIB::CEIL_F80, RTLIB::CEIL_F128,
3916
58
                                      RTLIB::CEIL_PPCF128));
3917
58
    break;
3918
80
  case ISD::FRINT:
3919
80
  case ISD::STRICT_FRINT:
3920
80
    Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3921
80
                                      RTLIB::RINT_F80, RTLIB::RINT_F128,
3922
80
                                      RTLIB::RINT_PPCF128));
3923
80
    break;
3924
98
  case ISD::FNEARBYINT:
3925
98
  case ISD::STRICT_FNEARBYINT:
3926
98
    Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3927
98
                                      RTLIB::NEARBYINT_F64,
3928
98
                                      RTLIB::NEARBYINT_F80,
3929
98
                                      RTLIB::NEARBYINT_F128,
3930
98
                                      RTLIB::NEARBYINT_PPCF128));
3931
98
    break;
3932
98
  case ISD::FROUND:
3933
70
  case ISD::STRICT_FROUND:
3934
70
    Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
3935
70
                                      RTLIB::ROUND_F64,
3936
70
                                      RTLIB::ROUND_F80,
3937
70
                                      RTLIB::ROUND_F128,
3938
70
                                      RTLIB::ROUND_PPCF128));
3939
70
    break;
3940
269
  case ISD::FPOWI:
3941
269
  case ISD::STRICT_FPOWI:
3942
269
    Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3943
269
                                      RTLIB::POWI_F80, RTLIB::POWI_F128,
3944
269
                                      RTLIB::POWI_PPCF128));
3945
269
    break;
3946
300
  case ISD::FPOW:
3947
300
  case ISD::STRICT_FPOW:
3948
300
    if (CanUseFiniteLibCall && 
DAG.getLibInfo().has(LibFunc_pow_finite)11
)
3949
4
      Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
3950
4
                                        RTLIB::POW_FINITE_F64,
3951
4
                                        RTLIB::POW_FINITE_F80,
3952
4
                                        RTLIB::POW_FINITE_F128,
3953
4
                                        RTLIB::POW_FINITE_PPCF128));
3954
296
    else
3955
296
      Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
3956
296
                                        RTLIB::POW_F80, RTLIB::POW_F128,
3957
296
                                        RTLIB::POW_PPCF128));
3958
300
    break;
3959
300
  case ISD::FDIV:
3960
23
    Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
3961
23
                                      RTLIB::DIV_F80, RTLIB::DIV_F128,
3962
23
                                      RTLIB::DIV_PPCF128));
3963
23
    break;
3964
300
  case ISD::FREM:
3965
219
  case ISD::STRICT_FREM:
3966
219
    Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
3967
219
                                      RTLIB::REM_F80, RTLIB::REM_F128,
3968
219
                                      RTLIB::REM_PPCF128));
3969
219
    break;
3970
241
  case ISD::FMA:
3971
241
  case ISD::STRICT_FMA:
3972
241
    Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
3973
241
                                      RTLIB::FMA_F80, RTLIB::FMA_F128,
3974
241
                                      RTLIB::FMA_PPCF128));
3975
241
    break;
3976
241
  case ISD::FADD:
3977
116
    Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
3978
116
                                      RTLIB::ADD_F80, RTLIB::ADD_F128,
3979
116
                                      RTLIB::ADD_PPCF128));
3980
116
    break;
3981
241
  case ISD::FMUL:
3982
224
    Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
3983
224
                                      RTLIB::MUL_F80, RTLIB::MUL_F128,
3984
224
                                      RTLIB::MUL_PPCF128));
3985
224
    break;
3986
565
  case ISD::FP16_TO_FP:
3987
565
    if (Node->getValueType(0) == MVT::f32) {
3988
565
      Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
3989
565
    }
3990
565
    break;
3991
551
  case ISD::FP_TO_FP16: {
3992
551
    RTLIB::Libcall LC =
3993
551
        RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
3994
551
    assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
3995
551
    Results.push_back(ExpandLibCall(LC, Node, false));
3996
551
    break;
3997
241
  }
3998
241
  case ISD::FSUB:
3999
118
    Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4000
118
                                      RTLIB::SUB_F80, RTLIB::SUB_F128,
4001
118
                                      RTLIB::SUB_PPCF128));
4002
118
    break;
4003
241
  case ISD::SREM:
4004
24
    Results.push_back(ExpandIntLibCall(Node, true,
4005
24
                                       RTLIB::SREM_I8,
4006
24
                                       RTLIB::SREM_I16, RTLIB::SREM_I32,
4007
24
                                       RTLIB::SREM_I64, RTLIB::SREM_I128));
4008
24
    break;
4009
241
  case ISD::UREM:
4010
68
    Results.push_back(ExpandIntLibCall(Node, false,
4011
68
                                       RTLIB::UREM_I8,
4012
68
                                       RTLIB::UREM_I16, RTLIB::UREM_I32,
4013
68
                                       RTLIB::UREM_I64, RTLIB::UREM_I128));
4014
68
    break;
4015
241
  case ISD::SDIV:
4016
64
    Results.push_back(ExpandIntLibCall(Node, true,
4017
64
                                       RTLIB::SDIV_I8,
4018
64
                                       RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4019
64
                                       RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4020
64
    break;
4021
241
  case ISD::UDIV:
4022
75
    Results.push_back(ExpandIntLibCall(Node, false,
4023
75
                                       RTLIB::UDIV_I8,
4024
75
                                       RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4025
75
                                       RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4026
75
    break;
4027
241
  case ISD::SDIVREM:
4028
14
  case ISD::UDIVREM:
4029
14
    // Expand into divrem libcall
4030
14
    ExpandDivRemLibCall(Node, Results);
4031
14
    break;
4032
37
  case ISD::MUL:
4033
37
    Results.push_back(ExpandIntLibCall(Node, false,
4034
37
                                       RTLIB::MUL_I8,
4035
37
                                       RTLIB::MUL_I16, RTLIB::MUL_I32,
4036
37
                                       RTLIB::MUL_I64, RTLIB::MUL_I128));
4037
37
    break;
4038
61
  case ISD::CTLZ_ZERO_UNDEF:
4039
61
    switch (Node->getSimpleValueType(0).SimpleTy) {
4040
61
    default:
4041
0
      llvm_unreachable("LibCall explicitly requested, but not available");
4042
61
    case MVT::i32:
4043
61
      Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4044
61
      break;
4045
61
    case MVT::i64:
4046
0
      Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4047
0
      break;
4048
61
    case MVT::i128:
4049
0
      Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4050
0
      break;
4051
61
    }
4052
61
    break;
4053
64.8k
  }
4054
64.8k
4055
64.8k
  // Replace the original node with the legalized result.
4056
64.8k
  if (!Results.empty()) {
4057
5.73k
    LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4058
5.73k
    ReplaceNode(Node, Results.data());
4059
5.73k
  } else
4060
64.8k
    LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4061
64.8k
}
4062
4063
// Determine the vector type to use in place of an original scalar element when
4064
// promoting equally sized vectors.
4065
static MVT getPromotedVectorElementType(const TargetLowering &TLI,
4066
13.7k
                                        MVT EltVT, MVT NewEltVT) {
4067
13.7k
  unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4068
13.7k
  MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4069
13.7k
  assert(TLI.isTypeLegal(MidVT) && "unexpected");
4070
13.7k
  return MidVT;
4071
13.7k
}
4072
4073
17.7k
void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4074
17.7k
  LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4075
17.7k
  SmallVector<SDValue, 8> Results;
4076
17.7k
  MVT OVT = Node->getSimpleValueType(0);
4077
17.7k
  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4078
17.7k
      
Node->getOpcode() == ISD::SINT_TO_FP17.5k
||
4079
17.7k
      
Node->getOpcode() == ISD::SETCC17.5k
||
4080
17.7k
      
Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT17.2k
||
4081
17.7k
      
Node->getOpcode() == ISD::INSERT_VECTOR_ELT5.48k
) {
4082
12.2k
    OVT = Node->getOperand(0).getSimpleValueType();
4083
12.2k
  }
4084
17.7k
  if (Node->getOpcode() == ISD::BR_CC)
4085
1
    OVT = Node->getOperand(2).getSimpleValueType();
4086
17.7k
  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4087
17.7k
  SDLoc dl(Node);
4088
17.7k
  SDValue Tmp1, Tmp2, Tmp3;
4089
17.7k
  switch (Node->getOpcode()) {
4090
17.7k
  case ISD::CTTZ:
4091
90
  case ISD::CTTZ_ZERO_UNDEF:
4092
90
  case ISD::CTLZ:
4093
90
  case ISD::CTLZ_ZERO_UNDEF:
4094
90
  case ISD::CTPOP:
4095
90
    // Zero extend the argument.
4096
90
    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4097
90
    if (Node->getOpcode() == ISD::CTTZ) {
4098
3
      // The count is the same in the promoted type except if the original
4099
3
      // value was zero.  This can be handled by setting the bit just off
4100
3
      // the top of the original type.
4101
3
      auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4102
3
                                        OVT.getSizeInBits());
4103
3
      Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4104
3
                         DAG.getConstant(TopBit, dl, NVT));
4105
3
    }
4106
90
    // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4107
90
    // already the correct result.
4108
90
    Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4109
90
    if (Node->getOpcode() == ISD::CTLZ ||
4110
90
        
Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF80
) {
4111
21
      // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4112
21
      Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4113
21
                          DAG.getConstant(NVT.getSizeInBits() -
4114
21
                                          OVT.getSizeInBits(), dl, NVT));
4115
21
    }
4116
90
    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4117
90
    break;
4118
90
  case ISD::BITREVERSE:
4119
3
  case ISD::BSWAP: {
4120
3
    unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4121
3
    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4122
3
    Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4123
3
    Tmp1 = DAG.getNode(
4124
3
        ISD::SRL, dl, NVT, Tmp1,
4125
3
        DAG.getConstant(DiffBits, dl,
4126
3
                        TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4127
3
4128
3
    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4129
3
    break;
4130
3
  }
4131
286
  case ISD::FP_TO_UINT:
4132
286
  case ISD::FP_TO_SINT:
4133
286
    Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4134
286
                                 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4135
286
    Results.push_back(Tmp1);
4136
286
    break;
4137
286
  case ISD::UINT_TO_FP:
4138
177
  case ISD::SINT_TO_FP:
4139
177
    Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4140
177
                                 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4141
177
    Results.push_back(Tmp1);
4142
177
    break;
4143
177
  case ISD::VAARG: {
4144
1
    SDValue Chain = Node->getOperand(0); // Get the chain.
4145
1
    SDValue Ptr = Node->getOperand(1); // Get the pointer.
4146
1
4147
1
    unsigned TruncOp;
4148
1
    if (OVT.isVector()) {
4149
0
      TruncOp = ISD::BITCAST;
4150
1
    } else {
4151
1
      assert(OVT.isInteger()
4152
1
        && "VAARG promotion is supported only for vectors or integer types");
4153
1
      TruncOp = ISD::TRUNCATE;
4154
1
    }
4155
1
4156
1
    // Perform the larger operation, then convert back
4157
1
    Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4158
1
             Node->getConstantOperandVal(3));
4159
1
    Chain = Tmp1.getValue(1);
4160
1
4161
1
    Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4162
1
4163
1
    // Modified the chain result - switch anything that used the old chain to
4164
1
    // use the new one.
4165
1
    DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4166
1
    DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4167
1
    if (UpdatedNodes) {
4168
0
      UpdatedNodes->insert(Tmp2.getNode());
4169
0
      UpdatedNodes->insert(Chain.getNode());
4170
0
    }
4171
1
    ReplacedNode(Node);
4172
1
    break;
4173
177
  }
4174
445
  case ISD::MUL:
4175
445
  case ISD::SDIV:
4176
445
  case ISD::SREM:
4177
445
  case ISD::UDIV:
4178
445
  case ISD::UREM:
4179
445
  case ISD::AND:
4180
445
  case ISD::OR:
4181
445
  case ISD::XOR: {
4182
445
    unsigned ExtOp, TruncOp;
4183
445
    if (OVT.isVector()) {
4184
444
      ExtOp   = ISD::BITCAST;
4185
444
      TruncOp = ISD::BITCAST;
4186
444
    } else {
4187
1
      assert(OVT.isInteger() && "Cannot promote logic operation");
4188
1
4189
1
      switch (Node->getOpcode()) {
4190
1
      default:
4191
1
        ExtOp = ISD::ANY_EXTEND;
4192
1
        break;
4193
1
      case ISD::SDIV:
4194
0
      case ISD::SREM:
4195
0
        ExtOp = ISD::SIGN_EXTEND;
4196
0
        break;
4197
0
      case ISD::UDIV:
4198
0
      case ISD::UREM:
4199
0
        ExtOp = ISD::ZERO_EXTEND;
4200
0
        break;
4201
1
      }
4202
1
      TruncOp = ISD::TRUNCATE;
4203
1
    }
4204
445
    // Promote each of the values to the new type.
4205
445
    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4206
445
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4207
445
    // Perform the larger operation, then convert back
4208
445
    Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4209
445
    Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4210
445
    break;
4211
445
  }
4212
445
  case ISD::UMUL_LOHI:
4213
0
  case ISD::SMUL_LOHI: {
4214
0
    // Promote to a multiply in a wider integer type.
4215
0
    unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4216
0
                                                         : ISD::SIGN_EXTEND;
4217
0
    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4218
0
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4219
0
    Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4220
0
4221
0
    auto &DL = DAG.getDataLayout();
4222
0
    unsigned OriginalSize = OVT.getScalarSizeInBits();
4223
0
    Tmp2 = DAG.getNode(
4224
0
        ISD::SRL, dl, NVT, Tmp1,
4225
0
        DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4226
0
    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4227
0
    Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4228
0
    break;
4229
0
  }
4230
467
  case ISD::SELECT: {
4231
467
    unsigned ExtOp, TruncOp;
4232
467
    if (Node->getValueType(0).isVector() ||
4233
467
        Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4234
203
      ExtOp   = ISD::BITCAST;
4235
203
      TruncOp = ISD::BITCAST;
4236
264
    } else if (Node->getValueType(0).isInteger()) {
4237
259
      ExtOp   = ISD::ANY_EXTEND;
4238
259
      TruncOp = ISD::TRUNCATE;
4239
259
    } else {
4240
5
      ExtOp   = ISD::FP_EXTEND;
4241
5
      TruncOp = ISD::FP_ROUND;
4242
5
    }
4243
467
    Tmp1 = Node->getOperand(0);
4244
467
    // Promote each of the values to the new type.
4245
467
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4246
467
    Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4247
467
    // Perform the larger operation, then round down.
4248
467
    Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4249
467
    Tmp1->setFlags(Node->getFlags());
4250
467
    if (TruncOp != ISD::FP_ROUND)
4251
462
      Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4252
5
    else
4253
5
      Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4254
5
                         DAG.getIntPtrConstant(0, dl));
4255
467
    Results.push_back(Tmp1);
4256
467
    break;
4257
0
  }
4258
1.19k
  case ISD::VECTOR_SHUFFLE: {
4259
1.19k
    ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4260
1.19k
4261
1.19k
    // Cast the two input vectors.
4262
1.19k
    Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4263
1.19k
    Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4264
1.19k
4265
1.19k
    // Convert the shuffle mask to the right # elements.
4266
1.19k
    Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4267
1.19k
    Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4268
1.19k
    Results.push_back(Tmp1);
4269
1.19k
    break;
4270
0
  }
4271
241
  case ISD::SETCC: {
4272
241
    unsigned ExtOp = ISD::FP_EXTEND;
4273
241
    if (NVT.isInteger()) {
4274
124
      ISD::CondCode CCCode =
4275
124
        cast<CondCodeSDNode>(Node->getOperand(2))->get();
4276
124
      ExtOp = isSignedIntSetCC(CCCode) ? 
ISD::SIGN_EXTEND0
: ISD::ZERO_EXTEND;
4277
124
    }
4278
241
    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4279
241
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4280
241
    Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
4281
241
                                  Tmp2, Node->getOperand(2), Node->getFlags()));
4282
241
    break;
4283
0
  }
4284
1
  case ISD::BR_CC: {
4285
1
    unsigned ExtOp = ISD::FP_EXTEND;
4286
1
    if (NVT.isInteger()) {
4287
0
      ISD::CondCode CCCode =
4288
0
        cast<CondCodeSDNode>(Node->getOperand(1))->get();
4289
0
      ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4290
0
    }
4291
1
    Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4292
1
    Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4293
1
    Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4294
1
                                  Node->getOperand(0), Node->getOperand(1),
4295
1
                                  Tmp1, Tmp2, Node->getOperand(4)));
4296
1
    break;
4297
0
  }
4298
283
  case ISD::FADD:
4299
283
  case ISD::FSUB:
4300
283
  case ISD::FMUL:
4301
283
  case ISD::FDIV:
4302
283
  case ISD::FREM:
4303
283
  case ISD::FMINNUM:
4304
283
  case ISD::FMAXNUM:
4305
283
  case ISD::FPOW:
4306
283
    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4307
283
    Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4308
283
    Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4309
283
                       Node->getFlags());
4310
283
    Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4311
283
                                  Tmp3, DAG.getIntPtrConstant(0, dl)));
4312
283
    break;
4313
283
  case ISD::FMA:
4314
25
    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4315
25
    Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4316
25
    Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4317
25
    Results.push_back(
4318
25
        DAG.getNode(ISD::FP_ROUND, dl, OVT,
4319
25
                    DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4320
25
                    DAG.getIntPtrConstant(0, dl)));
4321
25
    break;
4322
283
  case ISD::FCOPYSIGN:
4323
67
  case ISD::FPOWI: {
4324
67
    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4325
67
    Tmp2 = Node->getOperand(1);
4326
67
    Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4327
67
4328
67
    // fcopysign doesn't change anything but the sign bit, so
4329
67
    //   (fp_round (fcopysign (fpext a), b))
4330
67
    // is as precise as
4331
67
    //   (fp_round (fpext a))
4332
67
    // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4333
67
    const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4334
67
    Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4335
67
                                  Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4336
67
    break;
4337
67
  }
4338
484
  case ISD::FFLOOR:
4339
484
  case ISD::FCEIL:
4340
484
  case ISD::FRINT:
4341
484
  case ISD::FNEARBYINT:
4342
484
  case ISD::FROUND:
4343
484
  case ISD::FTRUNC:
4344
484
  case ISD::FNEG:
4345
484
  case ISD::FSQRT:
4346
484
  case ISD::FSIN:
4347
484
  case ISD::FCOS:
4348
484
  case ISD::FLOG:
4349
484
  case ISD::FLOG2:
4350
484
  case ISD::FLOG10:
4351
484
  case ISD::FABS:
4352
484
  case ISD::FEXP:
4353
484
  case ISD::FEXP2:
4354
484
    Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4355
484
    Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4356
484
    Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4357
484
                                  Tmp2, DAG.getIntPtrConstant(0, dl)));
4358
484
    break;
4359
484
  case ISD::BUILD_VECTOR: {
4360
395
    MVT EltVT = OVT.getVectorElementType();
4361
395
    MVT NewEltVT = NVT.getVectorElementType();
4362
395
4363
395
    // Handle bitcasts to a different vector type with the same total bit size
4364
395
    //
4365
395
    // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4366
395
    //  =>
4367
395
    //  v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4368
395
4369
395
    assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4370
395
           "Invalid promote type for build_vector");
4371
395
    assert(NewEltVT.bitsLT(EltVT) && "not handled");
4372
395
4373
395
    MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4374
395
4375
395
    SmallVector<SDValue, 8> NewOps;
4376
1.18k
    for (unsigned I = 0, E = Node->getNumOperands(); I != E; 
++I790
) {
4377
790
      SDValue Op = Node->getOperand(I);
4378
790
      NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4379
790
    }
4380
395
4381
395
    SDLoc SL(Node);
4382
395
    SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4383
395
    SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4384
395
    Results.push_back(CvtVec);
4385
395
    break;
4386
484
  }
4387
11.8k
  case ISD::EXTRACT_VECTOR_ELT: {
4388
11.8k
    MVT EltVT = OVT.getVectorElementType();
4389
11.8k
    MVT NewEltVT = NVT.getVectorElementType();
4390
11.8k
4391
11.8k
    // Handle bitcasts to a different vector type with the same total bit size.
4392
11.8k
    //
4393
11.8k
    // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4394
11.8k
    //  =>
4395
11.8k
    //  v4i32:castx = bitcast x:v2i64
4396
11.8k
    //
4397
11.8k
    // i64 = bitcast
4398
11.8k
    //   (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4399
11.8k
    //                       (i32 (extract_vector_elt castx, (2 * y + 1)))
4400
11.8k
    //
4401
11.8k
4402
11.8k
    assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4403
11.8k
           "Invalid promote type for extract_vector_elt");
4404
11.8k
    assert(NewEltVT.bitsLT(EltVT) && "not handled");
4405
11.8k
4406
11.8k
    MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4407
11.8k
    unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4408
11.8k
4409
11.8k
    SDValue Idx = Node->getOperand(1);
4410
11.8k
    EVT IdxVT = Idx.getValueType();
4411
11.8k
    SDLoc SL(Node);
4412
11.8k
    SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4413
11.8k
    SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4414
11.8k
4415
11.8k
    SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4416
11.8k
4417
11.8k
    SmallVector<SDValue, 8> NewOps;
4418
35.4k
    for (unsigned I = 0; I < NewEltsPerOldElt; 
++I23.6k
) {
4419
23.6k
      SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4420
23.6k
      SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4421
23.6k
4422
23.6k
      SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4423
23.6k
                                CastVec, TmpIdx);
4424
23.6k
      NewOps.push_back(Elt);
4425
23.6k
    }
4426
11.8k
4427
11.8k
    SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4428
11.8k
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4429
11.8k
    break;
4430
484
  }
4431
484
  case ISD::INSERT_VECTOR_ELT: {
4432
0
    MVT EltVT = OVT.getVectorElementType();
4433
0
    MVT NewEltVT = NVT.getVectorElementType();
4434
0
4435
0
    // Handle bitcasts to a different vector type with the same total bit size
4436
0
    //
4437
0
    // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4438
0
    //  =>
4439
0
    //  v4i32:castx = bitcast x:v2i64
4440
0
    //  v2i32:casty = bitcast y:i64
4441
0
    //
4442
0
    // v2i64 = bitcast
4443
0
    //   (v4i32 insert_vector_elt
4444
0
    //       (v4i32 insert_vector_elt v4i32:castx,
4445
0
    //                                (extract_vector_elt casty, 0), 2 * z),
4446
0
    //        (extract_vector_elt casty, 1), (2 * z + 1))
4447
0
4448
0
    assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4449
0
           "Invalid promote type for insert_vector_elt");
4450
0
    assert(NewEltVT.bitsLT(EltVT) && "not handled");
4451
0
4452
0
    MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4453
0
    unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4454
0
4455
0
    SDValue Val = Node->getOperand(1);
4456
0
    SDValue Idx = Node->getOperand(2);
4457
0
    EVT IdxVT = Idx.getValueType();
4458
0
    SDLoc SL(Node);
4459
0
4460
0
    SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4461
0
    SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4462
0
4463
0
    SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4464
0
    SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4465
0
4466
0
    SDValue NewVec = CastVec;
4467
0
    for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4468
0
      SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4469
0
      SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4470
0
4471
0
      SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4472
0
                                CastVal, IdxOffset);
4473
0
4474
0
      NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4475
0
                           NewVec, Elt, InEltIdx);
4476
0
    }
4477
0
4478
0
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4479
0
    break;
4480
484
  }
4481
1.56k
  case ISD::SCALAR_TO_VECTOR: {
4482
1.56k
    MVT EltVT = OVT.getVectorElementType();
4483
1.56k
    MVT NewEltVT = NVT.getVectorElementType();
4484
1.56k
4485
1.56k
    // Handle bitcasts to different vector type with the same total bit size.
4486
1.56k
    //
4487
1.56k
    // e.g. v2i64 = scalar_to_vector x:i64
4488
1.56k
    //   =>
4489
1.56k
    //  concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4490
1.56k
    //
4491
1.56k
4492
1.56k
    MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4493
1.56k
    SDValue Val = Node->getOperand(0);
4494
1.56k
    SDLoc SL(Node);
4495
1.56k
4496
1.56k
    SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4497
1.56k
    SDValue Undef = DAG.getUNDEF(MidVT);
4498
1.56k
4499
1.56k
    SmallVector<SDValue, 8> NewElts;
4500
1.56k
    NewElts.push_back(CastVal);
4501
3.12k
    for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; 
++I1.56k
)
4502
1.56k
      NewElts.push_back(Undef);
4503
1.56k
4504
1.56k
    SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4505
1.56k
    SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4506
1.56k
    Results.push_back(CvtVec);
4507
1.56k
    break;
4508
484
  }
4509
484
  case ISD::ATOMIC_SWAP: {
4510
23
    AtomicSDNode *AM = cast<AtomicSDNode>(Node);
4511
23
    SDLoc SL(Node);
4512
23
    SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
4513
23
    assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
4514
23
           "unexpected promotion type");
4515
23
    assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
4516
23
           "unexpected atomic_swap with illegal type");
4517
23
4518
23
    SDValue NewAtomic
4519
23
      = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
4520
23
                      DAG.getVTList(NVT, MVT::Other),
4521
23
                      { AM->getChain(), AM->getBasePtr(), CastVal },
4522
23
                      AM->getMemOperand());
4523
23
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
4524
23
    Results.push_back(NewAtomic.getValue(1));
4525
23
    break;
4526
17.7k
  }
4527
17.7k
  }
4528
17.7k
4529
17.7k
  // Replace the original node with the legalized result.
4530
17.7k
  if (!Results.empty()) {
4531
17.5k
    LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
4532
17.5k
    ReplaceNode(Node, Results.data());
4533
17.5k
  } else
4534
17.7k
    LLVM_DEBUG(dbgs() << "Could not promote node\n");
4535
17.7k
}
4536
4537
/// This is the entry point for the file.
4538
1.24M
void SelectionDAG::Legalize() {
4539
1.24M
  AssignTopologicalOrder();
4540
1.24M
4541
1.24M
  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4542
1.24M
  // Use a delete listener to remove nodes which were deleted during
4543
1.24M
  // legalization from LegalizeNodes. This is needed to handle the situation
4544
1.24M
  // where a new node is allocated by the object pool to the same address of a
4545
1.24M
  // previously deleted node.
4546
1.24M
  DAGNodeDeletedListener DeleteListener(
4547
1.24M
      *this,
4548
1.24M
      [&LegalizedNodes](SDNode *N, SDNode *E) 
{ LegalizedNodes.erase(N); }4.29k
);
4549
1.24M
4550
1.24M
  SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4551
1.24M
4552
1.24M
  // Visit all the nodes. We start in topological order, so that we see
4553
1.24M
  // nodes with their original operands intact. Legalization can produce
4554
1.24M
  // new nodes which may themselves need to be legalized. Iterate until all
4555
1.24M
  // nodes have been legalized.
4556
3.48M
  while (true) {
4557
3.48M
    bool AnyLegalized = false;
4558
79.9M
    for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4559
76.4M
      --NI;
4560
76.4M
4561
76.4M
      SDNode *N = &*NI;
4562
76.4M
      if (N->use_empty() && 
N != getRoot().getNode()5.12M
) {
4563
1.64M
        ++NI;
4564
1.64M
        DeleteNode(N);
4565
1.64M
        continue;
4566
1.64M
      }
4567
74.7M
4568
74.7M
      if (LegalizedNodes.insert(N).second) {
4569
28.2M
        AnyLegalized = true;
4570
28.2M
        Legalizer.LegalizeOp(N);
4571
28.2M
4572
28.2M
        if (N->use_empty() && 
N != getRoot().getNode()3.11M
) {
4573
1.86M
          ++NI;
4574
1.86M
          DeleteNode(N);
4575
1.86M
        }
4576
28.2M
      }
4577
74.7M
    }
4578
3.48M
    if (!AnyLegalized)
4579
1.24M
      break;
4580
3.48M
4581
3.48M
  }
4582
1.24M
4583
1.24M
  // Remove dead nodes now.
4584
1.24M
  RemoveDeadNodes();
4585
1.24M
}
4586
4587
bool SelectionDAG::LegalizeOp(SDNode *N,
4588
27.4M
                              SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4589
27.4M
  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4590
27.4M
  SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4591
27.4M
4592
27.4M
  // Directly insert the node in question, and legalize it. This will recurse
4593
27.4M
  // as needed through operands.
4594
27.4M
  LegalizedNodes.insert(N);
4595
27.4M
  Legalizer.LegalizeOp(N);
4596
27.4M
4597
27.4M
  return LegalizedNodes.count(N);
4598
27.4M
}