Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This implements routines for translating from LLVM IR into SelectionDAG IR.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "SelectionDAGBuilder.h"
14
#include "SDNodeDbgValue.h"
15
#include "llvm/ADT/APFloat.h"
16
#include "llvm/ADT/APInt.h"
17
#include "llvm/ADT/ArrayRef.h"
18
#include "llvm/ADT/BitVector.h"
19
#include "llvm/ADT/DenseMap.h"
20
#include "llvm/ADT/None.h"
21
#include "llvm/ADT/Optional.h"
22
#include "llvm/ADT/STLExtras.h"
23
#include "llvm/ADT/SmallPtrSet.h"
24
#include "llvm/ADT/SmallSet.h"
25
#include "llvm/ADT/SmallVector.h"
26
#include "llvm/ADT/StringRef.h"
27
#include "llvm/ADT/Triple.h"
28
#include "llvm/ADT/Twine.h"
29
#include "llvm/Analysis/AliasAnalysis.h"
30
#include "llvm/Analysis/BranchProbabilityInfo.h"
31
#include "llvm/Analysis/ConstantFolding.h"
32
#include "llvm/Analysis/EHPersonalities.h"
33
#include "llvm/Analysis/Loads.h"
34
#include "llvm/Analysis/MemoryLocation.h"
35
#include "llvm/Analysis/TargetLibraryInfo.h"
36
#include "llvm/Analysis/ValueTracking.h"
37
#include "llvm/Analysis/VectorUtils.h"
38
#include "llvm/CodeGen/Analysis.h"
39
#include "llvm/CodeGen/FunctionLoweringInfo.h"
40
#include "llvm/CodeGen/GCMetadata.h"
41
#include "llvm/CodeGen/ISDOpcodes.h"
42
#include "llvm/CodeGen/MachineBasicBlock.h"
43
#include "llvm/CodeGen/MachineFrameInfo.h"
44
#include "llvm/CodeGen/MachineFunction.h"
45
#include "llvm/CodeGen/MachineInstr.h"
46
#include "llvm/CodeGen/MachineInstrBuilder.h"
47
#include "llvm/CodeGen/MachineJumpTableInfo.h"
48
#include "llvm/CodeGen/MachineMemOperand.h"
49
#include "llvm/CodeGen/MachineModuleInfo.h"
50
#include "llvm/CodeGen/MachineOperand.h"
51
#include "llvm/CodeGen/MachineRegisterInfo.h"
52
#include "llvm/CodeGen/RuntimeLibcalls.h"
53
#include "llvm/CodeGen/SelectionDAG.h"
54
#include "llvm/CodeGen/SelectionDAGNodes.h"
55
#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
56
#include "llvm/CodeGen/StackMaps.h"
57
#include "llvm/CodeGen/SwiftErrorValueTracking.h"
58
#include "llvm/CodeGen/TargetFrameLowering.h"
59
#include "llvm/CodeGen/TargetInstrInfo.h"
60
#include "llvm/CodeGen/TargetLowering.h"
61
#include "llvm/CodeGen/TargetOpcodes.h"
62
#include "llvm/CodeGen/TargetRegisterInfo.h"
63
#include "llvm/CodeGen/TargetSubtargetInfo.h"
64
#include "llvm/CodeGen/ValueTypes.h"
65
#include "llvm/CodeGen/WinEHFuncInfo.h"
66
#include "llvm/IR/Argument.h"
67
#include "llvm/IR/Attributes.h"
68
#include "llvm/IR/BasicBlock.h"
69
#include "llvm/IR/CFG.h"
70
#include "llvm/IR/CallSite.h"
71
#include "llvm/IR/CallingConv.h"
72
#include "llvm/IR/Constant.h"
73
#include "llvm/IR/ConstantRange.h"
74
#include "llvm/IR/Constants.h"
75
#include "llvm/IR/DataLayout.h"
76
#include "llvm/IR/DebugInfoMetadata.h"
77
#include "llvm/IR/DebugLoc.h"
78
#include "llvm/IR/DerivedTypes.h"
79
#include "llvm/IR/Function.h"
80
#include "llvm/IR/GetElementPtrTypeIterator.h"
81
#include "llvm/IR/InlineAsm.h"
82
#include "llvm/IR/InstrTypes.h"
83
#include "llvm/IR/Instruction.h"
84
#include "llvm/IR/Instructions.h"
85
#include "llvm/IR/IntrinsicInst.h"
86
#include "llvm/IR/Intrinsics.h"
87
#include "llvm/IR/LLVMContext.h"
88
#include "llvm/IR/Metadata.h"
89
#include "llvm/IR/Module.h"
90
#include "llvm/IR/Operator.h"
91
#include "llvm/IR/PatternMatch.h"
92
#include "llvm/IR/Statepoint.h"
93
#include "llvm/IR/Type.h"
94
#include "llvm/IR/User.h"
95
#include "llvm/IR/Value.h"
96
#include "llvm/MC/MCContext.h"
97
#include "llvm/MC/MCSymbol.h"
98
#include "llvm/Support/AtomicOrdering.h"
99
#include "llvm/Support/BranchProbability.h"
100
#include "llvm/Support/Casting.h"
101
#include "llvm/Support/CodeGen.h"
102
#include "llvm/Support/CommandLine.h"
103
#include "llvm/Support/Compiler.h"
104
#include "llvm/Support/Debug.h"
105
#include "llvm/Support/ErrorHandling.h"
106
#include "llvm/Support/MachineValueType.h"
107
#include "llvm/Support/MathExtras.h"
108
#include "llvm/Support/raw_ostream.h"
109
#include "llvm/Target/TargetIntrinsicInfo.h"
110
#include "llvm/Target/TargetMachine.h"
111
#include "llvm/Target/TargetOptions.h"
112
#include "llvm/Transforms/Utils/Local.h"
113
#include <algorithm>
114
#include <cassert>
115
#include <cstddef>
116
#include <cstdint>
117
#include <cstring>
118
#include <iterator>
119
#include <limits>
120
#include <numeric>
121
#include <tuple>
122
#include <utility>
123
#include <vector>
124
125
using namespace llvm;
126
using namespace PatternMatch;
127
using namespace SwitchCG;
128
129
#define DEBUG_TYPE "isel"
130
131
/// LimitFloatPrecision - Generate low-precision inline sequences for
132
/// some float libcalls (6, 8 or 12 bits).
133
static unsigned LimitFloatPrecision;
134
135
static cl::opt<unsigned, true>
136
    LimitFPPrecision("limit-float-precision",
137
                     cl::desc("Generate low-precision inline sequences "
138
                              "for some float libcalls"),
139
                     cl::location(LimitFloatPrecision), cl::Hidden,
140
                     cl::init(0));
141
142
static cl::opt<unsigned> SwitchPeelThreshold(
143
    "switch-peel-threshold", cl::Hidden, cl::init(66),
144
    cl::desc("Set the case probability threshold for peeling the case from a "
145
             "switch statement. A value greater than 100 will void this "
146
             "optimization"));
147
148
// Limit the width of DAG chains. This is important in general to prevent
149
// DAG-based analysis from blowing up. For example, alias analysis and
150
// load clustering may not complete in reasonable time. It is difficult to
151
// recognize and avoid this situation within each individual analysis, and
152
// future analyses are likely to have the same behavior. Limiting DAG width is
153
// the safe approach and will be especially important with global DAGs.
154
//
155
// MaxParallelChains default is arbitrarily high to avoid affecting
156
// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
157
// sequence over this should have been converted to llvm.memcpy by the
158
// frontend. It is easy to induce this behavior with .ll code such as:
159
// %buffer = alloca [4096 x i8]
160
// %data = load [4096 x i8]* %argPtr
161
// store [4096 x i8] %data, [4096 x i8]* %buffer
162
static const unsigned MaxParallelChains = 64;
163
164
// Return the calling convention if the Value passed requires ABI mangling as it
165
// is a parameter to a function or a return value from a function which is not
166
// an intrinsic.
167
2.39k
static Optional<CallingConv::ID> getABIRegCopyCC(const Value *V) {
168
2.39k
  if (auto *R = dyn_cast<ReturnInst>(V))
169
0
    return R->getParent()->getParent()->getCallingConv();
170
2.39k
171
2.39k
  if (auto *CI = dyn_cast<CallInst>(V)) {
172
27
    const bool IsInlineAsm = CI->isInlineAsm();
173
27
    const bool IsIndirectFunctionCall =
174
27
        !IsInlineAsm && 
!CI->getCalledFunction()24
;
175
27
176
27
    // It is possible that the call instruction is an inline asm statement or an
177
27
    // indirect function call in which case the return value of
178
27
    // getCalledFunction() would be nullptr.
179
27
    const bool IsInstrinsicCall =
180
27
        !IsInlineAsm && 
!IsIndirectFunctionCall24
&&
181
27
        
CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic23
;
182
27
183
27
    if (!IsInlineAsm && 
!IsInstrinsicCall24
)
184
14
      return CI->getCallingConv();
185
2.37k
  }
186
2.37k
187
2.37k
  return None;
188
2.37k
}
189
190
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
191
                                      const SDValue *Parts, unsigned NumParts,
192
                                      MVT PartVT, EVT ValueVT, const Value *V,
193
                                      Optional<CallingConv::ID> CC);
194
195
/// getCopyFromParts - Create a value that contains the specified legal parts
196
/// combined into the value they represent.  If the parts combine to a type
197
/// larger than ValueVT then AssertOp can be used to specify whether the extra
198
/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
199
/// (ISD::AssertSext).
200
static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
201
                                const SDValue *Parts, unsigned NumParts,
202
                                MVT PartVT, EVT ValueVT, const Value *V,
203
                                Optional<CallingConv::ID> CC = None,
204
2.57M
                                Optional<ISD::NodeType> AssertOp = None) {
205
2.57M
  if (ValueVT.isVector())
206
302k
    return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
207
302k
                                  CC);
208
2.27M
209
2.27M
  assert(NumParts > 0 && "No parts to assemble!");
210
2.27M
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
211
2.27M
  SDValue Val = Parts[0];
212
2.27M
213
2.27M
  if (NumParts > 1) {
214
33.8k
    // Assemble the value from multiple parts.
215
33.8k
    if (ValueVT.isInteger()) {
216
31.0k
      unsigned PartBits = PartVT.getSizeInBits();
217
31.0k
      unsigned ValueBits = ValueVT.getSizeInBits();
218
31.0k
219
31.0k
      // Assemble the power of 2 part.
220
31.0k
      unsigned RoundParts =
221
31.0k
          (NumParts & (NumParts - 1)) ? 
1 << Log2_32(NumParts)470
:
NumParts30.5k
;
222
31.0k
      unsigned RoundBits = PartBits * RoundParts;
223
31.0k
      EVT RoundVT = RoundBits == ValueBits ?
224
30.0k
        ValueVT : 
EVT::getIntegerVT(*DAG.getContext(), RoundBits)1.02k
;
225
31.0k
      SDValue Lo, Hi;
226
31.0k
227
31.0k
      EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
228
31.0k
229
31.0k
      if (RoundParts > 2) {
230
2.01k
        Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
231
2.01k
                              PartVT, HalfVT, V);
232
2.01k
        Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
233
2.01k
                              RoundParts / 2, PartVT, HalfVT, V);
234
29.0k
      } else {
235
29.0k
        Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
236
29.0k
        Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
237
29.0k
      }
238
31.0k
239
31.0k
      if (DAG.getDataLayout().isBigEndian())
240
3.95k
        std::swap(Lo, Hi);
241
31.0k
242
31.0k
      Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
243
31.0k
244
31.0k
      if (RoundParts < NumParts) {
245
470
        // Assemble the trailing non-power-of-2 part.
246
470
        unsigned OddParts = NumParts - RoundParts;
247
470
        EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
248
470
        Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
249
470
                              OddVT, V, CC);
250
470
251
470
        // Combine the round and odd parts.
252
470
        Lo = Val;
253
470
        if (DAG.getDataLayout().isBigEndian())
254
5
          std::swap(Lo, Hi);
255
470
        EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
256
470
        Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
257
470
        Hi =
258
470
            DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
259
470
                        DAG.getConstant(Lo.getValueSizeInBits(), DL,
260
470
                                        TLI.getPointerTy(DAG.getDataLayout())));
261
470
        Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
262
470
        Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
263
470
      }
264
31.0k
    } else 
if (2.79k
PartVT.isFloatingPoint()2.79k
) {
265
176
      // FP split into multiple FP parts (for ppcf128)
266
176
      assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&
267
176
             "Unexpected split");
268
176
      SDValue Lo, Hi;
269
176
      Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
270
176
      Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
271
176
      if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
272
176
        std::swap(Lo, Hi);
273
176
      Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
274
2.61k
    } else {
275
2.61k
      // FP split into integer parts (soft fp)
276
2.61k
      assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
277
2.61k
             !PartVT.isVector() && "Unexpected split");
278
2.61k
      EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
279
2.61k
      Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
280
2.61k
    }
281
33.8k
  }
282
2.27M
283
2.27M
  // There is now one part, held in Val.  Correct it to match ValueVT.
284
2.27M
  // PartEVT is the type of the register class that holds the value.
285
2.27M
  // ValueVT is the type of the inline asm operation.
286
2.27M
  EVT PartEVT = Val.getValueType();
287
2.27M
288
2.27M
  if (PartEVT == ValueVT)
289
2.19M
    return Val;
290
75.7k
291
75.7k
  if (PartEVT.isInteger() && 
ValueVT.isFloatingPoint()74.5k
&&
292
75.7k
      
ValueVT.bitsLT(PartEVT)4.77k
) {
293
40
    // For an FP value in an integer part, we need to truncate to the right
294
40
    // width first.
295
40
    PartEVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
296
40
    Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
297
40
  }
298
75.7k
299
75.7k
  // Handle types that have the same size.
300
75.7k
  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
301
4.78k
    return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
302
70.9k
303
70.9k
  // Handle types with different sizes.
304
70.9k
  if (PartEVT.isInteger() && 
ValueVT.isInteger()69.7k
) {
305
69.7k
    if (ValueVT.bitsLT(PartEVT)) {
306
69.7k
      // For a truncate, see if we have any information to
307
69.7k
      // indicate whether the truncated bits will always be
308
69.7k
      // zero or sign-extension.
309
69.7k
      if (AssertOp.hasValue())
310
24.9k
        Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
311
24.9k
                          DAG.getValueType(ValueVT));
312
69.7k
      return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
313
69.7k
    }
314
13
    return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
315
13
  }
316
1.14k
317
1.14k
  
if (1.14k
PartEVT.isFloatingPoint()1.14k
&& ValueVT.isFloatingPoint()) {
318
1.14k
    // FP_ROUND's are always exact here.
319
1.14k
    if (ValueVT.bitsLT(Val.getValueType()))
320
1.14k
      return DAG.getNode(
321
1.14k
          ISD::FP_ROUND, DL, ValueVT, Val,
322
1.14k
          DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
323
0
324
0
    return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
325
0
  }
326
18.4E
327
18.4E
  // Handle MMX to a narrower integer type by bitcasting MMX to integer and
328
18.4E
  // then truncating.
329
18.4E
  if (PartEVT == MVT::x86mmx && 
ValueVT.isInteger()1
&&
330
18.4E
      
ValueVT.bitsLT(PartEVT)1
) {
331
1
    Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val);
332
1
    return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
333
1
  }
334
18.4E
335
18.4E
  report_fatal_error("Unknown mismatch in getCopyFromParts!");
336
18.4E
}
337
338
static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
339
12
                                              const Twine &ErrMsg) {
340
12
  const Instruction *I = dyn_cast_or_null<Instruction>(V);
341
12
  if (!V)
342
0
    return Ctx.emitError(ErrMsg);
343
12
344
12
  const char *AsmError = ", possible invalid constraint for vector type";
345
12
  if (const CallInst *CI = dyn_cast<CallInst>(I))
346
12
    if (isa<InlineAsm>(CI->getCalledValue()))
347
12
      return Ctx.emitError(I, ErrMsg + AsmError);
348
0
349
0
  return Ctx.emitError(I, ErrMsg);
350
0
}
351
352
/// getCopyFromPartsVector - Create a value that contains the specified legal
353
/// parts combined into the value they represent.  If the parts combine to a
354
/// type larger than ValueVT then AssertOp can be used to specify whether the
355
/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
356
/// ValueVT (ISD::AssertSext).
357
static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
358
                                      const SDValue *Parts, unsigned NumParts,
359
                                      MVT PartVT, EVT ValueVT, const Value *V,
360
302k
                                      Optional<CallingConv::ID> CallConv) {
361
302k
  assert(ValueVT.isVector() && "Not a vector value");
362
302k
  assert(NumParts > 0 && "No parts to assemble!");
363
302k
  const bool IsABIRegCopy = CallConv.hasValue();
364
302k
365
302k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
366
302k
  SDValue Val = Parts[0];
367
302k
368
302k
  // Handle a multi-element vector.
369
302k
  if (NumParts > 1) {
370
30.7k
    EVT IntermediateVT;
371
30.7k
    MVT RegisterVT;
372
30.7k
    unsigned NumIntermediates;
373
30.7k
    unsigned NumRegs;
374
30.7k
375
30.7k
    if (IsABIRegCopy) {
376
16.6k
      NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
377
16.6k
          *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
378
16.6k
          NumIntermediates, RegisterVT);
379
16.6k
    } else {
380
14.1k
      NumRegs =
381
14.1k
          TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
382
14.1k
                                     NumIntermediates, RegisterVT);
383
14.1k
    }
384
30.7k
385
30.7k
    assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
386
30.7k
    NumParts = NumRegs; // Silence a compiler warning.
387
30.7k
    assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
388
30.7k
    assert(RegisterVT.getSizeInBits() ==
389
30.7k
           Parts[0].getSimpleValueType().getSizeInBits() &&
390
30.7k
           "Part type sizes don't match!");
391
30.7k
392
30.7k
    // Assemble the parts into intermediate operands.
393
30.7k
    SmallVector<SDValue, 8> Ops(NumIntermediates);
394
30.7k
    if (NumIntermediates == NumParts) {
395
29.9k
      // If the register was not expanded, truncate or copy the value,
396
29.9k
      // as appropriate.
397
125k
      for (unsigned i = 0; i != NumParts; 
++i95.1k
)
398
95.1k
        Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
399
95.1k
                                  PartVT, IntermediateVT, V);
400
29.9k
    } else 
if (842
NumParts > 0842
) {
401
842
      // If the intermediate type was expanded, build the intermediate
402
842
      // operands from the parts.
403
842
      assert(NumParts % NumIntermediates == 0 &&
404
842
             "Must expand into a divisible number of parts!");
405
842
      unsigned Factor = NumParts / NumIntermediates;
406
2.70k
      for (unsigned i = 0; i != NumIntermediates; 
++i1.86k
)
407
1.86k
        Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
408
1.86k
                                  PartVT, IntermediateVT, V);
409
842
    }
410
30.7k
411
30.7k
    // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
412
30.7k
    // intermediate operands.
413
30.7k
    EVT BuiltVectorTy =
414
30.7k
        EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
415
30.7k
                         (IntermediateVT.isVector()
416
30.7k
                              ? 
IntermediateVT.getVectorNumElements() * NumParts23.5k
417
30.7k
                              : 
NumIntermediates7.19k
));
418
30.7k
    Val = DAG.getNode(IntermediateVT.isVector() ? 
ISD::CONCAT_VECTORS23.5k
419
30.7k
                                                : 
ISD::BUILD_VECTOR7.19k
,
420
30.7k
                      DL, BuiltVectorTy, Ops);
421
30.7k
  }
422
302k
423
302k
  // There is now one part, held in Val.  Correct it to match ValueVT.
424
302k
  EVT PartEVT = Val.getValueType();
425
302k
426
302k
  if (PartEVT == ValueVT)
427
291k
    return Val;
428
11.0k
429
11.0k
  if (PartEVT.isVector()) {
430
10.0k
    // If the element type of the source/dest vectors are the same, but the
431
10.0k
    // parts vector has more elements than the value vector, then we have a
432
10.0k
    // vector widening case (e.g. <2 x float> -> <4 x float>).  Extract the
433
10.0k
    // elements we want.
434
10.0k
    if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
435
3.37k
      assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&
436
3.37k
             "Cannot narrow, it would be a lossy transformation");
437
3.37k
      return DAG.getNode(
438
3.37k
          ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
439
3.37k
          DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
440
3.37k
    }
441
6.69k
442
6.69k
    // Vector/Vector bitcast.
443
6.69k
    if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
444
2.20k
      return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
445
4.49k
446
4.49k
    assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&
447
4.49k
      "Cannot handle this kind of promotion");
448
4.49k
    // Promoted vector extract
449
4.49k
    return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
450
4.49k
451
4.49k
  }
452
1.00k
453
1.00k
  // Trivial bitcast if the types are the same size and the destination
454
1.00k
  // vector type is legal.
455
1.00k
  if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
456
1.00k
      
TLI.isTypeLegal(ValueVT)829
)
457
23
    return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
458
979
459
979
  if (ValueVT.getVectorNumElements() != 1) {
460
194
     // Certain ABIs require that vectors are passed as integers. For vectors
461
194
     // are the same size, this is an obvious bitcast.
462
194
     if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
463
106
       return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
464
106
     } else 
if (88
ValueVT.getSizeInBits() < PartEVT.getSizeInBits()88
) {
465
80
       // Bitcast Val back the original type and extract the corresponding
466
80
       // vector we want.
467
80
       unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
468
80
       EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
469
80
                                           ValueVT.getVectorElementType(), Elts);
470
80
       Val = DAG.getBitcast(WiderVecType, Val);
471
80
       return DAG.getNode(
472
80
           ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
473
80
           DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
474
80
     }
475
8
476
8
     diagnosePossiblyInvalidConstraint(
477
8
         *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
478
8
     return DAG.getUNDEF(ValueVT);
479
8
  }
480
785
481
785
  // Handle cases such as i8 -> <1 x i1>
482
785
  EVT ValueSVT = ValueVT.getVectorElementType();
483
785
  if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
484
85
    Val = ValueVT.isFloatingPoint() ? 
DAG.getFPExtendOrRound(Val, DL, ValueSVT)6
485
85
                                    : 
DAG.getAnyExtOrTrunc(Val, DL, ValueSVT)79
;
486
785
487
785
  return DAG.getBuildVector(ValueVT, DL, Val);
488
785
}
489
490
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
491
                                 SDValue Val, SDValue *Parts, unsigned NumParts,
492
                                 MVT PartVT, const Value *V,
493
                                 Optional<CallingConv::ID> CallConv);
494
495
/// getCopyToParts - Create a series of nodes that contain the specified value
496
/// split into legal parts.  If the parts contain more bits than Val, then, for
497
/// integers, ExtendKind can be used to specify how to generate the extra bits.
498
static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
499
                           SDValue *Parts, unsigned NumParts, MVT PartVT,
500
                           const Value *V,
501
                           Optional<CallingConv::ID> CallConv = None,
502
2.08M
                           ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
503
2.08M
  EVT ValueVT = Val.getValueType();
504
2.08M
505
2.08M
  // Handle the vector case separately.
506
2.08M
  if (ValueVT.isVector())
507
160k
    return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
508
160k
                                CallConv);
509
1.92M
510
1.92M
  unsigned PartBits = PartVT.getSizeInBits();
511
1.92M
  unsigned OrigNumParts = NumParts;
512
1.92M
  assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&
513
1.92M
         "Copying to an illegal type!");
514
1.92M
515
1.92M
  if (NumParts == 0)
516
0
    return;
517
1.92M
518
1.92M
  assert(!ValueVT.isVector() && "Vector case handled elsewhere");
519
1.92M
  EVT PartEVT = PartVT;
520
1.92M
  if (PartEVT == ValueVT) {
521
1.81M
    assert(NumParts == 1 && "No-op copy with multiple parts!");
522
1.81M
    Parts[0] = Val;
523
1.81M
    return;
524
1.81M
  }
525
109k
526
109k
  if (NumParts * PartBits > ValueVT.getSizeInBits()) {
527
76.5k
    // If the parts cover more bits than the value has, promote the value.
528
76.5k
    if (PartVT.isFloatingPoint() && 
ValueVT.isFloatingPoint()669
) {
529
669
      assert(NumParts == 1 && "Do not know what to promote to!");
530
669
      Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
531
75.9k
    } else {
532
75.9k
      if (ValueVT.isFloatingPoint()) {
533
53
        // FP values need to be bitcast, then extended if they are being put
534
53
        // into a larger container.
535
53
        ValueVT = EVT::getIntegerVT(*DAG.getContext(),  ValueVT.getSizeInBits());
536
53
        Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
537
53
      }
538
75.9k
      assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
539
75.9k
             ValueVT.isInteger() &&
540
75.9k
             "Unknown mismatch!");
541
75.9k
      ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
542
75.9k
      Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
543
75.9k
      if (PartVT == MVT::x86mmx)
544
1
        Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
545
75.9k
    }
546
76.5k
  } else 
if (32.8k
PartBits == ValueVT.getSizeInBits()32.8k
) {
547
1.69k
    // Different types of the same size.
548
1.69k
    assert(NumParts == 1 && PartEVT != ValueVT);
549
1.69k
    Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
550
31.1k
  } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
551
186
    // If the parts cover less bits than value has, truncate the value.
552
186
    assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&
553
186
           ValueVT.isInteger() &&
554
186
           "Unknown mismatch!");
555
186
    ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
556
186
    Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
557
186
    if (PartVT == MVT::x86mmx)
558
0
      Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
559
186
  }
560
109k
561
109k
  // The value may have changed - recompute ValueVT.
562
109k
  ValueVT = Val.getValueType();
563
109k
  assert(NumParts * PartBits == ValueVT.getSizeInBits() &&
564
109k
         "Failed to tile the value with PartVT!");
565
109k
566
109k
  if (NumParts == 1) {
567
78.3k
    if (PartEVT != ValueVT) {
568
4
      diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
569
4
                                        "scalar-to-vector conversion failed");
570
4
      Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
571
4
    }
572
78.3k
573
78.3k
    Parts[0] = Val;
574
78.3k
    return;
575
78.3k
  }
576
31.0k
577
31.0k
  // Expand the value into multiple parts.
578
31.0k
  if (NumParts & (NumParts - 1)) {
579
159
    // The number of parts is not a power of 2.  Split off and copy the tail.
580
159
    assert(PartVT.isInteger() && ValueVT.isInteger() &&
581
159
           "Do not know what to expand to!");
582
159
    unsigned RoundParts = 1 << Log2_32(NumParts);
583
159
    unsigned RoundBits = RoundParts * PartBits;
584
159
    unsigned OddParts = NumParts - RoundParts;
585
159
    SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
586
159
      DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false));
587
159
588
159
    getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
589
159
                   CallConv);
590
159
591
159
    if (DAG.getDataLayout().isBigEndian())
592
3
      // The odd parts were reversed by getCopyToParts - unreverse them.
593
3
      std::reverse(Parts + RoundParts, Parts + NumParts);
594
159
595
159
    NumParts = RoundParts;
596
159
    ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
597
159
    Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
598
159
  }
599
31.0k
600
31.0k
  // The number of parts is a power of 2.  Repeatedly bisect the value using
601
31.0k
  // EXTRACT_ELEMENT.
602
31.0k
  Parts[0] = DAG.getNode(ISD::BITCAST, DL,
603
31.0k
                         EVT::getIntegerVT(*DAG.getContext(),
604
31.0k
                                           ValueVT.getSizeInBits()),
605
31.0k
                         Val);
606
31.0k
607
63.2k
  for (unsigned StepSize = NumParts; StepSize > 1; 
StepSize /= 232.2k
) {
608
66.1k
    for (unsigned i = 0; i < NumParts; 
i += StepSize33.9k
) {
609
33.9k
      unsigned ThisBits = StepSize * PartBits / 2;
610
33.9k
      EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
611
33.9k
      SDValue &Part0 = Parts[i];
612
33.9k
      SDValue &Part1 = Parts[i+StepSize/2];
613
33.9k
614
33.9k
      Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
615
33.9k
                          ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
616
33.9k
      Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
617
33.9k
                          ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
618
33.9k
619
33.9k
      if (ThisBits == PartBits && 
ThisVT != PartVT32.5k
) {
620
181
        Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
621
181
        Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
622
181
      }
623
33.9k
    }
624
32.2k
  }
625
31.0k
626
31.0k
  if (DAG.getDataLayout().isBigEndian())
627
2.52k
    std::reverse(Parts, Parts + OrigNumParts);
628
31.0k
}
629
630
static SDValue widenVectorToPartType(SelectionDAG &DAG,
631
5.07k
                                     SDValue Val, const SDLoc &DL, EVT PartVT) {
632
5.07k
  if (!PartVT.isVector())
633
102
    return SDValue();
634
4.97k
635
4.97k
  EVT ValueVT = Val.getValueType();
636
4.97k
  unsigned PartNumElts = PartVT.getVectorNumElements();
637
4.97k
  unsigned ValueNumElts = ValueVT.getVectorNumElements();
638
4.97k
  if (PartNumElts > ValueNumElts &&
639
4.97k
      
PartVT.getVectorElementType() == ValueVT.getVectorElementType()2.21k
) {
640
2.13k
    EVT ElementVT = PartVT.getVectorElementType();
641
2.13k
    // Vector widening case, e.g. <2 x float> -> <4 x float>.  Shuffle in
642
2.13k
    // undef elements.
643
2.13k
    SmallVector<SDValue, 16> Ops;
644
2.13k
    DAG.ExtractVectorElements(Val, Ops);
645
2.13k
    SDValue EltUndef = DAG.getUNDEF(ElementVT);
646
13.2k
    for (unsigned i = ValueNumElts, e = PartNumElts; i != e; 
++i11.0k
)
647
11.0k
      Ops.push_back(EltUndef);
648
2.13k
649
2.13k
    // FIXME: Use CONCAT for 2x -> 4x.
650
2.13k
    return DAG.getBuildVector(PartVT, DL, Ops);
651
2.13k
  }
652
2.83k
653
2.83k
  return SDValue();
654
2.83k
}
655
656
/// getCopyToPartsVector - Create a series of nodes that contain the specified
657
/// value split into legal parts.
658
static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
659
                                 SDValue Val, SDValue *Parts, unsigned NumParts,
660
                                 MVT PartVT, const Value *V,
661
160k
                                 Optional<CallingConv::ID> CallConv) {
662
160k
  EVT ValueVT = Val.getValueType();
663
160k
  assert(ValueVT.isVector() && "Not a vector");
664
160k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
665
160k
  const bool IsABIRegCopy = CallConv.hasValue();
666
160k
667
160k
  if (NumParts == 1) {
668
143k
    EVT PartEVT = PartVT;
669
143k
    if (PartEVT == ValueVT) {
670
138k
      // Nothing to do.
671
138k
    } else 
if (5.51k
PartVT.getSizeInBits() == ValueVT.getSizeInBits()5.51k
) {
672
712
      // Bitconvert vector->vector case.
673
712
      Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
674
4.80k
    } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) {
675
2.10k
      Val = Widened;
676
2.69k
    } else if (PartVT.isVector() &&
677
2.69k
               PartEVT.getVectorElementType().bitsGE(
678
2.59k
                 ValueVT.getVectorElementType()) &&
679
2.69k
               
PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()2.59k
) {
680
2.59k
681
2.59k
      // Promoted vector extract
682
2.59k
      Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
683
2.59k
    } else {
684
102
      if (ValueVT.getVectorNumElements() == 1) {
685
54
        Val = DAG.getNode(
686
54
            ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
687
54
            DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
688
54
      } else {
689
48
        assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&
690
48
               "lossy conversion of vector to scalar type");
691
48
        EVT IntermediateType =
692
48
            EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
693
48
        Val = DAG.getBitcast(IntermediateType, Val);
694
48
        Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
695
48
      }
696
102
    }
697
143k
698
143k
    assert(Val.getValueType() == PartVT && "Unexpected vector part value type");
699
143k
    Parts[0] = Val;
700
143k
    return;
701
143k
  }
702
16.1k
703
16.1k
  // Handle a multi-element vector.
704
16.1k
  EVT IntermediateVT;
705
16.1k
  MVT RegisterVT;
706
16.1k
  unsigned NumIntermediates;
707
16.1k
  unsigned NumRegs;
708
16.1k
  if (IsABIRegCopy) {
709
7.82k
    NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
710
7.82k
        *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
711
7.82k
        NumIntermediates, RegisterVT);
712
8.31k
  } else {
713
8.31k
    NumRegs =
714
8.31k
        TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
715
8.31k
                                   NumIntermediates, RegisterVT);
716
8.31k
  }
717
16.1k
718
16.1k
  assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!");
719
16.1k
  NumParts = NumRegs; // Silence a compiler warning.
720
16.1k
  assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!");
721
16.1k
722
16.1k
  unsigned IntermediateNumElts = IntermediateVT.isVector() ?
723
13.0k
    IntermediateVT.getVectorNumElements() : 
13.08k
;
724
16.1k
725
16.1k
  // Convert the vector to the appropiate type if necessary.
726
16.1k
  unsigned DestVectorNoElts = NumIntermediates * IntermediateNumElts;
727
16.1k
728
16.1k
  EVT BuiltVectorTy = EVT::getVectorVT(
729
16.1k
      *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
730
16.1k
  MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
731
16.1k
  if (ValueVT != BuiltVectorTy) {
732
275
    if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy))
733
34
      Val = Widened;
734
275
735
275
    Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
736
275
  }
737
16.1k
738
16.1k
  // Split the vector into intermediate operands.
739
16.1k
  SmallVector<SDValue, 8> Ops(NumIntermediates);
740
58.2k
  for (unsigned i = 0; i != NumIntermediates; 
++i42.0k
) {
741
42.0k
    if (IntermediateVT.isVector()) {
742
29.6k
      Ops[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
743
29.6k
                           DAG.getConstant(i * IntermediateNumElts, DL, IdxVT));
744
29.6k
    } else {
745
12.4k
      Ops[i] = DAG.getNode(
746
12.4k
          ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
747
12.4k
          DAG.getConstant(i, DL, IdxVT));
748
12.4k
    }
749
42.0k
  }
750
16.1k
751
16.1k
  // Split the intermediate operands into legal parts.
752
16.1k
  if (NumParts == NumIntermediates) {
753
15.9k
    // If the register was not expanded, promote or copy the value,
754
15.9k
    // as appropriate.
755
57.5k
    for (unsigned i = 0; i != NumParts; 
++i41.6k
)
756
41.6k
      getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
757
15.9k
  } else 
if (142
NumParts > 0142
) {
758
142
    // If the intermediate type was expanded, split each the value into
759
142
    // legal parts.
760
142
    assert(NumIntermediates != 0 && "division by zero");
761
142
    assert(NumParts % NumIntermediates == 0 &&
762
142
           "Must expand into a divisible number of parts!");
763
142
    unsigned Factor = NumParts / NumIntermediates;
764
615
    for (unsigned i = 0; i != NumIntermediates; 
++i473
)
765
473
      getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
766
473
                     CallConv);
767
142
  }
768
16.1k
}
769
770
RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
771
                           EVT valuevt, Optional<CallingConv::ID> CC)
772
    : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
773
92.7k
      RegCount(1, regs.size()), CallConv(CC) {}
774
775
RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
776
                           const DataLayout &DL, unsigned Reg, Type *Ty,
777
2.61M
                           Optional<CallingConv::ID> CC) {
778
2.61M
  ComputeValueVTs(TLI, DL, Ty, ValueVTs);
779
2.61M
780
2.61M
  CallConv = CC;
781
2.61M
782
2.61M
  for (EVT ValueVT : ValueVTs) {
783
2.61M
    unsigned NumRegs =
784
2.61M
        isABIMangled()
785
2.61M
            ? 
TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)16
786
2.61M
            : 
TLI.getNumRegisters(Context, ValueVT)2.61M
;
787
2.61M
    MVT RegisterVT =
788
2.61M
        isABIMangled()
789
2.61M
            ? 
TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)16
790
2.61M
            : 
TLI.getRegisterType(Context, ValueVT)2.61M
;
791
5.29M
    for (unsigned i = 0; i != NumRegs; 
++i2.67M
)
792
2.67M
      Regs.push_back(Reg + i);
793
2.61M
    RegVTs.push_back(RegisterVT);
794
2.61M
    RegCount.push_back(NumRegs);
795
2.61M
    Reg += NumRegs;
796
2.61M
  }
797
2.61M
}
798
799
SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
800
                                      FunctionLoweringInfo &FuncInfo,
801
                                      const SDLoc &dl, SDValue &Chain,
802
1.80M
                                      SDValue *Flag, const Value *V) const {
803
1.80M
  // A Value with type {} or [0 x %t] needs no registers.
804
1.80M
  if (ValueVTs.empty())
805
0
    return SDValue();
806
1.80M
807
1.80M
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
808
1.80M
809
1.80M
  // Assemble the legal parts into the final values.
810
1.80M
  SmallVector<SDValue, 4> Values(ValueVTs.size());
811
1.80M
  SmallVector<SDValue, 8> Parts;
812
3.61M
  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; 
++Value1.80M
) {
813
1.80M
    // Copy the legal parts from the registers.
814
1.80M
    EVT ValueVT = ValueVTs[Value];
815
1.80M
    unsigned NumRegs = RegCount[Value];
816
1.80M
    MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
817
14
                                          *DAG.getContext(),
818
14
                                          CallConv.getValue(), RegVTs[Value])
819
1.80M
                                    : 
RegVTs[Value]1.80M
;
820
1.80M
821
1.80M
    Parts.resize(NumRegs);
822
3.64M
    for (unsigned i = 0; i != NumRegs; 
++i1.84M
) {
823
1.84M
      SDValue P;
824
1.84M
      if (!Flag) {
825
1.83M
        P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
826
1.83M
      } else {
827
4.58k
        P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
828
4.58k
        *Flag = P.getValue(2);
829
4.58k
      }
830
1.84M
831
1.84M
      Chain = P.getValue(1);
832
1.84M
      Parts[i] = P;
833
1.84M
834
1.84M
      // If the source register was virtual and if we know something about it,
835
1.84M
      // add an assert node.
836
1.84M
      if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
837
1.84M
          
!RegisterVT.isInteger()1.84M
)
838
70.6k
        continue;
839
1.77M
840
1.77M
      const FunctionLoweringInfo::LiveOutInfo *LOI =
841
1.77M
        FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
842
1.77M
      if (!LOI)
843
738k
        continue;
844
1.03M
845
1.03M
      unsigned RegSize = RegisterVT.getScalarSizeInBits();
846
1.03M
      unsigned NumSignBits = LOI->NumSignBits;
847
1.03M
      unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
848
1.03M
849
1.03M
      if (NumZeroBits == RegSize) {
850
3.27k
        // The current value is a zero.
851
3.27k
        // Explicitly express that as it would be easier for
852
3.27k
        // optimizations to kick in.
853
3.27k
        Parts[i] = DAG.getConstant(0, dl, RegisterVT);
854
3.27k
        continue;
855
3.27k
      }
856
1.02M
857
1.02M
      // FIXME: We capture more information than the dag can represent.  For
858
1.02M
      // now, just use the tightest assertzext/assertsext possible.
859
1.02M
      bool isSExt;
860
1.02M
      EVT FromVT(MVT::Other);
861
1.02M
      if (NumZeroBits) {
862
125k
        FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
863
125k
        isSExt = false;
864
903k
      } else if (NumSignBits > 1) {
865
97.4k
        FromVT =
866
97.4k
            EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
867
97.4k
        isSExt = true;
868
806k
      } else {
869
806k
        continue;
870
806k
      }
871
222k
      // Add an assertion node.
872
222k
      assert(FromVT != MVT::Other);
873
222k
      Parts[i] = DAG.getNode(isSExt ? 
ISD::AssertSext97.4k
:
ISD::AssertZext125k
, dl,
874
222k
                             RegisterVT, P, DAG.getValueType(FromVT));
875
222k
    }
876
1.80M
877
1.80M
    Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
878
1.80M
                                     RegisterVT, ValueVT, V, CallConv);
879
1.80M
    Part += NumRegs;
880
1.80M
    Parts.clear();
881
1.80M
  }
882
1.80M
883
1.80M
  return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
884
1.80M
}
885
886
void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
887
                                 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
888
                                 const Value *V,
889
819k
                                 ISD::NodeType PreferredExtendType) const {
890
819k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
891
819k
  ISD::NodeType ExtendKind = PreferredExtendType;
892
819k
893
819k
  // Get the list of the values's legal parts.
894
819k
  unsigned NumRegs = Regs.size();
895
819k
  SmallVector<SDValue, 8> Parts(NumRegs);
896
1.64M
  for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; 
++Value822k
) {
897
822k
    unsigned NumParts = RegCount[Value];
898
822k
899
822k
    MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
900
2
                                          *DAG.getContext(),
901
2
                                          CallConv.getValue(), RegVTs[Value])
902
822k
                                    : 
RegVTs[Value]822k
;
903
822k
904
822k
    if (ExtendKind == ISD::ANY_EXTEND && 
TLI.isZExtFree(Val, RegisterVT)775k
)
905
44.4k
      ExtendKind = ISD::ZERO_EXTEND;
906
822k
907
822k
    getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
908
822k
                   NumParts, RegisterVT, V, CallConv, ExtendKind);
909
822k
    Part += NumParts;
910
822k
  }
911
819k
912
819k
  // Copy the parts into the registers.
913
819k
  SmallVector<SDValue, 8> Chains(NumRegs);
914
1.66M
  for (unsigned i = 0; i != NumRegs; 
++i842k
) {
915
842k
    SDValue Part;
916
842k
    if (!Flag) {
917
838k
      Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
918
838k
    } else {
919
4.79k
      Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
920
4.79k
      *Flag = Part.getValue(1);
921
4.79k
    }
922
842k
923
842k
    Chains[i] = Part.getValue(0);
924
842k
  }
925
819k
926
819k
  if (NumRegs == 1 || 
Flag17.7k
)
927
802k
    // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
928
802k
    // flagged to it. That is the CopyToReg nodes and the user are considered
929
802k
    // a single scheduling unit. If we create a TokenFactor and return it as
930
802k
    // chain, then the TokenFactor is both a predecessor (operand) of the
931
802k
    // user as well as a successor (the TF operands are flagged to the user).
932
802k
    // c1, f1 = CopyToReg
933
802k
    // c2, f2 = CopyToReg
934
802k
    // c3     = TokenFactor c1, c2
935
802k
    // ...
936
802k
    //        = op c3, ..., f2
937
802k
    Chain = Chains[NumRegs-1];
938
17.6k
  else
939
17.6k
    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
940
819k
}
941
942
void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
943
                                        unsigned MatchingIdx, const SDLoc &dl,
944
                                        SelectionDAG &DAG,
945
92.7k
                                        std::vector<SDValue> &Ops) const {
946
92.7k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
947
92.7k
948
92.7k
  unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
949
92.7k
  if (HasMatching)
950
435
    Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
951
92.3k
  else if (!Regs.empty() &&
952
92.3k
           TargetRegisterInfo::isVirtualRegister(Regs.front())) {
953
7.00k
    // Put the register class of the virtual registers in the flag word.  That
954
7.00k
    // way, later passes can recompute register class constraints for inline
955
7.00k
    // assembly as well as normal instructions.
956
7.00k
    // Don't do this for tied operands that can use the regclass information
957
7.00k
    // from the def.
958
7.00k
    const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
959
7.00k
    const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
960
7.00k
    Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
961
7.00k
  }
962
92.7k
963
92.7k
  SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
964
92.7k
  Ops.push_back(Res);
965
92.7k
966
92.7k
  if (Code == InlineAsm::Kind_Clobber) {
967
83.4k
    // Clobbers should always have a 1:1 mapping with registers, and may
968
83.4k
    // reference registers that have illegal (e.g. vector) types. Hence, we
969
83.4k
    // shouldn't try to apply any sort of splitting logic to them.
970
83.4k
    assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&
971
83.4k
           "No 1:1 mapping from clobbers to regs?");
972
83.4k
    unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
973
83.4k
    (void)SP;
974
166k
    for (unsigned I = 0, E = ValueVTs.size(); I != E; 
++I83.4k
) {
975
83.4k
      Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
976
83.4k
      assert(
977
83.4k
          (Regs[I] != SP ||
978
83.4k
           DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&
979
83.4k
          "If we clobbered the stack pointer, MFI should know about it.");
980
83.4k
    }
981
83.4k
    return;
982
83.4k
  }
983
9.24k
984
18.4k
  
for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); 9.24k
Value != e;
++Value9.24k
) {
985
9.24k
    unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
986
9.24k
    MVT RegisterVT = RegVTs[Value];
987
18.7k
    for (unsigned i = 0; i != NumRegs; 
++i9.46k
) {
988
9.46k
      assert(Reg < Regs.size() && "Mismatch in # registers expected");
989
9.46k
      unsigned TheReg = Regs[Reg++];
990
9.46k
      Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
991
9.46k
    }
992
9.24k
  }
993
9.24k
}
994
995
SmallVector<std::pair<unsigned, unsigned>, 4>
996
10
RegsForValue::getRegsAndSizes() const {
997
10
  SmallVector<std::pair<unsigned, unsigned>, 4> OutVec;
998
10
  unsigned I = 0;
999
14
  for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
1000
14
    unsigned RegCount = std::get<0>(CountAndVT);
1001
14
    MVT RegisterVT = std::get<1>(CountAndVT);
1002
14
    unsigned RegisterSize = RegisterVT.getSizeInBits();
1003
39
    for (unsigned E = I + RegCount; I != E; 
++I25
)
1004
25
      OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
1005
14
  }
1006
10
  return OutVec;
1007
10
}
1008
1009
void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
1010
278k
                               const TargetLibraryInfo *li) {
1011
278k
  AA = aa;
1012
278k
  GFI = gfi;
1013
278k
  LibInfo = li;
1014
278k
  DL = &DAG.getDataLayout();
1015
278k
  Context = DAG.getContext();
1016
278k
  LPadToCallSiteMap.clear();
1017
278k
  SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout());
1018
278k
}
1019
1020
1.24M
void SelectionDAGBuilder::clear() {
1021
1.24M
  NodeMap.clear();
1022
1.24M
  UnusedArgNodeMap.clear();
1023
1.24M
  PendingLoads.clear();
1024
1.24M
  PendingExports.clear();
1025
1.24M
  CurInst = nullptr;
1026
1.24M
  HasTailCall = false;
1027
1.24M
  SDNodeOrder = LowestSDNodeOrder;
1028
1.24M
  StatepointLowering.clear();
1029
1.24M
}
1030
1031
1.47M
void SelectionDAGBuilder::clearDanglingDebugInfo() {
1032
1.47M
  DanglingDebugInfoMap.clear();
1033
1.47M
}
1034
1035
1.27M
SDValue SelectionDAGBuilder::getRoot() {
1036
1.27M
  if (PendingLoads.empty())
1037
1.05M
    return DAG.getRoot();
1038
212k
1039
212k
  if (PendingLoads.size() == 1) {
1040
163k
    SDValue Root = PendingLoads[0];
1041
163k
    DAG.setRoot(Root);
1042
163k
    PendingLoads.clear();
1043
163k
    return Root;
1044
163k
  }
1045
48.6k
1046
48.6k
  // Otherwise, we have to make a token factor node.
1047
48.6k
  SDValue Root = DAG.getTokenFactor(getCurSDLoc(), PendingLoads);
1048
48.6k
  PendingLoads.clear();
1049
48.6k
  DAG.setRoot(Root);
1050
48.6k
  return Root;
1051
48.6k
}
1052
1053
2.14M
SDValue SelectionDAGBuilder::getControlRoot() {
1054
2.14M
  SDValue Root = DAG.getRoot();
1055
2.14M
1056
2.14M
  if (PendingExports.empty())
1057
1.65M
    return Root;
1058
483k
1059
483k
  // Turn all of the CopyToReg chains into one factored node.
1060
483k
  if (Root.getOpcode() != ISD::EntryToken) {
1061
229k
    unsigned i = 0, e = PendingExports.size();
1062
572k
    for (; i != e; 
++i342k
) {
1063
342k
      assert(PendingExports[i].getNode()->getNumOperands() > 1);
1064
342k
      if (PendingExports[i].getNode()->getOperand(0) == Root)
1065
0
        break;  // Don't add the root if we already indirectly depend on it.
1066
342k
    }
1067
229k
1068
229k
    if (i == e)
1069
229k
      PendingExports.push_back(Root);
1070
229k
  }
1071
483k
1072
483k
  Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1073
483k
                     PendingExports);
1074
483k
  PendingExports.clear();
1075
483k
  DAG.setRoot(Root);
1076
483k
  return Root;
1077
483k
}
1078
1079
6.77M
void SelectionDAGBuilder::visit(const Instruction &I) {
1080
6.77M
  // Set up outgoing PHI node register values before emitting the terminator.
1081
6.77M
  if (I.isTerminator()) {
1082
1.12M
    HandlePHINodesInSuccessorBlocks(I.getParent());
1083
1.12M
  }
1084
6.77M
1085
6.77M
  // Increase the SDNodeOrder if dealing with a non-debug instruction.
1086
6.77M
  if (!isa<DbgInfoIntrinsic>(I))
1087
6.77M
    ++SDNodeOrder;
1088
6.77M
1089
6.77M
  CurInst = &I;
1090
6.77M
1091
6.77M
  visit(I.getOpcode(), I);
1092
6.77M
1093
6.77M
  if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1094
238k
    // Propagate the fast-math-flags of this IR instruction to the DAG node that
1095
238k
    // maps to this instruction.
1096
238k
    // TODO: We could handle all flags (nsw, etc) here.
1097
238k
    // TODO: If an IR instruction maps to >1 node, only the final node will have
1098
238k
    //       flags set.
1099
238k
    if (SDNode *Node = getNodeForIRValue(&I)) {
1100
237k
      SDNodeFlags IncomingFlags;
1101
237k
      IncomingFlags.copyFMF(*FPMO);
1102
237k
      if (!Node->getFlags().isDefined())
1103
234k
        Node->setFlags(IncomingFlags);
1104
3.23k
      else
1105
3.23k
        Node->intersectFlagsWith(IncomingFlags);
1106
237k
    }
1107
238k
  }
1108
6.77M
1109
6.77M
  if (!I.isTerminator() && 
!HasTailCall5.65M
&&
1110
6.77M
      
!isStatepoint(&I)5.58M
) // statepoints handle their exports internally
1111
5.58M
    CopyToExportRegsIfNeeded(&I);
1112
6.77M
1113
6.77M
  CurInst = nullptr;
1114
6.77M
}
1115
1116
0
void SelectionDAGBuilder::visitPHI(const PHINode &) {
1117
0
  llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!");
1118
0
}
1119
1120
7.07M
void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1121
7.07M
  // Note: this doesn't use InstVisitor, because it has to work with
1122
7.07M
  // ConstantExpr's in addition to instructions.
1123
7.07M
  switch (Opcode) {
1124
7.07M
  
default: 0
llvm_unreachable0
("Unknown instruction type encountered!");
1125
7.07M
    // Build the switch statement using the Instruction.def file.
1126
7.07M
#define HANDLE_INST(NUM, OPCODE, CLASS) \
1127
7.07M
    
case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break7.07M
;
1128
7.07M
#include 
"llvm/IR/Instruction.def"259k
1129
7.07M
  }
1130
7.07M
}
1131
1132
void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1133
5.47k
                                                const DIExpression *Expr) {
1134
5.47k
  auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1135
40
    const DbgValueInst *DI = DDI.getDI();
1136
40
    DIVariable *DanglingVariable = DI->getVariable();
1137
40
    DIExpression *DanglingExpr = DI->getExpression();
1138
40
    if (DanglingVariable == Variable && 
Expr->fragmentsOverlap(DanglingExpr)4
) {
1139
4
      LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n");
1140
4
      return true;
1141
4
    }
1142
36
    return false;
1143
36
  };
1144
5.47k
1145
5.47k
  for (auto &DDIMI : DanglingDebugInfoMap) {
1146
26
    DanglingDebugInfoVector &DDIV = DDIMI.second;
1147
26
1148
26
    // If debug info is to be dropped, run it through final checks to see
1149
26
    // whether it can be salvaged.
1150
26
    for (auto &DDI : DDIV)
1151
20
      if (isMatchingDbgValue(DDI))
1152
2
        salvageUnresolvedDbgValue(DDI);
1153
26
1154
26
    DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1155
26
  }
1156
5.47k
}
1157
1158
// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1159
// generate the debug data structures now that we've seen its definition.
1160
void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1161
4.77M
                                                   SDValue Val) {
1162
4.77M
  auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1163
4.77M
  if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1164
4.77M
    return;
1165
14
1166
14
  DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1167
18
  for (auto &DDI : DDIV) {
1168
18
    const DbgValueInst *DI = DDI.getDI();
1169
18
    assert(DI && "Ill-formed DanglingDebugInfo");
1170
18
    DebugLoc dl = DDI.getdl();
1171
18
    unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1172
18
    unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1173
18
    DILocalVariable *Variable = DI->getVariable();
1174
18
    DIExpression *Expr = DI->getExpression();
1175
18
    assert(Variable->isValidLocationForIntrinsic(dl) &&
1176
18
           "Expected inlined-at fields to agree");
1177
18
    SDDbgValue *SDV;
1178
18
    if (Val.getNode()) {
1179
18
      // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
1180
18
      // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
1181
18
      // we couldn't resolve it directly when examining the DbgValue intrinsic
1182
18
      // in the first place we should not be more successful here). Unless we
1183
18
      // have some test case that prove this to be correct we should avoid
1184
18
      // calling EmitFuncArgumentDbgValue here.
1185
18
      if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1186
16
        LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
1187
16
                          << DbgSDNodeOrder << "] for:\n  " << *DI << "\n");
1188
16
        LLVM_DEBUG(dbgs() << "  By mapping to:\n    "; Val.dump());
1189
16
        // Increase the SDNodeOrder for the DbgValue here to make sure it is
1190
16
        // inserted after the definition of Val when emitting the instructions
1191
16
        // after ISel. An alternative could be to teach
1192
16
        // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1193
16
        LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()
1194
16
                   << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "
1195
16
                   << ValSDNodeOrder << "\n");
1196
16
        SDV = getDbgValue(Val, Variable, Expr, dl,
1197
16
                          std::max(DbgSDNodeOrder, ValSDNodeOrder));
1198
16
        DAG.AddDbgValue(SDV, Val.getNode(), false);
1199
16
      } else
1200
18
        LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
1201
18
                          << "in EmitFuncArgumentDbgValue\n");
1202
18
    } else {
1203
0
      LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1204
0
      auto Undef =
1205
0
          UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1206
0
      auto SDV =
1207
0
          DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
1208
0
      DAG.AddDbgValue(SDV, nullptr, false);
1209
0
    }
1210
18
  }
1211
14
  DDIV.clear();
1212
14
}
1213
1214
14
void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
1215
14
  Value *V = DDI.getDI()->getValue();
1216
14
  DILocalVariable *Var = DDI.getDI()->getVariable();
1217
14
  DIExpression *Expr = DDI.getDI()->getExpression();
1218
14
  DebugLoc DL = DDI.getdl();
1219
14
  DebugLoc InstDL = DDI.getDI()->getDebugLoc();
1220
14
  unsigned SDOrder = DDI.getSDNodeOrder();
1221
14
1222
14
  // Currently we consider only dbg.value intrinsics -- we tell the salvager
1223
14
  // that DW_OP_stack_value is desired.
1224
14
  assert(isa<DbgValueInst>(DDI.getDI()));
1225
14
  bool StackValue = true;
1226
14
1227
14
  // Can this Value can be encoded without any further work?
1228
14
  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder))
1229
0
    return;
1230
14
1231
14
  // Attempt to salvage back through as many instructions as possible. Bail if
1232
14
  // a non-instruction is seen, such as a constant expression or global
1233
14
  // variable. FIXME: Further work could recover those too.
1234
15
  
while (14
isa<Instruction>(V)) {
1235
2
    Instruction &VAsInst = *cast<Instruction>(V);
1236
2
    DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue);
1237
2
1238
2
    // If we cannot salvage any further, and haven't yet found a suitable debug
1239
2
    // expression, bail out.
1240
2
    if (!NewExpr)
1241
0
      break;
1242
2
1243
2
    // New value and expr now represent this debuginfo.
1244
2
    V = VAsInst.getOperand(0);
1245
2
    Expr = NewExpr;
1246
2
1247
2
    // Some kind of simplification occurred: check whether the operand of the
1248
2
    // salvaged debug expression can be encoded in this DAG.
1249
2
    if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) {
1250
1
      LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n  "
1251
1
                        << DDI.getDI() << "\nBy stripping back to:\n  " << V);
1252
1
      return;
1253
1
    }
1254
2
  }
1255
14
1256
14
  // This was the final opportunity to salvage this debug information, and it
1257
14
  // couldn't be done. Place an undef DBG_VALUE at this location to terminate
1258
14
  // any earlier variable location.
1259
14
  auto Undef = UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
1260
13
  auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
1261
13
  DAG.AddDbgValue(SDV, nullptr, false);
1262
13
1263
13
  LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n  " << DDI.getDI()
1264
13
                    << "\n");
1265
13
  LLVM_DEBUG(dbgs() << "  Last seen at:\n    " << *DDI.getDI()->getOperand(0)
1266
13
                    << "\n");
1267
13
}
1268
1269
bool SelectionDAGBuilder::handleDebugValue(const Value *V, DILocalVariable *Var,
1270
                                           DIExpression *Expr, DebugLoc dl,
1271
4.94k
                                           DebugLoc InstDL, unsigned Order) {
1272
4.94k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1273
4.94k
  SDDbgValue *SDV;
1274
4.94k
  if (isa<ConstantInt>(V) || 
isa<ConstantFP>(V)818
||
isa<UndefValue>(V)810
||
1275
4.94k
      
isa<ConstantPointerNull>(V)786
) {
1276
4.17k
    SDV = DAG.getConstantDbgValue(Var, Expr, V, dl, SDNodeOrder);
1277
4.17k
    DAG.AddDbgValue(SDV, nullptr, false);
1278
4.17k
    return true;
1279
4.17k
  }
1280
774
1281
774
  // If the Value is a frame index, we can create a FrameIndex debug value
1282
774
  // without relying on the DAG at all.
1283
774
  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1284
57
    auto SI = FuncInfo.StaticAllocaMap.find(AI);
1285
57
    if (SI != FuncInfo.StaticAllocaMap.end()) {
1286
56
      auto SDV =
1287
56
          DAG.getFrameIndexDbgValue(Var, Expr, SI->second,
1288
56
                                    /*IsIndirect*/ false, dl, SDNodeOrder);
1289
56
      // Do not attach the SDNodeDbgValue to an SDNode: this variable location
1290
56
      // is still available even if the SDNode gets optimized out.
1291
56
      DAG.AddDbgValue(SDV, nullptr, false);
1292
56
      return true;
1293
56
    }
1294
718
  }
1295
718
1296
718
  // Do not use getValue() in here; we don't want to generate code at
1297
718
  // this point if it hasn't been done yet.
1298
718
  SDValue N = NodeMap[V];
1299
718
  if (!N.getNode() && 
isa<Argument>(V)145
) // Check unused arguments map.
1300
61
    N = UnusedArgNodeMap[V];
1301
718
  if (N.getNode()) {
1302
627
    if (EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))
1303
283
      return true;
1304
344
    SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder);
1305
344
    DAG.AddDbgValue(SDV, N.getNode(), false);
1306
344
    return true;
1307
344
  }
1308
91
1309
91
  // Special rules apply for the first dbg.values of parameter variables in a
1310
91
  // function. Identify them by the fact they reference Argument Values, that
1311
91
  // they're parameters, and they are parameters of the current function. We
1312
91
  // need to let them dangle until they get an SDNode.
1313
91
  bool IsParamOfFunc = isa<Argument>(V) && 
Var->isParameter()7
&&
1314
91
                       
!InstDL.getInlinedAt()6
;
1315
91
  if (!IsParamOfFunc) {
1316
86
    // The value is not used in this block yet (or it would have an SDNode).
1317
86
    // We still want the value to appear for the user if possible -- if it has
1318
86
    // an associated VReg, we can refer to that instead.
1319
86
    auto VMI = FuncInfo.ValueMap.find(V);
1320
86
    if (VMI != FuncInfo.ValueMap.end()) {
1321
44
      unsigned Reg = VMI->second;
1322
44
      // If this is a PHI node, it may be split up into several MI PHI nodes
1323
44
      // (in FunctionLoweringInfo::set).
1324
44
      RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
1325
44
                       V->getType(), None);
1326
44
      if (RFV.occupiesMultipleRegs()) {
1327
7
        unsigned Offset = 0;
1328
7
        unsigned BitsToDescribe = 0;
1329
7
        if (auto VarSize = Var->getSizeInBits())
1330
7
          BitsToDescribe = *VarSize;
1331
7
        if (auto Fragment = Expr->getFragmentInfo())
1332
2
          BitsToDescribe = Fragment->SizeInBits;
1333
16
        for (auto RegAndSize : RFV.getRegsAndSizes()) {
1334
16
          unsigned RegisterSize = RegAndSize.second;
1335
16
          // Bail out if all bits are described already.
1336
16
          if (Offset >= BitsToDescribe)
1337
1
            break;
1338
15
          unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
1339
15
              ? 
BitsToDescribe - Offset2
1340
15
              : 
RegisterSize13
;
1341
15
          auto FragmentExpr = DIExpression::createFragmentExpression(
1342
15
              Expr, Offset, FragmentSize);
1343
15
          if (!FragmentExpr)
1344
0
              continue;
1345
15
          SDV = DAG.getVRegDbgValue(Var, *FragmentExpr, RegAndSize.first,
1346
15
                                    false, dl, SDNodeOrder);
1347
15
          DAG.AddDbgValue(SDV, nullptr, false);
1348
15
          Offset += RegisterSize;
1349
15
        }
1350
37
      } else {
1351
37
        SDV = DAG.getVRegDbgValue(Var, Expr, Reg, false, dl, SDNodeOrder);
1352
37
        DAG.AddDbgValue(SDV, nullptr, false);
1353
37
      }
1354
44
      return true;
1355
44
    }
1356
47
  }
1357
47
1358
47
  return false;
1359
47
}
1360
1361
1.19M
void SelectionDAGBuilder::resolveOrClearDbgInfo() {
1362
1.19M
  // Try to fixup any remaining dangling debug info -- and drop it if we can't.
1363
1.19M
  for (auto &Pair : DanglingDebugInfoMap)
1364
28
    for (auto &DDI : Pair.second)
1365
12
      salvageUnresolvedDbgValue(DDI);
1366
1.19M
  clearDanglingDebugInfo();
1367
1.19M
}
1368
1369
/// getCopyFromRegs - If there was virtual register allocated for the value V
1370
/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1371
4.62M
SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1372
4.62M
  DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1373
4.62M
  SDValue Result;
1374
4.62M
1375
4.62M
  if (It != FuncInfo.ValueMap.end()) {
1376
1.79M
    unsigned InReg = It->second;
1377
1.79M
1378
1.79M
    RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1379
1.79M
                     DAG.getDataLayout(), InReg, Ty,
1380
1.79M
                     None); // This is not an ABI copy.
1381
1.79M
    SDValue Chain = DAG.getEntryNode();
1382
1.79M
    Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1383
1.79M
                                 V);
1384
1.79M
    resolveDanglingDebugInfo(V, Result);
1385
1.79M
  }
1386
4.62M
1387
4.62M
  return Result;
1388
4.62M
}
1389
1390
/// getValue - Return an SDValue for the given Value.
1391
10.8M
SDValue SelectionDAGBuilder::getValue(const Value *V) {
1392
10.8M
  // If we already have an SDValue for this value, use it. It's important
1393
10.8M
  // to do this first, so that we don't create a CopyFromReg if we already
1394
10.8M
  // have a regular SDValue.
1395
10.8M
  SDValue &N = NodeMap[V];
1396
10.8M
  if (N.getNode()) 
return N6.18M
;
1397
4.62M
1398
4.62M
  // If there's a virtual register allocated and initialized for this
1399
4.62M
  // value, use it.
1400
4.62M
  if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1401
1.79M
    return copyFromReg;
1402
2.82M
1403
2.82M
  // Otherwise create a new SDValue and remember it.
1404
2.82M
  SDValue Val = getValueImpl(V);
1405
2.82M
  NodeMap[V] = Val;
1406
2.82M
  resolveDanglingDebugInfo(V, Val);
1407
2.82M
  return Val;
1408
2.82M
}
1409
1410
// Return true if SDValue exists for the given Value
1411
463
bool SelectionDAGBuilder::findValue(const Value *V) const {
1412
463
  return (NodeMap.find(V) != NodeMap.end()) ||
1413
463
    
(FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end())53
;
1414
463
}
1415
1416
/// getNonRegisterValue - Return an SDValue for the given Value, but
1417
/// don't look in FuncInfo.ValueMap for a virtual register.
1418
815k
SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1419
815k
  // If we already have an SDValue for this value, use it.
1420
815k
  SDValue &N = NodeMap[V];
1421
815k
  if (N.getNode()) {
1422
666k
    if (isa<ConstantSDNode>(N) || 
isa<ConstantFPSDNode>(N)647k
) {
1423
19.0k
      // Remove the debug location from the node as the node is about to be used
1424
19.0k
      // in a location which may differ from the original debug location.  This
1425
19.0k
      // is relevant to Constant and ConstantFP nodes because they can appear
1426
19.0k
      // as constant expressions inside PHI nodes.
1427
19.0k
      N->setDebugLoc(DebugLoc());
1428
19.0k
    }
1429
666k
    return N;
1430
666k
  }
1431
148k
1432
148k
  // Otherwise create a new SDValue and remember it.
1433
148k
  SDValue Val = getValueImpl(V);
1434
148k
  NodeMap[V] = Val;
1435
148k
  resolveDanglingDebugInfo(V, Val);
1436
148k
  return Val;
1437
148k
}
1438
1439
/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1440
/// Create an SDValue for the given value.
1441
2.97M
SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1442
2.97M
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1443
2.97M
1444
2.97M
  if (const Constant *C = dyn_cast<Constant>(V)) {
1445
2.79M
    EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1446
2.79M
1447
2.79M
    if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1448
1.37M
      return DAG.getConstant(*CI, getCurSDLoc(), VT);
1449
1.42M
1450
1.42M
    if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1451
764k
      return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1452
659k
1453
659k
    if (isa<ConstantPointerNull>(C)) {
1454
138k
      unsigned AS = V->getType()->getPointerAddressSpace();
1455
138k
      return DAG.getConstant(0, getCurSDLoc(),
1456
138k
                             TLI.getPointerTy(DAG.getDataLayout(), AS));
1457
138k
    }
1458
520k
1459
520k
    if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1460
55.7k
      return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1461
464k
1462
464k
    if (isa<UndefValue>(C) && 
!V->getType()->isAggregateType()47.0k
)
1463
45.6k
      return DAG.getUNDEF(VT);
1464
419k
1465
419k
    if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1466
296k
      visit(CE->getOpcode(), *CE);
1467
296k
      SDValue N1 = NodeMap[V];
1468
296k
      assert(N1.getNode() && "visit didn't populate the NodeMap!");
1469
296k
      return N1;
1470
296k
    }
1471
122k
1472
122k
    if (isa<ConstantStruct>(C) || 
isa<ConstantArray>(C)122k
) {
1473
247
      SmallVector<SDValue, 4> Constants;
1474
247
      for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1475
1.06k
           OI != OE; 
++OI821
) {
1476
821
        SDNode *Val = getValue(*OI).getNode();
1477
821
        // If the operand is an empty aggregate, there are no values.
1478
821
        if (!Val) 
continue1
;
1479
820
        // Add each leaf value from the operand to the Constants list
1480
820
        // to form a flattened list of all the values.
1481
1.64k
        
for (unsigned i = 0, e = Val->getNumValues(); 820
i != e;
++i827
)
1482
827
          Constants.push_back(SDValue(Val, i));
1483
820
      }
1484
247
1485
247
      return DAG.getMergeValues(Constants, getCurSDLoc());
1486
247
    }
1487
121k
1488
121k
    if (const ConstantDataSequential *CDS =
1489
84.7k
          dyn_cast<ConstantDataSequential>(C)) {
1490
84.7k
      SmallVector<SDValue, 4> Ops;
1491
532k
      for (unsigned i = 0, e = CDS->getNumElements(); i != e; 
++i447k
) {
1492
447k
        SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1493
447k
        // Add each leaf value from the operand to the Constants list
1494
447k
        // to form a flattened list of all the values.
1495
895k
        for (unsigned i = 0, e = Val->getNumValues(); i != e; 
++i447k
)
1496
447k
          Ops.push_back(SDValue(Val, i));
1497
447k
      }
1498
84.7k
1499
84.7k
      if (isa<ArrayType>(CDS->getType()))
1500
36
        return DAG.getMergeValues(Ops, getCurSDLoc());
1501
84.7k
      return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1502
84.7k
    }
1503
37.1k
1504
37.1k
    if (C->getType()->isStructTy() || 
C->getType()->isArrayTy()35.9k
) {
1505
1.51k
      assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&
1506
1.51k
             "Unknown struct or array constant!");
1507
1.51k
1508
1.51k
      SmallVector<EVT, 4> ValueVTs;
1509
1.51k
      ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1510
1.51k
      unsigned NumElts = ValueVTs.size();
1511
1.51k
      if (NumElts == 0)
1512
1
        return SDValue(); // empty struct
1513
1.51k
      SmallVector<SDValue, 4> Constants(NumElts);
1514
5.68k
      for (unsigned i = 0; i != NumElts; 
++i4.17k
) {
1515
4.17k
        EVT EltVT = ValueVTs[i];
1516
4.17k
        if (isa<UndefValue>(C))
1517
3.84k
          Constants[i] = DAG.getUNDEF(EltVT);
1518
334
        else if (EltVT.isFloatingPoint())
1519
30
          Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1520
304
        else
1521
304
          Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1522
4.17k
      }
1523
1.51k
1524
1.51k
      return DAG.getMergeValues(Constants, getCurSDLoc());
1525
1.51k
    }
1526
35.6k
1527
35.6k
    if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1528
135
      return DAG.getBlockAddress(BA, VT);
1529
35.5k
1530
35.5k
    VectorType *VecTy = cast<VectorType>(V->getType());
1531
35.5k
    unsigned NumElements = VecTy->getNumElements();
1532
35.5k
1533
35.5k
    // Now that we know the number and type of the elements, get that number of
1534
35.5k
    // elements into the Ops array based on what kind of constant it is.
1535
35.5k
    SmallVector<SDValue, 16> Ops;
1536
35.5k
    if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1537
36.1k
      for (unsigned i = 0; i != NumElements; 
++i32.3k
)
1538
32.3k
        Ops.push_back(getValue(CV->getOperand(i)));
1539
31.8k
    } else {
1540
31.8k
      assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!");
1541
31.8k
      EVT EltVT =
1542
31.8k
          TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1543
31.8k
1544
31.8k
      SDValue Op;
1545
31.8k
      if (EltVT.isFloatingPoint())
1546
4.13k
        Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1547
27.6k
      else
1548
27.6k
        Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1549
31.8k
      Ops.assign(NumElements, Op);
1550
31.8k
    }
1551
35.5k
1552
35.5k
    // Create a BUILD_VECTOR node.
1553
35.5k
    return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1554
35.5k
  }
1555
178k
1556
178k
  // If this is a static alloca, generate it as the frameindex instead of
1557
178k
  // computation.
1558
178k
  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1559
175k
    DenseMap<const AllocaInst*, int>::iterator SI =
1560
175k
      FuncInfo.StaticAllocaMap.find(AI);
1561
175k
    if (SI != FuncInfo.StaticAllocaMap.end())
1562
175k
      return DAG.getFrameIndex(SI->second,
1563
175k
                               TLI.getFrameIndexTy(DAG.getDataLayout()));
1564
2.38k
  }
1565
2.38k
1566
2.38k
  // If this is an instruction which fast-isel has deferred, select it now.
1567
2.38k
  if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1568
2.38k
    unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1569
2.38k
1570
2.38k
    RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1571
2.38k
                     Inst->getType(), getABIRegCopyCC(V));
1572
2.38k
    SDValue Chain = DAG.getEntryNode();
1573
2.38k
    return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1574
2.38k
  }
1575
2
1576
2
  llvm_unreachable("Can't get register for value!");
1577
2
}
1578
1579
168
void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1580
168
  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1581
168
  bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1582
168
  bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1583
168
  bool IsSEH = isAsynchronousEHPersonality(Pers);
1584
168
  bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1585
168
  MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1586
168
  if (!IsSEH)
1587
135
    CatchPadMBB->setIsEHScopeEntry();
1588
168
  // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1589
168
  if (IsMSVCCXX || 
IsCoreCLR94
)
1590
85
    CatchPadMBB->setIsEHFuncletEntry();
1591
168
  // Wasm does not need catchpads anymore
1592
168
  if (!IsWasmCXX)
1593
118
    DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1594
118
                            getControlRoot()));
1595
168
}
1596
1597
159
void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1598
159
  // Update machine-CFG edge.
1599
159
  MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1600
159
  FuncInfo.MBB->addSuccessor(TargetMBB);
1601
159
1602
159
  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1603
159
  bool IsSEH = isAsynchronousEHPersonality(Pers);
1604
159
  if (IsSEH) {
1605
31
    // If this is not a fall-through branch or optimizations are switched off,
1606
31
    // emit the branch.
1607
31
    if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1608
31
        
TM.getOptLevel() == CodeGenOpt::None22
)
1609
11
      DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1610
11
                              getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1611
31
    return;
1612
31
  }
1613
128
1614
128
  // Figure out the funclet membership for the catchret's successor.
1615
128
  // This will be used by the FuncletLayout pass to determine how to order the
1616
128
  // BB's.
1617
128
  // A 'catchret' returns to the outer scope's color.
1618
128
  Value *ParentPad = I.getCatchSwitchParentPad();
1619
128
  const BasicBlock *SuccessorColor;
1620
128
  if (isa<ConstantTokenNone>(ParentPad))
1621
119
    SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1622
9
  else
1623
9
    SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1624
128
  assert(SuccessorColor && "No parent funclet for catchret!");
1625
128
  MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1626
128
  assert(SuccessorColorMBB && "No MBB for SuccessorColor!");
1627
128
1628
128
  // Create the terminator node.
1629
128
  SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1630
128
                            getControlRoot(), DAG.getBasicBlock(TargetMBB),
1631
128
                            DAG.getBasicBlock(SuccessorColorMBB));
1632
128
  DAG.setRoot(Ret);
1633
128
}
1634
1635
66
void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1636
66
  // Don't emit any special code for the cleanuppad instruction. It just marks
1637
66
  // the start of an EH scope/funclet.
1638
66
  FuncInfo.MBB->setIsEHScopeEntry();
1639
66
  auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1640
66
  if (Pers != EHPersonality::Wasm_CXX) {
1641
46
    FuncInfo.MBB->setIsEHFuncletEntry();
1642
46
    FuncInfo.MBB->setIsCleanupFuncletEntry();
1643
46
  }
1644
66
}
1645
1646
// For wasm, there's alwyas a single catch pad attached to a catchswitch, and
1647
// the control flow always stops at the single catch pad, as it does for a
1648
// cleanup pad. In case the exception caught is not of the types the catch pad
1649
// catches, it will be rethrown by a rethrow.
1650
static void findWasmUnwindDestinations(
1651
    FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1652
    BranchProbability Prob,
1653
    SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1654
88
        &UnwindDests) {
1655
88
  while (EHPadBB) {
1656
76
    const Instruction *Pad = EHPadBB->getFirstNonPHI();
1657
76
    if (isa<CleanupPadInst>(Pad)) {
1658
23
      // Stop on cleanup pads.
1659
23
      UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1660
23
      UnwindDests.back().first->setIsEHScopeEntry();
1661
23
      break;
1662
53
    } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1663
53
      // Add the catchpad handlers to the possible destinations. We don't
1664
53
      // continue to the unwind destination of the catchswitch for wasm.
1665
53
      for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1666
53
        UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1667
53
        UnwindDests.back().first->setIsEHScopeEntry();
1668
53
      }
1669
53
      break;
1670
53
    } else {
1671
0
      continue;
1672
0
    }
1673
76
  }
1674
88
}
1675
1676
/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1677
/// many places it could ultimately go. In the IR, we have a single unwind
1678
/// destination, but in the machine CFG, we enumerate all the possible blocks.
1679
/// This function skips over imaginary basic blocks that hold catchswitch
1680
/// instructions, and finds all the "real" machine
1681
/// basic block destinations. As those destinations may not be successors of
1682
/// EHPadBB, here we also calculate the edge probability to those destinations.
1683
/// The passed-in Prob is the edge probability to EHPadBB.
1684
static void findUnwindDestinations(
1685
    FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1686
    BranchProbability Prob,
1687
    SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1688
6.42k
        &UnwindDests) {
1689
6.42k
  EHPersonality Personality =
1690
6.42k
    classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1691
6.42k
  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1692
6.42k
  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1693
6.42k
  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
1694
6.42k
  bool IsSEH = isAsynchronousEHPersonality(Personality);
1695
6.42k
1696
6.42k
  if (IsWasmCXX) {
1697
88
    findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
1698
88
    assert(UnwindDests.size() <= 1 &&
1699
88
           "There should be at most one unwind destination for wasm");
1700
88
    return;
1701
88
  }
1702
6.33k
1703
6.46k
  
while (6.33k
EHPadBB) {
1704
6.33k
    const Instruction *Pad = EHPadBB->getFirstNonPHI();
1705
6.33k
    BasicBlock *NewEHPadBB = nullptr;
1706
6.33k
    if (isa<LandingPadInst>(Pad)) {
1707
6.12k
      // Stop on landingpads. They are not funclets.
1708
6.12k
      UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1709
6.12k
      break;
1710
6.12k
    } else 
if (205
isa<CleanupPadInst>(Pad)205
) {
1711
70
      // Stop on cleanup pads. Cleanups are always funclet entries for all known
1712
70
      // personalities.
1713
70
      UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1714
70
      UnwindDests.back().first->setIsEHScopeEntry();
1715
70
      UnwindDests.back().first->setIsEHFuncletEntry();
1716
70
      break;
1717
135
    } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1718
135
      // Add the catchpad handlers to the possible destinations.
1719
146
      for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1720
146
        UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1721
146
        // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1722
146
        if (IsMSVCCXX || 
IsCoreCLR56
)
1723
101
          UnwindDests.back().first->setIsEHFuncletEntry();
1724
146
        if (!IsSEH)
1725
101
          UnwindDests.back().first->setIsEHScopeEntry();
1726
146
      }
1727
135
      NewEHPadBB = CatchSwitch->getUnwindDest();
1728
135
    } else {
1729
0
      continue;
1730
0
    }
1731
135
1732
135
    BranchProbabilityInfo *BPI = FuncInfo.BPI;
1733
135
    if (BPI && 
NewEHPadBB127
)
1734
28
      Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1735
135
    EHPadBB = NewEHPadBB;
1736
135
  }
1737
6.33k
}
1738
1739
53
void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1740
53
  // Update successor info.
1741
53
  SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1742
53
  auto UnwindDest = I.getUnwindDest();
1743
53
  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1744
53
  BranchProbability UnwindDestProb =
1745
53
      (BPI && 
UnwindDest45
)
1746
53
          ? 
BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)12
1747
53
          : 
BranchProbability::getZero()41
;
1748
53
  findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1749
53
  for (auto &UnwindDest : UnwindDests) {
1750
15
    UnwindDest.first->setIsEHPad();
1751
15
    addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1752
15
  }
1753
53
  FuncInfo.MBB->normalizeSuccProbs();
1754
53
1755
53
  // Create the terminator node.
1756
53
  SDValue Ret =
1757
53
      DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1758
53
  DAG.setRoot(Ret);
1759
53
}
1760
1761
0
void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1762
0
  report_fatal_error("visitCatchSwitch not yet implemented!");
1763
0
}
1764
1765
259k
void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1766
259k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1767
259k
  auto &DL = DAG.getDataLayout();
1768
259k
  SDValue Chain = getControlRoot();
1769
259k
  SmallVector<ISD::OutputArg, 8> Outs;
1770
259k
  SmallVector<SDValue, 8> OutVals;
1771
259k
1772
259k
  // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1773
259k
  // lower
1774
259k
  //
1775
259k
  //   %val = call <ty> @llvm.experimental.deoptimize()
1776
259k
  //   ret <ty> %val
1777
259k
  //
1778
259k
  // differently.
1779
259k
  if (I.getParent()->getTerminatingDeoptimizeCall()) {
1780
0
    LowerDeoptimizingReturn();
1781
0
    return;
1782
0
  }
1783
259k
1784
259k
  if (!FuncInfo.CanLowerReturn) {
1785
1.94k
    unsigned DemoteReg = FuncInfo.DemoteRegister;
1786
1.94k
    const Function *F = I.getParent()->getParent();
1787
1.94k
1788
1.94k
    // Emit a store of the return value through the virtual register.
1789
1.94k
    // Leave Outs empty so that LowerReturn won't try to load return
1790
1.94k
    // registers the usual way.
1791
1.94k
    SmallVector<EVT, 1> PtrValueVTs;
1792
1.94k
    ComputeValueVTs(TLI, DL,
1793
1.94k
                    F->getReturnType()->getPointerTo(
1794
1.94k
                        DAG.getDataLayout().getAllocaAddrSpace()),
1795
1.94k
                    PtrValueVTs);
1796
1.94k
1797
1.94k
    SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1798
1.94k
                                        DemoteReg, PtrValueVTs[0]);
1799
1.94k
    SDValue RetOp = getValue(I.getOperand(0));
1800
1.94k
1801
1.94k
    SmallVector<EVT, 4> ValueVTs, MemVTs;
1802
1.94k
    SmallVector<uint64_t, 4> Offsets;
1803
1.94k
    ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs,
1804
1.94k
                    &Offsets);
1805
1.94k
    unsigned NumValues = ValueVTs.size();
1806
1.94k
1807
1.94k
    SmallVector<SDValue, 4> Chains(NumValues);
1808
4.21k
    for (unsigned i = 0; i != NumValues; 
++i2.26k
) {
1809
2.26k
      // An aggregate return value cannot wrap around the address space, so
1810
2.26k
      // offsets to its parts don't wrap either.
1811
2.26k
      SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1812
2.26k
1813
2.26k
      SDValue Val = RetOp.getValue(i);
1814
2.26k
      if (MemVTs[i] != ValueVTs[i])
1815
0
        Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]);
1816
2.26k
      Chains[i] = DAG.getStore(Chain, getCurSDLoc(), Val,
1817
2.26k
          // FIXME: better loc info would be nice.
1818
2.26k
          Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1819
2.26k
    }
1820
1.94k
1821
1.94k
    Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1822
1.94k
                        MVT::Other, Chains);
1823
257k
  } else if (I.getNumOperands() != 0) {
1824
177k
    SmallVector<EVT, 4> ValueVTs;
1825
177k
    ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1826
177k
    unsigned NumValues = ValueVTs.size();
1827
177k
    if (NumValues) {
1828
177k
      SDValue RetOp = getValue(I.getOperand(0));
1829
177k
1830
177k
      const Function *F = I.getParent()->getParent();
1831
177k
1832
177k
      bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1833
177k
          I.getOperand(0)->getType(), F->getCallingConv(),
1834
177k
          /*IsVarArg*/ false);
1835
177k
1836
177k
      ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1837
177k
      if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1838
177k
                                          Attribute::SExt))
1839
4.12k
        ExtendKind = ISD::SIGN_EXTEND;
1840
173k
      else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1841
173k
                                               Attribute::ZExt))
1842
6.38k
        ExtendKind = ISD::ZERO_EXTEND;
1843
177k
1844
177k
      LLVMContext &Context = F->getContext();
1845
177k
      bool RetInReg = F->getAttributes().hasAttribute(
1846
177k
          AttributeList::ReturnIndex, Attribute::InReg);
1847
177k
1848
358k
      for (unsigned j = 0; j != NumValues; 
++j180k
) {
1849
180k
        EVT VT = ValueVTs[j];
1850
180k
1851
180k
        if (ExtendKind != ISD::ANY_EXTEND && 
VT.isInteger()10.5k
)
1852
10.5k
          VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1853
180k
1854
180k
        CallingConv::ID CC = F->getCallingConv();
1855
180k
1856
180k
        unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1857
180k
        MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1858
180k
        SmallVector<SDValue, 4> Parts(NumParts);
1859
180k
        getCopyToParts(DAG, getCurSDLoc(),
1860
180k
                       SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1861
180k
                       &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1862
180k
1863
180k
        // 'inreg' on function refers to return value
1864
180k
        ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1865
180k
        if (RetInReg)
1866
42
          Flags.setInReg();
1867
180k
1868
180k
        if (I.getOperand(0)->getType()->isPointerTy()) {
1869
9.07k
          Flags.setPointer();
1870
9.07k
          Flags.setPointerAddrSpace(
1871
9.07k
              cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace());
1872
9.07k
        }
1873
180k
1874
180k
        if (NeedsRegBlock) {
1875
2.28k
          Flags.setInConsecutiveRegs();
1876
2.28k
          if (j == NumValues - 1)
1877
2.11k
            Flags.setInConsecutiveRegsLast();
1878
2.28k
        }
1879
180k
1880
180k
        // Propagate extension type if any
1881
180k
        if (ExtendKind == ISD::SIGN_EXTEND)
1882
4.12k
          Flags.setSExt();
1883
176k
        else if (ExtendKind == ISD::ZERO_EXTEND)
1884
6.38k
          Flags.setZExt();
1885
180k
1886
378k
        for (unsigned i = 0; i < NumParts; 
++i198k
) {
1887
198k
          Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1888
198k
                                        VT, /*isfixed=*/true, 0, 0));
1889
198k
          OutVals.push_back(Parts[i]);
1890
198k
        }
1891
180k
      }
1892
177k
    }
1893
177k
  }
1894
259k
1895
259k
  // Push in swifterror virtual register as the last element of Outs. This makes
1896
259k
  // sure swifterror virtual register will be returned in the swifterror
1897
259k
  // physical register.
1898
259k
  const Function *F = I.getParent()->getParent();
1899
259k
  if (TLI.supportSwiftError() &&
1900
259k
      
F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)170k
) {
1901
112
    assert(SwiftError.getFunctionArg() && "Need a swift error argument");
1902
112
    ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1903
112
    Flags.setSwiftError();
1904
112
    Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1905
112
                                  EVT(TLI.getPointerTy(DL)) /*argvt*/,
1906
112
                                  true /*isfixed*/, 1 /*origidx*/,
1907
112
                                  0 /*partOffs*/));
1908
112
    // Create SDNode for the swifterror virtual register.
1909
112
    OutVals.push_back(
1910
112
        DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
1911
112
                            &I, FuncInfo.MBB, SwiftError.getFunctionArg()),
1912
112
                        EVT(TLI.getPointerTy(DL))));
1913
112
  }
1914
259k
1915
259k
  bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1916
259k
  CallingConv::ID CallConv =
1917
259k
    DAG.getMachineFunction().getFunction().getCallingConv();
1918
259k
  Chain = DAG.getTargetLoweringInfo().LowerReturn(
1919
259k
      Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1920
259k
1921
259k
  // Verify that the target's LowerReturn behaved as expected.
1922
259k
  assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&
1923
259k
         "LowerReturn didn't return a valid chain!");
1924
259k
1925
259k
  // Update the DAG with the new chain value resulting from return lowering.
1926
259k
  DAG.setRoot(Chain);
1927
259k
}
1928
1929
/// CopyToExportRegsIfNeeded - If the given value has virtual registers
1930
/// created for it, emit nodes to copy the value into the virtual
1931
/// registers.
1932
5.61M
void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1933
5.61M
  // Skip empty types
1934
5.61M
  if (V->getType()->isEmptyTy())
1935
16
    return;
1936
5.61M
1937
5.61M
  DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1938
5.61M
  if (VMI != FuncInfo.ValueMap.end()) {
1939
624k
    assert(!V->use_empty() && "Unused value assigned virtual registers!");
1940
624k
    CopyValueToVirtualRegister(V, VMI->second);
1941
624k
  }
1942
5.61M
}
1943
1944
/// ExportFromCurrentBlock - If this condition isn't known to be exported from
1945
/// the current basic block, add it to ValueMap now so that we'll get a
1946
/// CopyTo/FromReg.
1947
64.8k
void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1948
64.8k
  // No need to export constants.
1949
64.8k
  if (!isa<Instruction>(V) && 
!isa<Argument>(V)17.7k
)
return14.6k
;
1950
50.1k
1951
50.1k
  // Already exported?
1952
50.1k
  if (FuncInfo.isExportedInst(V)) 
return21.3k
;
1953
28.8k
1954
28.8k
  unsigned Reg = FuncInfo.InitializeRegForValue(V);
1955
28.8k
  CopyValueToVirtualRegister(V, Reg);
1956
28.8k
}
1957
1958
bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1959
67.5k
                                                     const BasicBlock *FromBB) {
1960
67.5k
  // The operands of the setcc have to be in this block.  We don't know
1961
67.5k
  // how to export them from some other block.
1962
67.5k
  if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1963
46.5k
    // Can export from current BB.
1964
46.5k
    if (VI->getParent() == FromBB)
1965
39.8k
      return true;
1966
6.68k
1967
6.68k
    // Is already exported, noop.
1968
6.68k
    return FuncInfo.isExportedInst(V);
1969
6.68k
  }
1970
21.0k
1971
21.0k
  // If this is an argument, we can export it if the BB is the entry block or
1972
21.0k
  // if it is already exported.
1973
21.0k
  if (isa<Argument>(V)) {
1974
2.06k
    if (FromBB == &FromBB->getParent()->getEntryBlock())
1975
365
      return true;
1976
1.69k
1977
1.69k
    // Otherwise, can only export this if it is already exported.
1978
1.69k
    return FuncInfo.isExportedInst(V);
1979
1.69k
  }
1980
19.0k
1981
19.0k
  // Otherwise, constants can always be exported.
1982
19.0k
  return true;
1983
19.0k
}
1984
1985
/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1986
BranchProbability
1987
SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1988
1.07M
                                        const MachineBasicBlock *Dst) const {
1989
1.07M
  BranchProbabilityInfo *BPI = FuncInfo.BPI;
1990
1.07M
  const BasicBlock *SrcBB = Src->getBasicBlock();
1991
1.07M
  const BasicBlock *DstBB = Dst->getBasicBlock();
1992
1.07M
  if (!BPI) {
1993
49
    // If BPI is not available, set the default probability as 1 / N, where N is
1994
49
    // the number of successors.
1995
49
    auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1996
49
    return BranchProbability(1, SuccSize);
1997
49
  }
1998
1.07M
  return BPI->getEdgeProbability(SrcBB, DstBB);
1999
1.07M
}
2000
2001
void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
2002
                                               MachineBasicBlock *Dst,
2003
1.15M
                                               BranchProbability Prob) {
2004
1.15M
  if (!FuncInfo.BPI)
2005
1.61k
    Src->addSuccessorWithoutProb(Dst);
2006
1.15M
  else {
2007
1.15M
    if (Prob.isUnknown())
2008
991k
      Prob = getEdgeProbability(Src, Dst);
2009
1.15M
    Src->addSuccessor(Dst, Prob);
2010
1.15M
  }
2011
1.15M
}
2012
2013
70.5k
static bool InBlock(const Value *V, const BasicBlock *BB) {
2014
70.5k
  if (const Instruction *I = dyn_cast<Instruction>(V))
2015
70.3k
    return I->getParent() == BB;
2016
249
  return true;
2017
249
}
2018
2019
/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
2020
/// This function emits a branch and is used at the leaves of an OR or an
2021
/// AND operator tree.
2022
void
2023
SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
2024
                                                  MachineBasicBlock *TBB,
2025
                                                  MachineBasicBlock *FBB,
2026
                                                  MachineBasicBlock *CurBB,
2027
                                                  MachineBasicBlock *SwitchBB,
2028
                                                  BranchProbability TProb,
2029
                                                  BranchProbability FProb,
2030
70.9k
                                                  bool InvertCond) {
2031
70.9k
  const BasicBlock *BB = CurBB->getBasicBlock();
2032
70.9k
2033
70.9k
  // If the leaf of the tree is a comparison, merge the condition into
2034
70.9k
  // the caseblock.
2035
70.9k
  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
2036
66.9k
    // The operands of the cmp have to be in this block.  We don't know
2037
66.9k
    // how to export them from some other block.  If this is the first block
2038
66.9k
    // of the sequence, no exporting is needed.
2039
66.9k
    if (CurBB == SwitchBB ||
2040
66.9k
        
(33.7k
isExportableFromCurrentBlock(BOp->getOperand(0), BB)33.7k
&&
2041
66.9k
         
isExportableFromCurrentBlock(BOp->getOperand(1), BB)33.7k
)) {
2042
66.9k
      ISD::CondCode Condition;
2043
66.9k
      if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
2044
63.5k
        ICmpInst::Predicate Pred =
2045
63.5k
            InvertCond ? 
IC->getInversePredicate()294
:
IC->getPredicate()63.2k
;
2046
63.5k
        Condition = getICmpCondCode(Pred);
2047
63.5k
      } else {
2048
3.37k
        const FCmpInst *FC = cast<FCmpInst>(Cond);
2049
3.37k
        FCmpInst::Predicate Pred =
2050
3.37k
            InvertCond ? 
FC->getInversePredicate()2
:
FC->getPredicate()3.37k
;
2051
3.37k
        Condition = getFCmpCondCode(Pred);
2052
3.37k
        if (TM.Options.NoNaNsFPMath)
2053
0
          Condition = getFCmpCodeWithoutNaN(Condition);
2054
3.37k
      }
2055
66.9k
2056
66.9k
      CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
2057
66.9k
                   TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2058
66.9k
      SL->SwitchCases.push_back(CB);
2059
66.9k
      return;
2060
66.9k
    }
2061
4.02k
  }
2062
4.02k
2063
4.02k
  // Create a CaseBlock record representing this branch.
2064
4.02k
  ISD::CondCode Opc = InvertCond ? 
ISD::SETNE197
:
ISD::SETEQ3.82k
;
2065
4.02k
  CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
2066
4.02k
               nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
2067
4.02k
  SL->SwitchCases.push_back(CB);
2068
4.02k
}
2069
2070
void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
2071
                                               MachineBasicBlock *TBB,
2072
                                               MachineBasicBlock *FBB,
2073
                                               MachineBasicBlock *CurBB,
2074
                                               MachineBasicBlock *SwitchBB,
2075
                                               Instruction::BinaryOps Opc,
2076
                                               BranchProbability TProb,
2077
                                               BranchProbability FProb,
2078
106k
                                               bool InvertCond) {
2079
106k
  // Skip over not part of the tree and remember to invert op and operands at
2080
106k
  // next level.
2081
106k
  Value *NotCond;
2082
106k
  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&
2083
106k
      
InBlock(NotCond, CurBB->getBasicBlock())482
) {
2084
363
    FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
2085
363
                         !InvertCond);
2086
363
    return;
2087
363
  }
2088
105k
2089
105k
  const Instruction *BOp = dyn_cast<Instruction>(Cond);
2090
105k
  // Compute the effective opcode for Cond, taking into account whether it needs
2091
105k
  // to be inverted, e.g.
2092
105k
  //   and (not (or A, B)), C
2093
105k
  // gets lowered as
2094
105k
  //   and (and (not A, not B), C)
2095
105k
  unsigned BOpc = 0;
2096
105k
  if (BOp) {
2097
105k
    BOpc = BOp->getOpcode();
2098
105k
    if (InvertCond) {
2099
581
      if (BOpc == Instruction::And)
2100
101
        BOpc = Instruction::Or;
2101
480
      else if (BOpc == Instruction::Or)
2102
111
        BOpc = Instruction::And;
2103
581
    }
2104
105k
  }
2105
105k
2106
105k
  // If this node is not part of the or/and tree, emit it as a branch.
2107
105k
  if (!BOp || 
!(105k
isa<BinaryOperator>(BOp)105k
||
isa<CmpInst>(BOp)67.3k
) ||
2108
105k
      
BOpc != unsigned(Opc)105k
||
!BOp->hasOneUse()37.3k
||
2109
105k
      
BOp->getParent() != CurBB->getBasicBlock()37.3k
||
2110
105k
      
!InBlock(BOp->getOperand(0), CurBB->getBasicBlock())35.1k
||
2111
105k
      
!InBlock(BOp->getOperand(1), CurBB->getBasicBlock())34.9k
) {
2112
70.9k
    EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
2113
70.9k
                                 TProb, FProb, InvertCond);
2114
70.9k
    return;
2115
70.9k
  }
2116
34.8k
2117
34.8k
  //  Create TmpBB after CurBB.
2118
34.8k
  MachineFunction::iterator BBI(CurBB);
2119
34.8k
  MachineFunction &MF = DAG.getMachineFunction();
2120
34.8k
  MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
2121
34.8k
  CurBB->getParent()->insert(++BBI, TmpBB);
2122
34.8k
2123
34.8k
  if (Opc == Instruction::Or) {
2124
9.20k
    // Codegen X | Y as:
2125
9.20k
    // BB1:
2126
9.20k
    //   jmp_if_X TBB
2127
9.20k
    //   jmp TmpBB
2128
9.20k
    // TmpBB:
2129
9.20k
    //   jmp_if_Y TBB
2130
9.20k
    //   jmp FBB
2131
9.20k
    //
2132
9.20k
2133
9.20k
    // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2134
9.20k
    // The requirement is that
2135
9.20k
    //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
2136
9.20k
    //     = TrueProb for original BB.
2137
9.20k
    // Assuming the original probabilities are A and B, one choice is to set
2138
9.20k
    // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
2139
9.20k
    // A/(1+B) and 2B/(1+B). This choice assumes that
2140
9.20k
    //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
2141
9.20k
    // Another choice is to assume TrueProb for BB1 equals to TrueProb for
2142
9.20k
    // TmpBB, but the math is more complicated.
2143
9.20k
2144
9.20k
    auto NewTrueProb = TProb / 2;
2145
9.20k
    auto NewFalseProb = TProb / 2 + FProb;
2146
9.20k
    // Emit the LHS condition.
2147
9.20k
    FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
2148
9.20k
                         NewTrueProb, NewFalseProb, InvertCond);
2149
9.20k
2150
9.20k
    // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
2151
9.20k
    SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
2152
9.20k
    BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2153
9.20k
    // Emit the RHS condition into TmpBB.
2154
9.20k
    FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2155
9.20k
                         Probs[0], Probs[1], InvertCond);
2156
25.6k
  } else {
2157
25.6k
    assert(Opc == Instruction::And && "Unknown merge op!");
2158
25.6k
    // Codegen X & Y as:
2159
25.6k
    // BB1:
2160
25.6k
    //   jmp_if_X TmpBB
2161
25.6k
    //   jmp FBB
2162
25.6k
    // TmpBB:
2163
25.6k
    //   jmp_if_Y TBB
2164
25.6k
    //   jmp FBB
2165
25.6k
    //
2166
25.6k
    //  This requires creation of TmpBB after CurBB.
2167
25.6k
2168
25.6k
    // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
2169
25.6k
    // The requirement is that
2170
25.6k
    //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
2171
25.6k
    //     = FalseProb for original BB.
2172
25.6k
    // Assuming the original probabilities are A and B, one choice is to set
2173
25.6k
    // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
2174
25.6k
    // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
2175
25.6k
    // TrueProb for BB1 * FalseProb for TmpBB.
2176
25.6k
2177
25.6k
    auto NewTrueProb = TProb + FProb / 2;
2178
25.6k
    auto NewFalseProb = FProb / 2;
2179
25.6k
    // Emit the LHS condition.
2180
25.6k
    FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
2181
25.6k
                         NewTrueProb, NewFalseProb, InvertCond);
2182
25.6k
2183
25.6k
    // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
2184
25.6k
    SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
2185
25.6k
    BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
2186
25.6k
    // Emit the RHS condition into TmpBB.
2187
25.6k
    FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
2188
25.6k
                         Probs[0], Probs[1], InvertCond);
2189
25.6k
  }
2190
34.8k
}
2191
2192
/// If the set of cases should be emitted as a series of branches, return true.
2193
/// If we should emit this as a bunch of and/or'd together conditions, return
2194
/// false.
2195
bool
2196
36.0k
SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
2197
36.0k
  if (Cases.size() != 2) 
return true3.26k
;
2198
32.8k
2199
32.8k
  // If this is two comparisons of the same values or'd or and'd together, they
2200
32.8k
  // will get folded into a single comparison, so don't emit two blocks.
2201
32.8k
  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
2202
32.8k
       
Cases[0].CmpRHS == Cases[1].CmpRHS1.40k
) ||
2203
32.8k
      
(32.7k
Cases[0].CmpRHS == Cases[1].CmpLHS32.7k
&&
2204
32.7k
       
Cases[0].CmpLHS == Cases[1].CmpRHS30
)) {
2205
9
    return false;
2206
9
  }
2207
32.7k
2208
32.7k
  // Handle: (X != null) | (Y != null) --> (X|Y) != 0
2209
32.7k
  // Handle: (X == null) & (Y == null) --> (X|Y) == 0
2210
32.7k
  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
2211
32.7k
      
Cases[0].CC == Cases[1].CC9.88k
&&
2212
32.7k
      
isa<Constant>(Cases[0].CmpRHS)7.07k
&&
2213
32.7k
      
cast<Constant>(Cases[0].CmpRHS)->isNullValue()6.99k
) {
2214
6.32k
    if (Cases[0].CC == ISD::SETEQ && 
Cases[0].TrueBB == Cases[1].ThisBB5.87k
)
2215
5.56k
      return false;
2216
761
    if (Cases[0].CC == ISD::SETNE && 
Cases[0].FalseBB == Cases[1].ThisBB168
)
2217
15
      return false;
2218
27.2k
  }
2219
27.2k
2220
27.2k
  return true;
2221
27.2k
}
2222
2223
825k
void SelectionDAGBuilder::visitBr(const BranchInst &I) {
2224
825k
  MachineBasicBlock *BrMBB = FuncInfo.MBB;
2225
825k
2226
825k
  // Update machine-CFG edges.
2227
825k
  MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
2228
825k
2229
825k
  if (I.isUnconditional()) {
2230
301k
    // Update machine-CFG edges.
2231
301k
    BrMBB->addSuccessor(Succ0MBB);
2232
301k
2233
301k
    // If this is not a fall-through branch or optimizations are switched off,
2234
301k
    // emit the branch.
2235
301k
    if (Succ0MBB != NextBlock(BrMBB) || 
TM.getOptLevel() == CodeGenOpt::None207k
)
2236
94.3k
      DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2237
94.3k
                              MVT::Other, getControlRoot(),
2238
94.3k
                              DAG.getBasicBlock(Succ0MBB)));
2239
301k
2240
301k
    return;
2241
301k
  }
2242
523k
2243
523k
  // If this condition is one of the special cases we handle, do special stuff
2244
523k
  // now.
2245
523k
  const Value *CondVal = I.getCondition();
2246
523k
  MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
2247
523k
2248
523k
  // If this is a series of conditions that are or'd or and'd together, emit
2249
523k
  // this as a sequence of branches instead of setcc's with and/or operations.
2250
523k
  // As long as jumps are not expensive, this should improve performance.
2251
523k
  // For example, instead of something like:
2252
523k
  //     cmp A, B
2253
523k
  //     C = seteq
2254
523k
  //     cmp D, E
2255
523k
  //     F = setle
2256
523k
  //     or C, F
2257
523k
  //     jnz foo
2258
523k
  // Emit:
2259
523k
  //     cmp A, B
2260
523k
  //     je foo
2261
523k
  //     cmp D, E
2262
523k
  //     jle foo
2263
523k
  if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2264
40.4k
    Instruction::BinaryOps Opcode = BOp->getOpcode();
2265
40.4k
    if (!DAG.getTargetLoweringInfo().isJumpExpensive() && 
BOp->hasOneUse()40.3k
&&
2266
40.4k
        
!I.getMetadata(LLVMContext::MD_unpredictable)36.2k
&&
2267
40.4k
        
(36.2k
Opcode == Instruction::And36.2k
||
Opcode == Instruction::Or8.79k
)) {
2268
36.0k
      FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2269
36.0k
                           Opcode,
2270
36.0k
                           getEdgeProbability(BrMBB, Succ0MBB),
2271
36.0k
                           getEdgeProbability(BrMBB, Succ1MBB),
2272
36.0k
                           /*InvertCond=*/false);
2273
36.0k
      // If the compares in later blocks need to use values not currently
2274
36.0k
      // exported from this block, export them now.  This block should always
2275
36.0k
      // be the first entry.
2276
36.0k
      assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!");
2277
36.0k
2278
36.0k
      // Allow some cases to be rejected.
2279
36.0k
      if (ShouldEmitAsBranches(SL->SwitchCases)) {
2280
59.7k
        for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; 
++i29.2k
) {
2281
29.2k
          ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS);
2282
29.2k
          ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS);
2283
29.2k
        }
2284
30.4k
2285
30.4k
        // Emit the branch for this block.
2286
30.4k
        visitSwitchCase(SL->SwitchCases[0], BrMBB);
2287
30.4k
        SL->SwitchCases.erase(SL->SwitchCases.begin());
2288
30.4k
        return;
2289
30.4k
      }
2290
5.58k
2291
5.58k
      // Okay, we decided not to do this, remove any inserted MBB's and clear
2292
5.58k
      // SwitchCases.
2293
11.1k
      
for (unsigned i = 1, e = SL->SwitchCases.size(); 5.58k
i != e;
++i5.58k
)
2294
5.58k
        FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB);
2295
5.58k
2296
5.58k
      SL->SwitchCases.clear();
2297
5.58k
    }
2298
40.4k
  }
2299
523k
2300
523k
  // Create a CaseBlock record representing this branch.
2301
523k
  CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2302
493k
               nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2303
493k
2304
493k
  // Use visitSwitchCase to actually insert the fast branch sequence for this
2305
493k
  // cond branch.
2306
493k
  visitSwitchCase(CB, BrMBB);
2307
493k
}
2308
2309
/// visitSwitchCase - Emits the necessary code to represent a single node in
2310
/// the binary search tree resulting from lowering a switch instruction.
2311
void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2312
564k
                                          MachineBasicBlock *SwitchBB) {
2313
564k
  SDValue Cond;
2314
564k
  SDValue CondLHS = getValue(CB.CmpLHS);
2315
564k
  SDLoc dl = CB.DL;
2316
564k
2317
564k
  if (CB.CC == ISD::SETTRUE) {
2318
17
    // Branch or fall through to TrueBB.
2319
17
    addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2320
17
    SwitchBB->normalizeSuccProbs();
2321
17
    if (CB.TrueBB != NextBlock(SwitchBB)) {
2322
14
      DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(),
2323
14
                              DAG.getBasicBlock(CB.TrueBB)));
2324
14
    }
2325
17
    return;
2326
17
  }
2327
564k
2328
564k
  auto &TLI = DAG.getTargetLoweringInfo();
2329
564k
  EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType());
2330
564k
2331
564k
  // Build the setcc now.
2332
564k
  if (!CB.CmpMHS) {
2333
563k
    // Fold "(X == true)" to X and "(X == false)" to !X to
2334
563k
    // handle common cases produced by branch lowering.
2335
563k
    if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2336
563k
        
CB.CC == ISD::SETEQ497k
)
2337
496k
      Cond = CondLHS;
2338
66.9k
    else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2339
66.9k
             
CB.CC == ISD::SETEQ4
) {
2340
4
      SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2341
4
      Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2342
66.9k
    } else {
2343
66.9k
      SDValue CondRHS = getValue(CB.CmpRHS);
2344
66.9k
2345
66.9k
      // If a pointer's DAG type is larger than its memory type then the DAG
2346
66.9k
      // values are zero-extended. This breaks signed comparisons so truncate
2347
66.9k
      // back to the underlying type before doing the compare.
2348
66.9k
      if (CondLHS.getValueType() != MemVT) {
2349
0
        CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT);
2350
0
        CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT);
2351
0
      }
2352
66.9k
      Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC);
2353
66.9k
    }
2354
563k
  } else {
2355
338
    assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now");
2356
338
2357
338
    const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2358
338
    const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2359
338
2360
338
    SDValue CmpOp = getValue(CB.CmpMHS);
2361
338
    EVT VT = CmpOp.getValueType();
2362
338
2363
338
    if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2364
1
      Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2365
1
                          ISD::SETLE);
2366
337
    } else {
2367
337
      SDValue SUB = DAG.getNode(ISD::SUB, dl,
2368
337
                                VT, CmpOp, DAG.getConstant(Low, dl, VT));
2369
337
      Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2370
337
                          DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2371
337
    }
2372
338
  }
2373
564k
2374
564k
  // Update successor info
2375
564k
  addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2376
564k
  // TrueBB and FalseBB are always different unless the incoming IR is
2377
564k
  // degenerate. This only happens when running llc on weird IR.
2378
564k
  if (CB.TrueBB != CB.FalseBB)
2379
564k
    addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2380
564k
  SwitchBB->normalizeSuccProbs();
2381
564k
2382
564k
  // If the lhs block is the next block, invert the condition so that we can
2383
564k
  // fall through to the lhs instead of the rhs block.
2384
564k
  if (CB.TrueBB == NextBlock(SwitchBB)) {
2385
234k
    std::swap(CB.TrueBB, CB.FalseBB);
2386
234k
    SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2387
234k
    Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2388
234k
  }
2389
564k
2390
564k
  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2391
564k
                               MVT::Other, getControlRoot(), Cond,
2392
564k
                               DAG.getBasicBlock(CB.TrueBB));
2393
564k
2394
564k
  // Insert the false branch. Do this even if it's a fall through branch,
2395
564k
  // this makes it easier to do DAG optimizations which require inverting
2396
564k
  // the branch condition.
2397
564k
  BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2398
564k
                       DAG.getBasicBlock(CB.FalseBB));
2399
564k
2400
564k
  DAG.setRoot(BrCond);
2401
564k
}
2402
2403
/// visitJumpTable - Emit JumpTable node in the current MBB
2404
2.26k
void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) {
2405
2.26k
  // Emit the code for the jump table
2406
2.26k
  assert(JT.Reg != -1U && "Should lower JT Header first!");
2407
2.26k
  EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2408
2.26k
  SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2409
2.26k
                                     JT.Reg, PTy);
2410
2.26k
  SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2411
2.26k
  SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2412
2.26k
                                    MVT::Other, Index.getValue(1),
2413
2.26k
                                    Table, Index);
2414
2.26k
  DAG.setRoot(BrJumpTable);
2415
2.26k
}
2416
2417
/// visitJumpTableHeader - This function emits necessary code to produce index
2418
/// in the JumpTable from switch case.
2419
void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT,
2420
                                               JumpTableHeader &JTH,
2421
2.26k
                                               MachineBasicBlock *SwitchBB) {
2422
2.26k
  SDLoc dl = getCurSDLoc();
2423
2.26k
2424
2.26k
  // Subtract the lowest switch case value from the value being switched on.
2425
2.26k
  SDValue SwitchOp = getValue(JTH.SValue);
2426
2.26k
  EVT VT = SwitchOp.getValueType();
2427
2.26k
  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2428
2.26k
                            DAG.getConstant(JTH.First, dl, VT));
2429
2.26k
2430
2.26k
  // The SDNode we just created, which holds the value being switched on minus
2431
2.26k
  // the smallest case value, needs to be copied to a virtual register so it
2432
2.26k
  // can be used as an index into the jump table in a subsequent basic block.
2433
2.26k
  // This value may be smaller or larger than the target's pointer type, and
2434
2.26k
  // therefore require extension or truncating.
2435
2.26k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2436
2.26k
  SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2437
2.26k
2438
2.26k
  unsigned JumpTableReg =
2439
2.26k
      FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2440
2.26k
  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2441
2.26k
                                    JumpTableReg, SwitchOp);
2442
2.26k
  JT.Reg = JumpTableReg;
2443
2.26k
2444
2.26k
  if (!JTH.OmitRangeCheck) {
2445
2.21k
    // Emit the range check for the jump table, and branch to the default block
2446
2.21k
    // for the switch statement if the value being switched on exceeds the
2447
2.21k
    // largest case in the switch.
2448
2.21k
    SDValue CMP = DAG.getSetCC(
2449
2.21k
        dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2450
2.21k
                                   Sub.getValueType()),
2451
2.21k
        Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2452
2.21k
2453
2.21k
    SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2454
2.21k
                                 MVT::Other, CopyTo, CMP,
2455
2.21k
                                 DAG.getBasicBlock(JT.Default));
2456
2.21k
2457
2.21k
    // Avoid emitting unnecessary branches to the next block.
2458
2.21k
    if (JT.MBB != NextBlock(SwitchBB))
2459
77
      BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2460
77
                           DAG.getBasicBlock(JT.MBB));
2461
2.21k
2462
2.21k
    DAG.setRoot(BrCond);
2463
2.21k
  } else {
2464
48
    // Avoid emitting unnecessary branches to the next block.
2465
48
    if (JT.MBB != NextBlock(SwitchBB))
2466
2
      DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo,
2467
2
                              DAG.getBasicBlock(JT.MBB)));
2468
46
    else
2469
46
      DAG.setRoot(CopyTo);
2470
48
  }
2471
2.26k
}
2472
2473
/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2474
/// variable if there exists one.
2475
static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2476
2.14k
                                 SDValue &Chain) {
2477
2.14k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2478
2.14k
  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2479
2.14k
  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2480
2.14k
  MachineFunction &MF = DAG.getMachineFunction();
2481
2.14k
  Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2482
2.14k
  MachineSDNode *Node =
2483
2.14k
      DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2484
2.14k
  if (Global) {
2485
2.12k
    MachinePointerInfo MPInfo(Global);
2486
2.12k
    auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2487
2.12k
                 MachineMemOperand::MODereferenceable;
2488
2.12k
    MachineMemOperand *MemRef = MF.getMachineMemOperand(
2489
2.12k
        MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy));
2490
2.12k
    DAG.setNodeMemRefs(Node, {MemRef});
2491
2.12k
  }
2492
2.14k
  if (PtrTy != PtrMemTy)
2493
0
    return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy);
2494
2.14k
  return SDValue(Node, 0);
2495
2.14k
}
2496
2497
/// Codegen a new tail for a stack protector check ParentMBB which has had its
2498
/// tail spliced into a stack protector check success bb.
2499
///
2500
/// For a high level explanation of how this fits into the stack protector
2501
/// generation see the comment on the declaration of class
2502
/// StackProtectorDescriptor.
2503
void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2504
486
                                                  MachineBasicBlock *ParentBB) {
2505
486
2506
486
  // First create the loads to the guard/stack slot for the comparison.
2507
486
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2508
486
  EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2509
486
  EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout());
2510
486
2511
486
  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2512
486
  int FI = MFI.getStackProtectorIndex();
2513
486
2514
486
  SDValue Guard;
2515
486
  SDLoc dl = getCurSDLoc();
2516
486
  SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2517
486
  const Module &M = *ParentBB->getParent()->getFunction().getParent();
2518
486
  unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2519
486
2520
486
  // Generate code to load the content of the guard slot.
2521
486
  SDValue GuardVal = DAG.getLoad(
2522
486
      PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr,
2523
486
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2524
486
      MachineMemOperand::MOVolatile);
2525
486
2526
486
  if (TLI.useStackGuardXorFP())
2527
145
    GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2528
486
2529
486
  // Retrieve guard check function, nullptr if instrumentation is inlined.
2530
486
  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
2531
84
    // The target provides a guard check function to validate the guard value.
2532
84
    // Generate a call to that function with the content of the guard slot as
2533
84
    // argument.
2534
84
    FunctionType *FnTy = GuardCheckFn->getFunctionType();
2535
84
    assert(FnTy->getNumParams() == 1 && "Invalid function signature");
2536
84
2537
84
    TargetLowering::ArgListTy Args;
2538
84
    TargetLowering::ArgListEntry Entry;
2539
84
    Entry.Node = GuardVal;
2540
84
    Entry.Ty = FnTy->getParamType(0);
2541
84
    if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
2542
84
      Entry.IsInReg = true;
2543
84
    Args.push_back(Entry);
2544
84
2545
84
    TargetLowering::CallLoweringInfo CLI(DAG);
2546
84
    CLI.setDebugLoc(getCurSDLoc())
2547
84
        .setChain(DAG.getEntryNode())
2548
84
        .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
2549
84
                   getValue(GuardCheckFn), std::move(Args));
2550
84
2551
84
    std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2552
84
    DAG.setRoot(Result.second);
2553
84
    return;
2554
84
  }
2555
402
2556
402
  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2557
402
  // Otherwise, emit a volatile load to retrieve the stack guard value.
2558
402
  SDValue Chain = DAG.getEntryNode();
2559
402
  if (TLI.useLoadStackGuardNode()) {
2560
268
    Guard = getLoadStackGuard(DAG, dl, Chain);
2561
268
  } else {
2562
134
    const Value *IRGuard = TLI.getSDagStackGuard(M);
2563
134
    SDValue GuardPtr = getValue(IRGuard);
2564
134
2565
134
    Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr,
2566
134
                        MachinePointerInfo(IRGuard, 0), Align,
2567
134
                        MachineMemOperand::MOVolatile);
2568
134
  }
2569
402
2570
402
  // Perform the comparison via a subtract/getsetcc.
2571
402
  EVT VT = Guard.getValueType();
2572
402
  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2573
402
2574
402
  SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2575
402
                                                        *DAG.getContext(),
2576
402
                                                        Sub.getValueType()),
2577
402
                             Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2578
402
2579
402
  // If the sub is not 0, then we know the guard/stackslot do not equal, so
2580
402
  // branch to failure MBB.
2581
402
  SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2582
402
                               MVT::Other, GuardVal.getOperand(0),
2583
402
                               Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2584
402
  // Otherwise branch to success MBB.
2585
402
  SDValue Br = DAG.getNode(ISD::BR, dl,
2586
402
                           MVT::Other, BrCond,
2587
402
                           DAG.getBasicBlock(SPD.getSuccessMBB()));
2588
402
2589
402
  DAG.setRoot(Br);
2590
402
}
2591
2592
/// Codegen the failure basic block for a stack protector check.
2593
///
2594
/// A failure stack protector machine basic block consists simply of a call to
2595
/// __stack_chk_fail().
2596
///
2597
/// For a high level explanation of how this fits into the stack protector
2598
/// generation see the comment on the declaration of class
2599
/// StackProtectorDescriptor.
2600
void
2601
392
SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2602
392
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2603
392
  SDValue Chain =
2604
392
      TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2605
392
                      None, false, getCurSDLoc(), false, false).second;
2606
392
  // On PS4, the "return address" must still be within the calling function,
2607
392
  // even if it's at the very end, so emit an explicit TRAP here.
2608
392
  // Passing 'true' for doesNotReturn above won't generate the trap for us.
2609
392
  if (TM.getTargetTriple().isPS4CPU())
2610
1
    Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain);
2611
392
2612
392
  DAG.setRoot(Chain);
2613
392
}
2614
2615
/// visitBitTestHeader - This function emits necessary code to produce value
2616
/// suitable for "bit tests"
2617
void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2618
286
                                             MachineBasicBlock *SwitchBB) {
2619
286
  SDLoc dl = getCurSDLoc();
2620
286
2621
286
  // Subtract the minimum value
2622
286
  SDValue SwitchOp = getValue(B.SValue);
2623
286
  EVT VT = SwitchOp.getValueType();
2624
286
  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2625
286
                            DAG.getConstant(B.First, dl, VT));
2626
286
2627
286
  // Check range
2628
286
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2629
286
  SDValue RangeCmp = DAG.getSetCC(
2630
286
      dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2631
286
                                 Sub.getValueType()),
2632
286
      Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2633
286
2634
286
  // Determine the type of the test operands.
2635
286
  bool UsePtrType = false;
2636
286
  if (!TLI.isTypeLegal(VT))
2637
0
    UsePtrType = true;
2638
286
  else {
2639
522
    for (unsigned i = 0, e = B.Cases.size(); i != e; 
++i236
)
2640
367
      if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2641
131
        // Switch table case range are encoded into series of masks.
2642
131
        // Just use pointer type, it's guaranteed to fit.
2643
131
        UsePtrType = true;
2644
131
        break;
2645
131
      }
2646
286
  }
2647
286
  if (UsePtrType) {
2648
131
    VT = TLI.getPointerTy(DAG.getDataLayout());
2649
131
    Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2650
131
  }
2651
286
2652
286
  B.RegVT = VT.getSimpleVT();
2653
286
  B.Reg = FuncInfo.CreateReg(B.RegVT);
2654
286
  SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2655
286
2656
286
  MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2657
286
2658
286
  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2659
286
  addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2660
286
  SwitchBB->normalizeSuccProbs();
2661
286
2662
286
  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2663
286
                                MVT::Other, CopyTo, RangeCmp,
2664
286
                                DAG.getBasicBlock(B.Default));
2665
286
2666
286
  // Avoid emitting unnecessary branches to the next block.
2667
286
  if (MBB != NextBlock(SwitchBB))
2668
28
    BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2669
28
                          DAG.getBasicBlock(MBB));
2670
286
2671
286
  DAG.setRoot(BrRange);
2672
286
}
2673
2674
/// visitBitTestCase - this function produces one "bit test"
2675
void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2676
                                           MachineBasicBlock* NextMBB,
2677
                                           BranchProbability BranchProbToNext,
2678
                                           unsigned Reg,
2679
                                           BitTestCase &B,
2680
410
                                           MachineBasicBlock *SwitchBB) {
2681
410
  SDLoc dl = getCurSDLoc();
2682
410
  MVT VT = BB.RegVT;
2683
410
  SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2684
410
  SDValue Cmp;
2685
410
  unsigned PopCount = countPopulation(B.Mask);
2686
410
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2687
410
  if (PopCount == 1) {
2688
110
    // Testing for a single bit; just compare the shift count with what it
2689
110
    // would need to be to shift a 1 bit in that position.
2690
110
    Cmp = DAG.getSetCC(
2691
110
        dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2692
110
        ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2693
110
        ISD::SETEQ);
2694
300
  } else if (PopCount == BB.Range) {
2695
28
    // There is only one zero bit in the range, test for it directly.
2696
28
    Cmp = DAG.getSetCC(
2697
28
        dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2698
28
        ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2699
28
        ISD::SETNE);
2700
272
  } else {
2701
272
    // Make desired shift
2702
272
    SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2703
272
                                    DAG.getConstant(1, dl, VT), ShiftOp);
2704
272
2705
272
    // Emit bit tests and jumps
2706
272
    SDValue AndOp = DAG.getNode(ISD::AND, dl,
2707
272
                                VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2708
272
    Cmp = DAG.getSetCC(
2709
272
        dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2710
272
        AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2711
272
  }
2712
410
2713
410
  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2714
410
  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2715
410
  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2716
410
  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2717
410
  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2718
410
  // one as they are relative probabilities (and thus work more like weights),
2719
410
  // and hence we need to normalize them to let the sum of them become one.
2720
410
  SwitchBB->normalizeSuccProbs();
2721
410
2722
410
  SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2723
410
                              MVT::Other, getControlRoot(),
2724
410
                              Cmp, DAG.getBasicBlock(B.TargetBB));
2725
410
2726
410
  // Avoid emitting unnecessary branches to the next block.
2727
410
  if (NextMBB != NextBlock(SwitchBB))
2728
170
    BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2729
170
                        DAG.getBasicBlock(NextMBB));
2730
410
2731
410
  DAG.setRoot(BrAnd);
2732
410
}
2733
2734
6.36k
void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2735
6.36k
  MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2736
6.36k
2737
6.36k
  // Retrieve successors. Look through artificial IR level blocks like
2738
6.36k
  // catchswitch for successors.
2739
6.36k
  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2740
6.36k
  const BasicBlock *EHPadBB = I.getSuccessor(1);
2741
6.36k
2742
6.36k
  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2743
6.36k
  // have to do anything here to lower funclet bundles.
2744
6.36k
  assert(!I.hasOperandBundlesOtherThan(
2745
6.36k
             {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2746
6.36k
         "Cannot lower invokes with arbitrary operand bundles yet!");
2747
6.36k
2748
6.36k
  const Value *Callee(I.getCalledValue());
2749
6.36k
  const Function *Fn = dyn_cast<Function>(Callee);
2750
6.36k
  if (isa<InlineAsm>(Callee))
2751
1
    visitInlineAsm(&I);
2752
6.36k
  else if (Fn && 
Fn->isIntrinsic()5.84k
) {
2753
14
    switch (Fn->getIntrinsicID()) {
2754
14
    default:
2755
0
      llvm_unreachable("Cannot invoke this intrinsic");
2756
14
    case Intrinsic::donothing:
2757
1
      // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2758
1
      break;
2759
14
    case Intrinsic::experimental_patchpoint_void:
2760
2
    case Intrinsic::experimental_patchpoint_i64:
2761
2
      visitPatchpoint(&I, EHPadBB);
2762
2
      break;
2763
8
    case Intrinsic::experimental_gc_statepoint:
2764
8
      LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2765
8
      break;
2766
3
    case Intrinsic::wasm_rethrow_in_catch: {
2767
3
      // This is usually done in visitTargetIntrinsic, but this intrinsic is
2768
3
      // special because it can be invoked, so we manually lower it to a DAG
2769
3
      // node here.
2770
3
      SmallVector<SDValue, 8> Ops;
2771
3
      Ops.push_back(getRoot()); // inchain
2772
3
      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2773
3
      Ops.push_back(
2774
3
          DAG.getTargetConstant(Intrinsic::wasm_rethrow_in_catch, getCurSDLoc(),
2775
3
                                TLI.getPointerTy(DAG.getDataLayout())));
2776
3
      SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain
2777
3
      DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops));
2778
3
      break;
2779
6.35k
    }
2780
6.35k
    }
2781
6.35k
  } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2782
0
    // Currently we do not lower any intrinsic calls with deopt operand bundles.
2783
0
    // Eventually we will support lowering the @llvm.experimental.deoptimize
2784
0
    // intrinsic, and right now there are no plans to support other intrinsics
2785
0
    // with deopt state.
2786
0
    LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2787
6.35k
  } else {
2788
6.35k
    LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2789
6.35k
  }
2790
6.36k
2791
6.36k
  // If the value of the invoke is used outside of its defining block, make it
2792
6.36k
  // available as a virtual register.
2793
6.36k
  // We already took care of the exported value for the statepoint instruction
2794
6.36k
  // during call to the LowerStatepoint.
2795
6.36k
  if (!isStatepoint(I)) {
2796
6.36k
    CopyToExportRegsIfNeeded(&I);
2797
6.36k
  }
2798
6.36k
2799
6.36k
  SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2800
6.36k
  BranchProbabilityInfo *BPI = FuncInfo.BPI;
2801
6.36k
  BranchProbability EHPadBBProb =
2802
6.36k
      BPI ? 
BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)6.21k
2803
6.36k
          : 
BranchProbability::getZero()153
;
2804
6.36k
  findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2805
6.36k
2806
6.36k
  // Update successor info.
2807
6.36k
  addSuccessorWithProb(InvokeMBB, Return);
2808
6.40k
  for (auto &UnwindDest : UnwindDests) {
2809
6.40k
    UnwindDest.first->setIsEHPad();
2810
6.40k
    addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2811
6.40k
  }
2812
6.36k
  InvokeMBB->normalizeSuccProbs();
2813
6.36k
2814
6.36k
  // Drop into normal successor.
2815
6.36k
  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
2816
6.36k
                          DAG.getBasicBlock(Return)));
2817
6.36k
}
2818
2819
5
void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
2820
5
  MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
2821
5
2822
5
  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2823
5
  // have to do anything here to lower funclet bundles.
2824
5
  assert(!I.hasOperandBundlesOtherThan(
2825
5
             {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
2826
5
         "Cannot lower callbrs with arbitrary operand bundles yet!");
2827
5
2828
5
  assert(isa<InlineAsm>(I.getCalledValue()) &&
2829
5
         "Only know how to handle inlineasm callbr");
2830
5
  visitInlineAsm(&I);
2831
5
2832
5
  // Retrieve successors.
2833
5
  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
2834
5
2835
5
  // Update successor info.
2836
5
  addSuccessorWithProb(CallBrMBB, Return);
2837
15
  for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; 
++i10
) {
2838
10
    MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
2839
10
    addSuccessorWithProb(CallBrMBB, Target);
2840
10
  }
2841
5
  CallBrMBB->normalizeSuccProbs();
2842
5
2843
5
  // Drop into default successor.
2844
5
  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2845
5
                          MVT::Other, getControlRoot(),
2846
5
                          DAG.getBasicBlock(Return)));
2847
5
}
2848
2849
0
void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2850
0
  llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!");
2851
0
}
2852
2853
3.55k
void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2854
3.55k
  assert(FuncInfo.MBB->isEHPad() &&
2855
3.55k
         "Call to landingpad not in landing pad!");
2856
3.55k
2857
3.55k
  // If there aren't registers to copy the values into (e.g., during SjLj
2858
3.55k
  // exceptions), then don't bother to create these DAG nodes.
2859
3.55k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2860
3.55k
  const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2861
3.55k
  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2862
3.55k
      
TLI.getExceptionSelectorRegister(PersonalityFn) == 0124
)
2863
124
    return;
2864
3.43k
2865
3.43k
  // If landingpad's return type is token type, we don't create DAG nodes
2866
3.43k
  // for its exception pointer and selector value. The extraction of exception
2867
3.43k
  // pointer or selector value from token type landingpads is not currently
2868
3.43k
  // supported.
2869
3.43k
  if (LP.getType()->isTokenTy())
2870
8
    return;
2871
3.42k
2872
3.42k
  SmallVector<EVT, 2> ValueVTs;
2873
3.42k
  SDLoc dl = getCurSDLoc();
2874
3.42k
  ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2875
3.42k
  assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported");
2876
3.42k
2877
3.42k
  // Get the two live-in registers as SDValues. The physregs have already been
2878
3.42k
  // copied into virtual registers.
2879
3.42k
  SDValue Ops[2];
2880
3.42k
  if (FuncInfo.ExceptionPointerVirtReg) {
2881
3.42k
    Ops[0] = DAG.getZExtOrTrunc(
2882
3.42k
        DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2883
3.42k
                           FuncInfo.ExceptionPointerVirtReg,
2884
3.42k
                           TLI.getPointerTy(DAG.getDataLayout())),
2885
3.42k
        dl, ValueVTs[0]);
2886
3.42k
  } else {
2887
0
    Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2888
0
  }
2889
3.42k
  Ops[1] = DAG.getZExtOrTrunc(
2890
3.42k
      DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2891
3.42k
                         FuncInfo.ExceptionSelectorVirtReg,
2892
3.42k
                         TLI.getPointerTy(DAG.getDataLayout())),
2893
3.42k
      dl, ValueVTs[1]);
2894
3.42k
2895
3.42k
  // Merge into one.
2896
3.42k
  SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2897
3.42k
                            DAG.getVTList(ValueVTs), Ops);
2898
3.42k
  setValue(&LP, Res);
2899
3.42k
}
2900
2901
void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2902
0
                                           MachineBasicBlock *Last) {
2903
0
  // Update JTCases.
2904
0
  for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i)
2905
0
    if (SL->JTCases[i].first.HeaderBB == First)
2906
0
      SL->JTCases[i].first.HeaderBB = Last;
2907
0
2908
0
  // Update BitTestCases.
2909
0
  for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i)
2910
0
    if (SL->BitTestCases[i].Parent == First)
2911
0
      SL->BitTestCases[i].Parent = Last;
2912
0
}
2913
2914
100
void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2915
100
  MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2916
100
2917
100
  // Update machine-CFG edges with unique successors.
2918
100
  SmallSet<BasicBlock*, 32> Done;
2919
387
  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; 
++i287
) {
2920
287
    BasicBlock *BB = I.getSuccessor(i);
2921
287
    bool Inserted = Done.insert(BB).second;
2922
287
    if (!Inserted)
2923
6
        continue;
2924
281
2925
281
    MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2926
281
    addSuccessorWithProb(IndirectBrMBB, Succ);
2927
281
  }
2928
100
  IndirectBrMBB->normalizeSuccProbs();
2929
100
2930
100
  DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2931
100
                          MVT::Other, getControlRoot(),
2932
100
                          getValue(I.getAddress())));
2933
100
}
2934
2935
26.1k
void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2936
26.1k
  if (!DAG.getTarget().Options.TrapUnreachable)
2937
1.67k
    return;
2938
24.4k
2939
24.4k
  // We may be able to ignore unreachable behind a noreturn call.
2940
24.4k
  if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
2941
24.3k
    const BasicBlock &BB = *I.getParent();
2942
24.3k
    if (&I != &BB.front()) {
2943
23.7k
      BasicBlock::const_iterator PredI =
2944
23.7k
        std::prev(BasicBlock::const_iterator(&I));
2945
23.7k
      if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2946
23.6k
        if (Call->doesNotReturn())
2947
22.8k
          return;
2948
1.63k
      }
2949
23.7k
    }
2950
24.3k
  }
2951
1.63k
2952
1.63k
  DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2953
1.63k
}
2954
2955
11.5k
void SelectionDAGBuilder::visitFSub(const User &I) {
2956
11.5k
  // -0.0 - X --> fneg
2957
11.5k
  Type *Ty = I.getType();
2958
11.5k
  if (isa<Constant>(I.getOperand(0)) &&
2959
11.5k
      
I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)6.45k
) {
2960
5.20k
    SDValue Op2 = getValue(I.getOperand(1));
2961
5.20k
    setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2962
5.20k
                             Op2.getValueType(), Op2));
2963
5.20k
    return;
2964
5.20k
  }
2965
6.35k
2966
6.35k
  visitBinary(I, ISD::FSUB);
2967
6.35k
}
2968
2969
/// Checks if the given instruction performs a vector reduction, in which case
2970
/// we have the freedom to alter the elements in the result as long as the
2971
/// reduction of them stays unchanged.
2972
609k
static bool isVectorReductionOp(const User *I) {
2973
609k
  const Instruction *Inst = dyn_cast<Instruction>(I);
2974
609k
  if (!Inst || 
!Inst->getType()->isVectorTy()608k
)
2975
492k
    return false;
2976
117k
2977
117k
  auto OpCode = Inst->getOpcode();
2978
117k
  switch (OpCode) {
2979
117k
  case Instruction::Add:
2980
69.1k
  case Instruction::Mul:
2981
69.1k
  case Instruction::And:
2982
69.1k
  case Instruction::Or:
2983
69.1k
  case Instruction::Xor:
2984
69.1k
    break;
2985
69.1k
  case Instruction::FAdd:
2986
23.1k
  case Instruction::FMul:
2987
23.1k
    if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2988
23.1k
      if (FPOp->getFastMathFlags().isFast())
2989
893
        break;
2990
22.2k
    LLVM_FALLTHROUGH;
2991
47.0k
  default:
2992
47.0k
    return false;
2993
69.9k
  }
2994
69.9k
2995
69.9k
  unsigned ElemNum = Inst->getType()->getVectorNumElements();
2996
69.9k
  // Ensure the reduction size is a power of 2.
2997
69.9k
  if (!isPowerOf2_32(ElemNum))
2998
257
    return false;
2999
69.7k
3000
69.7k
  unsigned ElemNumToReduce = ElemNum;
3001
69.7k
3002
69.7k
  // Do DFS search on the def-use chain from the given instruction. We only
3003
69.7k
  // allow four kinds of operations during the search until we reach the
3004
69.7k
  // instruction that extracts the first element from the vector:
3005
69.7k
  //
3006
69.7k
  //   1. The reduction operation of the same opcode as the given instruction.
3007
69.7k
  //
3008
69.7k
  //   2. PHI node.
3009
69.7k
  //
3010
69.7k
  //   3. ShuffleVector instruction together with a reduction operation that
3011
69.7k
  //      does a partial reduction.
3012
69.7k
  //
3013
69.7k
  //   4. ExtractElement that extracts the first element from the vector, and we
3014
69.7k
  //      stop searching the def-use chain here.
3015
69.7k
  //
3016
69.7k
  // 3 & 4 above perform a reduction on all elements of the vector. We push defs
3017
69.7k
  // from 1-3 to the stack to continue the DFS. The given instruction is not
3018
69.7k
  // a reduction operation if we meet any other instructions other than those
3019
69.7k
  // listed above.
3020
69.7k
3021
69.7k
  SmallVector<const User *, 16> UsersToVisit{Inst};
3022
69.7k
  SmallPtrSet<const User *, 16> Visited;
3023
69.7k
  bool ReduxExtracted = false;
3024
69.7k
3025
99.2k
  while (!UsersToVisit.empty()) {
3026
98.1k
    auto User = UsersToVisit.back();
3027
98.1k
    UsersToVisit.pop_back();
3028
98.1k
    if (!Visited.insert(User).second)
3029
4.34k
      continue;
3030
93.8k
3031
114k
    
for (const auto &U : User->users())93.8k
{
3032
114k
      auto Inst = dyn_cast<Instruction>(U);
3033
114k
      if (!Inst)
3034
0
        return false;
3035
114k
3036
114k
      if (Inst->getOpcode() == OpCode || 
isa<PHINode>(U)85.4k
) {
3037
42.4k
        if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
3038
499
          if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
3039
0
            return false;
3040
42.4k
        UsersToVisit.push_back(U);
3041
71.6k
      } else if (const ShuffleVectorInst *ShufInst =
3042
9.68k
                     dyn_cast<ShuffleVectorInst>(U)) {
3043
9.68k
        // Detect the following pattern: A ShuffleVector instruction together
3044
9.68k
        // with a reduction that do partial reduction on the first and second
3045
9.68k
        // ElemNumToReduce / 2 elements, and store the result in
3046
9.68k
        // ElemNumToReduce / 2 elements in another vector.
3047
9.68k
3048
9.68k
        unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
3049
9.68k
        if (ResultElements < ElemNum)
3050
112
          return false;
3051
9.57k
3052
9.57k
        if (ElemNumToReduce == 1)
3053
0
          return false;
3054
9.57k
        if (!isa<UndefValue>(U->getOperand(1)))
3055
1.23k
          return false;
3056
12.6k
        
for (unsigned i = 0; 8.34k
i < ElemNumToReduce / 2;
++i4.35k
)
3057
10.7k
          if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
3058
6.35k
            return false;
3059
13.4k
        
for (unsigned i = ElemNumToReduce / 2; 1.98k
i < ElemNum;
++i11.4k
)
3060
11.4k
          if (ShufInst->getMaskValue(i) != -1)
3061
8
            return false;
3062
1.98k
3063
1.98k
        // There is only one user of this ShuffleVector instruction, which
3064
1.98k
        // must be a reduction operation.
3065
1.98k
        
if (1.98k
!U->hasOneUse()1.98k
)
3066
0
          return false;
3067
1.98k
3068
1.98k
        auto U2 = dyn_cast<Instruction>(*U->user_begin());
3069
1.98k
        if (!U2 || U2->getOpcode() != OpCode)
3070
2
          return false;
3071
1.97k
3072
1.97k
        // Check operands of the reduction operation.
3073
1.97k
        if ((U2->getOperand(0) == U->getOperand(0) && 
U2->getOperand(1) == U1.96k
) ||
3074
1.97k
            
(16
U2->getOperand(1) == U->getOperand(0)16
&&
U2->getOperand(0) == U16
)) {
3075
1.97k
          UsersToVisit.push_back(U2);
3076
1.97k
          ElemNumToReduce /= 2;
3077
1.97k
        } else
3078
0
          return false;
3079
61.9k
      } else if (isa<ExtractElementInst>(U)) {
3080
4.29k
        // At this moment we should have reduced all elements in the vector.
3081
4.29k
        if (ElemNumToReduce != 1)
3082
3.30k
          return false;
3083
991
3084
991
        const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
3085
991
        if (!Val || !Val->isZero())
3086
0
          return false;
3087
991
3088
991
        ReduxExtracted = true;
3089
991
      } else
3090
57.6k
        return false;
3091
114k
    }
3092
93.8k
  }
3093
69.7k
  
return ReduxExtracted1.09k
;
3094
69.7k
}
3095
3096
66
void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) {
3097
66
  SDNodeFlags Flags;
3098
66
3099
66
  SDValue Op = getValue(I.getOperand(0));
3100
66
  SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(),
3101
66
                                    Op, Flags);
3102
66
  setValue(&I, UnNodeValue);
3103
66
}
3104
3105
609k
void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
3106
609k
  SDNodeFlags Flags;
3107
609k
  if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
3108
335k
    Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
3109
335k
    Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
3110
335k
  }
3111
609k
  if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
3112
2.76k
    Flags.setExact(ExactOp->isExact());
3113
2.76k
  }
3114
609k
  if (isVectorReductionOp(&I)) {
3115
991
    Flags.setVectorReduction(true);
3116
991
    LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n");
3117
991
  }
3118
609k
3119
609k
  SDValue Op1 = getValue(I.getOperand(0));
3120
609k
  SDValue Op2 = getValue(I.getOperand(1));
3121
609k
  SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
3122
609k
                                     Op1, Op2, Flags);
3123
609k
  setValue(&I, BinNodeValue);
3124
609k
}
3125
3126
117k
void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
3127
117k
  SDValue Op1 = getValue(I.getOperand(0));
3128
117k
  SDValue Op2 = getValue(I.getOperand(1));
3129
117k
3130
117k
  EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
3131
117k
      Op1.getValueType(), DAG.getDataLayout());
3132
117k
3133
117k
  // Coerce the shift amount to the right type if we can.
3134
117k
  if (!I.getType()->isVectorTy() && 
Op2.getValueType() != ShiftTy105k
) {
3135
67.3k
    unsigned ShiftSize = ShiftTy.getSizeInBits();
3136
67.3k
    unsigned Op2Size = Op2.getValueSizeInBits();
3137
67.3k
    SDLoc DL = getCurSDLoc();
3138
67.3k
3139
67.3k
    // If the operand is smaller than the shift count type, promote it.
3140
67.3k
    if (ShiftSize > Op2Size)
3141
23.0k
      Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
3142
44.3k
3143
44.3k
    // If the operand is larger than the shift count type but the shift
3144
44.3k
    // count type has enough bits to represent any shift value, truncate
3145
44.3k
    // it now. This is a common case and it exposes the truncate to
3146
44.3k
    // optimization early.
3147
44.3k
    else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
3148
44.3k
      Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
3149
9
    // Otherwise we'll need to temporarily settle for some other convenient
3150
9
    // type.  Type legalization will make adjustments once the shiftee is split.
3151
9
    else
3152
9
      Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
3153
67.3k
  }
3154
117k
3155
117k
  bool nuw = false;
3156
117k
  bool nsw = false;
3157
117k
  bool exact = false;
3158
117k
3159
117k
  if (Opcode == ISD::SRL || 
Opcode == ISD::SRA71.2k
||
Opcode == ISD::SHL61.4k
) {
3160
117k
3161
117k
    if (const OverflowingBinaryOperator *OFBinOp =
3162
61.4k
            dyn_cast<const OverflowingBinaryOperator>(&I)) {
3163
61.4k
      nuw = OFBinOp->hasNoUnsignedWrap();
3164
61.4k
      nsw = OFBinOp->hasNoSignedWrap();
3165
61.4k
    }
3166
117k
    if (const PossiblyExactOperator *ExactOp =
3167
56.1k
            dyn_cast<const PossiblyExactOperator>(&I))
3168
56.1k
      exact = ExactOp->isExact();
3169
117k
  }
3170
117k
  SDNodeFlags Flags;
3171
117k
  Flags.setExact(exact);
3172
117k
  Flags.setNoSignedWrap(nsw);
3173
117k
  Flags.setNoUnsignedWrap(nuw);
3174
117k
  SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
3175
117k
                            Flags);
3176
117k
  setValue(&I, Res);
3177
117k
}
3178
3179
3.27k
void SelectionDAGBuilder::visitSDiv(const User &I) {
3180
3.27k
  SDValue Op1 = getValue(I.getOperand(0));
3181
3.27k
  SDValue Op2 = getValue(I.getOperand(1));
3182
3.27k
3183
3.27k
  SDNodeFlags Flags;
3184
3.27k
  Flags.setExact(isa<PossiblyExactOperator>(&I) &&
3185
3.27k
                 cast<PossiblyExactOperator>(&I)->isExact());
3186
3.27k
  setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
3187
3.27k
                           Op2, Flags));
3188
3.27k
}
3189
3190
608k
void SelectionDAGBuilder::visitICmp(const User &I) {
3191
608k
  ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
3192
608k
  if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
3193
608k
    predicate = IC->getPredicate();
3194
353
  else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
3195
353
    predicate = ICmpInst::Predicate(IC->getPredicate());
3196
608k
  SDValue Op1 = getValue(I.getOperand(0));
3197
608k
  SDValue Op2 = getValue(I.getOperand(1));
3198
608k
  ISD::CondCode Opcode = getICmpCondCode(predicate);
3199
608k
3200
608k
  auto &TLI = DAG.getTargetLoweringInfo();
3201
608k
  EVT MemVT =
3202
608k
      TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3203
608k
3204
608k
  // If a pointer's DAG type is larger than its memory type then the DAG values
3205
608k
  // are zero-extended. This breaks signed comparisons so truncate back to the
3206
608k
  // underlying type before doing the compare.
3207
608k
  if (Op1.getValueType() != MemVT) {
3208
0
    Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT);
3209
0
    Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT);
3210
0
  }
3211
608k
3212
608k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3213
608k
                                                        I.getType());
3214
608k
  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
3215
608k
}
3216
3217
15.9k
void SelectionDAGBuilder::visitFCmp(const User &I) {
3218
15.9k
  FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
3219
15.9k
  if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
3220
15.9k
    predicate = FC->getPredicate();
3221
2
  else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
3222
2
    predicate = FCmpInst::Predicate(FC->getPredicate());
3223
15.9k
  SDValue Op1 = getValue(I.getOperand(0));
3224
15.9k
  SDValue Op2 = getValue(I.getOperand(1));
3225
15.9k
3226
15.9k
  ISD::CondCode Condition = getFCmpCondCode(predicate);
3227
15.9k
  auto *FPMO = dyn_cast<FPMathOperator>(&I);
3228
15.9k
  if ((FPMO && FPMO->hasNoNaNs()) || 
TM.Options.NoNaNsFPMath14.8k
)
3229
1.37k
    Condition = getFCmpCodeWithoutNaN(Condition);
3230
15.9k
3231
15.9k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3232
15.9k
                                                        I.getType());
3233
15.9k
  setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
3234
15.9k
}
3235
3236
// Check if the condition of the select has one use or two users that are both
3237
// selects with the same condition.
3238
14.6k
static bool hasOnlySelectUsers(const Value *Cond) {
3239
16.7k
  return llvm::all_of(Cond->users(), [](const Value *V) {
3240
16.7k
    return isa<SelectInst>(V);
3241
16.7k
  });
3242
14.6k
}
3243
3244
70.5k
void SelectionDAGBuilder::visitSelect(const User &I) {
3245
70.5k
  SmallVector<EVT, 4> ValueVTs;
3246
70.5k
  ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
3247
70.5k
                  ValueVTs);
3248
70.5k
  unsigned NumValues = ValueVTs.size();
3249
70.5k
  if (NumValues == 0) 
return7
;
3250
70.5k
3251
70.5k
  SmallVector<SDValue, 4> Values(NumValues);
3252
70.5k
  SDValue Cond     = getValue(I.getOperand(0));
3253
70.5k
  SDValue LHSVal   = getValue(I.getOperand(1));
3254
70.5k
  SDValue RHSVal   = getValue(I.getOperand(2));
3255
70.5k
  auto BaseOps = {Cond};
3256
70.5k
  ISD::NodeType OpCode = Cond.getValueType().isVector() ?
3257
42.9k
    
ISD::VSELECT27.6k
: ISD::SELECT;
3258
70.5k
3259
70.5k
  bool IsUnaryAbs = false;
3260
70.5k
3261
70.5k
  // Min/max matching is only viable if all output VTs are the same.
3262
70.5k
  if (is_splat(ValueVTs)) {
3263
70.5k
    EVT VT = ValueVTs[0];
3264
70.5k
    LLVMContext &Ctx = *DAG.getContext();
3265
70.5k
    auto &TLI = DAG.getTargetLoweringInfo();
3266
70.5k
3267
70.5k
    // We care about the legality of the operation after it has been type
3268
70.5k
    // legalized.
3269
85.1k
    while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
3270
85.1k
           
VT != TLI.getTypeToTransformTo(Ctx, VT)14.6k
)
3271
14.6k
      VT = TLI.getTypeToTransformTo(Ctx, VT);
3272
70.5k
3273
70.5k
    // If the vselect is legal, assume we want to leave this as a vector setcc +
3274
70.5k
    // vselect. Otherwise, if this is going to be scalarized, we want to see if
3275
70.5k
    // min/max is legal on the scalar type.
3276
70.5k
    bool UseScalarMinMax = VT.isVector() &&
3277
70.5k
      
!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)27.8k
;
3278
70.5k
3279
70.5k
    Value *LHS, *RHS;
3280
70.5k
    auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
3281
70.5k
    ISD::NodeType Opc = ISD::DELETED_NODE;
3282
70.5k
    switch (SPR.Flavor) {
3283
70.5k
    
case SPF_UMAX: Opc = ISD::UMAX; break5.69k
;
3284
70.5k
    
case SPF_UMIN: Opc = ISD::UMIN; break5.62k
;
3285
70.5k
    
case SPF_SMAX: Opc = ISD::SMAX; break5.44k
;
3286
70.5k
    
case SPF_SMIN: Opc = ISD::SMIN; break5.05k
;
3287
70.5k
    case SPF_FMINNUM:
3288
546
      switch (SPR.NaNBehavior) {
3289
546
      
case SPNB_NA: 0
llvm_unreachable0
("No NaN behavior for FP op?");
3290
546
      
case SPNB_RETURNS_NAN: Opc = ISD::FMINIMUM; break99
;
3291
546
      
case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break88
;
3292
546
      case SPNB_RETURNS_ANY: {
3293
359
        if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
3294
109
          Opc = ISD::FMINNUM;
3295
250
        else if (TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT))
3296
0
          Opc = ISD::FMINIMUM;
3297
250
        else if (UseScalarMinMax)
3298
3
          Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
3299
3
            ISD::FMINNUM : 
ISD::FMINIMUM0
;
3300
359
        break;
3301
546
      }
3302
546
      }
3303
546
      break;
3304
600
    case SPF_FMAXNUM:
3305
600
      switch (SPR.NaNBehavior) {
3306
600
      
case SPNB_NA: 0
llvm_unreachable0
("No NaN behavior for FP op?");
3307
600
      
case SPNB_RETURNS_NAN: Opc = ISD::FMAXIMUM; break147
;
3308
600
      
case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break71
;
3309
600
      case SPNB_RETURNS_ANY:
3310
382
3311
382
        if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
3312
131
          Opc = ISD::FMAXNUM;
3313
251
        else if (TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT))
3314
0
          Opc = ISD::FMAXIMUM;
3315
251
        else if (UseScalarMinMax)
3316
5
          Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
3317
3
            ISD::FMAXNUM : 
ISD::FMAXIMUM2
;
3318
382
        break;
3319
600
      }
3320
600
      break;
3321
1.04k
    case SPF_ABS:
3322
1.04k
      IsUnaryAbs = true;
3323
1.04k
      Opc = ISD::ABS;
3324
1.04k
      break;
3325
600
    case SPF_NABS:
3326
11
      // TODO: we need to produce sub(0, abs(X)).
3327
46.5k
    default: break;
3328
70.5k
    }
3329
70.5k
3330
70.5k
    if (!IsUnaryAbs && 
Opc != ISD::DELETED_NODE69.5k
&&
3331
70.5k
        
(22.4k
TLI.isOperationLegalOrCustom(Opc, VT)22.4k
||
3332
22.4k
         
(7.96k
UseScalarMinMax7.96k
&&
3333
7.96k
          
TLI.isOperationLegalOrCustom(Opc, VT.getScalarType())298
)) &&
3334
70.5k
        // If the underlying comparison instruction is used by any other
3335
70.5k
        // instruction, the consumed instructions won't be destroyed, so it is
3336
70.5k
        // not profitable to convert to a min/max.
3337
70.5k
        
hasOnlySelectUsers(cast<SelectInst>(I).getCondition())14.6k
) {
3338
14.5k
      OpCode = Opc;
3339
14.5k
      LHSVal = getValue(LHS);
3340
14.5k
      RHSVal = getValue(RHS);
3341
14.5k
      BaseOps = {};
3342
14.5k
    }
3343
70.5k
3344
70.5k
    if (IsUnaryAbs) {
3345
1.04k
      OpCode = Opc;
3346
1.04k
      LHSVal = getValue(LHS);
3347
1.04k
      BaseOps = {};
3348
1.04k
    }
3349
70.5k
  }
3350
70.5k
3351
70.5k
  if (IsUnaryAbs) {
3352
2.09k
    for (unsigned i = 0; i != NumValues; 
++i1.04k
) {
3353
1.04k
      Values[i] =
3354
1.04k
          DAG.getNode(OpCode, getCurSDLoc(),
3355
1.04k
                      LHSVal.getNode()->getValueType(LHSVal.getResNo() + i),
3356
1.04k
                      SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3357
1.04k
    }
3358
69.5k
  } else {
3359
139k
    for (unsigned i = 0; i != NumValues; 
++i69.5k
) {
3360
69.5k
      SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3361
69.5k
      Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3362
69.5k
      Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3363
69.5k
      Values[i] = DAG.getNode(
3364
69.5k
          OpCode, getCurSDLoc(),
3365
69.5k
          LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops);
3366
69.5k
    }
3367
69.5k
  }
3368
70.5k
3369
70.5k
  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3370
70.5k
                           DAG.getVTList(ValueVTs), Values));
3371
70.5k
}
3372
3373
101k
void SelectionDAGBuilder::visitTrunc(const User &I) {
3374
101k
  // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3375
101k
  SDValue N = getValue(I.getOperand(0));
3376
101k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3377
101k
                                                        I.getType());
3378
101k
  setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3379
101k
}
3380
3381
81.0k
void SelectionDAGBuilder::visitZExt(const User &I) {
3382
81.0k
  // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3383
81.0k
  // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3384
81.0k
  SDValue N = getValue(I.getOperand(0));
3385
81.0k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3386
81.0k
                                                        I.getType());
3387
81.0k
  setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3388
81.0k
}
3389
3390
61.9k
void SelectionDAGBuilder::visitSExt(const User &I) {
3391
61.9k
  // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3392
61.9k
  // SExt also can't be a cast to bool for same reason. So, nothing much to do
3393
61.9k
  SDValue N = getValue(I.getOperand(0));
3394
61.9k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3395
61.9k
                                                        I.getType());
3396
61.9k
  setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3397
61.9k
}
3398
3399
1.55k
void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3400
1.55k
  // FPTrunc is never a no-op cast, no need to check
3401
1.55k
  SDValue N = getValue(I.getOperand(0));
3402
1.55k
  SDLoc dl = getCurSDLoc();
3403
1.55k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3404
1.55k
  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3405
1.55k
  setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3406
1.55k
                           DAG.getTargetConstant(
3407
1.55k
                               0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3408
1.55k
}
3409
3410
4.69k
void SelectionDAGBuilder::visitFPExt(const User &I) {
3411
4.69k
  // FPExt is never a no-op cast, no need to check
3412
4.69k
  SDValue N = getValue(I.getOperand(0));
3413
4.69k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3414
4.69k
                                                        I.getType());
3415
4.69k
  setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3416
4.69k
}
3417
3418
2.67k
void SelectionDAGBuilder::visitFPToUI(const User &I) {
3419
2.67k
  // FPToUI is never a no-op cast, no need to check
3420
2.67k
  SDValue N = getValue(I.getOperand(0));
3421
2.67k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3422
2.67k
                                                        I.getType());
3423
2.67k
  setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3424
2.67k
}
3425
3426
3.61k
void SelectionDAGBuilder::visitFPToSI(const User &I) {
3427
3.61k
  // FPToSI is never a no-op cast, no need to check
3428
3.61k
  SDValue N = getValue(I.getOperand(0));
3429
3.61k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3430
3.61k
                                                        I.getType());
3431
3.61k
  setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3432
3.61k
}
3433
3434
6.31k
void SelectionDAGBuilder::visitUIToFP(const User &I) {
3435
6.31k
  // UIToFP is never a no-op cast, no need to check
3436
6.31k
  SDValue N = getValue(I.getOperand(0));
3437
6.31k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3438
6.31k
                                                        I.getType());
3439
6.31k
  setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3440
6.31k
}
3441
3442
26.1k
void SelectionDAGBuilder::visitSIToFP(const User &I) {
3443
26.1k
  // SIToFP is never a no-op cast, no need to check
3444
26.1k
  SDValue N = getValue(I.getOperand(0));
3445
26.1k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3446
26.1k
                                                        I.getType());
3447
26.1k
  setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3448
26.1k
}
3449
3450
61.1k
void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3451
61.1k
  // What to do depends on the size of the integer and the size of the pointer.
3452
61.1k
  // We can either truncate, zero extend, or no-op, accordingly.
3453
61.1k
  SDValue N = getValue(I.getOperand(0));
3454
61.1k
  auto &TLI = DAG.getTargetLoweringInfo();
3455
61.1k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3456
61.1k
                                                        I.getType());
3457
61.1k
  EVT PtrMemVT =
3458
61.1k
      TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType());
3459
61.1k
  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3460
61.1k
  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT);
3461
61.1k
  setValue(&I, N);
3462
61.1k
}
3463
3464
43.2k
void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3465
43.2k
  // What to do depends on the size of the integer and the size of the pointer.
3466
43.2k
  // We can either truncate, zero extend, or no-op, accordingly.
3467
43.2k
  SDValue N = getValue(I.getOperand(0));
3468
43.2k
  auto &TLI = DAG.getTargetLoweringInfo();
3469
43.2k
  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3470
43.2k
  EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
3471
43.2k
  N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT);
3472
43.2k
  N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT);
3473
43.2k
  setValue(&I, N);
3474
43.2k
}
3475
3476
705k
void SelectionDAGBuilder::visitBitCast(const User &I) {
3477
705k
  SDValue N = getValue(I.getOperand(0));
3478
705k
  SDLoc dl = getCurSDLoc();
3479
705k
  EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3480
705k
                                                        I.getType());
3481
705k
3482
705k
  // BitCast assures us that source and destination are the same size so this is
3483
705k
  // either a BITCAST or a no-op.
3484
705k
  if (DestVT != N.getValueType())
3485
44.1k
    setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3486
44.1k
                             DestVT, N)); // convert types.
3487
661k
  // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3488
661k
  // might fold any kind of constant expression to an integer constant and that
3489
661k
  // is not what we are looking for. Only recognize a bitcast of a genuine
3490
661k
  // constant integer as an opaque constant.
3491
661k
  else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3492
11.0k
    setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3493
11.0k
                                 /*isOpaque*/true));
3494
649k
  else
3495
649k
    setValue(&I, N);            // noop cast.
3496
705k
}
3497
3498
321
void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3499
321
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3500
321
  const Value *SV = I.getOperand(0);
3501
321
  SDValue N = getValue(SV);
3502
321
  EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3503
321
3504
321
  unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3505
321
  unsigned DestAS = I.getType()->getPointerAddressSpace();
3506
321
3507
321
  if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3508
228
    N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3509
321
3510
321
  setValue(&I, N);
3511
321
}
3512
3513
42.0k
void SelectionDAGBuilder::visitInsertElement(const User &I) {
3514
42.0k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3515
42.0k
  SDValue InVec = getValue(I.getOperand(0));
3516
42.0k
  SDValue InVal = getValue(I.getOperand(1));
3517
42.0k
  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3518
42.0k
                                     TLI.getVectorIdxTy(DAG.getDataLayout()));
3519
42.0k
  setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3520
42.0k
                           TLI.getValueType(DAG.getDataLayout(), I.getType()),
3521
42.0k
                           InVec, InVal, InIdx));
3522
42.0k
}
3523
3524
73.1k
void SelectionDAGBuilder::visitExtractElement(const User &I) {
3525
73.1k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3526
73.1k
  SDValue InVec = getValue(I.getOperand(0));
3527
73.1k
  SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3528
73.1k
                                     TLI.getVectorIdxTy(DAG.getDataLayout()));
3529
73.1k
  setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3530
73.1k
                           TLI.getValueType(DAG.getDataLayout(), I.getType()),
3531
73.1k
                           InVec, InIdx));
3532
73.1k
}
3533
3534
81.2k
void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3535
81.2k
  SDValue Src1 = getValue(I.getOperand(0));
3536
81.2k
  SDValue Src2 = getValue(I.getOperand(1));
3537
81.2k
  SDLoc DL = getCurSDLoc();
3538
81.2k
3539
81.2k
  SmallVector<int, 8> Mask;
3540
81.2k
  ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3541
81.2k
  unsigned MaskNumElts = Mask.size();
3542
81.2k
3543
81.2k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3544
81.2k
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3545
81.2k
  EVT SrcVT = Src1.getValueType();
3546
81.2k
  unsigned SrcNumElts = SrcVT.getVectorNumElements();
3547
81.2k
3548
81.2k
  if (SrcNumElts == MaskNumElts) {
3549
49.1k
    setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3550
49.1k
    return;
3551
49.1k
  }
3552
32.1k
3553
32.1k
  // Normalize the shuffle vector since mask and vector length don't match.
3554
32.1k
  if (SrcNumElts < MaskNumElts) {
3555
9.03k
    // Mask is longer than the source vectors. We can use concatenate vector to
3556
9.03k
    // make the mask and vectors lengths match.
3557
9.03k
3558
9.03k
    if (MaskNumElts % SrcNumElts == 0) {
3559
8.40k
      // Mask length is a multiple of the source vector length.
3560
8.40k
      // Check if the shuffle is some kind of concatenation of the input
3561
8.40k
      // vectors.
3562
8.40k
      unsigned NumConcat = MaskNumElts / SrcNumElts;
3563
8.40k
      bool IsConcat = true;
3564
8.40k
      SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3565
141k
      for (unsigned i = 0; i != MaskNumElts; 
++i133k
) {
3566
134k
        int Idx = Mask[i];
3567
134k
        if (Idx < 0)
3568
5.87k
          continue;
3569
128k
        // Ensure the indices in each SrcVT sized piece are sequential and that
3570
128k
        // the same source is used for the whole piece.
3571
128k
        if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3572
128k
            
(127k
ConcatSrcs[i / SrcNumElts] >= 0127k
&&
3573
127k
             
ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts)104k
)) {
3574
1.52k
          IsConcat = false;
3575
1.52k
          break;
3576
1.52k
        }
3577
127k
        // Remember which source this index came from.
3578
127k
        ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3579
127k
      }
3580
8.40k
3581
8.40k
      // The shuffle is concatenating multiple vectors together. Just emit
3582
8.40k
      // a CONCAT_VECTORS operation.
3583
8.40k
      if (IsConcat) {
3584
6.87k
        SmallVector<SDValue, 8> ConcatOps;
3585
22.3k
        for (auto Src : ConcatSrcs) {
3586
22.3k
          if (Src < 0)
3587
1.23k
            ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3588
21.0k
          else if (Src == 0)
3589
8.80k
            ConcatOps.push_back(Src1);
3590
12.2k
          else
3591
12.2k
            ConcatOps.push_back(Src2);
3592
22.3k
        }
3593
6.87k
        setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3594
6.87k
        return;
3595
6.87k
      }
3596
2.16k
    }
3597
2.16k
3598
2.16k
    unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3599
2.16k
    unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3600
2.16k
    EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3601
2.16k
                                    PaddedMaskNumElts);
3602
2.16k
3603
2.16k
    // Pad both vectors with undefs to make them the same length as the mask.
3604
2.16k
    SDValue UndefVal = DAG.getUNDEF(SrcVT);
3605
2.16k
3606
2.16k
    SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3607
2.16k
    SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3608
2.16k
    MOps1[0] = Src1;
3609
2.16k
    MOps2[0] = Src2;
3610
2.16k
3611
2.16k
    Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3612
2.16k
    Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3613
2.16k
3614
2.16k
    // Readjust mask for new input vector length.
3615
2.16k
    SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3616
35.5k
    for (unsigned i = 0; i != MaskNumElts; 
++i33.3k
) {
3617
33.3k
      int Idx = Mask[i];
3618
33.3k
      if (Idx >= (int)SrcNumElts)
3619
8.29k
        Idx -= SrcNumElts - PaddedMaskNumElts;
3620
33.3k
      MappedOps[i] = Idx;
3621
33.3k
    }
3622
2.16k
3623
2.16k
    SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3624
2.16k
3625
2.16k
    // If the concatenated vector was padded, extract a subvector with the
3626
2.16k
    // correct number of elements.
3627
2.16k
    if (MaskNumElts != PaddedMaskNumElts)
3628
636
      Result = DAG.getNode(
3629
636
          ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3630
636
          DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3631
2.16k
3632
2.16k
    setValue(&I, Result);
3633
2.16k
    return;
3634
2.16k
  }
3635
23.0k
3636
23.0k
  if (SrcNumElts > MaskNumElts) {
3637
23.0k
    // Analyze the access pattern of the vector to see if we can extract
3638
23.0k
    // two subvectors and do the shuffle.
3639
23.0k
    int StartIdx[2] = { -1, -1 };  // StartIdx to extract from
3640
23.0k
    bool CanExtract = true;
3641
76.0k
    for (int Idx : Mask) {
3642
76.0k
      unsigned Input = 0;
3643
76.0k
      if (Idx < 0)
3644
20
        continue;
3645
76.0k
3646
76.0k
      if (Idx >= (int)SrcNumElts) {
3647
2.58k
        Input = 1;
3648
2.58k
        Idx -= SrcNumElts;
3649
2.58k
      }
3650
76.0k
3651
76.0k
      // If all the indices come from the same MaskNumElts sized portion of
3652
76.0k
      // the sources we can use extract. Also make sure the extract wouldn't
3653
76.0k
      // extract past the end of the source.
3654
76.0k
      int NewStartIdx = alignDown(Idx, MaskNumElts);
3655
76.0k
      if (NewStartIdx + MaskNumElts > SrcNumElts ||
3656
76.0k
          
(75.9k
StartIdx[Input] >= 075.9k
&&
StartIdx[Input] != NewStartIdx52.8k
))
3657
3.67k
        CanExtract = false;
3658
76.0k
      // Make sure we always update StartIdx as we use it to track if all
3659
76.0k
      // elements are undef.
3660
76.0k
      StartIdx[Input] = NewStartIdx;
3661
76.0k
    }
3662
23.0k
3663
23.0k
    if (StartIdx[0] < 0 && 
StartIdx[1] < 0156
) {
3664
0
      setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3665
0
      return;
3666
0
    }
3667
23.0k
    if (CanExtract) {
3668
21.0k
      // Extract appropriate subvector and generate a vector shuffle
3669
63.2k
      for (unsigned Input = 0; Input < 2; 
++Input42.1k
) {
3670
42.1k
        SDValue &Src = Input == 0 ? 
Src121.0k
:
Src221.0k
;
3671
42.1k
        if (StartIdx[Input] < 0)
3672
20.9k
          Src = DAG.getUNDEF(VT);
3673
21.1k
        else {
3674
21.1k
          Src = DAG.getNode(
3675
21.1k
              ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3676
21.1k
              DAG.getConstant(StartIdx[Input], DL,
3677
21.1k
                              TLI.getVectorIdxTy(DAG.getDataLayout())));
3678
21.1k
        }
3679
42.1k
      }
3680
21.0k
3681
21.0k
      // Calculate new mask.
3682
21.0k
      SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3683
61.0k
      for (int &Idx : MappedOps) {
3684
61.0k
        if (Idx >= (int)SrcNumElts)
3685
2.52k
          Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3686
58.5k
        else if (Idx >= 0)
3687
58.5k
          Idx -= StartIdx[0];
3688
61.0k
      }
3689
21.0k
3690
21.0k
      setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3691
21.0k
      return;
3692
21.0k
    }
3693
2.00k
  }
3694
2.00k
3695
2.00k
  // We can't use either concat vectors or extract subvectors so fall back to
3696
2.00k
  // replacing the shuffle with extract and build vector.
3697
2.00k
  // to insert and build vector.
3698
2.00k
  EVT EltVT = VT.getVectorElementType();
3699
2.00k
  EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3700
2.00k
  SmallVector<SDValue,8> Ops;
3701
15.0k
  for (int Idx : Mask) {
3702
15.0k
    SDValue Res;
3703
15.0k
3704
15.0k
    if (Idx < 0) {
3705
8
      Res = DAG.getUNDEF(EltVT);
3706
15.0k
    } else {
3707
15.0k
      SDValue &Src = Idx < (int)SrcNumElts ? 
Src114.9k
:
Src267
;
3708
15.0k
      if (Idx >= (int)SrcNumElts) 
Idx -= SrcNumElts67
;
3709
15.0k
3710
15.0k
      Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3711
15.0k
                        EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3712
15.0k
    }
3713
15.0k
3714
15.0k
    Ops.push_back(Res);
3715
15.0k
  }
3716
2.00k
3717
2.00k
  setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3718
2.00k
}
3719
3720
3.61k
void SelectionDAGBuilder::visitInsertValue(const User &I) {
3721
3.61k
  ArrayRef<unsigned> Indices;
3722
3.61k
  if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3723
3.61k
    Indices = IV->getIndices();
3724
0
  else
3725
0
    Indices = cast<ConstantExpr>(&I)->getIndices();
3726
3.61k
3727
3.61k
  const Value *Op0 = I.getOperand(0);
3728
3.61k
  const Value *Op1 = I.getOperand(1);
3729
3.61k
  Type *AggTy = I.getType();
3730
3.61k
  Type *ValTy = Op1->getType();
3731
3.61k
  bool IntoUndef = isa<UndefValue>(Op0);
3732
3.61k
  bool FromUndef = isa<UndefValue>(Op1);
3733
3.61k
3734
3.61k
  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3735
3.61k
3736
3.61k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3737
3.61k
  SmallVector<EVT, 4> AggValueVTs;
3738
3.61k
  ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3739
3.61k
  SmallVector<EVT, 4> ValValueVTs;
3740
3.61k
  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3741
3.61k
3742
3.61k
  unsigned NumAggValues = AggValueVTs.size();
3743
3.61k
  unsigned NumValValues = ValValueVTs.size();
3744
3.61k
  SmallVector<SDValue, 4> Values(NumAggValues);
3745
3.61k
3746
3.61k
  // Ignore an insertvalue that produces an empty object
3747
3.61k
  if (!NumAggValues) {
3748
1
    setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3749
1
    return;
3750
1
  }
3751
3.61k
3752
3.61k
  SDValue Agg = getValue(Op0);
3753
3.61k
  unsigned i = 0;
3754
3.61k
  // Copy the beginning value(s) from the original aggregate.
3755
12.8k
  for (; i != LinearIndex; 
++i9.23k
)
3756
9.23k
    Values[i] = IntoUndef ? 
DAG.getUNDEF(AggValueVTs[i])90
:
3757
9.23k
                
SDValue(Agg.getNode(), Agg.getResNo() + i)9.14k
;
3758
3.61k
  // Copy values from the inserted value(s).
3759
3.61k
  if (NumValValues) {
3760
3.61k
    SDValue Val = getValue(Op1);
3761
7.25k
    for (; i != LinearIndex + NumValValues; 
++i3.63k
)
3762
3.63k
      Values[i] = FromUndef ? 
DAG.getUNDEF(AggValueVTs[i])13
:
3763
3.63k
                  
SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex)3.62k
;
3764
3.61k
  }
3765
3.61k
  // Copy remaining value(s) from the original aggregate.
3766
12.7k
  for (; i != NumAggValues; 
++i9.09k
)
3767
9.09k
    Values[i] = IntoUndef ? 
DAG.getUNDEF(AggValueVTs[i])1.84k
:
3768
9.09k
                
SDValue(Agg.getNode(), Agg.getResNo() + i)7.24k
;
3769
3.61k
3770
3.61k
  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3771
3.61k
                           DAG.getVTList(AggValueVTs), Values));
3772
3.61k
}
3773
3774
23.6k
void SelectionDAGBuilder::visitExtractValue(const User &I) {
3775
23.6k
  ArrayRef<unsigned> Indices;
3776
23.6k
  if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3777
23.6k
    Indices = EV->getIndices();
3778
1
  else
3779
1
    Indices = cast<ConstantExpr>(&I)->getIndices();
3780
23.6k
3781
23.6k
  const Value *Op0 = I.getOperand(0);
3782
23.6k
  Type *AggTy = Op0->getType();
3783
23.6k
  Type *ValTy = I.getType();
3784
23.6k
  bool OutOfUndef = isa<UndefValue>(Op0);
3785
23.6k
3786
23.6k
  unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3787
23.6k
3788
23.6k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3789
23.6k
  SmallVector<EVT, 4> ValValueVTs;
3790
23.6k
  ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3791
23.6k
3792
23.6k
  unsigned NumValValues = ValValueVTs.size();
3793
23.6k
3794
23.6k
  // Ignore a extractvalue that produces an empty object
3795
23.6k
  if (!NumValValues) {
3796
4
    setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3797
4
    return;
3798
4
  }
3799
23.6k
3800
23.6k
  SmallVector<SDValue, 4> Values(NumValValues);
3801
23.6k
3802
23.6k
  SDValue Agg = getValue(Op0);
3803
23.6k
  // Copy out the selected value(s).
3804
47.3k
  for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; 
++i23.7k
)
3805
23.7k
    Values[i - LinearIndex] =
3806
23.7k
      OutOfUndef ?
3807
15
        DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3808
23.7k
        
SDValue(Agg.getNode(), Agg.getResNo() + i)23.6k
;
3809
23.6k
3810
23.6k
  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3811
23.6k
                           DAG.getVTList(ValValueVTs), Values));
3812
23.6k
}
3813
3814
1.22M
void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3815
1.22M
  Value *Op0 = I.getOperand(0);
3816
1.22M
  // Note that the pointer operand may be a vector of pointers. Take the scalar
3817
1.22M
  // element which holds a pointer.
3818
1.22M
  unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3819
1.22M
  SDValue N = getValue(Op0);
3820
1.22M
  SDLoc dl = getCurSDLoc();
3821
1.22M
  auto &TLI = DAG.getTargetLoweringInfo();
3822
1.22M
  MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS);
3823
1.22M
  MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS);
3824
1.22M
3825
1.22M
  // Normalize Vector GEP - all scalar operands should be converted to the
3826
1.22M
  // splat vector.
3827
1.22M
  unsigned VectorWidth = I.getType()->isVectorTy() ?
3828
1.22M
    
cast<VectorType>(I.getType())->getVectorNumElements()389
: 0;
3829
1.22M
3830
1.22M
  if (VectorWidth && 
!N.getValueType().isVector()389
) {
3831
215
    LLVMContext &Context = *DAG.getContext();
3832
215
    EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3833
215
    N = DAG.getSplatBuildVector(VT, dl, N);
3834
215
  }
3835
1.22M
3836
1.22M
  for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3837
3.29M
       GTI != E; 
++GTI2.07M
) {
3838
2.07M
    const Value *Idx = GTI.getOperand();
3839
2.07M
    if (StructType *StTy = GTI.getStructTypeOrNull()) {
3840
438k
      unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3841
438k
      if (Field) {
3842
324k
        // N = N + Offset
3843
324k
        uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3844
324k
3845
324k
        // In an inbounds GEP with an offset that is nonnegative even when
3846
324k
        // interpreted as signed, assume there is no unsigned overflow.
3847
324k
        SDNodeFlags Flags;
3848
324k
        if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3849
308k
          Flags.setNoUnsignedWrap(true);
3850
324k
3851
324k
        N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3852
324k
                        DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3853
324k
      }
3854
1.63M
    } else {
3855
1.63M
      unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3856
1.63M
      MVT IdxTy = MVT::getIntegerVT(IdxSize);
3857
1.63M
      APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3858
1.63M
3859
1.63M
      // If this is a scalar constant or a splat vector of constants,
3860
1.63M
      // handle it quickly.
3861
1.63M
      const auto *CI = dyn_cast<ConstantInt>(Idx);
3862
1.63M
      if (!CI && 
isa<ConstantDataVector>(Idx)166k
&&
3863
1.63M
          
cast<ConstantDataVector>(Idx)->getSplatValue()62
)
3864
27
        CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3865
1.63M
3866
1.63M
      if (CI) {
3867
1.47M
        if (CI->isZero())
3868
852k
          continue;
3869
618k
        APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3870
618k
        LLVMContext &Context = *DAG.getContext();
3871
618k
        SDValue OffsVal = VectorWidth ?
3872
34
          DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3873
618k
          
DAG.getConstant(Offs, dl, IdxTy)618k
;
3874
618k
3875
618k
        // In an inbouds GEP with an offset that is nonnegative even when
3876
618k
        // interpreted as signed, assume there is no unsigned overflow.
3877
618k
        SDNodeFlags Flags;
3878
618k
        if (Offs.isNonNegative() && 
cast<GEPOperator>(I).isInBounds()579k
)
3879
390k
          Flags.setNoUnsignedWrap(true);
3880
618k
3881
618k
        OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType());
3882
618k
3883
618k
        N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3884
618k
        continue;
3885
618k
      }
3886
166k
3887
166k
      // N = N + Idx * ElementSize;
3888
166k
      SDValue IdxN = getValue(Idx);
3889
166k
3890
166k
      if (!IdxN.getValueType().isVector() && 
VectorWidth166k
) {
3891
22
        EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3892
22
        IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3893
22
      }
3894
166k
3895
166k
      // If the index is smaller or larger than intptr_t, truncate or extend
3896
166k
      // it.
3897
166k
      IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3898
166k
3899
166k
      // If this is a multiply by a power of two, turn it into a shl
3900
166k
      // immediately.  This is a very common case.
3901
166k
      if (ElementSize != 1) {
3902
85.1k
        if (ElementSize.isPowerOf2()) {
3903
69.5k
          unsigned Amt = ElementSize.logBase2();
3904
69.5k
          IdxN = DAG.getNode(ISD::SHL, dl,
3905
69.5k
                             N.getValueType(), IdxN,
3906
69.5k
                             DAG.getConstant(Amt, dl, IdxN.getValueType()));
3907
69.5k
        } else {
3908
15.5k
          SDValue Scale = DAG.getConstant(ElementSize.getZExtValue(), dl,
3909
15.5k
                                          IdxN.getValueType());
3910
15.5k
          IdxN = DAG.getNode(ISD::MUL, dl,
3911
15.5k
                             N.getValueType(), IdxN, Scale);
3912
15.5k
        }
3913
85.1k
      }
3914
166k
3915
166k
      N = DAG.getNode(ISD::ADD, dl,
3916
166k
                      N.getValueType(), N, IdxN);
3917
166k
    }
3918
2.07M
  }
3919
1.22M
3920
1.22M
  if (PtrMemTy != PtrTy && 
!cast<GEPOperator>(I).isInBounds()0
)
3921
0
    N = DAG.getPtrExtendInReg(N, dl, PtrMemTy);
3922
1.22M
3923
1.22M
  setValue(&I, N);
3924
1.22M
}
3925
3926
46.6k
void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3927
46.6k
  // If this is a fixed sized alloca in the entry block of the function,
3928
46.6k
  // allocate it statically on the stack.
3929
46.6k
  if (FuncInfo.StaticAllocaMap.count(&I))
3930
45.8k
    return;   // getValue will auto-populate this.
3931
796
3932
796
  SDLoc dl = getCurSDLoc();
3933
796
  Type *Ty = I.getAllocatedType();
3934
796
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3935
796
  auto &DL = DAG.getDataLayout();
3936
796
  uint64_t TySize = DL.getTypeAllocSize(Ty);
3937
796
  unsigned Align =
3938
796
      std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3939
796
3940
796
  SDValue AllocSize = getValue(I.getArraySize());
3941
796
3942
796
  EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3943
796
  if (AllocSize.getValueType() != IntPtr)
3944
210
    AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3945
796
3946
796
  AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3947
796
                          AllocSize,
3948
796
                          DAG.getConstant(TySize, dl, IntPtr));
3949
796
3950
796
  // Handle alignment.  If the requested alignment is less than or equal to
3951
796
  // the stack alignment, ignore it.  If the size is greater than or equal to
3952
796
  // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3953
796
  unsigned StackAlign =
3954
796
      DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3955
796
  if (Align <= StackAlign)
3956
706
    Align = 0;
3957
796
3958
796
  // Round the size of the allocation up to the stack alignment size
3959
796
  // by add SA-1 to the size. This doesn't overflow because we're computing
3960
796
  // an address inside an alloca.
3961
796
  SDNodeFlags Flags;
3962
796
  Flags.setNoUnsignedWrap(true);
3963
796
  AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3964
796
                          DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3965
796
3966
796
  // Mask out the low bits for alignment purposes.
3967
796
  AllocSize =
3968
796
      DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3969
796
                  DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3970
796
3971
796
  SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3972
796
  SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3973
796
  SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3974
796
  setValue(&I, DSA);
3975
796
  DAG.setRoot(DSA.getValue(1));
3976
796
3977
796
  assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects());
3978
796
}
3979
3980
621k
void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3981
621k
  if (I.isAtomic())
3982
1.66k
    return visitAtomicLoad(I);
3983
620k
3984
620k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3985
620k
  const Value *SV = I.getOperand(0);
3986
620k
  if (TLI.supportSwiftError()) {
3987
537k
    // Swifterror values can come from either a function parameter with
3988
537k
    // swifterror attribute or an alloca with swifterror attribute.
3989
537k
    if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3990
25.8k
      if (Arg->hasSwiftErrorAttr())
3991
6
        return visitLoadFromSwiftError(I);
3992
537k
    }
3993
537k
3994
537k
    if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3995
7.80k
      if (Alloca->isSwiftError())
3996
47
        return visitLoadFromSwiftError(I);
3997
620k
    }
3998
537k
  }
3999
620k
4000
620k
  SDValue Ptr = getValue(SV);
4001
620k
4002
620k
  Type *Ty = I.getType();
4003
620k
4004
620k
  bool isVolatile = I.isVolatile();
4005
620k
  bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
4006
620k
  bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
4007
620k
  bool isDereferenceable =
4008
620k
      isDereferenceablePointer(SV, I.getType(), DAG.getDataLayout());
4009
620k
  unsigned Alignment = I.getAlignment();
4010
620k
4011
620k
  AAMDNodes AAInfo;
4012
620k
  I.getAAMetadata(AAInfo);
4013
620k
  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4014
620k
4015
620k
  SmallVector<EVT, 4> ValueVTs, MemVTs;
4016
620k
  SmallVector<uint64_t, 4> Offsets;
4017
620k
  ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets);
4018
620k
  unsigned NumValues = ValueVTs.size();
4019
620k
  if (NumValues == 0)
4020
3
    return;
4021
620k
4022
620k
  SDValue Root;
4023
620k
  bool ConstantMemory = false;
4024
620k
  if (isVolatile || 
NumValues > MaxParallelChains597k
)
4025
22.5k
    // Serialize volatile loads with other side effects.
4026
22.5k
    Root = getRoot();
4027
597k
  else if (AA &&
4028
597k
           AA->pointsToConstantMemory(MemoryLocation(
4029
593k
               SV,
4030
593k
               LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4031
593k
               AAInfo))) {
4032
27.4k
    // Do not serialize (non-volatile) loads of constant memory with anything.
4033
27.4k
    Root = DAG.getEntryNode();
4034
27.4k
    ConstantMemory = true;
4035
570k
  } else {
4036
570k
    // Do not serialize non-volatile loads against each other.
4037
570k
    Root = DAG.getRoot();
4038
570k
  }
4039
620k
4040
620k
  SDLoc dl = getCurSDLoc();
4041
620k
4042
620k
  if (isVolatile)
4043
22.5k
    Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
4044
620k
4045
620k
  // An aggregate load cannot wrap around the address space, so offsets to its
4046
620k
  // parts don't wrap either.
4047
620k
  SDNodeFlags Flags;
4048
620k
  Flags.setNoUnsignedWrap(true);
4049
620k
4050
620k
  SmallVector<SDValue, 4> Values(NumValues);
4051
620k
  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4052
620k
  EVT PtrVT = Ptr.getValueType();
4053
620k
  unsigned ChainI = 0;
4054
1.24M
  for (unsigned i = 0; i != NumValues; 
++i, ++ChainI623k
) {
4055
623k
    // Serializing loads here may result in excessive register pressure, and
4056
623k
    // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
4057
623k
    // could recover a bit by hoisting nodes upward in the chain by recognizing
4058
623k
    // they are side-effect free or do not alias. The optimizer should really
4059
623k
    // avoid this case by converting large object/array copies to llvm.memcpy
4060
623k
    // (MaxParallelChains should always remain as failsafe).
4061
623k
    if (ChainI == MaxParallelChains) {
4062
0
      assert(PendingLoads.empty() && "PendingLoads must be serialized first");
4063
0
      SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4064
0
                                  makeArrayRef(Chains.data(), ChainI));
4065
0
      Root = Chain;
4066
0
      ChainI = 0;
4067
0
    }
4068
623k
    SDValue A = DAG.getNode(ISD::ADD, dl,
4069
623k
                            PtrVT, Ptr,
4070
623k
                            DAG.getConstant(Offsets[i], dl, PtrVT),
4071
623k
                            Flags);
4072
623k
    auto MMOFlags = MachineMemOperand::MONone;
4073
623k
    if (isVolatile)
4074
24.7k
      MMOFlags |= MachineMemOperand::MOVolatile;
4075
623k
    if (isNonTemporal)
4076
793
      MMOFlags |= MachineMemOperand::MONonTemporal;
4077
623k
    if (isInvariant)
4078
24.0k
      MMOFlags |= MachineMemOperand::MOInvariant;
4079
623k
    if (isDereferenceable)
4080
170k
      MMOFlags |= MachineMemOperand::MODereferenceable;
4081
623k
    MMOFlags |= TLI.getMMOFlags(I);
4082
623k
4083
623k
    SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A,
4084
623k
                            MachinePointerInfo(SV, Offsets[i]), Alignment,
4085
623k
                            MMOFlags, AAInfo, Ranges);
4086
623k
    Chains[ChainI] = L.getValue(1);
4087
623k
4088
623k
    if (MemVTs[i] != ValueVTs[i])
4089
0
      L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]);
4090
623k
4091
623k
    Values[i] = L;
4092
623k
  }
4093
620k
4094
620k
  if (!ConstantMemory) {
4095
592k
    SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4096
592k
                                makeArrayRef(Chains.data(), ChainI));
4097
592k
    if (isVolatile)
4098
22.5k
      DAG.setRoot(Chain);
4099
570k
    else
4100
570k
      PendingLoads.push_back(Chain);
4101
592k
  }
4102
620k
4103
620k
  setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
4104
620k
                           DAG.getVTList(ValueVTs), Values));
4105
620k
}
4106
4107
108
void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
4108
108
  assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4109
108
         "call visitStoreToSwiftError when backend supports swifterror");
4110
108
4111
108
  SmallVector<EVT, 4> ValueVTs;
4112
108
  SmallVector<uint64_t, 4> Offsets;
4113
108
  const Value *SrcV = I.getOperand(0);
4114
108
  ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4115
108
                  SrcV->getType(), ValueVTs, &Offsets);
4116
108
  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4117
108
         "expect a single EVT for swifterror");
4118
108
4119
108
  SDValue Src = getValue(SrcV);
4120
108
  // Create a virtual register, then update the virtual register.
4121
108
  unsigned VReg =
4122
108
      SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand());
4123
108
  // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
4124
108
  // Chain can be getRoot or getControlRoot.
4125
108
  SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
4126
108
                                      SDValue(Src.getNode(), Src.getResNo()));
4127
108
  DAG.setRoot(CopyNode);
4128
108
}
4129
4130
53
void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
4131
53
  assert(DAG.getTargetLoweringInfo().supportSwiftError() &&
4132
53
         "call visitLoadFromSwiftError when backend supports swifterror");
4133
53
4134
53
  assert(!I.isVolatile() &&
4135
53
         I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&
4136
53
         I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&
4137
53
         "Support volatile, non temporal, invariant for load_from_swift_error");
4138
53
4139
53
  const Value *SV = I.getOperand(0);
4140
53
  Type *Ty = I.getType();
4141
53
  AAMDNodes AAInfo;
4142
53
  I.getAAMetadata(AAInfo);
4143
53
  assert(
4144
53
      (!AA ||
4145
53
       !AA->pointsToConstantMemory(MemoryLocation(
4146
53
           SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
4147
53
           AAInfo))) &&
4148
53
      "load_from_swift_error should not be constant memory");
4149
53
4150
53
  SmallVector<EVT, 4> ValueVTs;
4151
53
  SmallVector<uint64_t, 4> Offsets;
4152
53
  ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
4153
53
                  ValueVTs, &Offsets);
4154
53
  assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&
4155
53
         "expect a single EVT for swifterror");
4156
53
4157
53
  // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
4158
53
  SDValue L = DAG.getCopyFromReg(
4159
53
      getRoot(), getCurSDLoc(),
4160
53
      SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]);
4161
53
4162
53
  setValue(&I, L);
4163
53
}
4164
4165
545k
void SelectionDAGBuilder::visitStore(const StoreInst &I) {
4166
545k
  if (I.isAtomic())
4167
6.85k
    return visitAtomicStore(I);
4168
539k
4169
539k
  const Value *SrcV = I.getOperand(0);
4170
539k
  const Value *PtrV = I.getOperand(1);
4171
539k
4172
539k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4173
539k
  if (TLI.supportSwiftError()) {
4174
482k
    // Swifterror values can come from either a function parameter with
4175
482k
    // swifterror attribute or an alloca with swifterror attribute.
4176
482k
    if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
4177
13.8k
      if (Arg->hasSwiftErrorAttr())
4178
45
        return visitStoreToSwiftError(I);
4179
482k
    }
4180
482k
4181
482k
    if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
4182
7.50k
      if (Alloca->isSwiftError())
4183
63
        return visitStoreToSwiftError(I);
4184
538k
    }
4185
482k
  }
4186
538k
4187
538k
  SmallVector<EVT, 4> ValueVTs, MemVTs;
4188
538k
  SmallVector<uint64_t, 4> Offsets;
4189
538k
  ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
4190
538k
                  SrcV->getType(), ValueVTs, &MemVTs, &Offsets);
4191
538k
  unsigned NumValues = ValueVTs.size();
4192
538k
  if (NumValues == 0)
4193
21
    return;
4194
538k
4195
538k
  // Get the lowered operands. Note that we do this after
4196
538k
  // checking if NumResults is zero, because with zero results
4197
538k
  // the operands won't have values in the map.
4198
538k
  SDValue Src = getValue(SrcV);
4199
538k
  SDValue Ptr = getValue(PtrV);
4200
538k
4201
538k
  SDValue Root = getRoot();
4202
538k
  SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
4203
538k
  SDLoc dl = getCurSDLoc();
4204
538k
  EVT PtrVT = Ptr.getValueType();
4205
538k
  unsigned Alignment = I.getAlignment();
4206
538k
  AAMDNodes AAInfo;
4207
538k
  I.getAAMetadata(AAInfo);
4208
538k
4209
538k
  auto MMOFlags = MachineMemOperand::MONone;
4210
538k
  if (I.isVolatile())
4211
20.1k
    MMOFlags |= MachineMemOperand::MOVolatile;
4212
538k
  if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
4213
833
    MMOFlags |= MachineMemOperand::MONonTemporal;
4214
538k
  MMOFlags |= TLI.getMMOFlags(I);
4215
538k
4216
538k
  // An aggregate load cannot wrap around the address space, so offsets to its
4217
538k
  // parts don't wrap either.
4218
538k
  SDNodeFlags Flags;
4219
538k
  Flags.setNoUnsignedWrap(true);
4220
538k
4221
538k
  unsigned ChainI = 0;
4222
1.08M
  for (unsigned i = 0; i != NumValues; 
++i, ++ChainI542k
) {
4223
542k
    // See visitLoad comments.
4224
542k
    if (ChainI == MaxParallelChains) {
4225
0
      SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4226
0
                                  makeArrayRef(Chains.data(), ChainI));
4227
0
      Root = Chain;
4228
0
      ChainI = 0;
4229
0
    }
4230
542k
    SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
4231
542k
                              DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
4232
542k
    SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i);
4233
542k
    if (MemVTs[i] != ValueVTs[i])
4234
0
      Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]);
4235
542k
    SDValue St =
4236
542k
        DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]),
4237
542k
                     Alignment, MMOFlags, AAInfo);
4238
542k
    Chains[ChainI] = St;
4239
542k
  }
4240
538k
4241
538k
  SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
4242
538k
                                  makeArrayRef(Chains.data(), ChainI));
4243
538k
  DAG.setRoot(StoreNode);
4244
538k
}
4245
4246
void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
4247
559
                                           bool IsCompressing) {
4248
559
  SDLoc sdl = getCurSDLoc();
4249
559
4250
559
  auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4251
559
                           unsigned& Alignment) {
4252
394
    // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
4253
394
    Src0 = I.getArgOperand(0);
4254
394
    Ptr = I.getArgOperand(1);
4255
394
    Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
4256
394
    Mask = I.getArgOperand(3);
4257
394
  };
4258
559
  auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4259
559
                           unsigned& Alignment) {
4260
165
    // llvm.masked.compressstore.*(Src0, Ptr, Mask)
4261
165
    Src0 = I.getArgOperand(0);
4262
165
    Ptr = I.getArgOperand(1);
4263
165
    Mask = I.getArgOperand(2);
4264
165
    Alignment = 0;
4265
165
  };
4266
559
4267
559
  Value  *PtrOperand, *MaskOperand, *Src0Operand;
4268
559
  unsigned Alignment;
4269
559
  if (IsCompressing)
4270
165
    getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4271
394
  else
4272
394
    getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4273
559
4274
559
  SDValue Ptr = getValue(PtrOperand);
4275
559
  SDValue Src0 = getValue(Src0Operand);
4276
559
  SDValue Mask = getValue(MaskOperand);
4277
559
4278
559
  EVT VT = Src0.getValueType();
4279
559
  if (!Alignment)
4280
165
    Alignment = DAG.getEVTAlignment(VT);
4281
559
4282
559
  AAMDNodes AAInfo;
4283
559
  I.getAAMetadata(AAInfo);
4284
559
4285
559
  MachineMemOperand *MMO =
4286
559
    DAG.getMachineFunction().
4287
559
    getMachineMemOperand(MachinePointerInfo(PtrOperand),
4288
559
                          MachineMemOperand::MOStore,  VT.getStoreSize(),
4289
559
                          Alignment, AAInfo);
4290
559
  SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
4291
559
                                         MMO, false /* Truncating */,
4292
559
                                         IsCompressing);
4293
559
  DAG.setRoot(StoreNode);
4294
559
  setValue(&I, StoreNode);
4295
559
}
4296
4297
// Get a uniform base for the Gather/Scatter intrinsic.
4298
// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
4299
// We try to represent it as a base pointer + vector of indices.
4300
// Usually, the vector of pointers comes from a 'getelementptr' instruction.
4301
// The first operand of the GEP may be a single pointer or a vector of pointers
4302
// Example:
4303
//   %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
4304
//  or
4305
//   %gep.ptr = getelementptr i32, i32* %ptr,        <8 x i32> %ind
4306
// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
4307
//
4308
// When the first GEP operand is a single pointer - it is the uniform base we
4309
// are looking for. If first operand of the GEP is a splat vector - we
4310
// extract the splat value and use it as a uniform base.
4311
// In all other cases the function returns 'false'.
4312
static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
4313
427
                           SDValue &Scale, SelectionDAGBuilder* SDB) {
4314
427
  SelectionDAG& DAG = SDB->DAG;
4315
427
  LLVMContext &Context = *DAG.getContext();
4316
427
4317
427
  assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type");
4318
427
  const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
4319
427
  if (!GEP)
4320
177
    return false;
4321
250
4322
250
  const Value *GEPPtr = GEP->getPointerOperand();
4323
250
  if (!GEPPtr->getType()->isVectorTy())
4324
152
    Ptr = GEPPtr;
4325
98
  else if (!(Ptr = getSplatValue(GEPPtr)))
4326
6
    return false;
4327
244
4328
244
  unsigned FinalIndex = GEP->getNumOperands() - 1;
4329
244
  Value *IndexVal = GEP->getOperand(FinalIndex);
4330
244
4331
244
  // Ensure all the other indices are 0.
4332
250
  for (unsigned i = 1; i < FinalIndex; 
++i6
) {
4333
18
    auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
4334
18
    if (!C || 
!C->isZero()6
)
4335
12
      return false;
4336
18
  }
4337
244
4338
244
  // The operands of the GEP may be defined in another basic block.
4339
244
  // In this case we'll not find nodes for the operands.
4340
244
  
if (232
!SDB->findValue(Ptr)232
||
!SDB->findValue(IndexVal)231
)
4341
17
    return false;
4342
215
4343
215
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4344
215
  const DataLayout &DL = DAG.getDataLayout();
4345
215
  Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
4346
215
                                SDB->getCurSDLoc(), TLI.getPointerTy(DL));
4347
215
  Base = SDB->getValue(Ptr);
4348
215
  Index = SDB->getValue(IndexVal);
4349
215
4350
215
  if (!Index.getValueType().isVector()) {
4351
6
    unsigned GEPWidth = GEP->getType()->getVectorNumElements();
4352
6
    EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
4353
6
    Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
4354
6
  }
4355
215
  return true;
4356
215
}
4357
4358
103
void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
4359
103
  SDLoc sdl = getCurSDLoc();
4360
103
4361
103
  // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
4362
103
  const Value *Ptr = I.getArgOperand(1);
4363
103
  SDValue Src0 = getValue(I.getArgOperand(0));
4364
103
  SDValue Mask = getValue(I.getArgOperand(3));
4365
103
  EVT VT = Src0.getValueType();
4366
103
  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
4367
103
  if (!Alignment)
4368
0
    Alignment = DAG.getEVTAlignment(VT);
4369
103
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4370
103
4371
103
  AAMDNodes AAInfo;
4372
103
  I.getAAMetadata(AAInfo);
4373
103
4374
103
  SDValue Base;
4375
103
  SDValue Index;
4376
103
  SDValue Scale;
4377
103
  const Value *BasePtr = Ptr;
4378
103
  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4379
103
4380
103
  const Value *MemOpBasePtr = UniformBase ? 
BasePtr37
:
nullptr66
;
4381
103
  MachineMemOperand *MMO = DAG.getMachineFunction().
4382
103
    getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
4383
103
                         MachineMemOperand::MOStore,  VT.getStoreSize(),
4384
103
                         Alignment, AAInfo);
4385
103
  if (!UniformBase) {
4386
66
    Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4387
66
    Index = getValue(Ptr);
4388
66
    Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4389
66
  }
4390
103
  SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4391
103
  SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4392
103
                                         Ops, MMO);
4393
103
  DAG.setRoot(Scatter);
4394
103
  setValue(&I, Scatter);
4395
103
}
4396
4397
632
void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4398
632
  SDLoc sdl = getCurSDLoc();
4399
632
4400
632
  auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4401
632
                           unsigned& Alignment) {
4402
381
    // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4403
381
    Ptr = I.getArgOperand(0);
4404
381
    Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4405
381
    Mask = I.getArgOperand(2);
4406
381
    Src0 = I.getArgOperand(3);
4407
381
  };
4408
632
  auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4409
632
                           unsigned& Alignment) {
4410
251
    // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4411
251
    Ptr = I.getArgOperand(0);
4412
251
    Alignment = 0;
4413
251
    Mask = I.getArgOperand(1);
4414
251
    Src0 = I.getArgOperand(2);
4415
251
  };
4416
632
4417
632
  Value  *PtrOperand, *MaskOperand, *Src0Operand;
4418
632
  unsigned Alignment;
4419
632
  if (IsExpanding)
4420
251
    getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4421
381
  else
4422
381
    getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4423
632
4424
632
  SDValue Ptr = getValue(PtrOperand);
4425
632
  SDValue Src0 = getValue(Src0Operand);
4426
632
  SDValue Mask = getValue(MaskOperand);
4427
632
4428
632
  EVT VT = Src0.getValueType();
4429
632
  if (!Alignment)
4430
251
    Alignment = DAG.getEVTAlignment(VT);
4431
632
4432
632
  AAMDNodes AAInfo;
4433
632
  I.getAAMetadata(AAInfo);
4434
632
  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4435
632
4436
632
  // Do not serialize masked loads of constant memory with anything.
4437
632
  bool AddToChain =
4438
632
      !AA || !AA->pointsToConstantMemory(MemoryLocation(
4439
632
                 PtrOperand,
4440
632
                 LocationSize::precise(
4441
632
                     DAG.getDataLayout().getTypeStoreSize(I.getType())),
4442
632
                 AAInfo));
4443
632
  SDValue InChain = AddToChain ? 
DAG.getRoot()631
:
DAG.getEntryNode()1
;
4444
632
4445
632
  MachineMemOperand *MMO =
4446
632
    DAG.getMachineFunction().
4447
632
    getMachineMemOperand(MachinePointerInfo(PtrOperand),
4448
632
                          MachineMemOperand::MOLoad,  VT.getStoreSize(),
4449
632
                          Alignment, AAInfo, Ranges);
4450
632
4451
632
  SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4452
632
                                   ISD::NON_EXTLOAD, IsExpanding);
4453
632
  if (AddToChain)
4454
631
    PendingLoads.push_back(Load.getValue(1));
4455
632
  setValue(&I, Load);
4456
632
}
4457
4458
324
void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4459
324
  SDLoc sdl = getCurSDLoc();
4460
324
4461
324
  // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4462
324
  const Value *Ptr = I.getArgOperand(0);
4463
324
  SDValue Src0 = getValue(I.getArgOperand(3));
4464
324
  SDValue Mask = getValue(I.getArgOperand(2));
4465
324
4466
324
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4467
324
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4468
324
  unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4469
324
  if (!Alignment)
4470
39
    Alignment = DAG.getEVTAlignment(VT);
4471
324
4472
324
  AAMDNodes AAInfo;
4473
324
  I.getAAMetadata(AAInfo);
4474
324
  const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4475
324
4476
324
  SDValue Root = DAG.getRoot();
4477
324
  SDValue Base;
4478
324
  SDValue Index;
4479
324
  SDValue Scale;
4480
324
  const Value *BasePtr = Ptr;
4481
324
  bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4482
324
  bool ConstantMemory = false;
4483
324
  if (UniformBase && 
AA178
&&
4484
324
      AA->pointsToConstantMemory(
4485
164
          MemoryLocation(BasePtr,
4486
164
                         LocationSize::precise(
4487
164
                             DAG.getDataLayout().getTypeStoreSize(I.getType())),
4488
164
                         AAInfo))) {
4489
5
    // Do not serialize (non-volatile) loads of constant memory with anything.
4490
5
    Root = DAG.getEntryNode();
4491
5
    ConstantMemory = true;
4492
5
  }
4493
324
4494
324
  MachineMemOperand *MMO =
4495
324
    DAG.getMachineFunction().
4496
324
    getMachineMemOperand(MachinePointerInfo(UniformBase ? 
BasePtr178
:
nullptr146
),
4497
324
                         MachineMemOperand::MOLoad,  VT.getStoreSize(),
4498
324
                         Alignment, AAInfo, Ranges);
4499
324
4500
324
  if (!UniformBase) {
4501
146
    Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4502
146
    Index = getValue(Ptr);
4503
146
    Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4504
146
  }
4505
324
  SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4506
324
  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4507
324
                                       Ops, MMO);
4508
324
4509
324
  SDValue OutChain = Gather.getValue(1);
4510
324
  if (!ConstantMemory)
4511
319
    PendingLoads.push_back(OutChain);
4512
324
  setValue(&I, Gather);
4513
324
}
4514
4515
2.59k
void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4516
2.59k
  SDLoc dl = getCurSDLoc();
4517
2.59k
  AtomicOrdering SuccessOrdering = I.getSuccessOrdering();
4518
2.59k
  AtomicOrdering FailureOrdering = I.getFailureOrdering();
4519
2.59k
  SyncScope::ID SSID = I.getSyncScopeID();
4520
2.59k
4521
2.59k
  SDValue InChain = getRoot();
4522
2.59k
4523
2.59k
  MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4524
2.59k
  SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4525
2.59k
4526
2.59k
  auto Alignment = DAG.getEVTAlignment(MemVT);
4527
2.59k
4528
2.59k
  auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
4529
2.59k
  if (I.isVolatile())
4530
696
    Flags |= MachineMemOperand::MOVolatile;
4531
2.59k
  Flags |= DAG.getTargetLoweringInfo().getMMOFlags(I);
4532
2.59k
4533
2.59k
  MachineFunction &MF = DAG.getMachineFunction();
4534
2.59k
  MachineMemOperand *MMO =
4535
2.59k
    MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4536
2.59k
                            Flags, MemVT.getStoreSize(), Alignment,
4537
2.59k
                            AAMDNodes(), nullptr, SSID, SuccessOrdering,
4538
2.59k
                            FailureOrdering);
4539
2.59k
4540
2.59k
  SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS,
4541
2.59k
                                   dl, MemVT, VTs, InChain,
4542
2.59k
                                   getValue(I.getPointerOperand()),
4543
2.59k
                                   getValue(I.getCompareOperand()),
4544
2.59k
                                   getValue(I.getNewValOperand()), MMO);
4545
2.59k
4546
2.59k
  SDValue OutChain = L.getValue(2);
4547
2.59k
4548
2.59k
  setValue(&I, L);
4549
2.59k
  DAG.setRoot(OutChain);
4550
2.59k
}
4551
4552
7.29k
void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4553
7.29k
  SDLoc dl = getCurSDLoc();
4554
7.29k
  ISD::NodeType NT;
4555
7.29k
  switch (I.getOperation()) {
4556
7.29k
  
default: 0
llvm_unreachable0
("Unknown atomicrmw operation");
4557
7.29k
  
case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break1.89k
;
4558
7.29k
  
case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break1.37k
;
4559
7.29k
  
case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break781
;
4560
7.29k
  
case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break536
;
4561
7.29k
  
case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break158
;
4562
7.29k
  
case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break526
;
4563
7.29k
  
case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break490
;
4564
7.29k
  
case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break371
;
4565
7.29k
  
case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break384
;
4566
7.29k
  
case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break376
;
4567
7.29k
  
case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break376
;
4568
7.29k
  
case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break34
;
4569
7.29k
  
case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break0
;
4570
7.29k
  }
4571
7.29k
  AtomicOrdering Ordering = I.getOrdering();
4572
7.29k
  SyncScope::ID SSID = I.getSyncScopeID();
4573
7.29k
4574
7.29k
  SDValue InChain = getRoot();
4575
7.29k
4576
7.29k
  auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
4577
7.29k
  auto Alignment = DAG.getEVTAlignment(MemVT);
4578
7.29k
4579
7.29k
  auto Flags = MachineMemOperand::MOLoad |  MachineMemOperand::MOStore;
4580
7.29k
  if (I.isVolatile())
4581
1.38k
    Flags |= MachineMemOperand::MOVolatile;
4582
7.29k
  Flags |= DAG.getTargetLoweringInfo().getMMOFlags(I);
4583
7.29k
4584
7.29k
  MachineFunction &MF = DAG.getMachineFunction();
4585
7.29k
  MachineMemOperand *MMO =
4586
7.29k
    MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags,
4587
7.29k
                            MemVT.getStoreSize(), Alignment, AAMDNodes(),
4588
7.29k
                            nullptr, SSID, Ordering);
4589
7.29k
4590
7.29k
  SDValue L =
4591
7.29k
    DAG.getAtomic(NT, dl, MemVT, InChain,
4592
7.29k
                  getValue(I.getPointerOperand()), getValue(I.getValOperand()),
4593
7.29k
                  MMO);
4594
7.29k
4595
7.29k
  SDValue OutChain = L.getValue(1);
4596
7.29k
4597
7.29k
  setValue(&I, L);
4598
7.29k
  DAG.setRoot(OutChain);
4599
7.29k
}
4600
4601
4.61k
void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4602
4.61k
  SDLoc dl = getCurSDLoc();
4603
4.61k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4604
4.61k
  SDValue Ops[3];
4605
4.61k
  Ops[0] = getRoot();
4606
4.61k
  Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4607
4.61k
                           TLI.getFenceOperandTy(DAG.getDataLayout()));
4608
4.61k
  Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4609
4.61k
                           TLI.getFenceOperandTy(DAG.getDataLayout()));
4610
4.61k
  DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4611
4.61k
}
4612
4613
1.66k
void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4614
1.66k
  SDLoc dl = getCurSDLoc();
4615
1.66k
  AtomicOrdering Order = I.getOrdering();
4616
1.66k
  SyncScope::ID SSID = I.getSyncScopeID();
4617
1.66k
4618
1.66k
  SDValue InChain = getRoot();
4619
1.66k
4620
1.66k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4621
1.66k
  EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4622
1.66k
  EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType());
4623
1.66k
4624
1.66k
  if (!TLI.supportsUnalignedAtomics() &&
4625
1.66k
      I.getAlignment() < MemVT.getSizeInBits() / 8)
4626
0
    report_fatal_error("Cannot generate unaligned atomic load");
4627
1.66k
4628
1.66k
  auto Flags = MachineMemOperand::MOLoad;
4629
1.66k
  if (I.isVolatile())
4630
2
    Flags |= MachineMemOperand::MOVolatile;
4631
1.66k
  if (I.getMetadata(LLVMContext::MD_invariant_load) != nullptr)
4632
4
    Flags |= MachineMemOperand::MOInvariant;
4633
1.66k
  if (isDereferenceablePointer(I.getPointerOperand(), I.getType(),
4634
1.66k
                               DAG.getDataLayout()))
4635
540
    Flags |= MachineMemOperand::MODereferenceable;
4636
1.66k
4637
1.66k
  Flags |= TLI.getMMOFlags(I);
4638
1.66k
4639
1.66k
  MachineMemOperand *MMO =
4640
1.66k
      DAG.getMachineFunction().
4641
1.66k
      getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4642
1.66k
                           Flags, MemVT.getStoreSize(),
4643
1.66k
                           I.getAlignment() ? I.getAlignment() :
4644
1.66k
                                              
DAG.getEVTAlignment(MemVT)0
,
4645
1.66k
                           AAMDNodes(), nullptr, SSID, Order);
4646
1.66k
4647
1.66k
  InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4648
1.66k
  SDValue L =
4649
1.66k
      DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
4650
1.66k
                    getValue(I.getPointerOperand()), MMO);
4651
1.66k
4652
1.66k
  SDValue OutChain = L.getValue(1);
4653
1.66k
  if (MemVT != VT)
4654
0
    L = DAG.getPtrExtOrTrunc(L, dl, VT);
4655
1.66k
4656
1.66k
  setValue(&I, L);
4657
1.66k
  DAG.setRoot(OutChain);
4658
1.66k
}
4659
4660
6.85k
void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4661
6.85k
  SDLoc dl = getCurSDLoc();
4662
6.85k
4663
6.85k
  AtomicOrdering Ordering = I.getOrdering();
4664
6.85k
  SyncScope::ID SSID = I.getSyncScopeID();
4665
6.85k
4666
6.85k
  SDValue InChain = getRoot();
4667
6.85k
4668
6.85k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4669
6.85k
  EVT MemVT =
4670
6.85k
      TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4671
6.85k
4672
6.85k
  if (I.getAlignment() < MemVT.getSizeInBits() / 8)
4673
0
    report_fatal_error("Cannot generate unaligned atomic store");
4674
6.85k
4675
6.85k
  auto Flags = MachineMemOperand::MOStore;
4676
6.85k
  if (I.isVolatile())
4677
121
    Flags |= MachineMemOperand::MOVolatile;
4678
6.85k
  Flags |= TLI.getMMOFlags(I);
4679
6.85k
4680
6.85k
  MachineFunction &MF = DAG.getMachineFunction();
4681
6.85k
  MachineMemOperand *MMO =
4682
6.85k
    MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags,
4683
6.85k
                            MemVT.getStoreSize(), I.getAlignment(), AAMDNodes(),
4684
6.85k
                            nullptr, SSID, Ordering);
4685
6.85k
4686
6.85k
  SDValue Val = getValue(I.getValueOperand());
4687
6.85k
  if (Val.getValueType() != MemVT)
4688
0
    Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT);
4689
6.85k
4690
6.85k
  SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4691
6.85k
                                   getValue(I.getPointerOperand()), Val, MMO);
4692
6.85k
4693
6.85k
4694
6.85k
  DAG.setRoot(OutChain);
4695
6.85k
}
4696
4697
/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4698
/// node.
4699
void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4700
137k
                                               unsigned Intrinsic) {
4701
137k
  // Ignore the callsite's attributes. A specific call site may be marked with
4702
137k
  // readnone, but the lowering code will expect the chain based on the
4703
137k
  // definition.
4704
137k
  const Function *F = I.getCalledFunction();
4705
137k
  bool HasChain = !F->doesNotAccessMemory();
4706
137k
  bool OnlyLoad = HasChain && 
F->onlyReadsMemory()80.7k
;
4707
137k
4708
137k
  // Build the operand list.
4709
137k
  SmallVector<SDValue, 8> Ops;
4710
137k
  if (HasChain) {  // If this intrinsic has side-effects, chainify it.
4711
80.7k
    if (OnlyLoad) {
4712
3.73k
      // We don't need to serialize loads against other loads.
4713
3.73k
      Ops.push_back(DAG.getRoot());
4714
76.9k
    } else {
4715
76.9k
      Ops.push_back(getRoot());
4716
76.9k
    }
4717
80.7k
  }
4718
137k
4719
137k
  // Info is set by getTgtMemInstrinsic
4720
137k
  TargetLowering::IntrinsicInfo Info;
4721
137k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4722
137k
  bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4723
137k
                                               DAG.getMachineFunction(),
4724
137k
                                               Intrinsic);
4725
137k
4726
137k
  // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4727
137k
  if (!IsTgtIntrinsic || 
Info.opc == ISD::INTRINSIC_VOID68.6k
||
4728
137k
      
Info.opc == ISD::INTRINSIC_W_CHAIN53.9k
)
4729
137k
    Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4730
137k
                                        TLI.getPointerTy(DAG.getDataLayout())));
4731
137k
4732
137k
  // Add all operands of the call to the operand list.
4733
381k
  for (unsigned i = 0, e = I.getNumArgOperands(); i != e; 
++i244k
) {
4734
244k
    SDValue Op = getValue(I.getArgOperand(i));
4735
244k
    Ops.push_back(Op);
4736
244k
  }
4737
137k
4738
137k
  SmallVector<EVT, 4> ValueVTs;
4739
137k
  ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4740
137k
4741
137k
  if (HasChain)
4742
80.7k
    ValueVTs.push_back(MVT::Other);
4743
137k
4744
137k
  SDVTList VTs = DAG.getVTList(ValueVTs);
4745
137k
4746
137k
  // Create the node.
4747
137k
  SDValue Result;
4748
137k
  if (IsTgtIntrinsic) {
4749
68.6k
    // This is target intrinsic that touches memory
4750
68.6k
    AAMDNodes AAInfo;
4751
68.6k
    I.getAAMetadata(AAInfo);
4752
68.6k
    Result =
4753
68.6k
        DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT,
4754
68.6k
                                MachinePointerInfo(Info.ptrVal, Info.offset),
4755
68.6k
                                Info.align, Info.flags, Info.size, AAInfo);
4756
68.6k
  } else 
if (68.3k
!HasChain68.3k
) {
4757
56.3k
    Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4758
56.3k
  } else 
if (12.0k
!I.getType()->isVoidTy()12.0k
) {
4759
2.23k
    Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4760
9.80k
  } else {
4761
9.80k
    Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4762
9.80k
  }
4763
137k
4764
137k
  if (HasChain) {
4765
80.7k
    SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4766
80.7k
    if (OnlyLoad)
4767
3.73k
      PendingLoads.push_back(Chain);
4768
76.9k
    else
4769
76.9k
      DAG.setRoot(Chain);
4770
80.7k
  }
4771
137k
4772
137k
  if (!I.getType()->isVoidTy()) {
4773
112k
    if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4774
24.2k
      EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4775
24.2k
      Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4776
24.2k
    } else
4777
88.2k
      Result = lowerRangeToAssertZExt(DAG, I, Result);
4778
112k
4779
112k
    setValue(&I, Result);
4780
112k
  }
4781
137k
}
4782
4783
/// GetSignificand - Get the significand and build it into a floating-point
4784
/// number with exponent of 1:
4785
///
4786
///   Op = (Op & 0x007fffff) | 0x3f800000;
4787
///
4788
/// where Op is the hexadecimal representation of floating point value.
4789
9
static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4790
9
  SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4791
9
                           DAG.getConstant(0x007fffff, dl, MVT::i32));
4792
9
  SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4793
9
                           DAG.getConstant(0x3f800000, dl, MVT::i32));
4794
9
  return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4795
9
}
4796
4797
/// GetExponent - Get the exponent:
4798
///
4799
///   (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4800
///
4801
/// where Op is the hexadecimal representation of floating point value.
4802
static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4803
9
                           const TargetLowering &TLI, const SDLoc &dl) {
4804
9
  SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4805
9
                           DAG.getConstant(0x7f800000, dl, MVT::i32));
4806
9
  SDValue t1 = DAG.getNode(
4807
9
      ISD::SRL, dl, MVT::i32, t0,
4808
9
      DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4809
9
  SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4810
9
                           DAG.getConstant(127, dl, MVT::i32));
4811
9
  return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4812
9
}
4813
4814
/// getF32Constant - Get 32-bit floating point constant.
4815
static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4816
97
                              const SDLoc &dl) {
4817
97
  return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4818
97
                           MVT::f32);
4819
97
}
4820
4821
static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4822
9
                                       SelectionDAG &DAG) {
4823
9
  // TODO: What fast-math-flags should be set on the floating-point nodes?
4824
9
4825
9
  //   IntegerPartOfX = ((int32_t)(t0);
4826
9
  SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4827
9
4828
9
  //   FractionalPartOfX = t0 - (float)IntegerPartOfX;
4829
9
  SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4830
9
  SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4831
9
4832
9
  //   IntegerPartOfX <<= 23;
4833
9
  IntegerPartOfX = DAG.getNode(
4834
9
      ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4835
9
      DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4836
9
                                  DAG.getDataLayout())));
4837
9
4838
9
  SDValue TwoToFractionalPartOfX;
4839
9
  if (LimitFloatPrecision <= 6) {
4840
3
    // For floating-point precision of 6:
4841
3
    //
4842
3
    //   TwoToFractionalPartOfX =
4843
3
    //     0.997535578f +
4844
3
    //       (0.735607626f + 0.252464424f * x) * x;
4845
3
    //
4846
3
    // error 0.0144103317, which is 6 bits
4847
3
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4848
3
                             getF32Constant(DAG, 0x3e814304, dl));
4849
3
    SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4850
3
                             getF32Constant(DAG, 0x3f3c50c8, dl));
4851
3
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4852
3
    TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4853
3
                                         getF32Constant(DAG, 0x3f7f5e7e, dl));
4854
6
  } else if (LimitFloatPrecision <= 12) {
4855
3
    // For floating-point precision of 12:
4856
3
    //
4857
3
    //   TwoToFractionalPartOfX =
4858
3
    //     0.999892986f +
4859
3
    //       (0.696457318f +
4860
3
    //         (0.224338339f + 0.792043434e-1f * x) * x) * x;
4861
3
    //
4862
3
    // error 0.000107046256, which is 13 to 14 bits
4863
3
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4864
3
                             getF32Constant(DAG, 0x3da235e3, dl));
4865
3
    SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4866
3
                             getF32Constant(DAG, 0x3e65b8f3, dl));
4867
3
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4868
3
    SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4869
3
                             getF32Constant(DAG, 0x3f324b07, dl));
4870
3
    SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4871
3
    TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4872
3
                                         getF32Constant(DAG, 0x3f7ff8fd, dl));
4873
3
  } else { // LimitFloatPrecision <= 18
4874
3
    // For floating-point precision of 18:
4875
3
    //
4876
3
    //   TwoToFractionalPartOfX =
4877
3
    //     0.999999982f +
4878
3
    //       (0.693148872f +
4879
3
    //         (0.240227044f +
4880
3
    //           (0.554906021e-1f +
4881
3
    //             (0.961591928e-2f +
4882
3
    //               (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4883
3
    // error 2.47208000*10^(-7), which is better than 18 bits
4884
3
    SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4885
3
                             getF32Constant(DAG, 0x3924b03e, dl));
4886
3
    SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4887
3
                             getF32Constant(DAG, 0x3ab24b87, dl));
4888
3
    SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4889
3
    SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4890
3
                             getF32Constant(DAG, 0x3c1d8c17, dl));
4891
3
    SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4892
3
    SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4893
3
                             getF32Constant(DAG, 0x3d634a1d, dl));
4894
3
    SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4895
3
    SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4896
3
                             getF32Constant(DAG, 0x3e75fe14, dl));
4897
3
    SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4898
3
    SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4899
3
                              getF32Constant(DAG, 0x3f317234, dl));
4900
3
    SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4901
3
    TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4902
3
                                         getF32Constant(DAG, 0x3f800000, dl));
4903
3
  }
4904
9
4905
9
  // Add the exponent into the result in integer domain.
4906
9
  SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4907
9
  return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4908
9
                     DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4909
9
}
4910
4911
/// expandExp - Lower an exp intrinsic. Handles the special sequences for
4912
/// limited-precision mode.
4913
static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4914
165
                         const TargetLowering &TLI) {
4915
165
  if (Op.getValueType() == MVT::f32 &&
4916
165
      
LimitFloatPrecision > 032
&&
LimitFloatPrecision <= 183
) {
4917
3
4918
3
    // Put the exponent in the right bit position for later addition to the
4919
3
    // final result:
4920
3
    //
4921
3
    //   #define LOG2OFe 1.4426950f
4922
3
    //   t0 = Op * LOG2OFe
4923
3
4924
3
    // TODO: What fast-math-flags should be set here?
4925
3
    SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4926
3
                             getF32Constant(DAG, 0x3fb8aa3b, dl));
4927
3
    return getLimitedPrecisionExp2(t0, dl, DAG);
4928
3
  }
4929
162
4930
162
  // No special expansion.
4931
162
  return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4932
162
}
4933
4934
/// expandLog - Lower a log intrinsic. Handles the special sequences for
4935
/// limited-precision mode.
4936
static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4937
107
                         const TargetLowering &TLI) {
4938
107
  // TODO: What fast-math-flags should be set on the floating-point nodes?
4939
107
4940
107
  if (Op.getValueType() == MVT::f32 &&
4941
107
      
LimitFloatPrecision > 028
&&
LimitFloatPrecision <= 183
) {
4942
3
    SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4943
3
4944
3
    // Scale the exponent by log(2) [0.69314718f].
4945
3
    SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4946
3
    SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4947
3
                                        getF32Constant(DAG, 0x3f317218, dl));
4948
3
4949
3
    // Get the significand and build it into a floating-point number with
4950
3
    // exponent of 1.
4951
3
    SDValue X = GetSignificand(DAG, Op1, dl);
4952
3
4953
3
    SDValue LogOfMantissa;
4954
3
    if (LimitFloatPrecision <= 6) {
4955
1
      // For floating-point precision of 6:
4956
1
      //
4957
1
      //   LogofMantissa =
4958
1
      //     -1.1609546f +
4959
1
      //       (1.4034025f - 0.23903021f * x) * x;
4960
1
      //
4961
1
      // error 0.0034276066, which is better than 8 bits
4962
1
      SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4963
1
                               getF32Constant(DAG, 0xbe74c456, dl));
4964
1
      SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4965
1
                               getF32Constant(DAG, 0x3fb3a2b1, dl));
4966
1
      SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4967
1
      LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4968
1
                                  getF32Constant(DAG, 0x3f949a29, dl));
4969
2
    } else if (LimitFloatPrecision <= 12) {
4970
1
      // For floating-point precision of 12:
4971
1
      //
4972
1
      //   LogOfMantissa =
4973
1
      //     -1.7417939f +
4974
1
      //       (2.8212026f +
4975
1
      //         (-1.4699568f +
4976
1
      //           (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4977
1
      //
4978
1
      // error 0.000061011436, which is 14 bits
4979
1
      SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4980
1
                               getF32Constant(DAG, 0xbd67b6d6, dl));
4981
1
      SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4982
1
                               getF32Constant(DAG, 0x3ee4f4b8, dl));
4983
1
      SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4984
1
      SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4985
1
                               getF32Constant(DAG, 0x3fbc278b, dl));
4986
1
      SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4987
1
      SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4988
1
                               getF32Constant(DAG, 0x40348e95, dl));
4989
1
      SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4990
1
      LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4991
1
                                  getF32Constant(DAG, 0x3fdef31a, dl));
4992
1
    } else { // LimitFloatPrecision <= 18
4993
1
      // For floating-point precision of 18:
4994
1
      //
4995
1
      //   LogOfMantissa =
4996
1
      //     -2.1072184f +
4997
1
      //       (4.2372794f +
4998
1
      //         (-3.7029485f +
4999
1
      //           (2.2781945f +
5000
1
      //             (-0.87823314f +
5001
1
      //               (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
5002
1
      //
5003
1
      // error 0.0000023660568, which is better than 18 bits
5004
1
      SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
5005
1
                               getF32Constant(DAG, 0xbc91e5ac, dl));
5006
1
      SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
5007
1
                               getF32Constant(DAG, 0x3e4350aa, dl));
5008
1
      SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
5009
1
      SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
5010
1
                               getF32Constant(DAG, 0x3f60d3e3, dl));
5011
1
      SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
5012
1
      SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
5013
1
                               getF32Constant(DAG, 0x4011cdf0, dl));
5014
1
      SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
5015
1
      SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
5016
1
                               getF32Constant(DAG, 0x406cfd1c, dl));
5017
1
      SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
5018
1
      SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
5019
1
                               getF32Constant(DAG, 0x408797cb, dl));
5020
1
      SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
5021
1
      LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
5022
1
                                  getF32Constant(DAG, 0x4006dcab, dl));
5023
1
    }
5024
3
5025
3
    return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
5026
3
  }
5027
104
5028
104
  // No special expansion.
5029
104
  return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
5030
104
}
5031
5032
/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
5033
/// limited-precision mode.
5034
static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
5035
104
                          const TargetLowering &TLI) {
5036
104
  // TODO: What fast-math-flags should be set on the floating-point nodes?
5037
104
5038
104
  if (Op.getValueType() == MVT::f32 &&
5039
104
      
LimitFloatPrecision > 032
&&
LimitFloatPrecision <= 183
) {
5040
3
    SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
5041
3
5042
3
    // Get the exponent.
5043
3
    SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
5044
3
5045
3
    // Get the significand and build it into a floating-point number with