Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
/// \file
9
/// This file implements the IRTranslator class.
10
//===----------------------------------------------------------------------===//
11
12
#include "llvm/CodeGen/GlobalISel/IRTranslator.h"
13
#include "llvm/ADT/PostOrderIterator.h"
14
#include "llvm/ADT/STLExtras.h"
15
#include "llvm/ADT/ScopeExit.h"
16
#include "llvm/ADT/SmallSet.h"
17
#include "llvm/ADT/SmallVector.h"
18
#include "llvm/Analysis/BranchProbabilityInfo.h"
19
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
20
#include "llvm/Analysis/ValueTracking.h"
21
#include "llvm/CodeGen/Analysis.h"
22
#include "llvm/CodeGen/FunctionLoweringInfo.h"
23
#include "llvm/CodeGen/GlobalISel/CallLowering.h"
24
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
25
#include "llvm/CodeGen/LowLevelType.h"
26
#include "llvm/CodeGen/MachineBasicBlock.h"
27
#include "llvm/CodeGen/MachineFrameInfo.h"
28
#include "llvm/CodeGen/MachineFunction.h"
29
#include "llvm/CodeGen/MachineInstrBuilder.h"
30
#include "llvm/CodeGen/MachineMemOperand.h"
31
#include "llvm/CodeGen/MachineOperand.h"
32
#include "llvm/CodeGen/MachineRegisterInfo.h"
33
#include "llvm/CodeGen/StackProtector.h"
34
#include "llvm/CodeGen/TargetFrameLowering.h"
35
#include "llvm/CodeGen/TargetLowering.h"
36
#include "llvm/CodeGen/TargetPassConfig.h"
37
#include "llvm/CodeGen/TargetRegisterInfo.h"
38
#include "llvm/CodeGen/TargetSubtargetInfo.h"
39
#include "llvm/IR/BasicBlock.h"
40
#include "llvm/IR/CFG.h"
41
#include "llvm/IR/Constant.h"
42
#include "llvm/IR/Constants.h"
43
#include "llvm/IR/DataLayout.h"
44
#include "llvm/IR/DebugInfo.h"
45
#include "llvm/IR/DerivedTypes.h"
46
#include "llvm/IR/Function.h"
47
#include "llvm/IR/GetElementPtrTypeIterator.h"
48
#include "llvm/IR/InlineAsm.h"
49
#include "llvm/IR/InstrTypes.h"
50
#include "llvm/IR/Instructions.h"
51
#include "llvm/IR/IntrinsicInst.h"
52
#include "llvm/IR/Intrinsics.h"
53
#include "llvm/IR/LLVMContext.h"
54
#include "llvm/IR/Metadata.h"
55
#include "llvm/IR/Type.h"
56
#include "llvm/IR/User.h"
57
#include "llvm/IR/Value.h"
58
#include "llvm/MC/MCContext.h"
59
#include "llvm/Pass.h"
60
#include "llvm/Support/Casting.h"
61
#include "llvm/Support/CodeGen.h"
62
#include "llvm/Support/Debug.h"
63
#include "llvm/Support/ErrorHandling.h"
64
#include "llvm/Support/LowLevelTypeImpl.h"
65
#include "llvm/Support/MathExtras.h"
66
#include "llvm/Support/raw_ostream.h"
67
#include "llvm/Target/TargetIntrinsicInfo.h"
68
#include "llvm/Target/TargetMachine.h"
69
#include <algorithm>
70
#include <cassert>
71
#include <cstdint>
72
#include <iterator>
73
#include <string>
74
#include <utility>
75
#include <vector>
76
77
#define DEBUG_TYPE "irtranslator"
78
79
using namespace llvm;
80
81
static cl::opt<bool>
82
    EnableCSEInIRTranslator("enable-cse-in-irtranslator",
83
                            cl::desc("Should enable CSE in irtranslator"),
84
                            cl::Optional, cl::init(false));
85
char IRTranslator::ID = 0;
86
87
102k
INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
88
102k
                false, false)
89
102k
INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
90
102k
INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
91
102k
INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
92
                false, false)
93
94
static void reportTranslationError(MachineFunction &MF,
95
                                   const TargetPassConfig &TPC,
96
                                   OptimizationRemarkEmitter &ORE,
97
294
                                   OptimizationRemarkMissed &R) {
98
294
  MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
99
294
100
294
  // Print the function name explicitly if we don't have a debug location (which
101
294
  // makes the diagnostic less useful) or if we're going to emit a raw error.
102
294
  if (!R.getLocation().isValid() || 
TPC.isGlobalISelAbortEnabled()0
)
103
294
    R << (" (in function: " + MF.getName() + ")").str();
104
294
105
294
  if (TPC.isGlobalISelAbortEnabled())
106
2
    report_fatal_error(R.getMsg());
107
292
  else
108
292
    ORE.emit(R);
109
294
}
110
111
7.00k
IRTranslator::IRTranslator() : MachineFunctionPass(ID) { }
112
113
#ifndef NDEBUG
114
namespace {
115
/// Verify that every instruction created has the same DILocation as the
116
/// instruction being translated.
117
class DILocationVerifier : public GISelChangeObserver {
118
  const Instruction *CurrInst = nullptr;
119
120
public:
121
  DILocationVerifier() = default;
122
  ~DILocationVerifier() = default;
123
124
  const Instruction *getCurrentInst() const { return CurrInst; }
125
  void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
126
127
  void erasingInstr(MachineInstr &MI) override {}
128
  void changingInstr(MachineInstr &MI) override {}
129
  void changedInstr(MachineInstr &MI) override {}
130
131
  void createdInstr(MachineInstr &MI) override {
132
    assert(getCurrentInst() && "Inserted instruction without a current MI");
133
134
    // Only print the check message if we're actually checking it.
135
#ifndef NDEBUG
136
    LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst
137
                      << " was copied to " << MI);
138
#endif
139
    // We allow insts in the entry block to have a debug loc line of 0 because
140
    // they could have originated from constants, and we don't want a jumpy
141
    // debug experience.
142
    assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
143
            MI.getDebugLoc().getLine() == 0) &&
144
           "Line info was not transferred to all instructions");
145
  }
146
};
147
} // namespace
148
#endif // ifndef NDEBUG
149
150
151
7.00k
void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
152
7.00k
  AU.addRequired<StackProtector>();
153
7.00k
  AU.addRequired<TargetPassConfig>();
154
7.00k
  AU.addRequired<GISelCSEAnalysisWrapperPass>();
155
7.00k
  getSelectionDAGFallbackAnalysisUsage(AU);
156
7.00k
  MachineFunctionPass::getAnalysisUsage(AU);
157
7.00k
}
158
159
IRTranslator::ValueToVRegInfo::VRegListT &
160
25.2k
IRTranslator::allocateVRegs(const Value &Val) {
161
25.2k
  assert(!VMap.contains(Val) && "Value already allocated in VMap");
162
25.2k
  auto *Regs = VMap.getVRegs(Val);
163
25.2k
  auto *Offsets = VMap.getOffsets(Val);
164
25.2k
  SmallVector<LLT, 4> SplitTys;
165
25.2k
  computeValueLLTs(*DL, *Val.getType(), SplitTys,
166
25.2k
                   Offsets->empty() ? 
Offsets1.52k
:
nullptr23.6k
);
167
52.5k
  for (unsigned i = 0; i < SplitTys.size(); 
++i27.3k
)
168
27.3k
    Regs->push_back(0);
169
25.2k
  return *Regs;
170
25.2k
}
171
172
26.1M
ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
173
26.1M
  auto VRegsIt = VMap.findVRegs(Val);
174
26.1M
  if (VRegsIt != VMap.vregs_end())
175
15.8M
    return *VRegsIt->second;
176
10.2M
177
10.2M
  if (Val.getType()->isVoidTy())
178
366k
    return *VMap.getVRegs(Val);
179
9.88M
180
9.88M
  // Create entry for this type.
181
9.88M
  auto *VRegs = VMap.getVRegs(Val);
182
9.88M
  auto *Offsets = VMap.getOffsets(Val);
183
9.88M
184
9.88M
  assert(Val.getType()->isSized() &&
185
9.88M
         "Don't know how to create an empty vreg");
186
9.88M
187
9.88M
  SmallVector<LLT, 4> SplitTys;
188
9.88M
  computeValueLLTs(*DL, *Val.getType(), SplitTys,
189
9.88M
                   Offsets->empty() ? 
Offsets1.59M
:
nullptr8.28M
);
190
9.88M
191
9.88M
  if (!isa<Constant>(Val)) {
192
7.74M
    for (auto Ty : SplitTys)
193
7.76M
      VRegs->push_back(MRI->createGenericVirtualRegister(Ty));
194
7.74M
    return *VRegs;
195
7.74M
  }
196
2.13M
197
2.13M
  if (Val.getType()->isAggregateType()) {
198
375
    // UndefValue, ConstantAggregateZero
199
375
    auto &C = cast<Constant>(Val);
200
375
    unsigned Idx = 0;
201
1.25k
    while (auto Elt = C.getAggregateElement(Idx++)) {
202
876
      auto EltRegs = getOrCreateVRegs(*Elt);
203
876
      llvm::copy(EltRegs, std::back_inserter(*VRegs));
204
876
    }
205
2.13M
  } else {
206
2.13M
    assert(SplitTys.size() == 1 && "unexpectedly split LLT");
207
2.13M
    VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));
208
2.13M
    bool Success = translate(cast<Constant>(Val), VRegs->front());
209
2.13M
    if (!Success) {
210
0
      OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
211
0
                                 MF->getFunction().getSubprogram(),
212
0
                                 &MF->getFunction().getEntryBlock());
213
0
      R << "unable to translate constant: " << ore::NV("Type", Val.getType());
214
0
      reportTranslationError(*MF, *TPC, *ORE, R);
215
0
      return *VRegs;
216
0
    }
217
2.13M
  }
218
2.13M
219
2.13M
  return *VRegs;
220
2.13M
}
221
222
150k
int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {
223
150k
  if (FrameIndices.find(&AI) != FrameIndices.end())
224
103k
    return FrameIndices[&AI];
225
47.4k
226
47.4k
  unsigned ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());
227
47.4k
  unsigned Size =
228
47.4k
      ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();
229
47.4k
230
47.4k
  // Always allocate at least one byte.
231
47.4k
  Size = std::max(Size, 1u);
232
47.4k
233
47.4k
  unsigned Alignment = AI.getAlignment();
234
47.4k
  if (!Alignment)
235
1.83k
    Alignment = DL->getABITypeAlignment(AI.getAllocatedType());
236
47.4k
237
47.4k
  int &FI = FrameIndices[&AI];
238
47.4k
  FI = MF->getFrameInfo().CreateStackObject(Size, Alignment, false, &AI);
239
47.4k
  return FI;
240
47.4k
}
241
242
2.11M
unsigned IRTranslator::getMemOpAlignment(const Instruction &I) {
243
2.11M
  unsigned Alignment = 0;
244
2.11M
  Type *ValTy = nullptr;
245
2.11M
  if (const StoreInst *SI = dyn_cast<StoreInst>(&I)) {
246
888k
    Alignment = SI->getAlignment();
247
888k
    ValTy = SI->getValueOperand()->getType();
248
1.22M
  } else if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
249
1.22M
    Alignment = LI->getAlignment();
250
1.22M
    ValTy = LI->getType();
251
1.22M
  } else 
if (const AtomicCmpXchgInst *25
AI25
= dyn_cast<AtomicCmpXchgInst>(&I)) {
252
3
    // TODO(PR27168): This instruction has no alignment attribute, but unlike
253
3
    // the default alignment for load/store, the default here is to assume
254
3
    // it has NATURAL alignment, not DataLayout-specified alignment.
255
3
    const DataLayout &DL = AI->getModule()->getDataLayout();
256
3
    Alignment = DL.getTypeStoreSize(AI->getCompareOperand()->getType());
257
3
    ValTy = AI->getCompareOperand()->getType();
258
22
  } else if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I)) {
259
22
    // TODO(PR27168): This instruction has no alignment attribute, but unlike
260
22
    // the default alignment for load/store, the default here is to assume
261
22
    // it has NATURAL alignment, not DataLayout-specified alignment.
262
22
    const DataLayout &DL = AI->getModule()->getDataLayout();
263
22
    Alignment = DL.getTypeStoreSize(AI->getValOperand()->getType());
264
22
    ValTy = AI->getType();
265
22
  } else {
266
0
    OptimizationRemarkMissed R("gisel-irtranslator", "", &I);
267
0
    R << "unable to translate memop: " << ore::NV("Opcode", &I);
268
0
    reportTranslationError(*MF, *TPC, *ORE, R);
269
0
    return 1;
270
0
  }
271
2.11M
272
2.11M
  return Alignment ? 
Alignment2.11M
:
DL->getABITypeAlignment(ValTy)3.80k
;
273
2.11M
}
274
275
8.43M
MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {
276
8.43M
  MachineBasicBlock *&MBB = BBToMBB[&BB];
277
8.43M
  assert(MBB && "BasicBlock was not encountered before");
278
8.43M
  return *MBB;
279
8.43M
}
280
281
76.6k
void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {
282
76.6k
  assert(NewPred && "new predecessor must be a real MachineBasicBlock");
283
76.6k
  MachinePreds[Edge].push_back(NewPred);
284
76.6k
}
285
286
bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
287
991k
                                     MachineIRBuilder &MIRBuilder) {
288
991k
  // Get or create a virtual register for each value.
289
991k
  // Unless the value is a Constant => loadimm cst?
290
991k
  // or inline constant each time?
291
991k
  // Creation of a virtual register needs to have a size.
292
991k
  Register Op0 = getOrCreateVReg(*U.getOperand(0));
293
991k
  Register Op1 = getOrCreateVReg(*U.getOperand(1));
294
991k
  Register Res = getOrCreateVReg(U);
295
991k
  uint16_t Flags = 0;
296
991k
  if (isa<Instruction>(U)) {
297
991k
    const Instruction &I = cast<Instruction>(U);
298
991k
    Flags = MachineInstr::copyFlagsFromInstruction(I);
299
991k
  }
300
991k
301
991k
  MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
302
991k
  return true;
303
991k
}
304
305
13.4k
bool IRTranslator::translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
306
13.4k
  // -0.0 - X --> G_FNEG
307
13.4k
  if (isa<Constant>(U.getOperand(0)) &&
308
13.4k
      
U.getOperand(0) == ConstantFP::getZeroValueForNegation(U.getType())4.14k
) {
309
2.91k
    Register Op1 = getOrCreateVReg(*U.getOperand(1));
310
2.91k
    Register Res = getOrCreateVReg(U);
311
2.91k
    uint16_t Flags = 0;
312
2.91k
    if (isa<Instruction>(U)) {
313
2.91k
      const Instruction &I = cast<Instruction>(U);
314
2.91k
      Flags = MachineInstr::copyFlagsFromInstruction(I);
315
2.91k
    }
316
2.91k
    // Negate the last operand of the FSUB
317
2.91k
    MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op1}, Flags);
318
2.91k
    return true;
319
2.91k
  }
320
10.5k
  return translateBinaryOp(TargetOpcode::G_FSUB, U, MIRBuilder);
321
10.5k
}
322
323
4
bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
324
4
  Register Op0 = getOrCreateVReg(*U.getOperand(0));
325
4
  Register Res = getOrCreateVReg(U);
326
4
  uint16_t Flags = 0;
327
4
  if (isa<Instruction>(U)) {
328
4
    const Instruction &I = cast<Instruction>(U);
329
4
    Flags = MachineInstr::copyFlagsFromInstruction(I);
330
4
  }
331
4
  MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op0}, Flags);
332
4
  return true;
333
4
}
334
335
bool IRTranslator::translateCompare(const User &U,
336
1.05M
                                    MachineIRBuilder &MIRBuilder) {
337
1.05M
  const CmpInst *CI = dyn_cast<CmpInst>(&U);
338
1.05M
  Register Op0 = getOrCreateVReg(*U.getOperand(0));
339
1.05M
  Register Op1 = getOrCreateVReg(*U.getOperand(1));
340
1.05M
  Register Res = getOrCreateVReg(U);
341
1.05M
  CmpInst::Predicate Pred =
342
1.05M
      CI ? 
CI->getPredicate()1.05M
: static_cast<CmpInst::Predicate>(
343
12
                                    cast<ConstantExpr>(U).getPredicate());
344
1.05M
  if (CmpInst::isIntPredicate(Pred))
345
1.04M
    MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
346
17.3k
  else if (Pred == CmpInst::FCMP_FALSE)
347
6
    MIRBuilder.buildCopy(
348
6
        Res, getOrCreateVReg(*Constant::getNullValue(CI->getType())));
349
17.3k
  else if (Pred == CmpInst::FCMP_TRUE)
350
6
    MIRBuilder.buildCopy(
351
6
        Res, getOrCreateVReg(*Constant::getAllOnesValue(CI->getType())));
352
17.3k
  else {
353
17.3k
    MIRBuilder.buildInstr(TargetOpcode::G_FCMP, {Res}, {Pred, Op0, Op1},
354
17.3k
                          MachineInstr::copyFlagsFromInstruction(*CI));
355
17.3k
  }
356
1.05M
357
1.05M
  return true;
358
1.05M
}
359
360
344k
bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {
361
344k
  const ReturnInst &RI = cast<ReturnInst>(U);
362
344k
  const Value *Ret = RI.getReturnValue();
363
344k
  if (Ret && 
DL->getTypeStoreSize(Ret->getType()) == 0256k
)
364
2
    Ret = nullptr;
365
344k
366
344k
  ArrayRef<Register> VRegs;
367
344k
  if (Ret)
368
256k
    VRegs = getOrCreateVRegs(*Ret);
369
344k
370
344k
  Register SwiftErrorVReg = 0;
371
344k
  if (CLI->supportSwiftError() && 
SwiftError.getFunctionArg()342k
) {
372
11
    SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(
373
11
        &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());
374
11
  }
375
344k
376
344k
  // The target may mess up with the insertion point, but
377
344k
  // this is not important as a return is the last instruction
378
344k
  // of the block anyway.
379
344k
  return CLI->lowerReturn(MIRBuilder, Ret, VRegs, SwiftErrorVReg);
380
344k
}
381
382
1.51M
bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {
383
1.51M
  const BranchInst &BrInst = cast<BranchInst>(U);
384
1.51M
  unsigned Succ = 0;
385
1.51M
  if (!BrInst.isUnconditional()) {
386
936k
    // We want a G_BRCOND to the true BB followed by an unconditional branch.
387
936k
    Register Tst = getOrCreateVReg(*BrInst.getCondition());
388
936k
    const BasicBlock &TrueTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ++));
389
936k
    MachineBasicBlock &TrueBB = getMBB(TrueTgt);
390
936k
    MIRBuilder.buildBrCond(Tst, TrueBB);
391
936k
  }
392
1.51M
393
1.51M
  const BasicBlock &BrTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ));
394
1.51M
  MachineBasicBlock &TgtBB = getMBB(BrTgt);
395
1.51M
  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
396
1.51M
397
1.51M
  // If the unconditional target is the layout successor, fallthrough.
398
1.51M
  if (!CurBB.isLayoutSuccessor(&TgtBB))
399
731k
    MIRBuilder.buildBr(TgtBB);
400
1.51M
401
1.51M
  // Link successors.
402
1.51M
  for (const BasicBlock *Succ : successors(&BrInst))
403
2.44M
    CurBB.addSuccessor(&getMBB(*Succ));
404
1.51M
  return true;
405
1.51M
}
406
407
void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,
408
                                        MachineBasicBlock *Dst,
409
77.1k
                                        BranchProbability Prob) {
410
77.1k
  if (!FuncInfo.BPI) {
411
77.1k
    Src->addSuccessorWithoutProb(Dst);
412
77.1k
    return;
413
77.1k
  }
414
0
  if (Prob.isUnknown())
415
0
    Prob = getEdgeProbability(Src, Dst);
416
0
  Src->addSuccessor(Dst, Prob);
417
0
}
418
419
BranchProbability
420
IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,
421
14.9k
                                 const MachineBasicBlock *Dst) const {
422
14.9k
  const BasicBlock *SrcBB = Src->getBasicBlock();
423
14.9k
  const BasicBlock *DstBB = Dst->getBasicBlock();
424
14.9k
  if (!FuncInfo.BPI) {
425
14.9k
    // If BPI is not available, set the default probability as 1 / N, where N is
426
14.9k
    // the number of successors.
427
14.9k
    auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
428
14.9k
    return BranchProbability(1, SuccSize);
429
14.9k
  }
430
0
  return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);
431
0
}
432
433
14.9k
bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {
434
14.9k
  using namespace SwitchCG;
435
14.9k
  // Extract cases from the switch.
436
14.9k
  const SwitchInst &SI = cast<SwitchInst>(U);
437
14.9k
  BranchProbabilityInfo *BPI = FuncInfo.BPI;
438
14.9k
  CaseClusterVector Clusters;
439
14.9k
  Clusters.reserve(SI.getNumCases());
440
46.3k
  for (auto &I : SI.cases()) {
441
46.3k
    MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());
442
46.3k
    assert(Succ && "Could not find successor mbb in mapping");
443
46.3k
    const ConstantInt *CaseVal = I.getCaseValue();
444
46.3k
    BranchProbability Prob =
445
46.3k
        BPI ? 
BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())0
446
46.3k
            : BranchProbability(1, SI.getNumCases() + 1);
447
46.3k
    Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
448
46.3k
  }
449
14.9k
450
14.9k
  MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());
451
14.9k
452
14.9k
  // Cluster adjacent cases with the same destination. We do this at all
453
14.9k
  // optimization levels because it's cheap to do and will make codegen faster
454
14.9k
  // if there are many clusters.
455
14.9k
  sortAndRangeify(Clusters);
456
14.9k
457
14.9k
  MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());
458
14.9k
459
14.9k
  // If there is only the default destination, jump there directly.
460
14.9k
  if (Clusters.empty()) {
461
0
    SwitchMBB->addSuccessor(DefaultMBB);
462
0
    if (DefaultMBB != SwitchMBB->getNextNode())
463
0
      MIB.buildBr(*DefaultMBB);
464
0
    return true;
465
0
  }
466
14.9k
467
14.9k
  SL->findJumpTables(Clusters, &SI, DefaultMBB);
468
14.9k
469
14.9k
  LLVM_DEBUG({
470
14.9k
    dbgs() << "Case clusters: ";
471
14.9k
    for (const CaseCluster &C : Clusters) {
472
14.9k
      if (C.Kind == CC_JumpTable)
473
14.9k
        dbgs() << "JT:";
474
14.9k
      if (C.Kind == CC_BitTests)
475
14.9k
        dbgs() << "BT:";
476
14.9k
477
14.9k
      C.Low->getValue().print(dbgs(), true);
478
14.9k
      if (C.Low != C.High) {
479
14.9k
        dbgs() << '-';
480
14.9k
        C.High->getValue().print(dbgs(), true);
481
14.9k
      }
482
14.9k
      dbgs() << ' ';
483
14.9k
    }
484
14.9k
    dbgs() << '\n';
485
14.9k
  });
486
14.9k
487
14.9k
  assert(!Clusters.empty());
488
14.9k
  SwitchWorkList WorkList;
489
14.9k
  CaseClusterIt First = Clusters.begin();
490
14.9k
  CaseClusterIt Last = Clusters.end() - 1;
491
14.9k
  auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);
492
14.9k
  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
493
14.9k
494
14.9k
  // FIXME: At the moment we don't do any splitting optimizations here like
495
14.9k
  // SelectionDAG does, so this worklist only has one entry.
496
29.9k
  while (!WorkList.empty()) {
497
14.9k
    SwitchWorkListItem W = WorkList.back();
498
14.9k
    WorkList.pop_back();
499
14.9k
    if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))
500
0
      return false;
501
14.9k
  }
502
14.9k
  return true;
503
14.9k
}
504
505
void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,
506
2.27k
                                 MachineBasicBlock *MBB) {
507
2.27k
  // Emit the code for the jump table
508
2.27k
  assert(JT.Reg != -1U && "Should lower JT Header first!");
509
2.27k
  MachineIRBuilder MIB(*MBB->getParent());
510
2.27k
  MIB.setMBB(*MBB);
511
2.27k
  MIB.setDebugLoc(CurBuilder->getDebugLoc());
512
2.27k
513
2.27k
  Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext());
514
2.27k
  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
515
2.27k
516
2.27k
  auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);
517
2.27k
  MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);
518
2.27k
}
519
520
bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,
521
                                       SwitchCG::JumpTableHeader &JTH,
522
2.27k
                                       MachineBasicBlock *HeaderBB) {
523
2.27k
  MachineIRBuilder MIB(*HeaderBB->getParent());
524
2.27k
  MIB.setMBB(*HeaderBB);
525
2.27k
  MIB.setDebugLoc(CurBuilder->getDebugLoc());
526
2.27k
527
2.27k
  const Value &SValue = *JTH.SValue;
528
2.27k
  // Subtract the lowest switch case value from the value being switched on.
529
2.27k
  const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);
530
2.27k
  Register SwitchOpReg = getOrCreateVReg(SValue);
531
2.27k
  auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);
532
2.27k
  auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);
533
2.27k
534
2.27k
  // This value may be smaller or larger than the target's pointer type, and
535
2.27k
  // therefore require extension or truncating.
536
2.27k
  Type *PtrIRTy = SValue.getType()->getPointerTo();
537
2.27k
  const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));
538
2.27k
  Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);
539
2.27k
540
2.27k
  JT.Reg = Sub.getReg(0);
541
2.27k
542
2.27k
  if (JTH.OmitRangeCheck) {
543
7
    if (JT.MBB != HeaderBB->getNextNode())
544
0
      MIB.buildBr(*JT.MBB);
545
7
    return true;
546
7
  }
547
2.26k
548
2.26k
  // Emit the range check for the jump table, and branch to the default block
549
2.26k
  // for the switch statement if the value being switched on exceeds the
550
2.26k
  // largest case in the switch.
551
2.26k
  auto Cst = getOrCreateVReg(
552
2.26k
      *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));
553
2.26k
  Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);
554
2.26k
  auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);
555
2.26k
556
2.26k
  auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);
557
2.26k
558
2.26k
  // Avoid emitting unnecessary branches to the next block.
559
2.26k
  if (JT.MBB != HeaderBB->getNextNode())
560
110
    BrCond = MIB.buildBr(*JT.MBB);
561
2.26k
  return true;
562
2.26k
}
563
564
void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
565
                                  MachineBasicBlock *SwitchBB,
566
28.7k
                                  MachineIRBuilder &MIB) {
567
28.7k
  Register CondLHS = getOrCreateVReg(*CB.CmpLHS);
568
28.7k
  Register Cond;
569
28.7k
  DebugLoc OldDbgLoc = MIB.getDebugLoc();
570
28.7k
  MIB.setDebugLoc(CB.DbgLoc);
571
28.7k
  MIB.setMBB(*CB.ThisBB);
572
28.7k
573
28.7k
  if (CB.PredInfo.NoCmp) {
574
4
    // Branch or fall through to TrueBB.
575
4
    addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
576
4
    addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
577
4
                      CB.ThisBB);
578
4
    CB.ThisBB->normalizeSuccProbs();
579
4
    if (CB.TrueBB != CB.ThisBB->getNextNode())
580
4
      MIB.buildBr(*CB.TrueBB);
581
4
    MIB.setDebugLoc(OldDbgLoc);
582
4
    return;
583
4
  }
584
28.7k
585
28.7k
  const LLT i1Ty = LLT::scalar(1);
586
28.7k
  // Build the compare.
587
28.7k
  if (!CB.CmpMHS) {
588
28.2k
    Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
589
28.2k
    Cond = MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
590
28.2k
  } else {
591
538
    assert(CB.PredInfo.Pred == CmpInst::ICMP_ULE &&
592
538
           "Can only handle ULE ranges");
593
538
594
538
    const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
595
538
    const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
596
538
597
538
    Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
598
538
    if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
599
0
      Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
600
0
      Cond =
601
0
          MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, CmpOpReg, CondRHS).getReg(0);
602
538
    } else {
603
538
      const LLT &CmpTy = MRI->getType(CmpOpReg);
604
538
      auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
605
538
      auto Diff = MIB.buildConstant(CmpTy, High - Low);
606
538
      Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);
607
538
    }
608
538
  }
609
28.7k
610
28.7k
  // Update successor info
611
28.7k
  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);
612
28.7k
613
28.7k
  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},
614
28.7k
                    CB.ThisBB);
615
28.7k
616
28.7k
  // TrueBB and FalseBB are always different unless the incoming IR is
617
28.7k
  // degenerate. This only happens when running llc on weird IR.
618
28.7k
  if (CB.TrueBB != CB.FalseBB)
619
28.7k
    addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);
620
28.7k
  CB.ThisBB->normalizeSuccProbs();
621
28.7k
622
28.7k
  //  if (SwitchBB->getBasicBlock() != CB.FalseBB->getBasicBlock())
623
28.7k
    addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
624
28.7k
                      CB.ThisBB);
625
28.7k
626
28.7k
  // If the lhs block is the next block, invert the condition so that we can
627
28.7k
  // fall through to the lhs instead of the rhs block.
628
28.7k
  if (CB.TrueBB == CB.ThisBB->getNextNode()) {
629
10.4k
    std::swap(CB.TrueBB, CB.FalseBB);
630
10.4k
    auto True = MIB.buildConstant(i1Ty, 1);
631
10.4k
    Cond = MIB.buildInstr(TargetOpcode::G_XOR, {i1Ty}, {Cond, True}, None)
632
10.4k
               .getReg(0);
633
10.4k
  }
634
28.7k
635
28.7k
  MIB.buildBrCond(Cond, *CB.TrueBB);
636
28.7k
  MIB.buildBr(*CB.FalseBB);
637
28.7k
  MIB.setDebugLoc(OldDbgLoc);
638
28.7k
}
639
640
bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
641
                                          MachineBasicBlock *SwitchMBB,
642
                                          MachineBasicBlock *CurMBB,
643
                                          MachineBasicBlock *DefaultMBB,
644
                                          MachineIRBuilder &MIB,
645
                                          MachineFunction::iterator BBI,
646
                                          BranchProbability UnhandledProbs,
647
                                          SwitchCG::CaseClusterIt I,
648
                                          MachineBasicBlock *Fallthrough,
649
2.27k
                                          bool FallthroughUnreachable) {
650
2.27k
  using namespace SwitchCG;
651
2.27k
  MachineFunction *CurMF = SwitchMBB->getParent();
652
2.27k
  // FIXME: Optimize away range check based on pivot comparisons.
653
2.27k
  JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
654
2.27k
  SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
655
2.27k
  BranchProbability DefaultProb = W.DefaultProb;
656
2.27k
657
2.27k
  // The jump block hasn't been inserted yet; insert it here.
658
2.27k
  MachineBasicBlock *JumpMBB = JT->MBB;
659
2.27k
  CurMF->insert(BBI, JumpMBB);
660
2.27k
661
2.27k
  // Since the jump table block is separate from the switch block, we need
662
2.27k
  // to keep track of it as a machine predecessor to the default block,
663
2.27k
  // otherwise we lose the phi edges.
664
2.27k
  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
665
2.27k
                    CurMBB);
666
2.27k
  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},
667
2.27k
                    JumpMBB);
668
2.27k
669
2.27k
  auto JumpProb = I->Prob;
670
2.27k
  auto FallthroughProb = UnhandledProbs;
671
2.27k
672
2.27k
  // If the default statement is a target of the jump table, we evenly
673
2.27k
  // distribute the default probability to successors of CurMBB. Also
674
2.27k
  // update the probability on the edge from JumpMBB to Fallthrough.
675
2.27k
  for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
676
2.27k
                                        SE = JumpMBB->succ_end();
677
17.3k
       SI != SE; 
++SI15.0k
) {
678
15.0k
    if (*SI == DefaultMBB) {
679
532
      JumpProb += DefaultProb / 2;
680
532
      FallthroughProb -= DefaultProb / 2;
681
532
      JumpMBB->setSuccProbability(SI, DefaultProb / 2);
682
532
      JumpMBB->normalizeSuccProbs();
683
14.5k
    } else {
684
14.5k
      // Also record edges from the jump table block to it's successors.
685
14.5k
      addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},
686
14.5k
                        JumpMBB);
687
14.5k
    }
688
15.0k
  }
689
2.27k
690
2.27k
  // Skip the range check if the fallthrough block is unreachable.
691
2.27k
  if (FallthroughUnreachable)
692
7
    JTH->OmitRangeCheck = true;
693
2.27k
694
2.27k
  if (!JTH->OmitRangeCheck)
695
2.26k
    addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
696
2.27k
  addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
697
2.27k
  CurMBB->normalizeSuccProbs();
698
2.27k
699
2.27k
  // The jump table header will be inserted in our current block, do the
700
2.27k
  // range check, and fall through to our fallthrough block.
701
2.27k
  JTH->HeaderBB = CurMBB;
702
2.27k
  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
703
2.27k
704
2.27k
  // If we're in the right place, emit the jump table header right now.
705
2.27k
  if (CurMBB == SwitchMBB) {
706
2.26k
    if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
707
0
      return false;
708
2.26k
    JTH->Emitted = true;
709
2.26k
  }
710
2.27k
  return true;
711
2.27k
}
712
bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
713
                                            Value *Cond,
714
                                            MachineBasicBlock *Fallthrough,
715
                                            bool FallthroughUnreachable,
716
                                            BranchProbability UnhandledProbs,
717
                                            MachineBasicBlock *CurMBB,
718
                                            MachineIRBuilder &MIB,
719
28.7k
                                            MachineBasicBlock *SwitchMBB) {
720
28.7k
  using namespace SwitchCG;
721
28.7k
  const Value *RHS, *LHS, *MHS;
722
28.7k
  CmpInst::Predicate Pred;
723
28.7k
  if (I->Low == I->High) {
724
28.2k
    // Check Cond == I->Low.
725
28.2k
    Pred = CmpInst::ICMP_EQ;
726
28.2k
    LHS = Cond;
727
28.2k
    RHS = I->Low;
728
28.2k
    MHS = nullptr;
729
28.2k
  } else {
730
538
    // Check I->Low <= Cond <= I->High.
731
538
    Pred = CmpInst::ICMP_ULE;
732
538
    LHS = I->Low;
733
538
    MHS = Cond;
734
538
    RHS = I->High;
735
538
  }
736
28.7k
737
28.7k
  // If Fallthrough is unreachable, fold away the comparison.
738
28.7k
  // The false probability is the sum of all unhandled cases.
739
28.7k
  CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,
740
28.7k
               CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);
741
28.7k
742
28.7k
  emitSwitchCase(CB, SwitchMBB, MIB);
743
28.7k
  return true;
744
28.7k
}
745
746
bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,
747
                                       Value *Cond,
748
                                       MachineBasicBlock *SwitchMBB,
749
                                       MachineBasicBlock *DefaultMBB,
750
14.9k
                                       MachineIRBuilder &MIB) {
751
14.9k
  using namespace SwitchCG;
752
14.9k
  MachineFunction *CurMF = FuncInfo.MF;
753
14.9k
  MachineBasicBlock *NextMBB = nullptr;
754
14.9k
  MachineFunction::iterator BBI(W.MBB);
755
14.9k
  if (++BBI != FuncInfo.MF->end())
756
14.9k
    NextMBB = &*BBI;
757
14.9k
758
14.9k
  if (EnableOpts) {
759
14.9k
    // Here, we order cases by probability so the most likely case will be
760
14.9k
    // checked first. However, two clusters can have the same probability in
761
14.9k
    // which case their relative ordering is non-deterministic. So we use Low
762
14.9k
    // as a tie-breaker as clusters are guaranteed to never overlap.
763
14.9k
    llvm::sort(W.FirstCluster, W.LastCluster + 1,
764
16.3k
               [](const CaseCluster &a, const CaseCluster &b) {
765
16.3k
                 return a.Prob != b.Prob
766
16.3k
                            ? 
a.Prob > b.Prob790
767
16.3k
                            : 
a.Low->getValue().slt(b.Low->getValue())15.5k
;
768
16.3k
               });
769
14.9k
770
14.9k
    // Rearrange the case blocks so that the last one falls through if possible
771
14.9k
    // without changing the order of probabilities.
772
20.7k
    for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {
773
15.3k
      --I;
774
15.3k
      if (I->Prob > W.LastCluster->Prob)
775
376
        break;
776
14.9k
      if (I->Kind == CC_Range && I->MBB == NextMBB) {
777
9.12k
        std::swap(*I, *W.LastCluster);
778
9.12k
        break;
779
9.12k
      }
780
14.9k
    }
781
14.9k
  }
782
14.9k
783
14.9k
  // Compute total probability.
784
14.9k
  BranchProbability DefaultProb = W.DefaultProb;
785
14.9k
  BranchProbability UnhandledProbs = DefaultProb;
786
46.0k
  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; 
++I31.0k
)
787
31.0k
    UnhandledProbs += I->Prob;
788
14.9k
789
14.9k
  MachineBasicBlock *CurMBB = W.MBB;
790
46.0k
  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; 
++I31.0k
) {
791
31.0k
    bool FallthroughUnreachable = false;
792
31.0k
    MachineBasicBlock *Fallthrough;
793
31.0k
    if (I == W.LastCluster) {
794
14.9k
      // For the last cluster, fall through to the default destination.
795
14.9k
      Fallthrough = DefaultMBB;
796
14.9k
      FallthroughUnreachable = isa<UnreachableInst>(
797
14.9k
          DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());
798
16.0k
    } else {
799
16.0k
      Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
800
16.0k
      CurMF->insert(BBI, Fallthrough);
801
16.0k
    }
802
31.0k
    UnhandledProbs -= I->Prob;
803
31.0k
804
31.0k
    switch (I->Kind) {
805
31.0k
    case CC_BitTests: {
806
0
      LLVM_DEBUG(dbgs() << "Switch to bit test optimization unimplemented");
807
0
      return false; // Bit tests currently unimplemented.
808
31.0k
    }
809
31.0k
    case CC_JumpTable: {
810
2.27k
      if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,
811
2.27k
                                  UnhandledProbs, I, Fallthrough,
812
2.27k
                                  FallthroughUnreachable)) {
813
0
        LLVM_DEBUG(dbgs() << "Failed to lower jump table");
814
0
        return false;
815
0
      }
816
2.27k
      break;
817
2.27k
    }
818
28.7k
    case CC_Range: {
819
28.7k
      if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,
820
28.7k
                                    FallthroughUnreachable, UnhandledProbs,
821
28.7k
                                    CurMBB, MIB, SwitchMBB)) {
822
0
        LLVM_DEBUG(dbgs() << "Failed to lower switch range");
823
0
        return false;
824
0
      }
825
28.7k
      break;
826
28.7k
    }
827
31.0k
    }
828
31.0k
    CurMBB = Fallthrough;
829
31.0k
  }
830
14.9k
831
14.9k
  return true;
832
14.9k
}
833
834
bool IRTranslator::translateIndirectBr(const User &U,
835
5
                                       MachineIRBuilder &MIRBuilder) {
836
5
  const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);
837
5
838
5
  const Register Tgt = getOrCreateVReg(*BrInst.getAddress());
839
5
  MIRBuilder.buildBrIndirect(Tgt);
840
5
841
5
  // Link successors.
842
5
  MachineBasicBlock &CurBB = MIRBuilder.getMBB();
843
5
  for (const BasicBlock *Succ : successors(&BrInst))
844
41
    CurBB.addSuccessor(&getMBB(*Succ));
845
5
846
5
  return true;
847
5
}
848
849
4.84M
static bool isSwiftError(const Value *V) {
850
4.84M
  if (auto Arg = dyn_cast<Argument>(V))
851
503k
    return Arg->hasSwiftErrorAttr();
852
4.33M
  if (auto AI = dyn_cast<AllocaInst>(V))
853
65.2k
    return AI->isSwiftError();
854
4.27M
  return false;
855
4.27M
}
856
857
1.22M
bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
858
1.22M
  const LoadInst &LI = cast<LoadInst>(U);
859
1.22M
860
1.22M
  auto Flags = LI.isVolatile() ? 
MachineMemOperand::MOVolatile2.45k
861
1.22M
                               : 
MachineMemOperand::MONone1.22M
;
862
1.22M
  Flags |= MachineMemOperand::MOLoad;
863
1.22M
864
1.22M
  if (DL->getTypeStoreSize(LI.getType()) == 0)
865
2
    return true;
866
1.22M
867
1.22M
  ArrayRef<Register> Regs = getOrCreateVRegs(LI);
868
1.22M
  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
869
1.22M
  Register Base = getOrCreateVReg(*LI.getPointerOperand());
870
1.22M
871
1.22M
  Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType());
872
1.22M
  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
873
1.22M
874
1.22M
  if (CLI->supportSwiftError() && 
isSwiftError(LI.getPointerOperand())1.22M
) {
875
5
    assert(Regs.size() == 1 && "swifterror should be single pointer");
876
5
    Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(),
877
5
                                                    LI.getPointerOperand());
878
5
    MIRBuilder.buildCopy(Regs[0], VReg);
879
5
    return true;
880
5
  }
881
1.22M
882
1.22M
  const MDNode *Ranges =
883
1.22M
      Regs.size() == 1 ? 
LI.getMetadata(LLVMContext::MD_range)1.22M
:
nullptr41
;
884
2.45M
  for (unsigned i = 0; i < Regs.size(); 
++i1.22M
) {
885
1.22M
    Register Addr;
886
1.22M
    MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
887
1.22M
888
1.22M
    MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8);
889
1.22M
    unsigned BaseAlign = getMemOpAlignment(LI);
890
1.22M
    auto MMO = MF->getMachineMemOperand(
891
1.22M
        Ptr, Flags, (MRI->getType(Regs[i]).getSizeInBits() + 7) / 8,
892
1.22M
        MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), Ranges,
893
1.22M
        LI.getSyncScopeID(), LI.getOrdering());
894
1.22M
    MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
895
1.22M
  }
896
1.22M
897
1.22M
  return true;
898
1.22M
}
899
900
888k
bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
901
888k
  const StoreInst &SI = cast<StoreInst>(U);
902
888k
  auto Flags = SI.isVolatile() ? 
MachineMemOperand::MOVolatile402
903
888k
                               : 
MachineMemOperand::MONone887k
;
904
888k
  Flags |= MachineMemOperand::MOStore;
905
888k
906
888k
  if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0)
907
2
    return true;
908
888k
909
888k
  ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
910
888k
  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
911
888k
  Register Base = getOrCreateVReg(*SI.getPointerOperand());
912
888k
913
888k
  Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType());
914
888k
  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
915
888k
916
888k
  if (CLI->supportSwiftError() && 
isSwiftError(SI.getPointerOperand())887k
) {
917
12
    assert(Vals.size() == 1 && "swifterror should be single pointer");
918
12
919
12
    Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),
920
12
                                                    SI.getPointerOperand());
921
12
    MIRBuilder.buildCopy(VReg, Vals[0]);
922
12
    return true;
923
12
  }
924
888k
925
1.77M
  
for (unsigned i = 0; 888k
i < Vals.size();
++i888k
) {
926
888k
    Register Addr;
927
888k
    MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8);
928
888k
929
888k
    MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8);
930
888k
    unsigned BaseAlign = getMemOpAlignment(SI);
931
888k
    auto MMO = MF->getMachineMemOperand(
932
888k
        Ptr, Flags, (MRI->getType(Vals[i]).getSizeInBits() + 7) / 8,
933
888k
        MinAlign(BaseAlign, Offsets[i] / 8), AAMDNodes(), nullptr,
934
888k
        SI.getSyncScopeID(), SI.getOrdering());
935
888k
    MIRBuilder.buildStore(Vals[i], Addr, *MMO);
936
888k
  }
937
888k
  return true;
938
888k
}
939
940
25.2k
static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {
941
25.2k
  const Value *Src = U.getOperand(0);
942
25.2k
  Type *Int32Ty = Type::getInt32Ty(U.getContext());
943
25.2k
944
25.2k
  // getIndexedOffsetInType is designed for GEPs, so the first index is the
945
25.2k
  // usual array element rather than looking into the actual aggregate.
946
25.2k
  SmallVector<Value *, 1> Indices;
947
25.2k
  Indices.push_back(ConstantInt::get(Int32Ty, 0));
948
25.2k
949
25.2k
  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {
950
24.0k
    for (auto Idx : EVI->indices())
951
24.0k
      Indices.push_back(ConstantInt::get(Int32Ty, Idx));
952
24.0k
  } else 
if (const InsertValueInst *1.21k
IVI1.21k
= dyn_cast<InsertValueInst>(&U)) {
953
1.21k
    for (auto Idx : IVI->indices())
954
1.46k
      Indices.push_back(ConstantInt::get(Int32Ty, Idx));
955
1.21k
  } else {
956
0
    for (unsigned i = 1; i < U.getNumOperands(); ++i)
957
0
      Indices.push_back(U.getOperand(i));
958
0
  }
959
25.2k
960
25.2k
  return 8 * static_cast<uint64_t>(
961
25.2k
                 DL.getIndexedOffsetInType(Src->getType(), Indices));
962
25.2k
}
963
964
bool IRTranslator::translateExtractValue(const User &U,
965
24.0k
                                         MachineIRBuilder &MIRBuilder) {
966
24.0k
  const Value *Src = U.getOperand(0);
967
24.0k
  uint64_t Offset = getOffsetFromIndices(U, *DL);
968
24.0k
  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
969
24.0k
  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);
970
24.0k
  unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();
971
24.0k
  auto &DstRegs = allocateVRegs(U);
972
24.0k
973
48.2k
  for (unsigned i = 0; i < DstRegs.size(); 
++i24.1k
)
974
24.1k
    DstRegs[i] = SrcRegs[Idx++];
975
24.0k
976
24.0k
  return true;
977
24.0k
}
978
979
bool IRTranslator::translateInsertValue(const User &U,
980
1.21k
                                        MachineIRBuilder &MIRBuilder) {
981
1.21k
  const Value *Src = U.getOperand(0);
982
1.21k
  uint64_t Offset = getOffsetFromIndices(U, *DL);
983
1.21k
  auto &DstRegs = allocateVRegs(U);
984
1.21k
  ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);
985
1.21k
  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);
986
1.21k
  ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));
987
1.21k
  auto InsertedIt = InsertedRegs.begin();
988
1.21k
989
4.38k
  for (unsigned i = 0; i < DstRegs.size(); 
++i3.17k
) {
990
3.17k
    if (DstOffsets[i] >= Offset && 
InsertedIt != InsertedRegs.end()2.20k
)
991
1.21k
      DstRegs[i] = *InsertedIt++;
992
1.96k
    else
993
1.96k
      DstRegs[i] = SrcRegs[i];
994
3.17k
  }
995
1.21k
996
1.21k
  return true;
997
1.21k
}
998
999
bool IRTranslator::translateSelect(const User &U,
1000
80.4k
                                   MachineIRBuilder &MIRBuilder) {
1001
80.4k
  Register Tst = getOrCreateVReg(*U.getOperand(0));
1002
80.4k
  ArrayRef<Register> ResRegs = getOrCreateVRegs(U);
1003
80.4k
  ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));
1004
80.4k
  ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
1005
80.4k
1006
80.4k
  const SelectInst &SI = cast<SelectInst>(U);
1007
80.4k
  uint16_t Flags = 0;
1008
80.4k
  if (const CmpInst *Cmp = dyn_cast<CmpInst>(SI.getCondition()))
1009
74.8k
    Flags = MachineInstr::copyFlagsFromInstruction(*Cmp);
1010
80.4k
1011
160k
  for (unsigned i = 0; i < ResRegs.size(); 
++i80.4k
) {
1012
80.4k
    MIRBuilder.buildInstr(TargetOpcode::G_SELECT, {ResRegs[i]},
1013
80.4k
                          {Tst, Op0Regs[i], Op1Regs[i]}, Flags);
1014
80.4k
  }
1015
80.4k
1016
80.4k
  return true;
1017
80.4k
}
1018
1019
bool IRTranslator::translateBitCast(const User &U,
1020
1.20M
                                    MachineIRBuilder &MIRBuilder) {
1021
1.20M
  // If we're bitcasting to the source type, we can reuse the source vreg.
1022
1.20M
  if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==
1023
1.20M
      getLLTForType(*U.getType(), *DL)) {
1024
1.20M
    Register SrcReg = getOrCreateVReg(*U.getOperand(0));
1025
1.20M
    auto &Regs = *VMap.getVRegs(U);
1026
1.20M
    // If we already assigned a vreg for this bitcast, we can't change that.
1027
1.20M
    // Emit a copy to satisfy the users we already emitted.
1028
1.20M
    if (!Regs.empty())
1029
43.0k
      MIRBuilder.buildCopy(Regs[0], SrcReg);
1030
1.16M
    else {
1031
1.16M
      Regs.push_back(SrcReg);
1032
1.16M
      VMap.getOffsets(U)->push_back(0);
1033
1.16M
    }
1034
1.20M
    return true;
1035
1.20M
  }
1036
1.71k
  return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);
1037
1.71k
}
1038
1039
bool IRTranslator::translateCast(unsigned Opcode, const User &U,
1040
610k
                                 MachineIRBuilder &MIRBuilder) {
1041
610k
  Register Op = getOrCreateVReg(*U.getOperand(0));
1042
610k
  Register Res = getOrCreateVReg(U);
1043
610k
  MIRBuilder.buildInstr(Opcode, {Res}, {Op});
1044
610k
  return true;
1045
610k
}
1046
1047
bool IRTranslator::translateGetElementPtr(const User &U,
1048
2.10M
                                          MachineIRBuilder &MIRBuilder) {
1049
2.10M
  // FIXME: support vector GEPs.
1050
2.10M
  if (U.getType()->isVectorTy())
1051
31
    return false;
1052
2.10M
1053
2.10M
  Value &Op0 = *U.getOperand(0);
1054
2.10M
  Register BaseReg = getOrCreateVReg(Op0);
1055
2.10M
  Type *PtrIRTy = Op0.getType();
1056
2.10M
  LLT PtrTy = getLLTForType(*PtrIRTy, *DL);
1057
2.10M
  Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy);
1058
2.10M
  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1059
2.10M
1060
2.10M
  int64_t Offset = 0;
1061
2.10M
  for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);
1062
6.07M
       GTI != E; 
++GTI3.97M
) {
1063
3.97M
    const Value *Idx = GTI.getOperand();
1064
3.97M
    if (StructType *StTy = GTI.getStructTypeOrNull()) {
1065
1.08M
      unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
1066
1.08M
      Offset += DL->getStructLayout(StTy)->getElementOffset(Field);
1067
1.08M
      continue;
1068
2.88M
    } else {
1069
2.88M
      uint64_t ElementSize = DL->getTypeAllocSize(GTI.getIndexedType());
1070
2.88M
1071
2.88M
      // If this is a scalar constant or a splat vector of constants,
1072
2.88M
      // handle it quickly.
1073
2.88M
      if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
1074
2.63M
        Offset += ElementSize * CI->getSExtValue();
1075
2.63M
        continue;
1076
2.63M
      }
1077
250k
1078
250k
      if (Offset != 0) {
1079
11.3k
        Register NewBaseReg = MRI->createGenericVirtualRegister(PtrTy);
1080
11.3k
        LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
1081
11.3k
        auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);
1082
11.3k
        MIRBuilder.buildGEP(NewBaseReg, BaseReg, OffsetMIB.getReg(0));
1083
11.3k
1084
11.3k
        BaseReg = NewBaseReg;
1085
11.3k
        Offset = 0;
1086
11.3k
      }
1087
250k
1088
250k
      Register IdxReg = getOrCreateVReg(*Idx);
1089
250k
      if (MRI->getType(IdxReg) != OffsetTy) {
1090
8
        Register NewIdxReg = MRI->createGenericVirtualRegister(OffsetTy);
1091
8
        MIRBuilder.buildSExtOrTrunc(NewIdxReg, IdxReg);
1092
8
        IdxReg = NewIdxReg;
1093
8
      }
1094
250k
1095
250k
      // N = N + Idx * ElementSize;
1096
250k
      // Avoid doing it for ElementSize of 1.
1097
250k
      Register GepOffsetReg;
1098
250k
      if (ElementSize != 1) {
1099
150k
        GepOffsetReg = MRI->createGenericVirtualRegister(OffsetTy);
1100
150k
        auto ElementSizeMIB = MIRBuilder.buildConstant(
1101
150k
            getLLTForType(*OffsetIRTy, *DL), ElementSize);
1102
150k
        MIRBuilder.buildMul(GepOffsetReg, ElementSizeMIB.getReg(0), IdxReg);
1103
150k
      } else
1104
100k
        GepOffsetReg = IdxReg;
1105
250k
1106
250k
      Register NewBaseReg = MRI->createGenericVirtualRegister(PtrTy);
1107
250k
      MIRBuilder.buildGEP(NewBaseReg, BaseReg, GepOffsetReg);
1108
250k
      BaseReg = NewBaseReg;
1109
250k
    }
1110
3.97M
  }
1111
2.10M
1112
2.10M
  if (Offset != 0) {
1113
1.52M
    auto OffsetMIB =
1114
1.52M
        MIRBuilder.buildConstant(getLLTForType(*OffsetIRTy, *DL), Offset);
1115
1.52M
    MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
1116
1.52M
    return true;
1117
1.52M
  }
1118
577k
1119
577k
  MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
1120
577k
  return true;
1121
577k
}
1122
1123
bool IRTranslator::translateMemFunc(const CallInst &CI,
1124
                                    MachineIRBuilder &MIRBuilder,
1125
35.5k
                                    Intrinsic::ID ID) {
1126
35.5k
1127
35.5k
  // If the source is undef, then just emit a nop.
1128
35.5k
  if (isa<UndefValue>(CI.getArgOperand(1)))
1129
3
    return true;
1130
35.5k
1131
35.5k
  ArrayRef<Register> Res;
1132
35.5k
  auto ICall = MIRBuilder.buildIntrinsic(ID, Res, true);
1133
142k
  for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; 
++AI106k
)
1134
106k
    ICall.addUse(getOrCreateVReg(**AI));
1135
35.5k
1136
35.5k
  unsigned DstAlign = 0, SrcAlign = 0;
1137
35.5k
  unsigned IsVol =
1138
35.5k
      cast<ConstantInt>(CI.getArgOperand(CI.getNumArgOperands() - 1))
1139
35.5k
          ->getZExtValue();
1140
35.5k
1141
35.5k
  if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {
1142
12.8k
    DstAlign = std::max<unsigned>(MCI->getDestAlignment(), 1);
1143
12.8k
    SrcAlign = std::max<unsigned>(MCI->getSourceAlignment(), 1);
1144
22.6k
  } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {
1145
508
    DstAlign = std::max<unsigned>(MMI->getDestAlignment(), 1);
1146
508
    SrcAlign = std::max<unsigned>(MMI->getSourceAlignment(), 1);
1147
22.1k
  } else {
1148
22.1k
    auto *MSI = cast<MemSetInst>(&CI);
1149
22.1k
    DstAlign = std::max<unsigned>(MSI->getDestAlignment(), 1);
1150
22.1k
  }
1151
35.5k
1152
35.5k
  // Create mem operands to store the alignment and volatile info.
1153
35.5k
  auto VolFlag = IsVol ? 
MachineMemOperand::MOVolatile0
: MachineMemOperand::MONone;
1154
35.5k
  ICall.addMemOperand(MF->getMachineMemOperand(
1155
35.5k
      MachinePointerInfo(CI.getArgOperand(0)),
1156
35.5k
      MachineMemOperand::MOStore | VolFlag, 1, DstAlign));
1157
35.5k
  if (ID != Intrinsic::memset)
1158
13.4k
    ICall.addMemOperand(MF->getMachineMemOperand(
1159
13.4k
        MachinePointerInfo(CI.getArgOperand(1)),
1160
13.4k
        MachineMemOperand::MOLoad | VolFlag, 1, SrcAlign));
1161
35.5k
1162
35.5k
  return true;
1163
35.5k
}
1164
1165
void IRTranslator::getStackGuard(Register DstReg,
1166
5.67k
                                 MachineIRBuilder &MIRBuilder) {
1167
5.67k
  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1168
5.67k
  MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
1169
5.67k
  auto MIB = MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD);
1170
5.67k
  MIB.addDef(DstReg);
1171
5.67k
1172
5.67k
  auto &TLI = *MF->getSubtarget().getTargetLowering();
1173
5.67k
  Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
1174
5.67k
  if (!Global)
1175
0
    return;
1176
5.67k
1177
5.67k
  MachinePointerInfo MPInfo(Global);
1178
5.67k
  auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1179
5.67k
               MachineMemOperand::MODereferenceable;
1180
5.67k
  MachineMemOperand *MemRef =
1181
5.67k
      MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
1182
5.67k
                               DL->getPointerABIAlignment(0));
1183
5.67k
  MIB.setMemRefs({MemRef});
1184
5.67k
}
1185
1186
bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
1187
2.00k
                                              MachineIRBuilder &MIRBuilder) {
1188
2.00k
  ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);
1189
2.00k
  MIRBuilder.buildInstr(Op)
1190
2.00k
      .addDef(ResRegs[0])
1191
2.00k
      .addDef(ResRegs[1])
1192
2.00k
      .addUse(getOrCreateVReg(*CI.getOperand(0)))
1193
2.00k
      .addUse(getOrCreateVReg(*CI.getOperand(1)));
1194
2.00k
1195
2.00k
  return true;
1196
2.00k
}
1197
1198
216k
unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
1199
216k
  switch (ID) {
1200
216k
    default:
1201
212k
      break;
1202
216k
    case Intrinsic::bswap:
1203
198
      return TargetOpcode::G_BSWAP;
1204
216k
    case Intrinsic::ceil:
1205
56
      return TargetOpcode::G_FCEIL;
1206
216k
    case Intrinsic::cos:
1207
209
      return TargetOpcode::G_FCOS;
1208
216k
    case Intrinsic::ctpop:
1209
36
      return TargetOpcode::G_CTPOP;
1210
216k
    case Intrinsic::exp:
1211
161
      return TargetOpcode::G_FEXP;
1212
216k
    case Intrinsic::exp2:
1213
29
      return TargetOpcode::G_FEXP2;
1214
216k
    case Intrinsic::fabs:
1215
1.35k
      return TargetOpcode::G_FABS;
1216
216k
    case Intrinsic::copysign:
1217
63
      return TargetOpcode::G_FCOPYSIGN;
1218
216k
    case Intrinsic::minnum:
1219
4
      return TargetOpcode::G_FMINNUM;
1220
216k
    case Intrinsic::maxnum:
1221
4
      return TargetOpcode::G_FMAXNUM;
1222
216k
    case Intrinsic::minimum:
1223
2
      return TargetOpcode::G_FMINIMUM;
1224
216k
    case Intrinsic::maximum:
1225
1
      return TargetOpcode::G_FMAXIMUM;
1226
216k
    case Intrinsic::canonicalize:
1227
2
      return TargetOpcode::G_FCANONICALIZE;
1228
216k
    case Intrinsic::floor:
1229
157
      return TargetOpcode::G_FFLOOR;
1230
216k
    case Intrinsic::fma:
1231
15
      return TargetOpcode::G_FMA;
1232
216k
    case Intrinsic::log:
1233
70
      return TargetOpcode::G_FLOG;
1234
216k
    case Intrinsic::log2:
1235
17
      return TargetOpcode::G_FLOG2;
1236
216k
    case Intrinsic::log10:
1237
70
      return TargetOpcode::G_FLOG10;
1238
216k
    case Intrinsic::nearbyint:
1239
17
      return TargetOpcode::G_FNEARBYINT;
1240
216k
    case Intrinsic::pow:
1241
78
      return TargetOpcode::G_FPOW;
1242
216k
    case Intrinsic::rint:
1243
38
      return TargetOpcode::G_FRINT;
1244
216k
    case Intrinsic::round:
1245
11
      return TargetOpcode::G_INTRINSIC_ROUND;
1246
216k
    case Intrinsic::sin:
1247
170
      return TargetOpcode::G_FSIN;
1248
216k
    case Intrinsic::sqrt:
1249
872
      return TargetOpcode::G_FSQRT;
1250
216k
    case Intrinsic::trunc:
1251
17
      return TargetOpcode::G_INTRINSIC_TRUNC;
1252
212k
  }
1253
212k
  return Intrinsic::not_intrinsic;
1254
212k
}
1255
1256
bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
1257
                                            Intrinsic::ID ID,
1258
216k
                                            MachineIRBuilder &MIRBuilder) {
1259
216k
1260
216k
  unsigned Op = getSimpleIntrinsicOpcode(ID);
1261
216k
1262
216k
  // Is this a simple intrinsic?
1263
216k
  if (Op == Intrinsic::not_intrinsic)
1264
212k
    return false;
1265
3.65k
1266
3.65k
  // Yes. Let's translate it.
1267
3.65k
  SmallVector<llvm::SrcOp, 4> VRegs;
1268
3.65k
  for (auto &Arg : CI.arg_operands())
1269
3.83k
    VRegs.push_back(getOrCreateVReg(*Arg));
1270
3.65k
1271
3.65k
  MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
1272
3.65k
                        MachineInstr::copyFlagsFromInstruction(CI));
1273
3.65k
  return true;
1274
3.65k
}
1275
1276
bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
1277
216k
                                           MachineIRBuilder &MIRBuilder) {
1278
216k
1279
216k
  // If this is a simple intrinsic (that is, we just need to add a def of
1280
216k
  // a vreg, and uses for each arg operand, then translate it.
1281
216k
  if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
1282
3.65k
    return true;
1283
212k
1284
212k
  switch (ID) {
1285
212k
  default:
1286
66.8k
    break;
1287
212k
  case Intrinsic::lifetime_start:
1288
100k
  case Intrinsic::lifetime_end: {
1289
100k
    // No stack colouring in O0, discard region information.
1290
100k
    if (MF->getTarget().getOptLevel() == CodeGenOpt::None)
1291
2
      return true;
1292
100k
1293
100k
    unsigned Op = ID == Intrinsic::lifetime_start ? 
TargetOpcode::LIFETIME_START43.4k
1294
100k
                                                  : 
TargetOpcode::LIFETIME_END57.5k
;
1295
100k
1296
100k
    // Get the underlying objects for the location passed on the lifetime
1297
100k
    // marker.
1298
100k
    SmallVector<const Value *, 4> Allocas;
1299
100k
    GetUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL);
1300
100k
1301
100k
    // Iterate over each underlying object, creating lifetime markers for each
1302
100k
    // static alloca. Quit if we find a non-static alloca.
1303
101k
    for (const Value *V : Allocas) {
1304
101k
      const AllocaInst *AI = dyn_cast<AllocaInst>(V);
1305
101k
      if (!AI)
1306
0
        continue;
1307
101k
1308
101k
      if (!AI->isStaticAlloca())
1309
2
        return true;
1310
101k
1311
101k
      MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
1312
101k
    }
1313
100k
    
return true100k
;
1314
100k
  }
1315
100k
  case Intrinsic::dbg_declare: {
1316
3
    const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
1317
3
    assert(DI.getVariable() && "Missing variable");
1318
3
1319
3
    const Value *Address = DI.getAddress();
1320
3
    if (!Address || isa<UndefValue>(Address)) {
1321
0
      LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n");
1322
0
      return true;
1323
0
    }
1324
3
1325
3
    assert(DI.getVariable()->isValidLocationForIntrinsic(
1326
3
               MIRBuilder.getDebugLoc()) &&
1327
3
           "Expected inlined-at fields to agree");
1328
3
    auto AI = dyn_cast<AllocaInst>(Address);
1329
3
    if (AI && 
AI->isStaticAlloca()2
) {
1330
1
      // Static allocas are tracked at the MF level, no need for DBG_VALUE
1331
1
      // instructions (in fact, they get ignored if they *do* exist).
1332
1
      MF->setVariableDbgInfo(DI.getVariable(), DI.getExpression(),
1333
1
                             getOrCreateFrameIndex(*AI), DI.getDebugLoc());
1334
2
    } else {
1335
2
      // A dbg.declare describes the address of a source variable, so lower it
1336
2
      // into an indirect DBG_VALUE.
1337
2
      MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
1338
2
                                       DI.getVariable(), DI.getExpression());
1339
2
    }
1340
3
    return true;
1341
3
  }
1342
3
  case Intrinsic::dbg_label: {
1343
0
    const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
1344
0
    assert(DI.getLabel() && "Missing label");
1345
0
1346
0
    assert(DI.getLabel()->isValidLocationForIntrinsic(
1347
0
               MIRBuilder.getDebugLoc()) &&
1348
0
           "Expected inlined-at fields to agree");
1349
0
1350
0
    MIRBuilder.buildDbgLabel(DI.getLabel());
1351
0
    return true;
1352
3
  }
1353
68
  case Intrinsic::vaend:
1354
68
    // No target I know of cares about va_end. Certainly no in-tree target
1355
68
    // does. Simplest intrinsic ever!
1356
68
    return true;
1357
67
  case Intrinsic::vastart: {
1358
67
    auto &TLI = *MF->getSubtarget().getTargetLowering();
1359
67
    Value *Ptr = CI.getArgOperand(0);
1360
67
    unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
1361
67
1362
67
    // FIXME: Get alignment
1363
67
    MIRBuilder.buildInstr(TargetOpcode::G_VASTART)
1364
67
        .addUse(getOrCreateVReg(*Ptr))
1365
67
        .addMemOperand(MF->getMachineMemOperand(
1366
67
            MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 1));
1367
67
    return true;
1368
3
  }
1369
10
  case Intrinsic::dbg_value: {
1370
10
    // This form of DBG_VALUE is target-independent.
1371
10
    const DbgValueInst &DI = cast<DbgValueInst>(CI);
1372
10
    const Value *V = DI.getValue();
1373
10
    assert(DI.getVariable()->isValidLocationForIntrinsic(
1374
10
               MIRBuilder.getDebugLoc()) &&
1375
10
           "Expected inlined-at fields to agree");
1376
10
    if (!V) {
1377
0
      // Currently the optimizer can produce this; insert an undef to
1378
0
      // help debugging.  Probably the optimizer should not do this.
1379
0
      MIRBuilder.buildIndirectDbgValue(0, DI.getVariable(), DI.getExpression());
1380
10
    } else if (const auto *CI = dyn_cast<Constant>(V)) {
1381
5
      MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression());
1382
5
    } else {
1383
5
      Register Reg = getOrCreateVReg(*V);
1384
5
      // FIXME: This does not handle register-indirect values at offset 0. The
1385
5
      // direct/indirect thing shouldn't really be handled by something as
1386
5
      // implicit as reg+noreg vs reg+imm in the first palce, but it seems
1387
5
      // pretty baked in right now.
1388
5
      MIRBuilder.buildDirectDbgValue(Reg, DI.getVariable(), DI.getExpression());
1389
5
    }
1390
10
    return true;
1391
3
  }
1392
1.40k
  case Intrinsic::uadd_with_overflow:
1393
1.40k
    return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);
1394
16
  case Intrinsic::sadd_with_overflow:
1395
16
    return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);
1396
10
  case Intrinsic::usub_with_overflow:
1397
10
    return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);
1398
11
  case Intrinsic::ssub_with_overflow:
1399
11
    return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);
1400
550
  case Intrinsic::umul_with_overflow:
1401
550
    return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
1402
12
  case Intrinsic::smul_with_overflow:
1403
12
    return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
1404
4
  case Intrinsic::fmuladd: {
1405
4
    const TargetMachine &TM = MF->getTarget();
1406
4
    const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1407
4
    Register Dst = getOrCreateVReg(CI);
1408
4
    Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
1409
4
    Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
1410
4
    Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
1411
4
    if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
1412
4
        
TLI.isFMAFasterThanFMulAndFAdd(TLI.getValueType(*DL, CI.getType()))3
) {
1413
1
      // TODO: Revisit this to see if we should move this part of the
1414
1
      // lowering to the combiner.
1415
1
      MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2},
1416
1
                            MachineInstr::copyFlagsFromInstruction(CI));
1417
3
    } else {
1418
3
      LLT Ty = getLLTForType(*CI.getType(), *DL);
1419
3
      auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, {Ty}, {Op0, Op1},
1420
3
                                        MachineInstr::copyFlagsFromInstruction(CI));
1421
3
      MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2},
1422
3
                            MachineInstr::copyFlagsFromInstruction(CI));
1423
3
    }
1424
4
    return true;
1425
3
  }
1426
35.5k
  case Intrinsic::memcpy:
1427
35.5k
  case Intrinsic::memmove:
1428
35.5k
  case Intrinsic::memset:
1429
35.5k
    return translateMemFunc(CI, MIRBuilder, ID);
1430
35.5k
  case Intrinsic::eh_typeid_for: {
1431
154
    GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0));
1432
154
    Register Reg = getOrCreateVReg(CI);
1433
154
    unsigned TypeID = MF->getTypeIDFor(GV);
1434
154
    MIRBuilder.buildConstant(Reg, TypeID);
1435
154
    return true;
1436
35.5k
  }
1437
35.5k
  case Intrinsic::objectsize: {
1438
5
    // If we don't know by now, we're never going to know.
1439
5
    const ConstantInt *Min = cast<ConstantInt>(CI.getArgOperand(1));
1440
5
1441
5
    MIRBuilder.buildConstant(getOrCreateVReg(CI), Min->isZero() ? 
-1ULL3
:
02
);
1442
5
    return true;
1443
35.5k
  }
1444
35.5k
  case Intrinsic::is_constant:
1445
2
    // If this wasn't constant-folded away by now, then it's not a
1446
2
    // constant.
1447
2
    MIRBuilder.buildConstant(getOrCreateVReg(CI), 0);
1448
2
    return true;
1449
35.5k
  case Intrinsic::stackguard:
1450
3.85k
    getStackGuard(getOrCreateVReg(CI), MIRBuilder);
1451
3.85k
    return true;
1452
35.5k
  case Intrinsic::stackprotector: {
1453
1.81k
    LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
1454
1.81k
    Register GuardVal = MRI->createGenericVirtualRegister(PtrTy);
1455
1.81k
    getStackGuard(GuardVal, MIRBuilder);
1456
1.81k
1457
1.81k
    AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));
1458
1.81k
    int FI = getOrCreateFrameIndex(*Slot);
1459
1.81k
    MF->getFrameInfo().setStackProtectorIndex(FI);
1460
1.81k
1461
1.81k
    MIRBuilder.buildStore(
1462
1.81k
        GuardVal, getOrCreateVReg(*Slot),
1463
1.81k
        *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
1464
1.81k
                                  MachineMemOperand::MOStore |
1465
1.81k
                                      MachineMemOperand::MOVolatile,
1466
1.81k
                                  PtrTy.getSizeInBits() / 8, 8));
1467
1.81k
    return true;
1468
35.5k
  }
1469
35.5k
  case Intrinsic::stacksave: {
1470
20
    // Save the stack pointer to the location provided by the intrinsic.
1471
20
    Register Reg = getOrCreateVReg(CI);
1472
20
    Register StackPtr = MF->getSubtarget()
1473
20
                            .getTargetLowering()
1474
20
                            ->getStackPointerRegisterToSaveRestore();
1475
20
1476
20
    // If the target doesn't specify a stack pointer, then fall back.
1477
20
    if (!StackPtr)
1478
0
      return false;
1479
20
1480
20
    MIRBuilder.buildCopy(Reg, StackPtr);
1481
20
    return true;
1482
20
  }
1483
20
  case Intrinsic::stackrestore: {
1484
20
    // Restore the stack pointer from the location provided by the intrinsic.
1485
20
    Register Reg = getOrCreateVReg(*CI.getArgOperand(0));
1486
20
    Register StackPtr = MF->getSubtarget()
1487
20
                            .getTargetLowering()
1488
20
                            ->getStackPointerRegisterToSaveRestore();
1489
20
1490
20
    // If the target doesn't specify a stack pointer, then fall back.
1491
20
    if (!StackPtr)
1492
0
      return false;
1493
20
1494
20
    MIRBuilder.buildCopy(StackPtr, Reg);
1495
20
    return true;
1496
20
  }
1497
1.47k
  case Intrinsic::cttz:
1498
1.47k
  case Intrinsic::ctlz: {
1499
1.47k
    ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
1500
1.47k
    bool isTrailing = ID == Intrinsic::cttz;
1501
1.47k
    unsigned Opcode = isTrailing
1502
1.47k
                          ? 
Cst->isZero() 10
?
TargetOpcode::G_CTTZ4
1503
10
                                          : 
TargetOpcode::G_CTTZ_ZERO_UNDEF6
1504
1.47k
                          : 
Cst->isZero() 1.46k
?
TargetOpcode::G_CTLZ39
1505
1.46k
                                          : 
TargetOpcode::G_CTLZ_ZERO_UNDEF1.42k
;
1506
1.47k
    MIRBuilder.buildInstr(Opcode)
1507
1.47k
        .addDef(getOrCreateVReg(CI))
1508
1.47k
        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
1509
1.47k
    return true;
1510
1.47k
  }
1511
1.47k
  case Intrinsic::invariant_start: {
1512
49
    LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
1513
49
    Register Undef = MRI->createGenericVirtualRegister(PtrTy);
1514
49
    MIRBuilder.buildUndef(Undef);
1515
49
    return true;
1516
1.47k
  }
1517
1.47k
  case Intrinsic::invariant_end:
1518
2
    return true;
1519
1.47k
  case Intrinsic::assume:
1520
21
  case Intrinsic::var_annotation:
1521
21
  case Intrinsic::sideeffect:
1522
21
    // Discard annotate attributes, assumptions, and artificial side-effects.
1523
21
    return true;
1524
66.8k
  }
1525
66.8k
  return false;
1526
66.8k
}
1527
1528
bool IRTranslator::translateInlineAsm(const CallInst &CI,
1529
40
                                      MachineIRBuilder &MIRBuilder) {
1530
40
  const InlineAsm &IA = cast<InlineAsm>(*CI.getCalledValue());
1531
40
  if (!IA.getConstraintString().empty())
1532
36
    return false;
1533
4
1534
4
  unsigned ExtraInfo = 0;
1535
4
  if (IA.hasSideEffects())
1536
2
    ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1537
4
  if (IA.getDialect() == InlineAsm::AD_Intel)
1538
0
    ExtraInfo |= InlineAsm::Extra_AsmDialect;
1539
4
1540
4
  MIRBuilder.buildInstr(TargetOpcode::INLINEASM)
1541
4
    .addExternalSymbol(IA.getAsmString().c_str())
1542
4
    .addImm(ExtraInfo);
1543
4
1544
4
  return true;
1545
4
}
1546
1547
1.41M
bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
1548
1.41M
  const CallInst &CI = cast<CallInst>(U);
1549
1.41M
  auto TII = MF->getTarget().getIntrinsicInfo();
1550
1.41M
  const Function *F = CI.getCalledFunction();
1551
1.41M
1552
1.41M
  // FIXME: support Windows dllimport function calls.
1553
1.41M
  if (F && 
F->hasDLLImportStorageClass()1.34M
)
1554
1
    return false;
1555
1.41M
1556
1.41M
  if (CI.isInlineAsm())
1557
40
    return translateInlineAsm(CI, MIRBuilder);
1558
1.41M
1559
1.41M
  Intrinsic::ID ID = Intrinsic::not_intrinsic;
1560
1.41M
  if (F && 
F->isIntrinsic()1.34M
) {
1561
216k
    ID = F->getIntrinsicID();
1562
216k
    if (TII && 
ID == Intrinsic::not_intrinsic0
)
1563
0
      ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
1564
216k
  }
1565
1.41M
1566
1.41M
  if (!F || 
!F->isIntrinsic()1.34M
||
ID == Intrinsic::not_intrinsic216k
) {
1567
1.19M
    ArrayRef<Register> Res = getOrCreateVRegs(CI);
1568
1.19M
1569
1.19M
    SmallVector<ArrayRef<Register>, 8> Args;
1570
1.19M
    Register SwiftInVReg = 0;
1571
1.19M
    Register SwiftErrorVReg = 0;
1572
2.67M
    for (auto &Arg: CI.arg_operands()) {
1573
2.67M
      if (CLI->supportSwiftError() && 
isSwiftError(Arg)2.67M
) {
1574
12
        assert(SwiftInVReg == 0 && "Expected only one swift error argument");
1575
12
        LLT Ty = getLLTForType(*Arg->getType(), *DL);
1576
12
        SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
1577
12
        MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
1578
12
                                              &CI, &MIRBuilder.getMBB(), Arg));
1579
12
        Args.emplace_back(makeArrayRef(SwiftInVReg));
1580
12
        SwiftErrorVReg =
1581
12
            SwiftError.getOrCreateVRegDefAt(&CI, &MIRBuilder.getMBB(), Arg);
1582
12
        continue;
1583
12
      }
1584
2.67M
      Args.push_back(getOrCreateVRegs(*Arg));
1585
2.67M
    }
1586
1.19M
1587
1.19M
    MF->getFrameInfo().setHasCalls(true);
1588
1.19M
    bool Success =
1589
1.19M
        CLI->lowerCall(MIRBuilder, &CI, Res, Args, SwiftErrorVReg,
1590
1.19M
                       [&]() 
{ return getOrCreateVReg(*CI.getCalledValue()); }63.5k
);
1591
1.19M
1592
1.19M
    return Success;
1593
1.19M
  }
1594
216k
1595
216k
  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
1596
216k
1597
216k
  if (translateKnownIntrinsic(CI, ID, MIRBuilder))
1598
149k
    return true;
1599
66.8k
1600
66.8k
  ArrayRef<Register> ResultRegs;
1601
66.8k
  if (!CI.getType()->isVoidTy())
1602
49.7k
    ResultRegs = getOrCreateVRegs(CI);
1603
66.8k
1604
66.8k
  // Ignore the callsite attributes. Backend code is most likely not expecting
1605
66.8k
  // an intrinsic to sometimes have side effects and sometimes not.
1606
66.8k
  MachineInstrBuilder MIB =
1607
66.8k
      MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory());
1608
66.8k
  if (isa<FPMathOperator>(CI))
1609
275
    MIB->copyIRFlags(CI);
1610
66.8k
1611
123k
  for (auto &Arg : CI.arg_operands()) {
1612
123k
    // Some intrinsics take metadata parameters. Reject them.
1613
123k
    if (isa<MetadataAsValue>(Arg))
1614
3
      return false;
1615
123k
    ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg);
1616
123k
    if (VRegs.size() > 1)
1617
0
      return false;
1618
123k
    MIB.addUse(VRegs[0]);
1619
123k
  }
1620
66.8k
1621
66.8k
  // Add a MachineMemOperand if it is a target mem intrinsic.
1622
66.8k
  const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
1623
66.8k
  TargetLowering::IntrinsicInfo Info;
1624
66.8k
  // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
1625
66.8k
  if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
1626
58.2k
    unsigned Align = Info.align;
1627
58.2k
    if (Align == 0)
1628
13.5k
      Align = DL->getABITypeAlignment(Info.memVT.getTypeForEVT(F->getContext()));
1629
58.2k
1630
58.2k
    uint64_t Size = Info.memVT.getStoreSize();
1631
58.2k
    MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
1632
58.2k
                                               Info.flags, Size, Align));
1633
58.2k
  }
1634
66.8k
1635
66.8k
  return true;
1636
66.8k
}
1637
1638
bool IRTranslator::translateInvoke(const User &U,
1639
24.3k
                                   MachineIRBuilder &MIRBuilder) {
1640
24.3k
  const InvokeInst &I = cast<InvokeInst>(U);
1641
24.3k
  MCContext &Context = MF->getContext();
1642
24.3k
1643
24.3k
  const BasicBlock *ReturnBB = I.getSuccessor(0);
1644
24.3k
  const BasicBlock *EHPadBB = I.getSuccessor(1);
1645
24.3k
1646
24.3k
  const Value *Callee = I.getCalledValue();
1647
24.3k
  const Function *Fn = dyn_cast<Function>(Callee);
1648
24.3k
  if (isa<InlineAsm>(Callee))
1649
0
    return false;
1650
24.3k
1651
24.3k
  // FIXME: support invoking patchpoint and statepoint intrinsics.
1652
24.3k
  if (Fn && 
Fn->isIntrinsic()20.6k
)
1653
0
    return false;
1654
24.3k
1655
24.3k
  // FIXME: support whatever these are.
1656
24.3k
  if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
1657
0
    return false;
1658
24.3k
1659
24.3k
  // FIXME: support Windows exception handling.
1660
24.3k
  if (!isa<LandingPadInst>(EHPadBB->front()))
1661
75
    return false;
1662
24.3k
1663
24.3k
  // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
1664
24.3k
  // the region covered by the try.
1665
24.3k
  MCSymbol *BeginSymbol = Context.createTempSymbol();
1666
24.3k
  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
1667
24.3k
1668
24.3k
  ArrayRef<Register> Res;
1669
24.3k
  if (!I.getType()->isVoidTy())
1670
17.6k
    Res = getOrCreateVRegs(I);
1671
24.3k
  SmallVector<ArrayRef<Register>, 8> Args;
1672
24.3k
  Register SwiftErrorVReg = 0;
1673
24.3k
  Register SwiftInVReg = 0;
1674
48.9k
  for (auto &Arg : I.arg_operands()) {
1675
48.9k
    if (CLI->supportSwiftError() && isSwiftError(Arg)) {
1676
0
      assert(SwiftInVReg == 0 && "Expected only one swift error argument");
1677
0
      LLT Ty = getLLTForType(*Arg->getType(), *DL);
1678
0
      SwiftInVReg = MRI->createGenericVirtualRegister(Ty);
1679
0
      MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(
1680
0
                                            &I, &MIRBuilder.getMBB(), Arg));
1681
0
      Args.push_back(makeArrayRef(SwiftInVReg));
1682
0
      SwiftErrorVReg =
1683
0
          SwiftError.getOrCreateVRegDefAt(&I, &MIRBuilder.getMBB(), Arg);
1684
0
      continue;
1685
0
    }
1686
48.9k
1687
48.9k
    Args.push_back(getOrCreateVRegs(*Arg));
1688
48.9k
  }
1689
24.3k
1690
24.3k
  if (!CLI->lowerCall(MIRBuilder, &I, Res, Args, SwiftErrorVReg,
1691
24.3k
                      [&]() 
{ return getOrCreateVReg(*I.getCalledValue()); }3.74k
))
1692
2
    return false;
1693
24.3k
1694
24.3k
  MCSymbol *EndSymbol = Context.createTempSymbol();
1695
24.3k
  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
1696
24.3k
1697
24.3k
  // FIXME: track probabilities.
1698
24.3k
  MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
1699
24.3k
                    &ReturnMBB = getMBB(*ReturnBB);
1700
24.3k
  MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
1701
24.3k
  MIRBuilder.getMBB().addSuccessor(&ReturnMBB);
1702
24.3k
  MIRBuilder.getMBB().addSuccessor(&EHPadMBB);
1703
24.3k
  MIRBuilder.buildBr(ReturnMBB);
1704
24.3k
1705
24.3k
  return true;
1706
24.3k
}
1707
1708
bool IRTranslator::translateCallBr(const User &U,
1709
0
                                   MachineIRBuilder &MIRBuilder) {
1710
0
  // FIXME: Implement this.
1711
0
  return false;
1712
0
}
1713
1714
bool IRTranslator::translateLandingPad(const User &U,
1715
14.0k
                                       MachineIRBuilder &MIRBuilder) {
1716
14.0k
  const LandingPadInst &LP = cast<LandingPadInst>(U);
1717
14.0k
1718
14.0k
  MachineBasicBlock &MBB = MIRBuilder.getMBB();
1719
14.0k
1720
14.0k
  MBB.setIsEHPad();
1721
14.0k
1722
14.0k
  // If there aren't registers to copy the values into (e.g., during SjLj
1723
14.0k
  // exceptions), then don't bother.
1724
14.0k
  auto &TLI = *MF->getSubtarget().getTargetLowering();
1725
14.0k
  const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();
1726
14.0k
  if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
1727
14.0k
      
TLI.getExceptionSelectorRegister(PersonalityFn) == 00
)
1728
0
    return true;
1729
14.0k
1730
14.0k
  // If landingpad's return type is token type, we don't create DAG nodes
1731
14.0k
  // for its exception pointer and selector value. The extraction of exception
1732
14.0k
  // pointer or selector value from token type landingpads is not currently
1733
14.0k
  // supported.
1734
14.0k
  if (LP.getType()->isTokenTy())
1735
0
    return true;
1736
14.0k
1737
14.0k
  // Add a label to mark the beginning of the landing pad.  Deletion of the
1738
14.0k
  // landing pad can thus be detected via the MachineModuleInfo.
1739
14.0k
  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
1740
14.0k
    .addSym(MF->addLandingPad(&MBB));
1741
14.0k
1742
14.0k
  LLT Ty = getLLTForType(*LP.getType(), *DL);
1743
14.0k
  Register Undef = MRI->createGenericVirtualRegister(Ty);
1744
14.0k
  MIRBuilder.buildUndef(Undef);
1745
14.0k
1746
14.0k
  SmallVector<LLT, 2> Tys;
1747
14.0k
  for (Type *Ty : cast<StructType>(LP.getType())->elements())
1748
28.1k
    Tys.push_back(getLLTForType(*Ty, *DL));
1749
14.0k
  assert(Tys.size() == 2 && "Only two-valued landingpads are supported");
1750
14.0k
1751
14.0k
  // Mark exception register as live in.
1752
14.0k
  Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn);
1753
14.0k
  if (!ExceptionReg)
1754
0
    return false;
1755
14.0k
1756
14.0k
  MBB.addLiveIn(ExceptionReg);
1757
14.0k
  ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);
1758
14.0k
  MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);
1759
14.0k
1760
14.0k
  Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn);
1761
14.0k
  if (!SelectorReg)
1762
0
    return false;
1763
14.0k
1764
14.0k
  MBB.addLiveIn(SelectorReg);
1765
14.0k
  Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);
1766
14.0k
  MIRBuilder.buildCopy(PtrVReg, SelectorReg);
1767
14.0k
  MIRBuilder.buildCast(ResRegs[1], PtrVReg);
1768
14.0k
1769
14.0k
  return true;
1770
14.0k
}
1771
1772
bool IRTranslator::translateAlloca(const User &U,
1773
47.5k
                                   MachineIRBuilder &MIRBuilder) {
1774
47.5k
  auto &AI = cast<AllocaInst>(U);
1775
47.5k
1776
47.5k
  if (AI.isSwiftError())
1777
7
    return true;
1778
47.5k
1779
47.5k
  if (AI.isStaticAlloca()) {
1780
47.4k
    Register Res = getOrCreateVReg(AI);
1781
47.4k
    int FI = getOrCreateFrameIndex(AI);
1782
47.4k
    MIRBuilder.buildFrameIndex(Res, FI);
1783
47.4k
    return true;
1784
47.4k
  }
1785
103
1786
103
  // FIXME: support stack probing for Windows.
1787
103
  if (MF->getTarget().getTargetTriple().isOSWindows())
1788
1
    return false;
1789
102
1790
102
  // Now we're in the harder dynamic case.
1791
102
  Type *Ty = AI.getAllocatedType();
1792
102
  unsigned Align =
1793
102
      std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI.getAlignment());
1794
102
1795
102
  Register NumElts = getOrCreateVReg(*AI.getArraySize());
1796
102
1797
102
  Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());
1798
102
  LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);
1799
102
  if (MRI->getType(NumElts) != IntPtrTy) {
1800
9
    Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);
1801
9
    MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);
1802
9
    NumElts = ExtElts;
1803
9
  }
1804
102
1805
102
  Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);
1806
102
  Register TySize =
1807
102
      getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, -DL->getTypeAllocSize(Ty)));
1808
102
  MIRBuilder.buildMul(AllocSize, NumElts, TySize);
1809
102
1810
102
  LLT PtrTy = getLLTForType(*AI.getType(), *DL);
1811
102
  auto &TLI = *MF->getSubtarget().getTargetLowering();
1812
102
  Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1813
102
1814
102
  Register SPTmp = MRI->createGenericVirtualRegister(PtrTy);
1815
102
  MIRBuilder.buildCopy(SPTmp, SPReg);
1816
102
1817
102
  Register AllocTmp = MRI->createGenericVirtualRegister(PtrTy);
1818
102
  MIRBuilder.buildGEP(AllocTmp, SPTmp, AllocSize);
1819
102
1820
102
  // Handle alignment. We have to realign if the allocation granule was smaller
1821
102
  // than stack alignment, or the specific alloca requires more than stack
1822
102
  // alignment.
1823
102
  unsigned StackAlign =
1824
102
      MF->getSubtarget().getFrameLowering()->getStackAlignment();
1825
102
  Align = std::max(Align, StackAlign);
1826
102
  if (Align > StackAlign || 
DL->getTypeAllocSize(Ty) % StackAlign != 085
) {
1827
97
    // Round the size of the allocation up to the stack alignment size
1828
97
    // by add SA-1 to the size. This doesn't overflow because we're computing
1829
97
    // an address inside an alloca.
1830
97
    Register AlignedAlloc = MRI->createGenericVirtualRegister(PtrTy);
1831
97
    MIRBuilder.buildPtrMask(AlignedAlloc, AllocTmp, Log2_32(Align));
1832
97
    AllocTmp = AlignedAlloc;
1833
97
  }
1834
102
1835
102
  MIRBuilder.buildCopy(SPReg, AllocTmp);
1836
102
  MIRBuilder.buildCopy(getOrCreateVReg(AI), AllocTmp);
1837
102
1838
102
  MF->getFrameInfo().CreateVariableSizedObject(Align ? Align : 
10
, &AI);
1839
102
  assert(MF->getFrameInfo().hasVarSizedObjects());
1840
102
  return true;
1841
102
}
1842
1843
87
bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
1844
87
  // FIXME: We may need more info about the type. Because of how LLT works,
1845
87
  // we're completely discarding the i64/double distinction here (amongst
1846
87
  // others). Fortunately the ABIs I know of where that matters don't use va_arg
1847
87
  // anyway but that's not guaranteed.
1848
87
  MIRBuilder.buildInstr(TargetOpcode::G_VAARG)
1849
87
    .addDef(getOrCreateVReg(U))
1850
87
    .addUse(getOrCreateVReg(*U.getOperand(0)))
1851
87
    .addImm(DL->getABITypeAlignment(U.getType()));
1852
87
  return true;
1853
87
}
1854
1855
bool IRTranslator::translateInsertElement(const User &U,
1856
13.1k
                                          MachineIRBuilder &MIRBuilder) {
1857
13.1k
  // If it is a <1 x Ty> vector, use the scalar as it is
1858
13.1k
  // not a legal vector type in LLT.
1859
13.1k
  if (U.getType()->getVectorNumElements() == 1) {
1860
10
    Register Elt = getOrCreateVReg(*U.getOperand(1));
1861
10
    auto &Regs = *VMap.getVRegs(U);
1862
10
    if (Regs.empty()) {
1863
10
      Regs.push_back(Elt);
1864
10
      VMap.getOffsets(U)->push_back(0);
1865
10
    } else {
1866
0
      MIRBuilder.buildCopy(Regs[0], Elt);
1867
0
    }
1868
10
    return true;
1869
10
  }
1870
13.1k
1871
13.1k
  Register Res = getOrCreateVReg(U);
1872
13.1k
  Register Val = getOrCreateVReg(*U.getOperand(0));
1873
13.1k
  Register Elt = getOrCreateVReg(*U.getOperand(1));
1874
13.1k
  Register Idx = getOrCreateVReg(*U.getOperand(2));
1875
13.1k
  MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);
1876
13.1k
  return true;
1877
13.1k
}
1878
1879
bool IRTranslator::translateExtractElement(const User &U,
1880
6.11k
                                           MachineIRBuilder &MIRBuilder) {
1881
6.11k
  // If it is a <1 x Ty> vector, use the scalar as it is
1882
6.11k
  // not a legal vector type in LLT.
1883
6.11k
  if (U.getOperand(0)->getType()->getVectorNumElements() == 1) {
1884
1.06k
    Register Elt = getOrCreateVReg(*U.getOperand(0));
1885
1.06k
    auto &Regs = *VMap.getVRegs(U);
1886
1.06k
    if (Regs.empty()) {
1887
1.06k
      Regs.push_back(Elt);
1888
1.06k
      VMap.getOffsets(U)->push_back(0);
1889
1.06k
    } else {
1890
0
      MIRBuilder.buildCopy(Regs[0], Elt);
1891
0
    }
1892
1.06k
    return true;
1893
1.06k
  }
1894
5.05k
  Register Res = getOrCreateVReg(U);
1895
5.05k
  Register Val = getOrCreateVReg(*U.getOperand(0));
1896
5.05k
  const auto &TLI = *MF->getSubtarget().getTargetLowering();
1897
5.05k
  unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits();
1898
5.05k
  Register Idx;
1899
5.05k
  if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {
1900
5.05k
    if (CI->getBitWidth() != PreferredVecIdxWidth) {
1901
4.90k
      APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth);
1902
4.90k
      auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);
1903
4.90k
      Idx = getOrCreateVReg(*NewIdxCI);
1904
4.90k
    }
1905
5.05k
  }
1906
5.05k
  if (!Idx)
1907
155
    Idx = getOrCreateVReg(*U.getOperand(1));
1908
5.05k
  if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {
1909
2
    const LLT &VecIdxTy = LLT::scalar(PreferredVecIdxWidth);
1910
2
    Idx = MIRBuilder.buildSExtOrTrunc(VecIdxTy, Idx)->getOperand(0).getReg();
1911
2
  }
1912
5.05k
  MIRBuilder.buildExtractVectorElement(Res, Val, Idx);
1913
5.05k
  return true;
1914
5.05k
}
1915
1916
bool IRTranslator::translateShuffleVector(const User &U,
1917
28.3k
                                          MachineIRBuilder &MIRBuilder) {
1918
28.3k
  MIRBuilder.buildInstr(TargetOpcode::G_SHUFFLE_VECTOR)
1919
28.3k
      .addDef(getOrCreateVReg(U))
1920
28.3k
      .addUse(getOrCreateVReg(*U.getOperand(0)))
1921
28.3k
      .addUse(getOrCreateVReg(*U.getOperand(1)))
1922
28.3k
      .addUse(getOrCreateVReg(*U.getOperand(2)));
1923
28.3k
  return true;
1924
28.3k
}
1925
1926
533k
bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {
1927
533k
  const PHINode &PI = cast<PHINode>(U);
1928
533k
1929
533k
  SmallVector<MachineInstr *, 4> Insts;
1930
535k
  for (auto Reg : getOrCreateVRegs(PI)) {
1931
535k
    auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});
1932
535k
    Insts.push_back(MIB.getInstr());
1933
535k
  }
1934
533k
1935
533k
  PendingPHIs.emplace_back(&PI, std::move(Insts));
1936
533k
  return true;
1937
533k
}
1938
1939
bool IRTranslator::translateAtomicCmpXchg(const User &U,
1940
3
                                          MachineIRBuilder &MIRBuilder) {
1941
3
  const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);
1942
3
1943
3
  if (I.isWeak())
1944
0
    return false;
1945
3
1946
3
  auto Flags = I.isVolatile() ? 
MachineMemOperand::MOVolatile0
1947
3
                              : MachineMemOperand::MONone;
1948
3
  Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1949
3
1950
3
  Type *ResType = I.getType();
1951
3
  Type *ValType = ResType->Type::getStructElementType(0);
1952
3
1953
3
  auto Res = getOrCreateVRegs(I);
1954
3
  Register OldValRes = Res[0];
1955
3
  Register SuccessRes = Res[1];
1956
3
  Register Addr = getOrCreateVReg(*I.getPointerOperand());
1957
3
  Register Cmp = getOrCreateVReg(*I.getCompareOperand());
1958
3
  Register NewVal = getOrCreateVReg(*I.getNewValOperand());
1959
3
1960
3
  MIRBuilder.buildAtomicCmpXchgWithSuccess(
1961
3
      OldValRes, SuccessRes, Addr, Cmp, NewVal,
1962
3
      *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
1963
3
                                Flags, DL->getTypeStoreSize(ValType),
1964
3
                                getMemOpAlignment(I), AAMDNodes(), nullptr,
1965
3
                                I.getSyncScopeID(), I.getSuccessOrdering(),
1966
3
                                I.getFailureOrdering()));
1967
3
  return true;
1968
3
}
1969
1970
bool IRTranslator::translateAtomicRMW(const User &U,
1971
22
                                      MachineIRBuilder &MIRBuilder) {
1972
22
  const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
1973
22
1974
22
  auto Flags = I.isVolatile() ? 
MachineMemOperand::MOVolatile0
1975
22
                              : MachineMemOperand::MONone;
1976
22
  Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1977
22
1978
22
  Type *ResType = I.getType();
1979
22
1980
22
  Register Res = getOrCreateVReg(I);
1981
22
  Register Addr = getOrCreateVReg(*I.getPointerOperand());
1982
22
  Register Val = getOrCreateVReg(*I.getValOperand());
1983
22
1984
22
  unsigned Opcode = 0;
1985
22
  switch (I.getOperation()) {
1986
22
  default:
1987
0
    llvm_unreachable("Unknown atomicrmw op");
1988
22
    
return false0
;
1989
22
  case AtomicRMWInst::Xchg:
1990
2
    Opcode = TargetOpcode::G_ATOMICRMW_XCHG;
1991
2
    break;
1992
22
  case AtomicRMWInst::Add:
1993
2
    Opcode = TargetOpcode::G_ATOMICRMW_ADD;
1994
2
    break;
1995
22
  case AtomicRMWInst::Sub:
1996
2
    Opcode = TargetOpcode::G_ATOMICRMW_SUB;
1997
2
    break;
1998
22
  case AtomicRMWInst::And:
1999
2
    Opcode = TargetOpcode::G_ATOMICRMW_AND;
2000
2
    break;
2001
22
  case AtomicRMWInst::Nand:
2002
2
    Opcode = TargetOpcode::G_ATOMICRMW_NAND;
2003
2
    break;
2004
22
  case AtomicRMWInst::Or:
2005
2
    Opcode = TargetOpcode::G_ATOMICRMW_OR;
2006
2
    break;
2007
22
  case AtomicRMWInst::Xor:
2008
2
    Opcode = TargetOpcode::G_ATOMICRMW_XOR;
2009
2
    break;
2010
22
  case AtomicRMWInst::Max:
2011
2
    Opcode = TargetOpcode::G_ATOMICRMW_MAX;
2012
2
    break;
2013
22
  case AtomicRMWInst::Min:
2014
2
    Opcode = TargetOpcode::G_ATOMICRMW_MIN;
2015
2
    break;
2016
22
  case AtomicRMWInst::UMax:
2017
2
    Opcode = TargetOpcode::G_ATOMICRMW_UMAX;
2018
2
    break;
2019
22
  case AtomicRMWInst::UMin:
2020
2
    Opcode = TargetOpcode::G_ATOMICRMW_UMIN;
2021
2
    break;
2022
22
  }
2023
22
2024
22
  MIRBuilder.buildAtomicRMW(
2025
22
      Opcode, Res, Addr, Val,
2026
22
      *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
2027
22
                                Flags, DL->getTypeStoreSize(ResType),
2028
22
                                getMemOpAlignment(I), AAMDNodes(), nullptr,
2029
22
                                I.getSyncScopeID(), I.getOrdering()));
2030
22
  return true;
2031
22
}
2032
2033
bool IRTranslator::translateFence(const User &U,
2034
1.62k
                                  MachineIRBuilder &MIRBuilder) {
2035
1.62k
  const FenceInst &Fence = cast<FenceInst>(U);
2036
1.62k
  MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
2037
1.62k
                        Fence.getSyncScopeID());
2038
1.62k
  return true;
2039
1.62k
}
2040
2041
237k
void IRTranslator::finishPendingPhis() {
2042
#ifndef NDEBUG
2043
  DILocationVerifier Verifier;
2044
  GISelObserverWrapper WrapperObserver(&Verifier);
2045
  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2046
#endif // ifndef NDEBUG
2047
532k
  for (auto &Phi : PendingPHIs) {
2048
532k
    const PHINode *PI = Phi.first;
2049
532k
    ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
2050
532k
    MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();
2051
532k
    EntryBuilder->setDebugLoc(PI->getDebugLoc());
2052
#ifndef NDEBUG
2053
    Verifier.setCurrentInst(PI);
2054
#endif // ifndef NDEBUG
2055
2056
532k
    SmallSet<const MachineBasicBlock *, 16> SeenPreds;
2057
1.81M
    for (unsigned i = 0; i < PI->getNumIncomingValues(); 
++i1.28M
) {
2058
1.28M
      auto IRPred = PI->getIncomingBlock(i);
2059
1.28M
      ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
2060
1.29M
      for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) {
2061
1.29M
        if (SeenPreds.count(Pred) || 
!PhiMBB->isPredecessor(Pred)1.28M
)
2062
6.91k
          continue;
2063
1.28M
        SeenPreds.insert(Pred);
2064
2.57M
        for (unsigned j = 0; j < ValRegs.size(); 
++j1.28M
) {
2065
1.28M
          MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);
2066
1.28M
          MIB.addUse(ValRegs[j]);
2067
1.28M
          MIB.addMBB(Pred);
2068
1.28M
        }
2069
1.28M
      }
2070
1.28M
    }
2071
532k
  }
2072
237k
}
2073
2074
bool IRTranslator::valueIsSplit(const Value &V,
2075
0
                                SmallVectorImpl<uint64_t> *Offsets) {
2076
0
  SmallVector<LLT, 4> SplitTys;
2077
0
  if (Offsets && !Offsets->empty())
2078
0
    Offsets->clear();
2079
0
  computeValueLLTs(*DL, *V.getType(), SplitTys, Offsets);
2080
0
  return SplitTys.size() > 1;
2081
0
}
2082
2083
11.6M
bool IRTranslator::translate(const Instruction &Inst) {
2084
11.6M
  CurBuilder->setDebugLoc(Inst.getDebugLoc());
2085
11.6M
  // We only emit constants into the entry block from here. To prevent jumpy
2086
11.6M
  // debug behaviour set the line to 0.
2087
11.6M
  if (const DebugLoc &DL = Inst.getDebugLoc())
2088
29
    EntryBuilder->setDebugLoc(
2089
29
        DebugLoc::get(0, 0, DL.getScope(), DL.getInlinedAt()));
2090
11.6M
  else
2091
11.6M
    EntryBuilder->setDebugLoc(DebugLoc());
2092
11.6M
2093
11.6M
  switch (Inst.getOpcode()) {
2094
11.6M
#define HANDLE_INST(NUM, OPCODE, CLASS)                                        \
2095
11.6M
  case Instruction::OPCODE:                                                    \
2096
11.6M
    return translate##OPCODE(Inst, *CurBuilder.get());
2097
11.6M
#include "llvm/IR/Instruction.def"
2098
11.6M
  default:
2099
0
    return false;
2100
11.6M
  }
2101
11.6M
}
2102
2103
2.13M
bool IRTranslator::translate(const Constant &C, Register Reg) {
2104
2.13M
  if (auto CI = dyn_cast<ConstantInt>(&C))
2105
956k
    EntryBuilder->buildConstant(Reg, *CI);
2106
1.17M
  else if (auto CF = dyn_cast<ConstantFP>(&C))
2107
28.4k
    EntryBuilder->buildFConstant(Reg, *CF);
2108
1.14M
  else if (isa<UndefValue>(C))
2109
5.25k
    EntryBuilder->buildUndef(Reg);
2110
1.14M
  else if (isa<ConstantPointerNull>(C)) {
2111
114k
    // As we are trying to build a constant val of 0 into a pointer,
2112
114k
    // insert a cast to make them correct with respect to types.
2113
114k
    unsigned NullSize = DL->getTypeSizeInBits(C.getType());
2114
114k
    auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize);
2115
114k
    auto *ZeroVal = ConstantInt::get(ZeroTy, 0);
2116
114k
    Register ZeroReg = getOrCreateVReg(*ZeroVal);
2117
114k
    EntryBuilder->buildCast(Reg, ZeroReg);
2118
1.02M
  } else if (auto GV = dyn_cast<GlobalValue>(&C))
2119
498k
    EntryBuilder->buildGlobalValue(Reg, GV);
2120
530k
  else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
2121
10.3k
    if (!CAZ->getType()->isVectorTy())
2122
0
      return false;
2123
10.3k
    // Return the scalar if it is a <1 x Ty> vector.
2124
10.3k
    if (CAZ->getNumElements() == 1)
2125
2
      return translate(*CAZ->getElementValue(0u), Reg);
2126
10.3k
    SmallVector<Register, 4> Ops;
2127
39.8k
    for (unsigned i = 0; i < CAZ->getNumElements(); 
++i29.5k
) {
2128
29.5k
      Constant &Elt = *CAZ->getElementValue(i);
2129
29.5k
      Ops.push_back(getOrCreateVReg(Elt));
2130
29.5k
    }
2131
10.3k
    EntryBuilder->buildBuildVector(Reg, Ops);
2132
520k
  } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
2133
36.4k
    // Return the scalar if it is a <1 x Ty> vector.
2134
36.4k
    if (CV->getNumElements() == 1)
2135
290
      return translate(*CV->getElementAsConstant(0), Reg);
2136
36.1k
    SmallVector<Register, 4> Ops;
2137
180k
    for (unsigned i = 0; i < CV->getNumElements(); 
++i144k
) {
2138
144k
      Constant &Elt = *CV->getElementAsConstant(i);
2139
144k
      Ops.push_back(getOrCreateVReg(Elt));
2140
144k
    }
2141
36.1k
    EntryBuilder->buildBuildVector(Reg, Ops);
2142
483k
  } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
2143
482k
    switch(CE->getOpcode()) {
2144
482k
#define HANDLE_INST(NUM, OPCODE, CLASS)                                        \
2145
482k
  case Instruction::OPCODE:                                                    \
2146
482k
    return translate##OPCODE(*CE, *EntryBuilder.get());
2147
482k
#include "llvm/IR/Instruction.def"
2148
482k
    default:
2149
0
      return false;
2150
1.38k
    }
2151
1.38k
  } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
2152
1.38k
    if (CV->getNumOperands() == 1)
2153
0
      return translate(*CV->getOperand(0), Reg);
2154
1.38k
    SmallVector<Register, 4> Ops;
2155
7.96k
    for (unsigned i = 0; i < CV->getNumOperands(); 
++i6.57k
) {
2156
6.57k
      Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
2157
6.57k
    }
2158
1.38k
    EntryBuilder->buildBuildVector(Reg, Ops);
2159
1.38k
  } else 
if (auto *2
BA2
= dyn_cast<BlockAddress>(&C)) {
2160
2
    EntryBuilder->buildBlockAddress(Reg, BA);
2161
2
  } else
2162
0
    return false;
2163
1.65M
2164
1.65M
  return true;
2165
1.65M
}
2166
2167
1.90M
void IRTranslator::finalizeBasicBlock() {
2168
1.90M
  for (auto &JTCase : SL->JTCases) {
2169
2.27k
    // Emit header first, if it wasn't already emitted.
2170
2.27k
    if (!JTCase.first.Emitted)
2171
2
      emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);
2172
2.27k
2173
2.27k
    emitJumpTable(JTCase.second, JTCase.second.MBB);
2174
2.27k
  }
2175
1.90M
  SL->JTCases.clear();
2176
1.90M
}
2177
2178
237k
void IRTranslator::finalizeFunction() {
2179
237k
  // Release the memory used by the different maps we
2180
237k
  // needed during the translation.
2181
237k
  PendingPHIs.clear();
2182
237k
  VMap.reset();
2183
237k
  FrameIndices.clear();
2184
237k
  MachinePreds.clear();
2185
237k
  // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
2186
237k
  // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
2187
237k
  // destroying it twice (in ~IRTranslator() and ~LLVMContext())
2188
237k
  EntryBuilder.reset();
2189
237k
  CurBuilder.reset();
2190
237k
  FuncInfo.clear();
2191
237k
}
2192
2193
237k
bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
2194
237k
  MF = &CurMF;
2195
237k
  const Function &F = MF->getFunction();
2196
237k
  if (F.empty())
2197
0
    return false;
2198
237k
  GISelCSEAnalysisWrapper &Wrapper =
2199
237k
      getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
2200
237k
  // Set the CSEConfig and run the analysis.
2201
237k
  GISelCSEInfo *CSEInfo = nullptr;
2202
237k
  TPC = &getAnalysis<TargetPassConfig>();
2203
237k
  bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
2204
237k
                       ? 
EnableCSEInIRTranslator37
2205
237k
                       : 
TPC->isGISelCSEEnabled()237k
;
2206
237k
2207
237k
  if (EnableCSE) {
2208
237k
    EntryBuilder = make_unique<CSEMIRBuilder>(CurMF);
2209
237k
    CSEInfo = &Wrapper.get(TPC->getCSEConfig());
2210
237k
    EntryBuilder->setCSEInfo(CSEInfo);
2211
237k
    CurBuilder = make_unique<CSEMIRBuilder>(CurMF);
2212
237k
    CurBuilder->setCSEInfo(CSEInfo);
2213
237k
  } else {
2214
36
    EntryBuilder = make_unique<MachineIRBuilder>();
2215
36
    CurBuilder = make_unique<MachineIRBuilder>();
2216
36
  }
2217
237k
  CLI = MF->getSubtarget().getCallLowering();
2218
237k
  CurBuilder->setMF(*MF);
2219
237k
  EntryBuilder->setMF(*MF);
2220
237k
  MRI = &MF->getRegInfo();
2221
237k
  DL = &F.getParent()->getDataLayout();
2222
237k
  ORE = llvm::make_unique<OptimizationRemarkEmitter>(&F);
2223
237k
  FuncInfo.MF = MF;
2224
237k
  FuncInfo.BPI = nullptr;
2225
237k
  const auto &TLI = *MF->getSubtarget().getTargetLowering();
2226
237k
  const TargetMachine &TM = MF->getTarget();
2227
237k
  SL = make_unique<GISelSwitchLowering>(this, FuncInfo);
2228
237k
  SL->init(TLI, TM, *DL);
2229
237k
2230
237k
  EnableOpts = TM.getOptLevel() != CodeGenOpt::None && 
!skipFunction(F)236k
;
2231
237k
2232
237k
  assert(PendingPHIs.empty() && "stale PHIs");
2233
237k
2234
237k
  if (!DL->isLittleEndian()) {
2235
46
    // Currently we don't properly handle big endian code.
2236
46
    OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2237
46
                               F.getSubprogram(), &F.getEntryBlock());
2238
46
    R << "unable to translate in big endian mode";
2239
46
    reportTranslationError(*MF, *TPC, *ORE, R);
2240
46
  }
2241
237k
2242
237k
  // Release the per-function state when we return, whether we succeeded or not.
2243
237k
  auto FinalizeOnReturn = make_scope_exit([this]() 
{ finalizeFunction(); }237k
);
2244
237k
2245
237k
  // Setup a separate basic-block for the arguments and constants
2246
237k
  MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
2247
237k
  MF->push_back(EntryBB);
2248
237k
  EntryBuilder->setMBB(*EntryBB);
2249
237k
2250
237k
  DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
2251
237k
  SwiftError.setFunction(CurMF);
2252
237k
  SwiftError.createEntriesInEntryBlock(DbgLoc);
2253
237k
2254
237k
  // Create all blocks, in IR order, to preserve the layout.
2255
1.92M
  for (const BasicBlock &BB: F) {
2256
1.92M
    auto *&MBB = BBToMBB[&BB];
2257
1.92M
2258
1.92M
    MBB = MF->CreateMachineBasicBlock(&BB);
2259
1.92M
    MF->push_back(MBB);
2260
1.92M
2261
1.92M
    if (BB.hasAddressTaken())
2262
42
      MBB->setHasAddressTaken();
2263
1.92M
  }
2264
237k
2265
237k
  // Make our arguments/constants entry block fallthrough to the IR entry block.
2266
237k
  EntryBB->addSuccessor(&getMBB(F.front()));
2267
237k
2268
237k
  // Lower the actual args into this basic block.
2269
237k
  SmallVector<ArrayRef<Register>, 8> VRegArgs;
2270
561k
  for (const Argument &Arg: F.args()) {
2271
561k
    if (DL->getTypeStoreSize(Arg.getType()) == 0)
2272
3
      continue; // Don't handle zero sized types.
2273
561k
    ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
2274
561k
    VRegArgs.push_back(VRegs);
2275
561k
2276
561k
    if (Arg.hasSwiftErrorAttr()) {
2277
10
      assert(VRegs.size() == 1 && "Too many vregs for Swift error");
2278
10
      SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);
2279
10
    }
2280
561k
  }
2281
237k
2282
237k
  // We don't currently support translating swifterror or swiftself functions.
2283
561k
  for (auto &Arg : F.args()) {
2284
561k
    if (Arg.hasSwiftSelfAttr()) {
2285
2
      OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2286
2
                                 F.getSubprogram(), &F.getEntryBlock());
2287
2
      R << "unable to lower arguments due to swiftself: "
2288
2
        << ore::NV("Prototype", F.getType());
2289
2
      reportTranslationError(*MF, *TPC, *ORE, R);
2290
2
      return false;
2291
2
    }
2292
561k
  }
2293
237k
2294
237k
  
if (237k
!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs)237k
) {
2295
62
    OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2296
62
                               F.getSubprogram(), &F.getEntryBlock());
2297
62
    R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
2298
62
    reportTranslationError(*MF, *TPC, *ORE, R);
2299
62
    return false;
2300
62
  }
2301
237k
2302
237k
  // Need to visit defs before uses when translating instructions.
2303
237k
  GISelObserverWrapper WrapperObserver;
2304
237k
  if (EnableCSE && 
CSEInfo237k
)
2305
237k
    WrapperObserver.addObserver(CSEInfo);
2306
237k
  {
2307
237k
    ReversePostOrderTraversal<const Function *> RPOT(&F);
2308
#ifndef NDEBUG
2309
    DILocationVerifier Verifier;
2310
    WrapperObserver.addObserver(&Verifier);
2311
#endif // ifndef NDEBUG
2312
    RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
2313
1.90M
    for (const BasicBlock *BB : RPOT) {
2314
1.90M
      MachineBasicBlock &MBB = getMBB(*BB);
2315
1.90M
      // Set the insertion point of all the following translations to
2316
1.90M
      // the end of this basic block.
2317
1.90M
      CurBuilder->setMBB(MBB);
2318
1.90M
2319
11.6M
      for (const Instruction &Inst : *BB) {
2320
#ifndef NDEBUG
2321
        Verifier.setCurrentInst(&Inst);
2322
#endif // ifndef NDEBUG
2323
11.6M
        if (translate(Inst))
2324
11.6M
          continue;
2325
184
2326
184
        OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
2327
184
                                   Inst.getDebugLoc(), BB);
2328
184
        R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);
2329
184
2330
184
        if (ORE->allowExtraAnalysis("gisel-irtranslator")) {
2331
41
          std::string InstStrStorage;
2332
41
          raw_string_ostream InstStr(InstStrStorage);
2333
41
          InstStr << Inst;
2334
41
2335
41
          R << ": '" << InstStr.str() << "'";
2336
41
        }
2337
184
2338
184
        reportTranslationError(*MF, *TPC, *ORE, R);
2339
184
        return false;
2340
184
      }
2341
1.90M
2342
1.90M
      finalizeBasicBlock();
2343
1.90M
    }
2344
#ifndef NDEBUG
2345
    WrapperObserver.removeObserver(&Verifier);
2346
#endif
2347
  }
2348
237k
2349
237k
  finishPendingPhis();
2350
237k
2351
237k
  SwiftError.propagateVRegs();
2352
237k
2353
237k
  // Merge the argument lowering and constants block with its single
2354
237k
  // successor, the LLVM-IR entry block.  We want the basic block to
2355
237k
  // be maximal.
2356
237k
  assert(EntryBB->succ_size() == 1 &&
2357
237k
         "Custom BB used for lowering should have only one successor");
2358
237k
  // Get the successor of the current entry block.
2359
237k
  MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();
2360
237k
  assert(NewEntryBB.pred_size() == 1 &&
2361
237k
         "LLVM-IR entry block has a predecessor!?");
2362
237k
  // Move all the instruction from the current entry block to the
2363
237k
  // new entry block.
2364
237k
  NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),
2365
237k
                    EntryBB->end());
2366
237k
2367
237k
  // Update the live-in information for the new entry block.
2368
237k
  for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())
2369
544k
    NewEntryBB.addLiveIn(LiveIn);
2370
237k
  NewEntryBB.sortUniqueLiveIns();
2371
237k
2372
237k
  // Get rid of the now empty basic block.
2373
237k
  EntryBB->removeSuccessor(&NewEntryBB);
2374
237k
  MF->remove(EntryBB);
2375
237k
  MF->DeleteMachineBasicBlock(EntryBB);
2376
237k
2377
237k
  assert(&MF->front() == &NewEntryBB &&
2378
237k
         "New entry wasn't next in the list of basic block!");
2379
237k
2380
237k
  // Initialize stack protector information.
2381
237k
  StackProtector &SP = getAnalysis<StackProtector>();
2382
237k
  SP.copyToMachineFrameInfo(MF->getFrameInfo());
2383
237k
2384
237k
  return false;
2385
237k
}