Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- FastISel.cpp - Implementation of the FastISel class ----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains the implementation of the FastISel class.
10
//
11
// "Fast" instruction selection is designed to emit very poor code quickly.
12
// Also, it is not designed to be able to do much lowering, so most illegal
13
// types (e.g. i64 on 32-bit targets) and operations are not supported.  It is
14
// also not intended to be able to do much optimization, except in a few cases
15
// where doing optimizations reduces overall compile time.  For example, folding
16
// constants into immediate fields is often done, because it's cheap and it
17
// reduces the number of instructions later phases have to examine.
18
//
19
// "Fast" instruction selection is able to fail gracefully and transfer
20
// control to the SelectionDAG selector for operations that it doesn't
21
// support.  In many cases, this allows us to avoid duplicating a lot of
22
// the complicated lowering logic that SelectionDAG currently has.
23
//
24
// The intended use for "fast" instruction selection is "-O0" mode
25
// compilation, where the quality of the generated code is irrelevant when
26
// weighed against the speed at which the code can be generated.  Also,
27
// at -O0, the LLVM optimizers are not running, and this makes the
28
// compile time of codegen a much higher portion of the overall compile
29
// time.  Despite its limitations, "fast" instruction selection is able to
30
// handle enough code on its own to provide noticeable overall speedups
31
// in -O0 compiles.
32
//
33
// Basic operations are supported in a target-independent way, by reading
34
// the same instruction descriptions that the SelectionDAG selector reads,
35
// and identifying simple arithmetic operations that can be directly selected
36
// from simple operators.  More complicated operations currently require
37
// target-specific code.
38
//
39
//===----------------------------------------------------------------------===//
40
41
#include "llvm/CodeGen/FastISel.h"
42
#include "llvm/ADT/APFloat.h"
43
#include "llvm/ADT/APSInt.h"
44
#include "llvm/ADT/DenseMap.h"
45
#include "llvm/ADT/Optional.h"
46
#include "llvm/ADT/SmallPtrSet.h"
47
#include "llvm/ADT/SmallString.h"
48
#include "llvm/ADT/SmallVector.h"
49
#include "llvm/ADT/Statistic.h"
50
#include "llvm/Analysis/BranchProbabilityInfo.h"
51
#include "llvm/Analysis/TargetLibraryInfo.h"
52
#include "llvm/CodeGen/Analysis.h"
53
#include "llvm/CodeGen/FunctionLoweringInfo.h"
54
#include "llvm/CodeGen/ISDOpcodes.h"
55
#include "llvm/CodeGen/MachineBasicBlock.h"
56
#include "llvm/CodeGen/MachineFrameInfo.h"
57
#include "llvm/CodeGen/MachineInstr.h"
58
#include "llvm/CodeGen/MachineInstrBuilder.h"
59
#include "llvm/CodeGen/MachineMemOperand.h"
60
#include "llvm/CodeGen/MachineModuleInfo.h"
61
#include "llvm/CodeGen/MachineOperand.h"
62
#include "llvm/CodeGen/MachineRegisterInfo.h"
63
#include "llvm/CodeGen/StackMaps.h"
64
#include "llvm/CodeGen/TargetInstrInfo.h"
65
#include "llvm/CodeGen/TargetLowering.h"
66
#include "llvm/CodeGen/TargetSubtargetInfo.h"
67
#include "llvm/CodeGen/ValueTypes.h"
68
#include "llvm/IR/Argument.h"
69
#include "llvm/IR/Attributes.h"
70
#include "llvm/IR/BasicBlock.h"
71
#include "llvm/IR/CallSite.h"
72
#include "llvm/IR/CallingConv.h"
73
#include "llvm/IR/Constant.h"
74
#include "llvm/IR/Constants.h"
75
#include "llvm/IR/DataLayout.h"
76
#include "llvm/IR/DebugInfo.h"
77
#include "llvm/IR/DebugLoc.h"
78
#include "llvm/IR/DerivedTypes.h"
79
#include "llvm/IR/Function.h"
80
#include "llvm/IR/GetElementPtrTypeIterator.h"
81
#include "llvm/IR/GlobalValue.h"
82
#include "llvm/IR/InlineAsm.h"
83
#include "llvm/IR/InstrTypes.h"
84
#include "llvm/IR/Instruction.h"
85
#include "llvm/IR/Instructions.h"
86
#include "llvm/IR/IntrinsicInst.h"
87
#include "llvm/IR/LLVMContext.h"
88
#include "llvm/IR/Mangler.h"
89
#include "llvm/IR/Metadata.h"
90
#include "llvm/IR/Operator.h"
91
#include "llvm/IR/PatternMatch.h"
92
#include "llvm/IR/Type.h"
93
#include "llvm/IR/User.h"
94
#include "llvm/IR/Value.h"
95
#include "llvm/MC/MCContext.h"
96
#include "llvm/MC/MCInstrDesc.h"
97
#include "llvm/MC/MCRegisterInfo.h"
98
#include "llvm/Support/Casting.h"
99
#include "llvm/Support/Debug.h"
100
#include "llvm/Support/ErrorHandling.h"
101
#include "llvm/Support/MachineValueType.h"
102
#include "llvm/Support/MathExtras.h"
103
#include "llvm/Support/raw_ostream.h"
104
#include "llvm/Target/TargetMachine.h"
105
#include "llvm/Target/TargetOptions.h"
106
#include <algorithm>
107
#include <cassert>
108
#include <cstdint>
109
#include <iterator>
110
#include <utility>
111
112
using namespace llvm;
113
using namespace PatternMatch;
114
115
#define DEBUG_TYPE "isel"
116
117
// FIXME: Remove this after the feature has proven reliable.
118
static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values",
119
                                     cl::init(true), cl::Hidden,
120
                                     cl::desc("Sink local values in FastISel"));
121
122
STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by "
123
                                         "target-independent selector");
124
STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by "
125
                                    "target-specific selector");
126
STATISTIC(NumFastIselDead, "Number of dead insts removed on failure");
127
128
/// Set the current block to which generated machine instructions will be
129
/// appended.
130
15.8k
void FastISel::startNewBlock() {
131
15.8k
  assert(LocalValueMap.empty() &&
132
15.8k
         "local values should be cleared after finishing a BB");
133
15.8k
134
15.8k
  // Instructions are appended to FuncInfo.MBB. If the basic block already
135
15.8k
  // contains labels or copies, use the last instruction as the last local
136
15.8k
  // value.
137
15.8k
  EmitStartPt = nullptr;
138
15.8k
  if (!FuncInfo.MBB->empty())
139
290
    EmitStartPt = &FuncInfo.MBB->back();
140
15.8k
  LastLocalValue = EmitStartPt;
141
15.8k
}
142
143
/// Flush the local CSE map and sink anything we can.
144
15.8k
void FastISel::finishBasicBlock() { flushLocalValueMap(); }
145
146
13.4k
bool FastISel::lowerArguments() {
147
13.4k
  if (!FuncInfo.CanLowerReturn)
148
152
    // Fallback to SDISel argument lowering code to deal with sret pointer
149
152
    // parameter.
150
152
    return false;
151
13.2k
152
13.2k
  if (!fastLowerArguments())
153
8.09k
    return false;
154
5.18k
155
5.18k
  // Enter arguments into ValueMap for uses in non-entry BBs.
156
5.18k
  for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(),
157
5.18k
                                    E = FuncInfo.Fn->arg_end();
158
11.4k
       I != E; 
++I6.31k
) {
159
6.31k
    DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I);
160
6.31k
    assert(VI != LocalValueMap.end() && "Missed an argument?");
161
6.31k
    FuncInfo.ValueMap[&*I] = VI->second;
162
6.31k
  }
163
5.18k
  return true;
164
5.18k
}
165
166
/// Return the defined register if this instruction defines exactly one
167
/// virtual register and uses no other virtual registers. Otherwise return 0.
168
3.73k
static unsigned findSinkableLocalRegDef(MachineInstr &MI) {
169
3.73k
  unsigned RegDef = 0;
170
11.0k
  for (const MachineOperand &MO : MI.operands()) {
171
11.0k
    if (!MO.isReg())
172
4.12k
      continue;
173
6.91k
    if (MO.isDef()) {
174
4.21k
      if (RegDef)
175
481
        return 0;
176
3.73k
      RegDef = MO.getReg();
177
3.73k
    } else 
if (2.69k
TargetRegisterInfo::isVirtualRegister(MO.getReg())2.69k
) {
178
453
      // This is another use of a vreg. Don't try to sink it.
179
453
      return 0;
180
453
    }
181
6.91k
  }
182
3.73k
  
return RegDef2.80k
;
183
3.73k
}
184
185
18.1k
void FastISel::flushLocalValueMap() {
186
18.1k
  // Try to sink local values down to their first use so that we can give them a
187
18.1k
  // better debug location. This has the side effect of shrinking local value
188
18.1k
  // live ranges, which helps out fast regalloc.
189
18.1k
  if (SinkLocalValues && LastLocalValue != EmitStartPt) {
190
2.72k
    // Sink local value materialization instructions between EmitStartPt and
191
2.72k
    // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to
192
2.72k
    // avoid inserting into the range that we're iterating over.
193
2.72k
    MachineBasicBlock::reverse_iterator RE =
194
2.72k
        EmitStartPt ? 
MachineBasicBlock::reverse_iterator(EmitStartPt)576
195
2.72k
                    : 
FuncInfo.MBB->rend()2.14k
;
196
2.72k
    MachineBasicBlock::reverse_iterator RI(LastLocalValue);
197
2.72k
198
2.72k
    InstOrderMap OrderMap;
199
7.53k
    for (; RI != RE;) {
200
4.81k
      MachineInstr &LocalMI = *RI;
201
4.81k
      ++RI;
202
4.81k
      bool Store = true;
203
4.81k
      if (!LocalMI.isSafeToMove(nullptr, Store))
204
1.07k
        continue;
205
3.73k
      unsigned DefReg = findSinkableLocalRegDef(LocalMI);
206
3.73k
      if (DefReg == 0)
207
934
        continue;
208
2.80k
209
2.80k
      sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap);
210
2.80k
    }
211
2.72k
  }
212
18.1k
213
18.1k
  LocalValueMap.clear();
214
18.1k
  LastLocalValue = EmitStartPt;
215
18.1k
  recomputeInsertPt();
216
18.1k
  SavedInsertPt = FuncInfo.InsertPt;
217
18.1k
  LastFlushPoint = FuncInfo.InsertPt;
218
18.1k
}
219
220
static bool isRegUsedByPhiNodes(unsigned DefReg,
221
2.76k
                                FunctionLoweringInfo &FuncInfo) {
222
2.76k
  for (auto &P : FuncInfo.PHINodesToUpdate)
223
76
    if (P.second == DefReg)
224
37
      return true;
225
2.76k
  
return false2.72k
;
226
2.76k
}
227
228
/// Build a map of instruction orders. Return the first terminator and its
229
/// order. Consider EH_LABEL instructions to be terminators as well, since local
230
/// values for phis after invokes must be materialized before the call.
231
void FastISel::InstOrderMap::initialize(
232
1.92k
    MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) {
233
1.92k
  unsigned Order = 0;
234
17.2k
  for (MachineInstr &I : *MBB) {
235
17.2k
    if (!FirstTerminator &&
236
17.2k
        
(17.1k
I.isTerminator()17.1k
||
(15.7k
I.isEHLabel()15.7k
&&
&I != &MBB->front()6
))) {
237
1.41k
      FirstTerminator = &I;
238
1.41k
      FirstTerminatorOrder = Order;
239
1.41k
    }
240
17.2k
    Orders[&I] = Order++;
241
17.2k
242
17.2k
    // We don't need to order instructions past the last flush point.
243
17.2k
    if (I.getIterator() == LastFlushPoint)
244
737
      break;
245
17.2k
  }
246
1.92k
}
247
248
void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI,
249
                                             unsigned DefReg,
250
2.80k
                                             InstOrderMap &OrderMap) {
251
2.80k
  // If this register is used by a register fixup, MRI will not contain all
252
2.80k
  // the uses until after register fixups, so don't attempt to sink or DCE
253
2.80k
  // this instruction. Register fixups typically come from no-op cast
254
2.80k
  // instructions, which replace the cast instruction vreg with the local
255
2.80k
  // value vreg.
256
2.80k
  if (FuncInfo.RegsWithFixups.count(DefReg))
257
37
    return;
258
2.76k
259
2.76k
  // We can DCE this instruction if there are no uses and it wasn't a
260
2.76k
  // materialized for a successor PHI node.
261
2.76k
  bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo);
262
2.76k
  if (!UsedByPHI && 
MRI.use_nodbg_empty(DefReg)2.72k
) {
263
129
    if (EmitStartPt == &LocalMI)
264
0
      EmitStartPt = EmitStartPt->getPrevNode();
265
129
    LLVM_DEBUG(dbgs() << "removing dead local value materialization "
266
129
                      << LocalMI);
267
129
    OrderMap.Orders.erase(&LocalMI);
268
129
    LocalMI.eraseFromParent();
269
129
    return;
270
129
  }
271
2.63k
272
2.63k
  // Number the instructions if we haven't yet so we can efficiently find the
273
2.63k
  // earliest use.
274
2.63k
  if (OrderMap.Orders.empty())
275
1.92k
    OrderMap.initialize(FuncInfo.MBB, LastFlushPoint);
276
2.63k
277
2.63k
  // Find the first user in the BB.
278
2.63k
  MachineInstr *FirstUser = nullptr;
279
2.63k
  unsigned FirstOrder = std::numeric_limits<unsigned>::max();
280
2.84k
  for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) {
281
2.84k
    auto I = OrderMap.Orders.find(&UseInst);
282
2.84k
    assert(I != OrderMap.Orders.end() &&
283
2.84k
           "local value used by instruction outside local region");
284
2.84k
    unsigned UseOrder = I->second;
285
2.84k
    if (UseOrder < FirstOrder) {
286
2.63k
      FirstOrder = UseOrder;
287
2.63k
      FirstUser = &UseInst;
288
2.63k
    }
289
2.84k
  }
290
2.63k
291
2.63k
  // The insertion point will be the first terminator or the first user,
292
2.63k
  // whichever came first. If there was no terminator, this must be a
293
2.63k
  // fallthrough block and the insertion point is the end of the block.
294
2.63k
  MachineBasicBlock::instr_iterator SinkPos;
295
2.63k
  if (UsedByPHI && 
OrderMap.FirstTerminatorOrder < FirstOrder37
) {
296
33
    FirstOrder = OrderMap.FirstTerminatorOrder;
297
33
    SinkPos = OrderMap.FirstTerminator->getIterator();
298
2.60k
  } else if (FirstUser) {
299
2.59k
    SinkPos = FirstUser->getIterator();
300
2.59k
  } else {
301
4
    assert(UsedByPHI && "must be users if not used by a phi");
302
4
    SinkPos = FuncInfo.MBB->instr_end();
303
4
  }
304
2.63k
305
2.63k
  // Collect all DBG_VALUEs before the new insertion position so that we can
306
2.63k
  // sink them.
307
2.63k
  SmallVector<MachineInstr *, 1> DbgValues;
308
2.85k
  for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) {
309
2.85k
    if (!DbgVal.isDebugValue())
310
2.84k
      continue;
311
14
    unsigned UseOrder = OrderMap.Orders[&DbgVal];
312
14
    if (UseOrder < FirstOrder)
313
14
      DbgValues.push_back(&DbgVal);
314
14
  }
315
2.63k
316
2.63k
  // Sink LocalMI before SinkPos and assign it the same DebugLoc.
317
2.63k
  LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI);
318
2.63k
  FuncInfo.MBB->remove(&LocalMI);
319
2.63k
  FuncInfo.MBB->insert(SinkPos, &LocalMI);
320
2.63k
  if (SinkPos != FuncInfo.MBB->end())
321
2.63k
    LocalMI.setDebugLoc(SinkPos->getDebugLoc());
322
2.63k
323
2.63k
  // Sink any debug values that we've collected.
324
2.63k
  for (MachineInstr *DI : DbgValues) {
325
14
    FuncInfo.MBB->remove(DI);
326
14
    FuncInfo.MBB->insert(SinkPos, DI);
327
14
  }
328
2.63k
}
329
330
13.0k
bool FastISel::hasTrivialKill(const Value *V) {
331
13.0k
  // Don't consider constants or arguments to have trivial kills.
332
13.0k
  const Instruction *I = dyn_cast<Instruction>(V);
333
13.0k
  if (!I)
334
6.79k
    return false;
335
6.26k
336
6.26k
  // No-op casts are trivially coalesced by fast-isel.
337
6.26k
  if (const auto *Cast = dyn_cast<CastInst>(I))
338
1.06k
    if (Cast->isNoopCast(DL) && 
!hasTrivialKill(Cast->getOperand(0))455
)
339
316
      return false;
340
5.94k
341
5.94k
  // Even the value might have only one use in the LLVM IR, it is possible that
342
5.94k
  // FastISel might fold the use into another instruction and now there is more
343
5.94k
  // than one use at the Machine Instruction level.
344
5.94k
  unsigned Reg = lookUpRegForValue(V);
345
5.94k
  if (Reg && 
!MRI.use_empty(Reg)5.78k
)
346
210
    return false;
347
5.73k
348
5.73k
  // GEPs with all zero indices are trivially coalesced by fast-isel.
349
5.73k
  if (const auto *GEP = dyn_cast<GetElementPtrInst>(I))
350
56
    if (GEP->hasAllZeroIndices() && 
!hasTrivialKill(GEP->getOperand(0))24
)
351
9
      return false;
352
5.73k
353
5.73k
  // Only instructions with a single use in the same basic block are considered
354
5.73k
  // to have trivial kills.
355
5.73k
  return I->hasOneUse() &&
356
5.73k
         
!(5.22k
I->getOpcode() == Instruction::BitCast5.22k
||
357
5.22k
           
I->getOpcode() == Instruction::PtrToInt5.19k
||
358
5.22k
           
I->getOpcode() == Instruction::IntToPtr5.18k
) &&
359
5.73k
         
cast<Instruction>(*I->user_begin())->getParent() == I->getParent()5.18k
;
360
5.73k
}
361
362
41.6k
unsigned FastISel::getRegForValue(const Value *V) {
363
41.6k
  EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true);
364
41.6k
  // Don't handle non-simple values in FastISel.
365
41.6k
  if (!RealVT.isSimple())
366
9
    return 0;
367
41.6k
368
41.6k
  // Ignore illegal types. We must do this before looking up the value
369
41.6k
  // in ValueMap because Arguments are given virtual registers regardless
370
41.6k
  // of whether FastISel can handle them.
371
41.6k
  MVT VT = RealVT.getSimpleVT();
372
41.6k
  if (!TLI.isTypeLegal(VT)) {
373
3.54k
    // Handle integer promotions, though, because they're common and easy.
374
3.54k
    if (VT == MVT::i1 || 
VT == MVT::i82.25k
||
VT == MVT::i161.29k
)
375
3.04k
      VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT();
376
497
    else
377
497
      return 0;
378
41.1k
  }
379
41.1k
380
41.1k
  // Look up the value to see if we already have a register for it.
381
41.1k
  unsigned Reg = lookUpRegForValue(V);
382
41.1k
  if (Reg)
383
19.1k
    return Reg;
384
21.9k
385
21.9k
  // In bottom-up mode, just create the virtual register which will be used
386
21.9k
  // to hold the value. It will be materialized later.
387
21.9k
  if (isa<Instruction>(V) &&
388
21.9k
      
(17.6k
!isa<AllocaInst>(V)17.6k
||
389
17.6k
       
!FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))478
))
390
17.1k
    return FuncInfo.InitializeRegForValue(V);
391
4.77k
392
4.77k
  SavePoint SaveInsertPt = enterLocalValueArea();
393
4.77k
394
4.77k
  // Materialize the value in a register. Emit any instructions in the
395
4.77k
  // local value area.
396
4.77k
  Reg = materializeRegForValue(V, VT);
397
4.77k
398
4.77k
  leaveLocalValueArea(SaveInsertPt);
399
4.77k
400
4.77k
  return Reg;
401
4.77k
}
402
403
1.40k
unsigned FastISel::materializeConstant(const Value *V, MVT VT) {
404
1.40k
  unsigned Reg = 0;
405
1.40k
  if (const auto *CI = dyn_cast<ConstantInt>(V)) {
406
144
    if (CI->getValue().getActiveBits() <= 64)
407
144
      Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
408
1.25k
  } else if (isa<AllocaInst>(V))
409
464
    Reg = fastMaterializeAlloca(cast<AllocaInst>(V));
410
795
  else if (isa<ConstantPointerNull>(V))
411
39
    // Translate this as an integer zero so that it can be
412
39
    // local-CSE'd with actual integer zeros.
413
39
    Reg = getRegForValue(
414
39
        Constant::getNullValue(DL.getIntPtrType(V->getContext())));
415
756
  else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
416
9
    if (CF->isNullValue())
417
1
      Reg = fastMaterializeFloatZero(CF);
418
8
    else
419
8
      // Try to emit the constant directly.
420
8
      Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF);
421
9
422
9
    if (!Reg) {
423
1
      // Try to emit the constant by using an integer constant with a cast.
424
1
      const APFloat &Flt = CF->getValueAPF();
425
1
      EVT IntVT = TLI.getPointerTy(DL);
426
1
      uint32_t IntBitWidth = IntVT.getSizeInBits();
427
1
      APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false);
428
1
      bool isExact;
429
1
      (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact);
430
1
      if (isExact) {
431
1
        unsigned IntegerReg =
432
1
            getRegForValue(ConstantInt::get(V->getContext(), SIntVal));
433
1
        if (IntegerReg != 0)
434
1
          Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg,
435
1
                           /*Kill=*/false);
436
1
      }
437
1
    }
438
747
  } else if (const auto *Op = dyn_cast<Operator>(V)) {
439
280
    if (!selectOperator(Op, Op->getOpcode()))
440
67
      if (!isa<Instruction>(Op) ||
441
67
          
!fastSelectInstruction(cast<Instruction>(Op))0
)
442
67
        return 0;
443
213
    Reg = lookUpRegForValue(Op);
444
467
  } else if (isa<UndefValue>(V)) {
445
192
    Reg = createResultReg(TLI.getRegClassFor(VT));
446
192
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
447
192
            TII.get(TargetOpcode::IMPLICIT_DEF), Reg);
448
192
  }
449
1.40k
  
return Reg1.33k
;
450
1.40k
}
451
452
/// Helper for getRegForValue. This function is called when the value isn't
453
/// already available in a register and must be materialized with new
454
/// instructions.
455
4.77k
unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) {
456
4.77k
  unsigned Reg = 0;
457
4.77k
  // Give the target-specific code a try first.
458
4.77k
  if (isa<Constant>(V))
459
4.30k
    Reg = fastMaterializeConstant(cast<Constant>(V));
460
4.77k
461
4.77k
  // If target-specific code couldn't or didn't want to handle the value, then
462
4.77k
  // give target-independent code a try.
463
4.77k
  if (!Reg)
464
1.40k
    Reg = materializeConstant(V, VT);
465
4.77k
466
4.77k
  // Don't cache constant materializations in the general ValueMap.
467
4.77k
  // To do so would require tracking what uses they dominate.
468
4.77k
  if (Reg) {
469
4.42k
    LocalValueMap[V] = Reg;
470
4.42k
    LastLocalValue = MRI.getVRegDef(Reg);
471
4.42k
  }
472
4.77k
  return Reg;
473
4.77k
}
474
475
48.0k
unsigned FastISel::lookUpRegForValue(const Value *V) {
476
48.0k
  // Look up the value to see if we already have a register for it. We
477
48.0k
  // cache values defined by Instructions across blocks, and other values
478
48.0k
  // only locally. This is because Instructions already have the SSA
479
48.0k
  // def-dominates-use requirement enforced.
480
48.0k
  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V);
481
48.0k
  if (I != FuncInfo.ValueMap.end())
482
23.3k
    return I->second;
483
24.7k
  return LocalValueMap[V];
484
24.7k
}
485
486
18.7k
void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) {
487
18.7k
  if (!isa<Instruction>(I)) {
488
6.52k
    LocalValueMap[I] = Reg;
489
6.52k
    return;
490
6.52k
  }
491
12.2k
492
12.2k
  unsigned &AssignedReg = FuncInfo.ValueMap[I];
493
12.2k
  if (AssignedReg == 0)
494
320
    // Use the new register.
495
320
    AssignedReg = Reg;
496
11.9k
  else if (Reg != AssignedReg) {
497
11.9k
    // Arrange for uses of AssignedReg to be replaced by uses of Reg.
498
23.9k
    for (unsigned i = 0; i < NumRegs; 
i++12.0k
) {
499
12.0k
      FuncInfo.RegFixups[AssignedReg + i] = Reg + i;
500
12.0k
      FuncInfo.RegsWithFixups.insert(Reg + i);
501
12.0k
    }
502
11.9k
503
11.9k
    AssignedReg = Reg;
504
11.9k
  }
505
12.2k
}
506
507
141
std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) {
508
141
  unsigned IdxN = getRegForValue(Idx);
509
141
  if (IdxN == 0)
510
9
    // Unhandled operand. Halt "fast" selection and bail.
511
9
    return std::pair<unsigned, bool>(0, false);
512
132
513
132
  bool IdxNIsKill = hasTrivialKill(Idx);
514
132
515
132
  // If the index is smaller or larger than intptr_t, truncate or extend it.
516
132
  MVT PtrVT = TLI.getPointerTy(DL);
517
132
  EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false);
518
132
  if (IdxVT.bitsLT(PtrVT)) {
519
12
    IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN,
520
12
                      IdxNIsKill);
521
12
    IdxNIsKill = true;
522
120
  } else if (IdxVT.bitsGT(PtrVT)) {
523
9
    IdxN =
524
9
        fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill);
525
9
    IdxNIsKill = true;
526
9
  }
527
132
  return std::pair<unsigned, bool>(IdxN, IdxNIsKill);
528
132
}
529
530
120k
void FastISel::recomputeInsertPt() {
531
120k
  if (getLastLocalValue()) {
532
59.0k
    FuncInfo.InsertPt = getLastLocalValue();
533
59.0k
    FuncInfo.MBB = FuncInfo.InsertPt->getParent();
534
59.0k
    ++FuncInfo.InsertPt;
535
59.0k
  } else
536
61.2k
    FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI();
537
120k
538
120k
  // Now skip past any EH_LABELs, which must remain at the beginning.
539
120k
  while (FuncInfo.InsertPt != FuncInfo.MBB->end() &&
540
120k
         
FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL86.9k
)
541
43
    ++FuncInfo.InsertPt;
542
120k
}
543
544
void FastISel::removeDeadCode(MachineBasicBlock::iterator I,
545
721
                              MachineBasicBlock::iterator E) {
546
721
  assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 &&
547
721
         "Invalid iterator!");
548
1.57k
  while (I != E) {
549
851
    if (LastFlushPoint == I)
550
98
      LastFlushPoint = E;
551
851
    if (SavedInsertPt == I)
552
175
      SavedInsertPt = E;
553
851
    if (EmitStartPt == I)
554
0
      EmitStartPt = E.isValid() ? &*E : nullptr;
555
851
    if (LastLocalValue == I)
556
0
      LastLocalValue = E.isValid() ? &*E : nullptr;
557
851
558
851
    MachineInstr *Dead = &*I;
559
851
    ++I;
560
851
    Dead->eraseFromParent();
561
851
    ++NumFastIselDead;
562
851
  }
563
721
  recomputeInsertPt();
564
721
}
565
566
4.84k
FastISel::SavePoint FastISel::enterLocalValueArea() {
567
4.84k
  MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt;
568
4.84k
  DebugLoc OldDL = DbgLoc;
569
4.84k
  recomputeInsertPt();
570
4.84k
  DbgLoc = DebugLoc();
571
4.84k
  SavePoint SP = {OldInsertPt, OldDL};
572
4.84k
  return SP;
573
4.84k
}
574
575
4.84k
void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) {
576
4.84k
  if (FuncInfo.InsertPt != FuncInfo.MBB->begin())
577
4.62k
    LastLocalValue = &*std::prev(FuncInfo.InsertPt);
578
4.84k
579
4.84k
  // Restore the previous insert position.
580
4.84k
  FuncInfo.InsertPt = OldInsertPt.InsertPt;
581
4.84k
  DbgLoc = OldInsertPt.DL;
582
4.84k
}
583
584
2.06k
bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) {
585
2.06k
  EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true);
586
2.06k
  if (VT == MVT::Other || !VT.isSimple())
587
0
    // Unhandled type. Halt "fast" selection and bail.
588
0
    return false;
589
2.06k
590
2.06k
  // We only handle legal types. For example, on x86-32 the instruction
591
2.06k
  // selector contains all of the 64-bit instructions from x86-64,
592
2.06k
  // under the assumption that i64 won't be used if the target doesn't
593
2.06k
  // support it.
594
2.06k
  if (!TLI.isTypeLegal(VT)) {
595
148
    // MVT::i1 is special. Allow AND, OR, or XOR because they
596
148
    // don't require additional zeroing, which makes them easy.
597
148
    if (VT == MVT::i1 && 
(41
ISDOpcode == ISD::AND41
||
ISDOpcode == ISD::OR36
||
598
41
                          
ISDOpcode == ISD::XOR28
))
599
34
      VT = TLI.getTypeToTransformTo(I->getContext(), VT);
600
114
    else
601
114
      return false;
602
1.95k
  }
603
1.95k
604
1.95k
  // Check if the first operand is a constant, and handle it as "ri".  At -O0,
605
1.95k
  // we don't have anything that canonicalizes operand order.
606
1.95k
  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0)))
607
66
    if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) {
608
35
      unsigned Op1 = getRegForValue(I->getOperand(1));
609
35
      if (!Op1)
610
0
        return false;
611
35
      bool Op1IsKill = hasTrivialKill(I->getOperand(1));
612
35
613
35
      unsigned ResultReg =
614
35
          fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill,
615
35
                       CI->getZExtValue(), VT.getSimpleVT());
616
35
      if (!ResultReg)
617
0
        return false;
618
35
619
35
      // We successfully emitted code for the given LLVM Instruction.
620
35
      updateValueMap(I, ResultReg);
621
35
      return true;
622
35
    }
623
1.91k
624
1.91k
  unsigned Op0 = getRegForValue(I->getOperand(0));
625
1.91k
  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
626
29
    return false;
627
1.88k
  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
628
1.88k
629
1.88k
  // Check if the second operand is a constant and handle it appropriately.
630
1.88k
  if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) {
631
470
    uint64_t Imm = CI->getSExtValue();
632
470
633
470
    // Transform "sdiv exact X, 8" -> "sra X, 3".
634
470
    if (ISDOpcode == ISD::SDIV && 
isa<BinaryOperator>(I)5
&&
635
470
        
cast<BinaryOperator>(I)->isExact()5
&&
isPowerOf2_64(Imm)4
) {
636
4
      Imm = Log2_64(Imm);
637
4
      ISDOpcode = ISD::SRA;
638
4
    }
639
470
640
470
    // Transform "urem x, pow2" -> "and x, pow2-1".
641
470
    if (ISDOpcode == ISD::UREM && 
isa<BinaryOperator>(I)5
&&
642
470
        
isPowerOf2_64(Imm)5
) {
643
3
      --Imm;
644
3
      ISDOpcode = ISD::AND;
645
3
    }
646
470
647
470
    unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0,
648
470
                                      Op0IsKill, Imm, VT.getSimpleVT());
649
470
    if (!ResultReg)
650
16
      return false;
651
454
652
454
    // We successfully emitted code for the given LLVM Instruction.
653
454
    updateValueMap(I, ResultReg);
654
454
    return true;
655
454
  }
656
1.41k
657
1.41k
  unsigned Op1 = getRegForValue(I->getOperand(1));
658
1.41k
  if (!Op1) // Unhandled operand. Halt "fast" selection and bail.
659
71
    return false;
660
1.34k
  bool Op1IsKill = hasTrivialKill(I->getOperand(1));
661
1.34k
662
1.34k
  // Now we have both operands in registers. Emit the instruction.
663
1.34k
  unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(),
664
1.34k
                                   ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill);
665
1.34k
  if (!ResultReg)
666
252
    // Target-specific code wasn't able to find a machine opcode for
667
252
    // the given ISD opcode and type. Halt "fast" selection and bail.
668
252
    return false;
669
1.09k
670
1.09k
  // We successfully emitted code for the given LLVM Instruction.
671
1.09k
  updateValueMap(I, ResultReg);
672
1.09k
  return true;
673
1.09k
}
674
675
325
bool FastISel::selectGetElementPtr(const User *I) {
676
325
  unsigned N = getRegForValue(I->getOperand(0));
677
325
  if (!N) // Unhandled operand. Halt "fast" selection and bail.
678
48
    return false;
679
277
  bool NIsKill = hasTrivialKill(I->getOperand(0));
680
277
681
277
  // Keep a running tab of the total offset to coalesce multiple N = N + Offset
682
277
  // into a single N = N + TotalOffset.
683
277
  uint64_t TotalOffs = 0;
684
277
  // FIXME: What's a good SWAG number for MaxOffs?
685
277
  uint64_t MaxOffs = 2048;
686
277
  MVT VT = TLI.getPointerTy(DL);
687
277
  for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I);
688
836
       GTI != E; 
++GTI559
) {
689
577
    const Value *Idx = GTI.getOperand();
690
577
    if (StructType *StTy = GTI.getStructTypeOrNull()) {
691
53
      uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue();
692
53
      if (Field) {
693
27
        // N = N + Offset
694
27
        TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field);
695
27
        if (TotalOffs >= MaxOffs) {
696
0
          N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
697
0
          if (!N) // Unhandled operand. Halt "fast" selection and bail.
698
0
            return false;
699
0
          NIsKill = true;
700
0
          TotalOffs = 0;
701
0
        }
702
27
      }
703
524
    } else {
704
524
      Type *Ty = GTI.getIndexedType();
705
524
706
524
      // If this is a constant subscript, handle it quickly.
707
524
      if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
708
467
        if (CI->isZero())
709
318
          continue;
710
149
        // N = N + Offset
711
149
        uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue();
712
149
        TotalOffs += DL.getTypeAllocSize(Ty) * IdxN;
713
149
        if (TotalOffs >= MaxOffs) {
714
12
          N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
715
12
          if (!N) // Unhandled operand. Halt "fast" selection and bail.
716
0
            return false;
717
12
          NIsKill = true;
718
12
          TotalOffs = 0;
719
12
        }
720
149
        continue;
721
57
      }
722
57
      if (TotalOffs) {
723
0
        N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
724
0
        if (!N) // Unhandled operand. Halt "fast" selection and bail.
725
0
          return false;
726
0
        NIsKill = true;
727
0
        TotalOffs = 0;
728
0
      }
729
57
730
57
      // N = N + Idx * ElementSize;
731
57
      uint64_t ElementSize = DL.getTypeAllocSize(Ty);
732
57
      std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx);
733
57
      unsigned IdxN = Pair.first;
734
57
      bool IdxNIsKill = Pair.second;
735
57
      if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
736
15
        return false;
737
42
738
42
      if (ElementSize != 1) {
739
18
        IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT);
740
18
        if (!IdxN) // Unhandled operand. Halt "fast" selection and bail.
741
3
          return false;
742
15
        IdxNIsKill = true;
743
15
      }
744
42
      N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill);
745
39
      if (!N) // Unhandled operand. Halt "fast" selection and bail.
746
0
        return false;
747
39
    }
748
577
  }
749
277
  
if (259
TotalOffs259
) {
750
96
    N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT);
751
96
    if (!N) // Unhandled operand. Halt "fast" selection and bail.
752
0
      return false;
753
259
  }
754
259
755
259
  // We successfully emitted code for the given LLVM Instruction.
756
259
  updateValueMap(I, N);
757
259
  return true;
758
259
}
759
760
bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
761
65
                                   const CallInst *CI, unsigned StartIdx) {
762
184
  for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; 
++i119
) {
763
119
    Value *Val = CI->getArgOperand(i);
764
119
    // Check for constants and encode them with a StackMaps::ConstantOp prefix.
765
119
    if (const auto *C = dyn_cast<ConstantInt>(Val)) {
766
18
      Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
767
18
      Ops.push_back(MachineOperand::CreateImm(C->getSExtValue()));
768
101
    } else if (isa<ConstantPointerNull>(Val)) {
769
0
      Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp));
770
0
      Ops.push_back(MachineOperand::CreateImm(0));
771
101
    } else if (auto *AI = dyn_cast<AllocaInst>(Val)) {
772
8
      // Values coming from a stack location also require a special encoding,
773
8
      // but that is added later on by the target specific frame index
774
8
      // elimination implementation.
775
8
      auto SI = FuncInfo.StaticAllocaMap.find(AI);
776
8
      if (SI != FuncInfo.StaticAllocaMap.end())
777
8
        Ops.push_back(MachineOperand::CreateFI(SI->second));
778
0
      else
779
0
        return false;
780
93
    } else {
781
93
      unsigned Reg = getRegForValue(Val);
782
93
      if (!Reg)
783
0
        return false;
784
93
      Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
785
93
    }
786
119
  }
787
65
  return true;
788
65
}
789
790
26
bool FastISel::selectStackmap(const CallInst *I) {
791
26
  // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>,
792
26
  //                                  [live variables...])
793
26
  assert(I->getCalledFunction()->getReturnType()->isVoidTy() &&
794
26
         "Stackmap cannot return a value.");
795
26
796
26
  // The stackmap intrinsic only records the live variables (the arguments
797
26
  // passed to it) and emits NOPS (if requested). Unlike the patchpoint
798
26
  // intrinsic, this won't be lowered to a function call. This means we don't
799
26
  // have to worry about calling conventions and target-specific lowering code.
800
26
  // Instead we perform the call lowering right here.
801
26
  //
802
26
  // CALLSEQ_START(0, 0...)
803
26
  // STACKMAP(id, nbytes, ...)
804
26
  // CALLSEQ_END(0, 0)
805
26
  //
806
26
  SmallVector<MachineOperand, 32> Ops;
807
26
808
26
  // Add the <id> and <numBytes> constants.
809
26
  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
810
26
         "Expected a constant integer.");
811
26
  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
812
26
  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
813
26
814
26
  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
815
26
         "Expected a constant integer.");
816
26
  const auto *NumBytes =
817
26
      cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
818
26
  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
819
26
820
26
  // Push live variables for the stack map (skipping the first two arguments
821
26
  // <id> and <numBytes>).
822
26
  if (!addStackMapLiveVars(Ops, I, 2))
823
0
    return false;
824
26
825
26
  // We are not adding any register mask info here, because the stackmap doesn't
826
26
  // clobber anything.
827
26
828
26
  // Add scratch registers as implicit def and early clobber.
829
26
  CallingConv::ID CC = I->getCallingConv();
830
26
  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
831
82
  for (unsigned i = 0; ScratchRegs[i]; 
++i56
)
832
56
    Ops.push_back(MachineOperand::CreateReg(
833
56
        ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
834
56
        /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
835
26
836
26
  // Issue CALLSEQ_START
837
26
  unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
838
26
  auto Builder =
839
26
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown));
840
26
  const MCInstrDesc &MCID = Builder.getInstr()->getDesc();
841
89
  for (unsigned I = 0, E = MCID.getNumOperands(); I < E; 
++I63
)
842
63
    Builder.addImm(0);
843
26
844
26
  // Issue STACKMAP.
845
26
  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
846
26
                                    TII.get(TargetOpcode::STACKMAP));
847
26
  for (auto const &MO : Ops)
848
205
    MIB.add(MO);
849
26
850
26
  // Issue CALLSEQ_END
851
26
  unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
852
26
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp))
853
26
      .addImm(0)
854
26
      .addImm(0);
855
26
856
26
  // Inform the Frame Information that we have a stackmap in this function.
857
26
  FuncInfo.MF->getFrameInfo().setHasStackMap();
858
26
859
26
  return true;
860
26
}
861
862
/// Lower an argument list according to the target calling convention.
863
///
864
/// This is a helper for lowering intrinsics that follow a target calling
865
/// convention or require stack pointer adjustment. Only a subset of the
866
/// intrinsic's operands need to participate in the calling convention.
867
bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
868
                                 unsigned NumArgs, const Value *Callee,
869
39
                                 bool ForceRetVoidTy, CallLoweringInfo &CLI) {
870
39
  ArgListTy Args;
871
39
  Args.reserve(NumArgs);
872
39
873
39
  // Populate the argument list.
874
39
  ImmutableCallSite CS(CI);
875
154
  for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; 
++ArgI115
) {
876
115
    Value *V = CI->getOperand(ArgI);
877
115
878
115
    assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
879
115
880
115
    ArgListEntry Entry;
881
115
    Entry.Val = V;
882
115
    Entry.Ty = V->getType();
883
115
    Entry.setAttributes(&CS, ArgI);
884
115
    Args.push_back(Entry);
885
115
  }
886
39
887
39
  Type *RetTy = ForceRetVoidTy ? 
Type::getVoidTy(CI->getType()->getContext())1
888
39
                               : 
CI->getType()38
;
889
39
  CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs);
890
39
891
39
  return lowerCallTo(CLI);
892
39
}
893
894
FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee(
895
    const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy,
896
16
    StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) {
897
16
  SmallString<32> MangledName;
898
16
  Mangler::getNameWithPrefix(MangledName, Target, DL);
899
16
  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
900
16
  return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs);
901
16
}
902
903
39
bool FastISel::selectPatchpoint(const CallInst *I) {
904
39
  // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
905
39
  //                                                 i32 <numBytes>,
906
39
  //                                                 i8* <target>,
907
39
  //                                                 i32 <numArgs>,
908
39
  //                                                 [Args...],
909
39
  //                                                 [live variables...])
910
39
  CallingConv::ID CC = I->getCallingConv();
911
39
  bool IsAnyRegCC = CC == CallingConv::AnyReg;
912
39
  bool HasDef = !I->getType()->isVoidTy();
913
39
  Value *Callee = I->getOperand(PatchPointOpers::TargetPos)->stripPointerCasts();
914
39
915
39
  // Get the real number of arguments participating in the call <numArgs>
916
39
  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) &&
917
39
         "Expected a constant integer.");
918
39
  const auto *NumArgsVal =
919
39
      cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos));
920
39
  unsigned NumArgs = NumArgsVal->getZExtValue();
921
39
922
39
  // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
923
39
  // This includes all meta-operands up to but not including CC.
924
39
  unsigned NumMetaOpers = PatchPointOpers::CCPos;
925
39
  assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs &&
926
39
         "Not enough arguments provided to the patchpoint intrinsic");
927
39
928
39
  // For AnyRegCC the arguments are lowered later on manually.
929
39
  unsigned NumCallArgs = IsAnyRegCC ? 
01
:
NumArgs38
;
930
39
  CallLoweringInfo CLI;
931
39
  CLI.setIsPatchPoint();
932
39
  if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI))
933
0
    return false;
934
39
935
39
  assert(CLI.Call && "No call instruction specified.");
936
39
937
39
  SmallVector<MachineOperand, 32> Ops;
938
39
939
39
  // Add an explicit result reg if we use the anyreg calling convention.
940
39
  if (IsAnyRegCC && 
HasDef1
) {
941
0
    assert(CLI.NumResultRegs == 0 && "Unexpected result register.");
942
0
    CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64));
943
0
    CLI.NumResultRegs = 1;
944
0
    Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*isDef=*/true));
945
0
  }
946
39
947
39
  // Add the <id> and <numBytes> constants.
948
39
  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) &&
949
39
         "Expected a constant integer.");
950
39
  const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos));
951
39
  Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue()));
952
39
953
39
  assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) &&
954
39
         "Expected a constant integer.");
955
39
  const auto *NumBytes =
956
39
      cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos));
957
39
  Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue()));
958
39
959
39
  // Add the call target.
960
39
  if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) {
961
18
    uint64_t CalleeConstAddr =
962
18
      cast<ConstantInt>(C->getOperand(0))->getZExtValue();
963
18
    Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
964
21
  } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) {
965
4
    if (C->getOpcode() == Instruction::IntToPtr) {
966
4
      uint64_t CalleeConstAddr =
967
4
        cast<ConstantInt>(C->getOperand(0))->getZExtValue();
968
4
      Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr));
969
4
    } else
970
4
      
llvm_unreachable0
("Unsupported ConstantExpr.");
971
17
  } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) {
972
4
    Ops.push_back(MachineOperand::CreateGA(GV, 0));
973
13
  } else if (isa<ConstantPointerNull>(Callee))
974
13
    Ops.push_back(MachineOperand::CreateImm(0));
975
13
  else
976
13
    
llvm_unreachable0
("Unsupported callee address.");
977
39
978
39
  // Adjust <numArgs> to account for any arguments that have been passed on
979
39
  // the stack instead.
980
39
  unsigned NumCallRegArgs = IsAnyRegCC ? 
NumArgs1
:
CLI.OutRegs.size()38
;
981
39
  Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs));
982
39
983
39
  // Add the calling convention
984
39
  Ops.push_back(MachineOperand::CreateImm((unsigned)CC));
985
39
986
39
  // Add the arguments we omitted previously. The register allocator should
987
39
  // place these in any free register.
988
39
  if (IsAnyRegCC) {
989
3
    for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; 
++i2
) {
990
2
      unsigned Reg = getRegForValue(I->getArgOperand(i));
991
2
      if (!Reg)
992
0
        return false;
993
2
      Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
994
2
    }
995
1
  }
996
39
997
39
  // Push the arguments from the call instruction.
998
39
  for (auto Reg : CLI.OutRegs)
999
51
    Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false));
1000
39
1001
39
  // Push live variables for the stack map.
1002
39
  if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs))
1003
0
    return false;
1004
39
1005
39
  // Push the register mask info.
1006
39
  Ops.push_back(MachineOperand::CreateRegMask(
1007
39
      TRI.getCallPreservedMask(*FuncInfo.MF, CC)));
1008
39
1009
39
  // Add scratch registers as implicit def and early clobber.
1010
39
  const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC);
1011
132
  for (unsigned i = 0; ScratchRegs[i]; 
++i93
)
1012
93
    Ops.push_back(MachineOperand::CreateReg(
1013
93
        ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false,
1014
93
        /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true));
1015
39
1016
39
  // Add implicit defs (return values).
1017
39
  for (auto Reg : CLI.InRegs)
1018
21
    Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/true,
1019
21
                                            /*isImp=*/true));
1020
39
1021
39
  // Insert the patchpoint instruction before the call generated by the target.
1022
39
  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc,
1023
39
                                    TII.get(TargetOpcode::PATCHPOINT));
1024
39
1025
39
  for (auto &MO : Ops)
1026
441
    MIB.add(MO);
1027
39
1028
39
  MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1029
39
1030
39
  // Delete the original call instruction.
1031
39
  CLI.Call->eraseFromParent();
1032
39
1033
39
  // Inform the Frame Information that we have a patchpoint in this function.
1034
39
  FuncInfo.MF->getFrameInfo().setHasPatchPoint();
1035
39
1036
39
  if (CLI.NumResultRegs)
1037
21
    updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs);
1038
39
  return true;
1039
39
}
1040
1041
0
bool FastISel::selectXRayCustomEvent(const CallInst *I) {
1042
0
  const auto &Triple = TM.getTargetTriple();
1043
0
  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1044
0
    return true; // don't do anything to this instruction.
1045
0
  SmallVector<MachineOperand, 8> Ops;
1046
0
  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1047
0
                                          /*isDef=*/false));
1048
0
  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1049
0
                                          /*isDef=*/false));
1050
0
  MachineInstrBuilder MIB =
1051
0
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1052
0
              TII.get(TargetOpcode::PATCHABLE_EVENT_CALL));
1053
0
  for (auto &MO : Ops)
1054
0
    MIB.add(MO);
1055
0
1056
0
  // Insert the Patchable Event Call instruction, that gets lowered properly.
1057
0
  return true;
1058
0
}
1059
1060
0
bool FastISel::selectXRayTypedEvent(const CallInst *I) {
1061
0
  const auto &Triple = TM.getTargetTriple();
1062
0
  if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
1063
0
    return true; // don't do anything to this instruction.
1064
0
  SmallVector<MachineOperand, 8> Ops;
1065
0
  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)),
1066
0
                                          /*isDef=*/false));
1067
0
  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)),
1068
0
                                          /*isDef=*/false));
1069
0
  Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)),
1070
0
                                          /*isDef=*/false));
1071
0
  MachineInstrBuilder MIB =
1072
0
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1073
0
              TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL));
1074
0
  for (auto &MO : Ops)
1075
0
    MIB.add(MO);
1076
0
1077
0
  // Insert the Patchable Typed Event Call instruction, that gets lowered properly.
1078
0
  return true;
1079
0
}
1080
1081
/// Returns an AttributeList representing the attributes applied to the return
1082
/// value of the given call.
1083
2.19k
static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) {
1084
2.19k
  SmallVector<Attribute::AttrKind, 2> Attrs;
1085
2.19k
  if (CLI.RetSExt)
1086
54
    Attrs.push_back(Attribute::SExt);
1087
2.19k
  if (CLI.RetZExt)
1088
45
    Attrs.push_back(Attribute::ZExt);
1089
2.19k
  if (CLI.IsInReg)
1090
0
    Attrs.push_back(Attribute::InReg);
1091
2.19k
1092
2.19k
  return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
1093
2.19k
                            Attrs);
1094
2.19k
}
1095
1096
bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName,
1097
34
                           unsigned NumArgs) {
1098
34
  MCContext &Ctx = MF->getContext();
1099
34
  SmallString<32> MangledName;
1100
34
  Mangler::getNameWithPrefix(MangledName, SymName, DL);
1101
34
  MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName);
1102
34
  return lowerCallTo(CI, Sym, NumArgs);
1103
34
}
1104
1105
bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
1106
34
                           unsigned NumArgs) {
1107
34
  ImmutableCallSite CS(CI);
1108
34
1109
34
  FunctionType *FTy = CS.getFunctionType();
1110
34
  Type *RetTy = CS.getType();
1111
34
1112
34
  ArgListTy Args;
1113
34
  Args.reserve(NumArgs);
1114
34
1115
34
  // Populate the argument list.
1116
34
  // Attributes for args start at offset 1, after the return attribute.
1117
136
  for (unsigned ArgI = 0; ArgI != NumArgs; 
++ArgI102
) {
1118
102
    Value *V = CI->getOperand(ArgI);
1119
102
1120
102
    assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
1121
102
1122
102
    ArgListEntry Entry;
1123
102
    Entry.Val = V;
1124
102
    Entry.Ty = V->getType();
1125
102
    Entry.setAttributes(&CS, ArgI);
1126
102
    Args.push_back(Entry);
1127
102
  }
1128
34
  TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args);
1129
34
1130
34
  CallLoweringInfo CLI;
1131
34
  CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs);
1132
34
1133
34
  return lowerCallTo(CLI);
1134
34
}
1135
1136
2.19k
bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
1137
2.19k
  // Handle the incoming return values from the call.
1138
2.19k
  CLI.clearIns();
1139
2.19k
  SmallVector<EVT, 4> RetTys;
1140
2.19k
  ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys);
1141
2.19k
1142
2.19k
  SmallVector<ISD::OutputArg, 4> Outs;
1143
2.19k
  GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL);
1144
2.19k
1145
2.19k
  bool CanLowerReturn = TLI.CanLowerReturn(
1146
2.19k
      CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext());
1147
2.19k
1148
2.19k
  // FIXME: sret demotion isn't supported yet - bail out.
1149
2.19k
  if (!CanLowerReturn)
1150
5
    return false;
1151
2.19k
1152
3.14k
  
for (unsigned I = 0, E = RetTys.size(); 2.19k
I != E;
++I951
) {
1153
951
    EVT VT = RetTys[I];
1154
951
    MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
1155
951
    unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
1156
1.92k
    for (unsigned i = 0; i != NumRegs; 
++i971
) {
1157
971
      ISD::InputArg MyFlags;
1158
971
      MyFlags.VT = RegisterVT;
1159
971
      MyFlags.ArgVT = VT;
1160
971
      MyFlags.Used = CLI.IsReturnValueUsed;
1161
971
      if (CLI.RetSExt)
1162
54
        MyFlags.Flags.setSExt();
1163
971
      if (CLI.RetZExt)
1164
45
        MyFlags.Flags.setZExt();
1165
971
      if (CLI.IsInReg)
1166
0
        MyFlags.Flags.setInReg();
1167
971
      CLI.Ins.push_back(MyFlags);
1168
971
    }
1169
951
  }
1170
2.19k
1171
2.19k
  // Handle all of the outgoing arguments.
1172
2.19k
  CLI.clearOuts();
1173
4.46k
  for (auto &Arg : CLI.getArgs()) {
1174
4.46k
    Type *FinalType = Arg.Ty;
1175
4.46k
    if (Arg.IsByVal)
1176
35
      FinalType = cast<PointerType>(Arg.Ty)->getElementType();
1177
4.46k
    bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters(
1178
4.46k
        FinalType, CLI.CallConv, CLI.IsVarArg);
1179
4.46k
1180
4.46k
    ISD::ArgFlagsTy Flags;
1181
4.46k
    if (Arg.IsZExt)
1182
186
      Flags.setZExt();
1183
4.46k
    if (Arg.IsSExt)
1184
56
      Flags.setSExt();
1185
4.46k
    if (Arg.IsInReg)
1186
16
      Flags.setInReg();
1187
4.46k
    if (Arg.IsSRet)
1188
72
      Flags.setSRet();
1189
4.46k
    if (Arg.IsSwiftSelf)
1190
24
      Flags.setSwiftSelf();
1191
4.46k
    if (Arg.IsSwiftError)
1192
5
      Flags.setSwiftError();
1193
4.46k
    if (Arg.IsByVal)
1194
35
      Flags.setByVal();
1195
4.46k
    if (Arg.IsInAlloca) {
1196
6
      Flags.setInAlloca();
1197
6
      // Set the byval flag for CCAssignFn callbacks that don't know about
1198
6
      // inalloca. This way we can know how many bytes we should've allocated
1199
6
      // and how many bytes a callee cleanup function will pop.  If we port
1200
6
      // inalloca to more targets, we'll have to add custom inalloca handling in
1201
6
      // the various CC lowering callbacks.
1202
6
      Flags.setByVal();
1203
6
    }
1204
4.46k
    if (Arg.IsByVal || 
Arg.IsInAlloca4.43k
) {
1205
41
      PointerType *Ty = cast<PointerType>(Arg.Ty);
1206
41
      Type *ElementTy = Ty->getElementType();
1207
41
      unsigned FrameSize =
1208
41
          DL.getTypeAllocSize(Arg.ByValType ? 
Arg.ByValType35
:
ElementTy6
);
1209
41
1210
41
      // For ByVal, alignment should come from FE. BE will guess if this info
1211
41
      // is not there, but there are cases it cannot get right.
1212
41
      unsigned FrameAlign = Arg.Alignment;
1213
41
      if (!FrameAlign)
1214
12
        FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL);
1215
41
      Flags.setByValSize(FrameSize);
1216
41
      Flags.setByValAlign(FrameAlign);
1217
41
    }
1218
4.46k
    if (Arg.IsNest)
1219
1
      Flags.setNest();
1220
4.46k
    if (NeedsRegBlock)
1221
10
      Flags.setInConsecutiveRegs();
1222
4.46k
    unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty);
1223
4.46k
    Flags.setOrigAlign(OriginalAlignment);
1224
4.46k
1225
4.46k
    CLI.OutVals.push_back(Arg.Val);
1226
4.46k
    CLI.OutFlags.push_back(Flags);
1227
4.46k
  }
1228
2.19k
1229
2.19k
  if (!fastLowerCall(CLI))
1230
969
    return false;
1231
1.22k
1232
1.22k
  // Set all unused physreg defs as dead.
1233
1.22k
  assert(CLI.Call && "No call instruction specified.");
1234
1.22k
  CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI);
1235
1.22k
1236
1.22k
  if (CLI.NumResultRegs && 
CLI.CS379
)
1237
342
    updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs);
1238
1.22k
1239
1.22k
  // Set labels for heapallocsite call.
1240
1.22k
  if (CLI.CS && 
CLI.CS->getInstruction()->getMetadata("heapallocsite")1.16k
) {
1241
3
    const MDNode *MD = CLI.CS->getInstruction()->getMetadata("heapallocsite");
1242
3
    MF->addCodeViewHeapAllocSite(CLI.Call, MD);
1243
3
  }
1244
1.22k
1245
1.22k
  return true;
1246
1.22k
}
1247
1248
2.10k
bool FastISel::lowerCall(const CallInst *CI) {
1249
2.10k
  ImmutableCallSite CS(CI);
1250
2.10k
1251
2.10k
  FunctionType *FuncTy = CS.getFunctionType();
1252
2.10k
  Type *RetTy = CS.getType();
1253
2.10k
1254
2.10k
  ArgListTy Args;
1255
2.10k
  ArgListEntry Entry;
1256
2.10k
  Args.reserve(CS.arg_size());
1257
2.10k
1258
2.10k
  for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1259
6.33k
       i != e; 
++i4.23k
) {
1260
4.23k
    Value *V = *i;
1261
4.23k
1262
4.23k
    // Skip empty types
1263
4.23k
    if (V->getType()->isEmptyTy())
1264
0
      continue;
1265
4.23k
1266
4.23k
    Entry.Val = V;
1267
4.23k
    Entry.Ty = V->getType();
1268
4.23k
1269
4.23k
    // Skip the first return-type Attribute to get to params.
1270
4.23k
    Entry.setAttributes(&CS, i - CS.arg_begin());
1271
4.23k
    Args.push_back(Entry);
1272
4.23k
  }
1273
2.10k
1274
2.10k
  // Check if target-independent constraints permit a tail call here.
1275
2.10k
  // Target-dependent constraints are checked within fastLowerCall.
1276
2.10k
  bool IsTailCall = CI->isTailCall();
1277
2.10k
  if (IsTailCall && 
!isInTailCallPosition(CS, TM)383
)
1278
246
    IsTailCall = false;
1279
2.10k
1280
2.10k
  CallLoweringInfo CLI;
1281
2.10k
  CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS)
1282
2.10k
      .setTailCall(IsTailCall);
1283
2.10k
1284
2.10k
  return lowerCallTo(CLI);
1285
2.10k
}
1286
1287
5.67k
bool FastISel::selectCall(const User *I) {
1288
5.67k
  const CallInst *Call = cast<CallInst>(I);
1289
5.67k
1290
5.67k
  // Handle simple inline asms.
1291
5.67k
  if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) {
1292
140
    // If the inline asm has side effects, then make sure that no local value
1293
140
    // lives across by flushing the local value map.
1294
140
    if (IA->hasSideEffects())
1295
126
      flushLocalValueMap();
1296
140
1297
140
    // Don't attempt to handle constraints.
1298
140
    if (!IA->getConstraintString().empty())
1299
135
      return false;
1300
5
1301
5
    unsigned ExtraInfo = 0;
1302
5
    if (IA->hasSideEffects())
1303
5
      ExtraInfo |= InlineAsm::Extra_HasSideEffects;
1304
5
    if (IA->isAlignStack())
1305
0
      ExtraInfo |= InlineAsm::Extra_IsAlignStack;
1306
5
1307
5
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1308
5
            TII.get(TargetOpcode::INLINEASM))
1309
5
        .addExternalSymbol(IA->getAsmString().c_str())
1310
5
        .addImm(ExtraInfo);
1311
5
    return true;
1312
5
  }
1313
5.53k
1314
5.53k
  // Handle intrinsic function calls.
1315
5.53k
  if (const auto *II = dyn_cast<IntrinsicInst>(Call))
1316
3.42k
    return selectIntrinsicCall(II);
1317
2.10k
1318
2.10k
  // Usually, it does not make sense to initialize a value,
1319
2.10k
  // make an unrelated function call and use the value, because
1320
2.10k
  // it tends to be spilled on the stack. So, we move the pointer
1321
2.10k
  // to the last local value to the beginning of the block, so that
1322
2.10k
  // all the values which have already been materialized,
1323
2.10k
  // appear after the call. It also makes sense to skip intrinsics
1324
2.10k
  // since they tend to be inlined.
1325
2.10k
  flushLocalValueMap();
1326
2.10k
1327
2.10k
  return lowerCall(Call);
1328
2.10k
}
1329
1330
3.42k
bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
1331
3.42k
  switch (II->getIntrinsicID()) {
1332
3.42k
  default:
1333
2.75k
    break;
1334
3.42k
  // At -O0 we don't care about the lifetime intrinsics.
1335
3.42k
  case Intrinsic::lifetime_start:
1336
17
  case Intrinsic::lifetime_end:
1337
17
  // The donothing intrinsic does, well, nothing.
1338
17
  case Intrinsic::donothing:
1339
17
  // Neither does the sideeffect intrinsic.
1340
17
  case Intrinsic::sideeffect:
1341
17
  // Neither does the assume intrinsic; it's also OK not to codegen its operand.
1342
17
  case Intrinsic::assume:
1343
17
    return true;
1344
487
  case Intrinsic::dbg_declare: {
1345
487
    const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
1346
487
    assert(DI->getVariable() && "Missing variable");
1347
487
    if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1348
0
      LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1349
0
      return true;
1350
0
    }
1351
487
1352
487
    const Value *Address = DI->getAddress();
1353
487
    if (!Address || isa<UndefValue>(Address)) {
1354
4
      LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1355
4
      return true;
1356
4
    }
1357
483
1358
483
    // Byval arguments with frame indices were already handled after argument
1359
483
    // lowering and before isel.
1360
483
    const auto *Arg =
1361
483
        dyn_cast<Argument>(Address->stripInBoundsConstantOffsets());
1362
483
    if (Arg && 
FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX25
)
1363
483
      
return true10
;
1364
473
1365
473
    Optional<MachineOperand> Op;
1366
473
    if (unsigned Reg = lookUpRegForValue(Address))
1367
35
      Op = MachineOperand::CreateReg(Reg, false);
1368
473
1369
473
    // If we have a VLA that has a "use" in a metadata node that's then used
1370
473
    // here but it has no other uses, then we have a problem. E.g.,
1371
473
    //
1372
473
    //   int foo (const int *x) {
1373
473
    //     char a[*x];
1374
473
    //     return 0;
1375
473
    //   }
1376
473
    //
1377
473
    // If we assign 'a' a vreg and fast isel later on has to use the selection
1378
473
    // DAG isel, it will want to copy the value to the vreg. However, there are
1379
473
    // no uses, which goes counter to what selection DAG isel expects.
1380
473
    if (!Op && 
!Address->use_empty()438
&&
isa<Instruction>(Address)399
&&
1381
473
        
(399
!isa<AllocaInst>(Address)399
||
1382
399
         
!FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))398
))
1383
1
      Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address),
1384
1
                                     false);
1385
473
1386
473
    if (Op) {
1387
36
      assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) &&
1388
36
             "Expected inlined-at fields to agree");
1389
36
      // A dbg.declare describes the address of a source variable, so lower it
1390
36
      // into an indirect DBG_VALUE.
1391
36
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1392
36
              TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true,
1393
36
              *Op, DI->getVariable(), DI->getExpression());
1394
437
    } else {
1395
437
      // We can't yet handle anything else here because it would require
1396
437
      // generating code, thus altering codegen because of debug info.
1397
437
      LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1398
437
    }
1399
473
    return true;
1400
473
  }
1401
473
  case Intrinsic::dbg_value: {
1402
85
    // This form of DBG_VALUE is target-independent.
1403
85
    const DbgValueInst *DI = cast<DbgValueInst>(II);
1404
85
    const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
1405
85
    const Value *V = DI->getValue();
1406
85
    assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) &&
1407
85
           "Expected inlined-at fields to agree");
1408
85
    if (!V) {
1409
0
      // Currently the optimizer can produce this; insert an undef to
1410
0
      // help debugging.  Probably the optimizer should not do this.
1411
0
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U,
1412
0
              DI->getVariable(), DI->getExpression());
1413
85
    } else if (const auto *CI = dyn_cast<ConstantInt>(V)) {
1414
16
      if (CI->getBitWidth() > 64)
1415
0
        BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1416
0
            .addCImm(CI)
1417
0
            .addImm(0U)
1418
0
            .addMetadata(DI->getVariable())
1419
0
            .addMetadata(DI->getExpression());
1420
16
      else
1421
16
        BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1422
16
            .addImm(CI->getZExtValue())
1423
16
            .addImm(0U)
1424
16
            .addMetadata(DI->getVariable())
1425
16
            .addMetadata(DI->getExpression());
1426
69
    } else if (const auto *CF = dyn_cast<ConstantFP>(V)) {
1427
0
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
1428
0
          .addFPImm(CF)
1429
0
          .addImm(0U)
1430
0
          .addMetadata(DI->getVariable())
1431
0
          .addMetadata(DI->getExpression());
1432
69
    } else if (unsigned Reg = lookUpRegForValue(V)) {
1433
53
      // FIXME: This does not handle register-indirect values at offset 0.
1434
53
      bool IsIndirect = false;
1435
53
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg,
1436
53
              DI->getVariable(), DI->getExpression());
1437
53
    } else {
1438
16
      // We can't yet handle anything else here because it would require
1439
16
      // generating code, thus altering codegen because of debug info.
1440
16
      LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1441
16
    }
1442
85
    return true;
1443
473
  }
1444
473
  case Intrinsic::dbg_label: {
1445
5
    const DbgLabelInst *DI = cast<DbgLabelInst>(II);
1446
5
    assert(DI->getLabel() && "Missing label");
1447
5
    if (!FuncInfo.MF->getMMI().hasDebugInfo()) {
1448
0
      LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
1449
0
      return true;
1450
0
    }
1451
5
1452
5
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1453
5
            TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
1454
5
    return true;
1455
5
  }
1456
5
  case Intrinsic::objectsize: {
1457
3
    ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1));
1458
3
    unsigned long long Res = CI->isZero() ? 
-1ULL1
:
02
;
1459
3
    Constant *ResCI = ConstantInt::get(II->getType(), Res);
1460
3
    unsigned ResultReg = getRegForValue(ResCI);
1461
3
    if (!ResultReg)
1462
0
      return false;
1463
3
    updateValueMap(II, ResultReg);
1464
3
    return true;
1465
3
  }
1466
3
  case Intrinsic::is_constant: {
1467
2
    Constant *ResCI = ConstantInt::get(II->getType(), 0);
1468
2
    unsigned ResultReg = getRegForValue(ResCI);
1469
2
    if (!ResultReg)
1470
0
      return false;
1471
2
    updateValueMap(II, ResultReg);
1472
2
    return true;
1473
2
  }
1474
8
  case Intrinsic::launder_invariant_group:
1475
8
  case Intrinsic::strip_invariant_group:
1476
8
  case Intrinsic::expect: {
1477
8
    unsigned ResultReg = getRegForValue(II->getArgOperand(0));
1478
8
    if (!ResultReg)
1479
0
      return false;
1480
8
    updateValueMap(II, ResultReg);
1481
8
    return true;
1482
8
  }
1483
26
  case Intrinsic::experimental_stackmap:
1484
26
    return selectStackmap(II);
1485
39
  case Intrinsic::experimental_patchpoint_void:
1486
39
  case Intrinsic::experimental_patchpoint_i64:
1487
39
    return selectPatchpoint(II);
1488
39
1489
39
  case Intrinsic::xray_customevent:
1490
0
    return selectXRayCustomEvent(II);
1491
39
  case Intrinsic::xray_typedevent:
1492
0
    return selectXRayTypedEvent(II);
1493
2.75k
  }
1494
2.75k
1495
2.75k
  return fastLowerIntrinsicCall(II);
1496
2.75k
}
1497
1498
1.22k
bool FastISel::selectCast(const User *I, unsigned Opcode) {
1499
1.22k
  EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1500
1.22k
  EVT DstVT = TLI.getValueType(DL, I->getType());
1501
1.22k
1502
1.22k
  if (SrcVT == MVT::Other || !SrcVT.isSimple() || 
DstVT == MVT::Other1.22k
||
1503
1.22k
      
!DstVT.isSimple()1.22k
)
1504
2
    // Unhandled type. Halt "fast" selection and bail.
1505
2
    return false;
1506
1.22k
1507
1.22k
  // Check if the destination type is legal.
1508
1.22k
  if (!TLI.isTypeLegal(DstVT))
1509
172
    return false;
1510
1.05k
1511
1.05k
  // Check if the source operand is legal.
1512
1.05k
  if (!TLI.isTypeLegal(SrcVT))
1513
596
    return false;
1514
455
1515
455
  unsigned InputReg = getRegForValue(I->getOperand(0));
1516
455
  if (!InputReg)
1517
5
    // Unhandled operand.  Halt "fast" selection and bail.
1518
5
    return false;
1519
450
1520
450
  bool InputRegIsKill = hasTrivialKill(I->getOperand(0));
1521
450
1522
450
  unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(),
1523
450
                                  Opcode, InputReg, InputRegIsKill);
1524
450
  if (!ResultReg)
1525
111
    return false;
1526
339
1527
339
  updateValueMap(I, ResultReg);
1528
339
  return true;
1529
339
}
1530
1531
4.49k
bool FastISel::selectBitCast(const User *I) {
1532
4.49k
  // If the bitcast doesn't change the type, just use the operand value.
1533
4.49k
  if (I->getType() == I->getOperand(0)->getType()) {
1534
3
    unsigned Reg = getRegForValue(I->getOperand(0));
1535
3
    if (!Reg)
1536
0
      return false;
1537
3
    updateValueMap(I, Reg);
1538
3
    return true;
1539
3
  }
1540
4.49k
1541
4.49k
  // Bitcasts of other values become reg-reg copies or BITCAST operators.
1542
4.49k
  EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1543
4.49k
  EVT DstEVT = TLI.getValueType(DL, I->getType());
1544
4.49k
  if (SrcEVT == MVT::Other || DstEVT == MVT::Other ||
1545
4.49k
      !TLI.isTypeLegal(SrcEVT) || 
!TLI.isTypeLegal(DstEVT)4.45k
)
1546
45
    // Unhandled type. Halt "fast" selection and bail.
1547
45
    return false;
1548
4.44k
1549
4.44k
  MVT SrcVT = SrcEVT.getSimpleVT();
1550
4.44k
  MVT DstVT = DstEVT.getSimpleVT();
1551
4.44k
  unsigned Op0 = getRegForValue(I->getOperand(0));
1552
4.44k
  if (!Op0) // Unhandled operand. Halt "fast" selection and bail.
1553
16
    return false;
1554
4.43k
  bool Op0IsKill = hasTrivialKill(I->getOperand(0));
1555
4.43k
1556
4.43k
  // First, try to perform the bitcast by inserting a reg-reg copy.
1557
4.43k
  unsigned ResultReg = 0;
1558
4.43k
  if (SrcVT == DstVT) {
1559
414
    const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT);
1560
414
    const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
1561
414
    // Don't attempt a cross-class copy. It will likely fail.
1562
414
    if (SrcClass == DstClass) {
1563
414
      ResultReg = createResultReg(DstClass);
1564
414
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
1565
414
              TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0);
1566
414
    }
1567
414
  }
1568
4.43k
1569
4.43k
  // If the reg-reg copy failed, select a BITCAST opcode.
1570
4.43k
  if (!ResultReg)
1571
4.01k
    ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill);
1572
4.43k
1573
4.43k
  if (!ResultReg)
1574
3.99k
    return false;
1575
434
1576
434
  updateValueMap(I, ResultReg);
1577
434
  return true;
1578
434
}
1579
1580
// Remove local value instructions starting from the instruction after
1581
// SavedLastLocalValue to the current function insert point.
1582
void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue)
1583
1.14k
{
1584
1.14k
  MachineInstr *CurLastLocalValue = getLastLocalValue();
1585
1.14k
  if (CurLastLocalValue != SavedLastLocalValue) {
1586
25
    // Find the first local value instruction to be deleted.
1587
25
    // This is the instruction after SavedLastLocalValue if it is non-NULL.
1588
25
    // Otherwise it's the first instruction in the block.
1589
25
    MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue);
1590
25
    if (SavedLastLocalValue)
1591
4
      ++FirstDeadInst;
1592
21
    else
1593
21
      FirstDeadInst = FuncInfo.MBB->getFirstNonPHI();
1594
25
    setLastLocalValue(SavedLastLocalValue);
1595
25
    removeDeadCode(FirstDeadInst, FuncInfo.InsertPt);
1596
25
  }
1597
1.14k
}
1598
1599
41.2k
bool FastISel::selectInstruction(const Instruction *I) {
1600
41.2k
  MachineInstr *SavedLastLocalValue = getLastLocalValue();
1601
41.2k
  // Just before the terminator instruction, insert instructions to
1602
41.2k
  // feed PHI nodes in successor blocks.
1603
41.2k
  if (I->isTerminator()) {
1604
15.8k
    if (!handlePHINodesInSuccessorBlocks(I->getParent())) {
1605
68
      // PHI node handling may have generated local value instructions,
1606
68
      // even though it failed to handle all PHI nodes.
1607
68
      // We remove these instructions because SelectionDAGISel will generate
1608
68
      // them again.
1609
68
      removeDeadLocalValueCode(SavedLastLocalValue);
1610
68
      return false;
1611
68
    }
1612
41.1k
  }
1613
41.1k
1614
41.1k
  // FastISel does not handle any operand bundles except OB_funclet.
1615
41.1k
  if (ImmutableCallSite CS = ImmutableCallSite(I))
1616
5.87k
    
for (unsigned i = 0, e = CS.getNumOperandBundles(); 5.86k
i != e;
++i13
)
1617
13
      if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet)
1618
0
        return false;
1619
41.1k
1620
41.1k
  DbgLoc = I->getDebugLoc();
1621
41.1k
1622
41.1k
  SavedInsertPt = FuncInfo.InsertPt;
1623
41.1k
1624
41.1k
  if (const auto *Call = dyn_cast<CallInst>(I)) {
1625
5.71k
    const Function *F = Call->getCalledFunction();
1626
5.71k
    LibFunc Func;
1627
5.71k
1628
5.71k
    // As a special case, don't handle calls to builtin library functions that
1629
5.71k
    // may be translated directly to target instructions.
1630
5.71k
    if (F && 
!F->hasLocalLinkage()5.43k
&&
F->hasName()5.36k
&&
1631
5.71k
        
LibInfo->getLibFunc(F->getName(), Func)5.36k
&&
1632
5.71k
        
LibInfo->hasOptimizedCodeGen(Func)125
)
1633
10
      return false;
1634
5.70k
1635
5.70k
    // Don't handle Intrinsic::trap if a trap function is specified.
1636
5.70k
    if (F && 
F->getIntrinsicID() == Intrinsic::trap5.42k
&&
1637
5.70k
        
Call->hasFnAttr("trap-func-name")16
)
1638
0
      return false;
1639
41.1k
  }
1640
41.1k
1641
41.1k
  // First, try doing target-independent selection.
1642
41.1k
  if (!SkipTargetIndependentISel) {
1643
35.6k
    if (selectOperator(I, I->getOpcode())) {
1644
5.28k
      ++NumFastIselSuccessIndependent;
1645
5.28k
      DbgLoc = DebugLoc();
1646
5.28k
      return true;
1647
5.28k
    }
1648
30.3k
    // Remove dead code.
1649
30.3k
    recomputeInsertPt();
1650
30.3k
    if (SavedInsertPt != FuncInfo.InsertPt)
1651
61
      removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1652
30.3k
    SavedInsertPt = FuncInfo.InsertPt;
1653
30.3k
  }
1654
41.1k
  // Next, try calling the target to attempt to handle the instruction.
1655
41.1k
  
if (35.8k
fastSelectInstruction(I)35.8k
) {
1656
26.7k
    ++NumFastIselSuccessTarget;
1657
26.7k
    DbgLoc = DebugLoc();
1658
26.7k
    return true;
1659
26.7k
  }
1660
9.15k
  // Remove dead code.
1661
9.15k
  recomputeInsertPt();
1662
9.15k
  if (SavedInsertPt != FuncInfo.InsertPt)
1663
17
    removeDeadCode(FuncInfo.InsertPt, SavedInsertPt);
1664
9.15k
1665
9.15k
  DbgLoc = DebugLoc();
1666
9.15k
  // Undo phi node updates, because they will be added again by SelectionDAG.
1667
9.15k
  if (I->isTerminator()) {
1668
1.07k
    // PHI node handling may have generated local value instructions.
1669
1.07k
    // We remove them because SelectionDAGISel will generate them again.
1670
1.07k
    removeDeadLocalValueCode(SavedLastLocalValue);
1671
1.07k
    FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
1672
1.07k
  }
1673
9.15k
  return false;
1674
9.15k
}
1675
1676
/// Emit an unconditional branch to the given block, unless it is the immediate
1677
/// (fall-through) successor, and update the CFG.
1678
void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
1679
1.81k
                              const DebugLoc &DbgLoc) {
1680
1.81k
  if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
1681
1.81k
      
FuncInfo.MBB->isLayoutSuccessor(MSucc)1.52k
) {
1682
1.10k
    // For more accurate line information if this is the only instruction
1683
1.10k
    // in the block then emit it, otherwise we have the unconditional
1684
1.10k
    // fall-through case, which needs no instructions.
1685
1.10k
  } else {
1686
714
    // The unconditional branch case.
1687
714
    TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
1688
714
                     SmallVector<MachineOperand, 0>(), DbgLoc);
1689
714
  }
1690
1.81k
  if (FuncInfo.BPI) {
1691
271
    auto BranchProbability = FuncInfo.BPI->getEdgeProbability(
1692
271
        FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock());
1693
271
    FuncInfo.MBB->addSuccessor(MSucc, BranchProbability);
1694
271
  } else
1695
1.54k
    FuncInfo.MBB->addSuccessorWithoutProb(MSucc);
1696
1.81k
}
1697
1698
void FastISel::finishCondBranch(const BasicBlock *BranchBB,
1699
                                MachineBasicBlock *TrueMBB,
1700
710
                                MachineBasicBlock *FalseMBB) {
1701
710
  // Add TrueMBB as successor unless it is equal to the FalseMBB: This can
1702
710
  // happen in degenerate IR and MachineIR forbids to have a block twice in the
1703
710
  // successor/predecessor lists.
1704
710
  if (TrueMBB != FalseMBB) {
1705
706
    if (FuncInfo.BPI) {
1706
176
      auto BranchProbability =
1707
176
          FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock());
1708
176
      FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability);
1709
176
    } else
1710
530
      FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB);
1711
706
  }
1712
710
1713
710
  fastEmitBranch(FalseMBB, DbgLoc);
1714
710
}
1715
1716
/// Emit an FNeg operation.
1717
54
bool FastISel::selectFNeg(const User *I, const Value *In) {
1718
54
  unsigned OpReg = getRegForValue(In);
1719
54
  if (!OpReg)
1720
0
    return false;
1721
54
  bool OpRegIsKill = hasTrivialKill(In);
1722
54
1723
54
  // If the target has ISD::FNEG, use it.
1724
54
  EVT VT = TLI.getValueType(DL, I->getType());
1725
54
  unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG,
1726
54
                                  OpReg, OpRegIsKill);
1727
54
  if (ResultReg) {
1728
5
    updateValueMap(I, ResultReg);
1729
5
    return true;
1730
5
  }
1731
49
1732
49
  // Bitcast the value to integer, twiddle the sign bit with xor,
1733
49
  // and then bitcast it back to floating-point.
1734
49
  if (VT.getSizeInBits() > 64)
1735
40
    return false;
1736
9
  EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits());
1737
9
  if (!TLI.isTypeLegal(IntVT))
1738
1
    return false;
1739
8
1740
8
  unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(),
1741
8
                               ISD::BITCAST, OpReg, OpRegIsKill);
1742
8
  if (!IntReg)
1743
0
    return false;
1744
8
1745
8
  unsigned IntResultReg = fastEmit_ri_(
1746
8
      IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true,
1747
8
      UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT());
1748
8
  if (!IntResultReg)
1749
0
    return false;
1750
8
1751
8
  ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST,
1752
8
                         IntResultReg, /*IsKill=*/true);
1753
8
  if (!ResultReg)
1754
0
    return false;
1755
8
1756
8
  updateValueMap(I, ResultReg);
1757
8
  return true;
1758
8
}
1759
1760
513
bool FastISel::selectExtractValue(const User *U) {
1761
513
  const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U);
1762
513
  if (!EVI)
1763
0
    return false;
1764
513
1765
513
  // Make sure we only try to handle extracts with a legal result.  But also
1766
513
  // allow i1 because it's easy.
1767
513
  EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true);
1768
513
  if (!RealVT.isSimple())
1769
0
    return false;
1770
513
  MVT VT = RealVT.getSimpleVT();
1771
513
  if (!TLI.isTypeLegal(VT) && 
VT != MVT::i1210
)
1772
5
    return false;
1773
508
1774
508
  const Value *Op0 = EVI->getOperand(0);
1775
508
  Type *AggTy = Op0->getType();
1776
508
1777
508
  // Get the base result register.
1778
508
  unsigned ResultReg;
1779
508
  DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0);
1780
508
  if (I != FuncInfo.ValueMap.end())
1781
247
    ResultReg = I->second;
1782
261
  else if (isa<Instruction>(Op0))
1783
258
    ResultReg = FuncInfo.InitializeRegForValue(Op0);
1784
3
  else
1785
3
    return false; // fast-isel can't handle aggregate constants at the moment
1786
505
1787
505
  // Get the actual result register, which is an offset from the base register.
1788
505
  unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices());
1789
505
1790
505
  SmallVector<EVT, 4> AggValueVTs;
1791
505
  ComputeValueVTs(TLI, DL, AggTy, AggValueVTs);
1792
505
1793
884
  for (unsigned i = 0; i < VTIndex; 
i++379
)
1794
379
    ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]);
1795
505
1796
505
  updateValueMap(EVI, ResultReg);
1797
505
  return true;
1798
505
}
1799
1800
37.0k
bool FastISel::selectOperator(const User *I, unsigned Opcode) {
1801
37.0k
  switch (Opcode) {
1802
37.0k
  case Instruction::Add:
1803
571
    return selectBinaryOp(I, ISD::ADD);
1804
37.0k
  case Instruction::FAdd:
1805
261
    return selectBinaryOp(I, ISD::FADD);
1806
37.0k
  case Instruction::Sub:
1807
133
    return selectBinaryOp(I, ISD::SUB);
1808
37.0k
  case Instruction::FSub: {
1809
75
    // FNeg is currently represented in LLVM IR as a special case of FSub.
1810
75
    Value *X;
1811
75
    if (match(I, m_FNeg(m_Value(X))))
1812
52
       return selectFNeg(I, X);
1813
23
    return selectBinaryOp(I, ISD::FSUB);
1814
23
  }
1815
109
  case Instruction::Mul:
1816
109
    return selectBinaryOp(I, ISD::MUL);
1817
29
  case Instruction::FMul:
1818
29
    return selectBinaryOp(I, ISD::FMUL);
1819
23
  case Instruction::SDiv:
1820
21
    return selectBinaryOp(I, ISD::SDIV);
1821
26
  case Instruction::UDiv:
1822
26
    return selectBinaryOp(I, ISD::UDIV);
1823
23
  case Instruction::FDiv:
1824
23
    return selectBinaryOp(I, ISD::FDIV);
1825
44
  case Instruction::SRem:
1826
44
    return selectBinaryOp(I, ISD::SREM);
1827
23
  case Instruction::URem:
1828
22
    return selectBinaryOp(I, ISD::UREM);
1829
23
  case Instruction::FRem:
1830
1
    return selectBinaryOp(I, ISD::FREM);
1831
75
  case Instruction::Shl:
1832
75
    return selectBinaryOp(I, ISD::SHL);
1833
63
  case Instruction::LShr:
1834
63
    return selectBinaryOp(I, ISD::SRL);
1835
60
  case Instruction::AShr:
1836
60
    return selectBinaryOp(I, ISD::SRA);
1837
232
  case Instruction::And:
1838
232
    return selectBinaryOp(I, ISD::AND);
1839
150
  case Instruction::Or:
1840
150
    return selectBinaryOp(I, ISD::OR);
1841
196
  case Instruction::Xor:
1842
196
    return selectBinaryOp(I, ISD::XOR);
1843
23
1844
23
  case Instruction::FNeg:
1845
2
    return selectFNeg(I, I->getOperand(0));
1846
23
1847
325
  case Instruction::GetElementPtr:
1848
325
    return selectGetElementPtr(I);
1849
23
1850
1.46k
  case Instruction::Br: {
1851
1.46k
    const BranchInst *BI = cast<BranchInst>(I);
1852
1.46k
1853
1.46k
    if (BI->isUnconditional()) {
1854
847
      const BasicBlock *LLVMSucc = BI->getSuccessor(0);
1855
847
      MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc];
1856
847
      fastEmitBranch(MSucc, BI->getDebugLoc());
1857
847
      return true;
1858
847
    }
1859
617
1860
617
    // Conditional branches are not handed yet.
1861
617
    // Halt "fast" selection and bail.
1862
617
    return false;
1863
617
  }
1864
617
1865
617
  case Instruction::Unreachable:
1866
109
    if (TM.Options.TrapUnreachable)
1867
62
      return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0;
1868
47
    else
1869
47
      return true;
1870
0
1871
16
  case Instruction::Alloca:
1872
16
    // FunctionLowering has the static-sized case covered.
1873
16
    if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I)))
1874
0
      return true;
1875
16
1876
16
    // Dynamic-sized alloca is not handled yet.
1877
16
    return false;
1878
16
1879
5.62k
  case Instruction::Call:
1880
5.62k
    // On AIX, call lowering uses the DAG-ISEL path currently so that the
1881
5.62k
    // callee of the direct function call instruction will be mapped to the
1882
5.62k
    // symbol for the function's entry point, which is distinct from the
1883
5.62k
    // function descriptor symbol. The latter is the symbol whose XCOFF symbol
1884
5.62k
    // name is the C-linkage name of the source level function.
1885
5.62k
    if (TM.getTargetTriple().isOSAIX())
1886
0
      return false;
1887
5.62k
    return selectCall(I);
1888
5.62k
1889
5.62k
  case Instruction::BitCast:
1890
4.44k
    return selectBitCast(I);
1891
5.62k
1892
5.62k
  case Instruction::FPToSI:
1893
34
    return selectCast(I, ISD::FP_TO_SINT);
1894
5.62k
  case Instruction::ZExt:
1895
386
    return selectCast(I, ISD::ZERO_EXTEND);
1896
5.62k
  case Instruction::SExt:
1897
485
    return selectCast(I, ISD::SIGN_EXTEND);
1898
5.62k
  case Instruction::Trunc:
1899
143
    return selectCast(I, ISD::TRUNCATE);
1900
5.62k
  case Instruction::SIToFP:
1901
109
    return selectCast(I, ISD::SINT_TO_FP);
1902
5.62k
1903
5.62k
  case Instruction::IntToPtr: // Deliberate fall-through.
1904
88
  case Instruction::PtrToInt: {
1905
88
    EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
1906
88
    EVT DstVT = TLI.getValueType(DL, I->getType());
1907
88
    if (DstVT.bitsGT(SrcVT))
1908
3
      return selectCast(I, ISD::ZERO_EXTEND);
1909
85
    if (DstVT.bitsLT(SrcVT))
1910
18
      return selectCast(I, ISD::TRUNCATE);
1911
67
    unsigned Reg = getRegForValue(I->getOperand(0));
1912
67
    if (!Reg)
1913
8
      return false;
1914
59
    updateValueMap(I, Reg);
1915
59
    return true;
1916
59
  }
1917
59
1918
513
  case Instruction::ExtractValue:
1919
513
    return selectExtractValue(I);
1920
59
1921
59
  case Instruction::PHI:
1922
0
    llvm_unreachable("FastISel shouldn't visit PHI nodes!");
1923
59
1924
21.1k
  default:
1925
21.1k
    // Unhandled instruction. Halt "fast" selection and bail.
1926
21.1k
    return false;
1927
37.0k
  }
1928
37.0k
}
1929
1930
FastISel::FastISel(FunctionLoweringInfo &FuncInfo,
1931
                   const TargetLibraryInfo *LibInfo,
1932
                   bool SkipTargetIndependentISel)
1933
    : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()),
1934
      MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()),
1935
      TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()),
1936
      TII(*MF->getSubtarget().getInstrInfo()),
1937
      TLI(*MF->getSubtarget().getTargetLowering()),
1938
      TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo),
1939
13.4k
      SkipTargetIndependentISel(SkipTargetIndependentISel) {}
1940
1941
13.4k
FastISel::~FastISel() = default;
1942
1943
0
bool FastISel::fastLowerArguments() { return false; }
1944
1945
617
bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; }
1946
1947
191
bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) {
1948
191
  return false;
1949
191
}
1950
1951
58
unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; }
1952
1953
unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/,
1954
0
                              bool /*Op0IsKill*/) {
1955
0
  return 0;
1956
0
}
1957
1958
unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/,
1959
                               bool /*Op0IsKill*/, unsigned /*Op1*/,
1960
0
                               bool /*Op1IsKill*/) {
1961
0
  return 0;
1962
0
}
1963
1964
0
unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) {
1965
0
  return 0;
1966
0
}
1967
1968
unsigned FastISel::fastEmit_f(MVT, MVT, unsigned,
1969
0
                              const ConstantFP * /*FPImm*/) {
1970
0
  return 0;
1971
0
}
1972
1973
unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/,
1974
22
                               bool /*Op0IsKill*/, uint64_t /*Imm*/) {
1975
22
  return 0;
1976
22
}
1977
1978
/// This method is a wrapper of fastEmit_ri. It first tries to emit an
1979
/// instruction with an immediate operand using fastEmit_ri.
1980
/// If that fails, it materializes the immediate into a register and try
1981
/// fastEmit_rr instead.
1982
unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0,
1983
1.04k
                                bool Op0IsKill, uint64_t Imm, MVT ImmType) {
1984
1.04k
  // If this is a multiply by a power of two, emit this as a shift left.
1985
1.04k
  if (Opcode == ISD::MUL && 
isPowerOf2_64(Imm)61
) {
1986
37
    Opcode = ISD::SHL;
1987
37
    Imm = Log2_64(Imm);
1988
1.00k
  } else if (Opcode == ISD::UDIV && 
isPowerOf2_64(Imm)4
) {
1989
3
    // div x, 8 -> srl x, 3
1990
3
    Opcode = ISD::SRL;
1991
3
    Imm = Log2_64(Imm);
1992
3
  }
1993
1.04k
1994
1.04k
  // Horrible hack (to be removed), check to make sure shift amounts are
1995
1.04k
  // in-range.
1996
1.04k
  if ((Opcode == ISD::SHL || 
Opcode == ISD::SRA986
||
Opcode == ISD::SRL967
) &&
1997
1.04k
      
Imm >= VT.getSizeInBits()100
)
1998
4
    return 0;
1999
1.03k
2000
1.03k
  // First check if immediate type is legal. If not, we can't use the ri form.
2001
1.03k
  unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm);
2002
1.03k
  if (ResultReg)
2003
952
    return ResultReg;
2004
87
  unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm);
2005
87
  bool IsImmKill = true;
2006
87
  if (!MaterialReg) {
2007
48
    // This is a bit ugly/slow, but failing here means falling out of
2008
48
    // fast-isel, which would be very slow.
2009
48
    IntegerType *ITy =
2010
48
        IntegerType::get(FuncInfo.Fn->getContext(), VT.getSizeInBits());
2011
48
    MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm));
2012
48
    if (!MaterialReg)
2013
0
      return 0;
2014
48
    // FIXME: If the materialized register here has no uses yet then this
2015
48
    // will be the first use and we should be able to mark it as killed.
2016
48
    // However, the local value area for materialising constant expressions
2017
48
    // grows down, not up, which means that any constant expressions we generate
2018
48
    // later which also use 'Imm' could be after this instruction and therefore
2019
48
    // after this kill.
2020
48
    IsImmKill = false;
2021
48
  }
2022
87
  return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill);
2023
87
}
2024
2025
22.5k
unsigned FastISel::createResultReg(const TargetRegisterClass *RC) {
2026
22.5k
  return MRI.createVirtualRegister(RC);
2027
22.5k
}
2028
2029
unsigned FastISel::constrainOperandRegClass(const MCInstrDesc &II, unsigned Op,
2030
17.2k
                                            unsigned OpNum) {
2031
17.2k
  if (TargetRegisterInfo::isVirtualRegister(Op)) {
2032
11.6k
    const TargetRegisterClass *RegClass =
2033
11.6k
        TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF);
2034
11.6k
    if (!MRI.constrainRegClass(Op, RegClass)) {
2035
0
      // If it's not legal to COPY between the register classes, something
2036
0
      // has gone very wrong before we got here.
2037
0
      unsigned NewOp = createResultReg(RegClass);
2038
0
      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2039
0
              TII.get(TargetOpcode::COPY), NewOp).addReg(Op);
2040
0
      return NewOp;
2041
0
    }
2042
17.2k
  }
2043
17.2k
  return Op;
2044
17.2k
}
2045
2046
unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode,
2047
406
                                 const TargetRegisterClass *RC) {
2048
406
  unsigned ResultReg = createResultReg(RC);
2049
406
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2050
406
2051
406
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg);
2052
406
  return ResultReg;
2053
406
}
2054
2055
unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode,
2056
                                  const TargetRegisterClass *RC, unsigned Op0,
2057
823
                                  bool Op0IsKill) {
2058
823
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2059
823
2060
823
  unsigned ResultReg = createResultReg(RC);
2061
823
  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2062
823
2063
823
  if (II.getNumDefs() >= 1)
2064
799
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2065
799
        .addReg(Op0, getKillRegState(Op0IsKill));
2066
24
  else {
2067
24
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2068
24
        .addReg(Op0, getKillRegState(Op0IsKill));
2069
24
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2070
24
            TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2071
24
  }
2072
823
2073
823
  return ResultReg;
2074
823
}
2075
2076
unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode,
2077
                                   const TargetRegisterClass *RC, unsigned Op0,
2078
                                   bool Op0IsKill, unsigned Op1,
2079
1.29k
                                   bool Op1IsKill) {
2080
1.29k
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2081
1.29k
2082
1.29k
  unsigned ResultReg = createResultReg(RC);
2083
1.29k
  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2084
1.29k
  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2085
1.29k
2086
1.29k
  if (II.getNumDefs() >= 1)
2087
1.29k
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2088
1.29k
        .addReg(Op0, getKillRegState(Op0IsKill))
2089
1.29k
        .addReg(Op1, getKillRegState(Op1IsKill));
2090
0
  else {
2091
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2092
0
        .addReg(Op0, getKillRegState(Op0IsKill))
2093
0
        .addReg(Op1, getKillRegState(Op1IsKill));
2094
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2095
0
            TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2096
0
  }
2097
1.29k
  return ResultReg;
2098
1.29k
}
2099
2100
unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode,
2101
                                    const TargetRegisterClass *RC, unsigned Op0,
2102
                                    bool Op0IsKill, unsigned Op1,
2103
                                    bool Op1IsKill, unsigned Op2,
2104
59
                                    bool Op2IsKill) {
2105
59
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2106
59
2107
59
  unsigned ResultReg = createResultReg(RC);
2108
59
  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2109
59
  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2110
59
  Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2);
2111
59
2112
59
  if (II.getNumDefs() >= 1)
2113
59
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2114
59
        .addReg(Op0, getKillRegState(Op0IsKill))
2115
59
        .addReg(Op1, getKillRegState(Op1IsKill))
2116
59
        .addReg(Op2, getKillRegState(Op2IsKill));
2117
0
  else {
2118
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2119
0
        .addReg(Op0, getKillRegState(Op0IsKill))
2120
0
        .addReg(Op1, getKillRegState(Op1IsKill))
2121
0
        .addReg(Op2, getKillRegState(Op2IsKill));
2122
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2123
0
            TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2124
0
  }
2125
59
  return ResultReg;
2126
59
}
2127
2128
unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode,
2129
                                   const TargetRegisterClass *RC, unsigned Op0,
2130
1.06k
                                   bool Op0IsKill, uint64_t Imm) {
2131
1.06k
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2132
1.06k
2133
1.06k
  unsigned ResultReg = createResultReg(RC);
2134
1.06k
  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2135
1.06k
2136
1.06k
  if (II.getNumDefs() >= 1)
2137
1.06k
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2138
1.06k
        .addReg(Op0, getKillRegState(Op0IsKill))
2139
1.06k
        .addImm(Imm);
2140
0
  else {
2141
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2142
0
        .addReg(Op0, getKillRegState(Op0IsKill))
2143
0
        .addImm(Imm);
2144
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2145
0
            TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2146
0
  }
2147
1.06k
  return ResultReg;
2148
1.06k
}
2149
2150
unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode,
2151
                                    const TargetRegisterClass *RC, unsigned Op0,
2152
                                    bool Op0IsKill, uint64_t Imm1,
2153
361
                                    uint64_t Imm2) {
2154
361
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2155
361
2156
361
  unsigned ResultReg = createResultReg(RC);
2157
361
  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2158
361
2159
361
  if (II.getNumDefs() >= 1)
2160
361
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2161
361
        .addReg(Op0, getKillRegState(Op0IsKill))
2162
361
        .addImm(Imm1)
2163
361
        .addImm(Imm2);
2164
0
  else {
2165
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2166
0
        .addReg(Op0, getKillRegState(Op0IsKill))
2167
0
        .addImm(Imm1)
2168
0
        .addImm(Imm2);
2169
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2170
0
            TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2171
0
  }
2172
361
  return ResultReg;
2173
361
}
2174
2175
unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode,
2176
                                  const TargetRegisterClass *RC,
2177
8
                                  const ConstantFP *FPImm) {
2178
8
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2179
8
2180
8
  unsigned ResultReg = createResultReg(RC);
2181
8
2182
8
  if (II.getNumDefs() >= 1)
2183
8
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2184
8
        .addFPImm(FPImm);
2185
0
  else {
2186
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2187
0
        .addFPImm(FPImm);
2188
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2189
0
            TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2190
0
  }
2191
8
  return ResultReg;
2192
8
}
2193
2194
unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode,
2195
                                    const TargetRegisterClass *RC, unsigned Op0,
2196
                                    bool Op0IsKill, unsigned Op1,
2197
355
                                    bool Op1IsKill, uint64_t Imm) {
2198
355
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2199
355
2200
355
  unsigned ResultReg = createResultReg(RC);
2201
355
  Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs());
2202
355
  Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1);
2203
355
2204
355
  if (II.getNumDefs() >= 1)
2205
355
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2206
355
        .addReg(Op0, getKillRegState(Op0IsKill))
2207
355
        .addReg(Op1, getKillRegState(Op1IsKill))
2208
355
        .addImm(Imm);
2209
0
  else {
2210
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II)
2211
0
        .addReg(Op0, getKillRegState(Op0IsKill))
2212
0
        .addReg(Op1, getKillRegState(Op1IsKill))
2213
0
        .addImm(Imm);
2214
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2215
0
            TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2216
0
  }
2217
355
  return ResultReg;
2218
355
}
2219
2220
unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode,
2221
739
                                  const TargetRegisterClass *RC, uint64_t Imm) {
2222
739
  unsigned ResultReg = createResultReg(RC);
2223
739
  const MCInstrDesc &II = TII.get(MachineInstOpcode);
2224
739
2225
739
  if (II.getNumDefs() >= 1)
2226
739
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg)
2227
739
        .addImm(Imm);
2228
0
  else {
2229
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addImm(Imm);
2230
0
    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
2231
0
            TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]);
2232
0
  }
2233
739
  return ResultReg;
2234
739
}
2235
2236
unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0,
2237
178
                                              bool Op0IsKill, uint32_t Idx) {
2238
178
  unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT));
2239
178
  assert(TargetRegisterInfo::isVirtualRegister(Op0) &&
2240
178
         "Cannot yet extract from physregs");
2241
178
  const TargetRegisterClass *RC = MRI.getRegClass(Op0);
2242
178
  MRI.constrainRegClass(Op0, TRI.getSubClassWithSubReg(RC, Idx));
2243
178
  BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY),
2244
178
          ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx);
2245
178
  return ResultReg;
2246
178
}
2247
2248
/// Emit MachineInstrs to compute the value of Op with all but the least
2249
/// significant bit set to zero.
2250
355
unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) {
2251
355
  return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1);
2252
355
}
2253
2254
/// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks.
2255
/// Emit code to ensure constants are copied into registers when needed.
2256
/// Remember the virtual registers that need to be added to the Machine PHI
2257
/// nodes as input.  We cannot just directly add them, because expansion
2258
/// might result in multiple MBB's for one BB.  As such, the start of the
2259
/// BB might correspond to a different MBB than the end.
2260
15.8k
bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
2261
15.8k
  const Instruction *TI = LLVMBB->getTerminator();
2262
15.8k
2263
15.8k
  SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
2264
15.8k
  FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size();
2265
15.8k
2266
15.8k
  // Check successor nodes' PHI nodes that expect a constant to be available
2267
15.8k
  // from this block.
2268
19.3k
  for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; 
++succ3.48k
) {
2269
3.54k
    const BasicBlock *SuccBB = TI->getSuccessor(succ);
2270
3.54k
    if (!isa<PHINode>(SuccBB->begin()))
2271
3.15k
      continue;
2272
396
    MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
2273
396
2274
396
    // If this terminator has multiple identical successors (common for
2275
396
    // switches), only handle each succ once.
2276
396
    if (!SuccsHandled.insert(SuccMBB).second)
2277
1
      continue;
2278
395
2279
395
    MachineBasicBlock::iterator MBBI = SuccMBB->begin();
2280
395
2281
395
    // At this point we know that there is a 1-1 correspondence between LLVM PHI
2282
395
    // nodes and Machine PHI nodes, but the incoming operands have not been
2283
395
    // emitted yet.
2284
430
    for (const PHINode &PN : SuccBB->phis()) {
2285
430
      // Ignore dead phi's.
2286
430
      if (PN.use_empty())
2287
8
        continue;
2288
422
2289
422
      // Only handle legal types. Two interesting things to note here. First,
2290
422
      // by bailing out early, we may leave behind some dead instructions,
2291
422
      // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its
2292
422
      // own moves. Second, this check is necessary because FastISel doesn't
2293
422
      // use CreateRegs to create registers, so it always creates
2294
422
      // exactly one register for each non-void instruction.
2295
422
      EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true);
2296
422
      if (VT == MVT::Other || !TLI.isTypeLegal(VT)) {
2297
89
        // Handle integer promotions, though, because they're common and easy.
2298
89
        if (!(VT == MVT::i1 || 
VT == MVT::i869
||
VT == MVT::i1667
)) {
2299
65
          FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
2300
65
          return false;
2301
65
        }
2302
357
      }
2303
357
2304
357
      const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
2305
357
2306
357
      // Set the DebugLoc for the copy. Prefer the location of the operand
2307
357
      // if there is one; use the location of the PHI otherwise.
2308
357
      DbgLoc = PN.getDebugLoc();
2309
357
      if (const auto *Inst = dyn_cast<Instruction>(PHIOp))
2310
267
        DbgLoc = Inst->getDebugLoc();
2311
357
2312
357
      unsigned Reg = getRegForValue(PHIOp);
2313
357
      if (!Reg) {
2314
3
        FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate);
2315
3
        return false;
2316
3
      }
2317
354
      FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg));
2318
354
      DbgLoc = DebugLoc();
2319
354
    }
2320
395
  }
2321
15.8k
2322
15.8k
  
return true15.8k
;
2323
15.8k
}
2324
2325
2.61k
bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) {
2326
2.61k
  assert(LI->hasOneUse() &&
2327
2.61k
         "tryToFoldLoad expected a LoadInst with a single use");
2328
2.61k
  // We know that the load has a single use, but don't know what it is.  If it
2329
2.61k
  // isn't one of the folded instructions, then we can't succeed here.  Handle
2330
2.61k
  // this by scanning the single-use users of the load until we get to FoldInst.
2331
2.61k
  unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs.
2332
2.61k
2333
2.61k
  const Instruction *TheUser = LI->user_back();
2334
3.25k
  while (TheUser != FoldInst && // Scan up until we find FoldInst.
2335
3.25k
         // Stay in the right block.
2336
3.25k
         
TheUser->getParent() == FoldInst->getParent()1.16k
&&
2337
3.25k
         
--MaxUsers1.07k
) { // Don't scan too far.
2338
1.06k
    // If there are multiple or no uses of this instruction, then bail out.
2339
1.06k
    if (!TheUser->hasOneUse())
2340
428
      return false;
2341
641
2342
641
    TheUser = TheUser->user_back();
2343
641
  }
2344
2.61k
2345
2.61k
  // If we didn't find the fold instruction, then we failed to collapse the
2346
2.61k
  // sequence.
2347
2.61k
  
if (2.18k
TheUser != FoldInst2.18k
)
2348
96
    return false;
2349
2.09k
2350
2.09k
  // Don't try to fold volatile loads.  Target has to deal with alignment
2351
2.09k
  // constraints.
2352
2.09k
  if (LI->isVolatile())
2353
70
    return false;
2354
2.02k
2355
2.02k
  // Figure out which vreg this is going into.  If there is no assigned vreg yet
2356
2.02k
  // then there actually was no reference to it.  Perhaps the load is referenced
2357
2.02k
  // by a dead instruction.
2358
2.02k
  unsigned LoadReg = getRegForValue(LI);
2359
2.02k
  if (!LoadReg)
2360
0
    return false;
2361
2.02k
2362
2.02k
  // We can't fold if this vreg has no uses or more than one use.  Multiple uses
2363
2.02k
  // may mean that the instruction got lowered to multiple MIs, or the use of
2364
2.02k
  // the loaded value ended up being multiple operands of the result.
2365
2.02k
  if (!MRI.hasOneUse(LoadReg))
2366
49
    return false;
2367
1.97k
2368
1.97k
  MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(LoadReg);
2369
1.97k
  MachineInstr *User = RI->getParent();
2370
1.97k
2371
1.97k
  // Set the insertion point properly.  Folding the load can cause generation of
2372
1.97k
  // other random instructions (like sign extends) for addressing modes; make
2373
1.97k
  // sure they get inserted in a logical place before the new instruction.
2374
1.97k
  FuncInfo.InsertPt = User;
2375
1.97k
  FuncInfo.MBB = User->getParent();
2376
1.97k
2377
1.97k
  // Ask the target to try folding the load.
2378
1.97k
  return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI);
2379
1.97k
}
2380
2381
105
bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) {
2382
105
  // Must be an add.
2383
105
  if (!isa<AddOperator>(Add))
2384
89
    return false;
2385
16
  // Type size needs to match.
2386
16
  if (DL.getTypeSizeInBits(GEP->getType()) !=
2387
16
      DL.getTypeSizeInBits(Add->getType()))
2388
5
    return false;
2389
11
  // Must be in the same basic block.
2390
11
  if (isa<Instruction>(Add) &&
2391
11
      FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB)
2392
1
    return false;
2393
10
  // Must have a constant operand.
2394
10
  return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1));
2395
10
}
2396
2397
MachineMemOperand *
2398
4.47k
FastISel::createMachineMemOperandFor(const Instruction *I) const {
2399
4.47k
  const Value *Ptr;
2400
4.47k
  Type *ValTy;
2401
4.47k
  unsigned Alignment;
2402
4.47k
  MachineMemOperand::Flags Flags;
2403
4.47k
  bool IsVolatile;
2404
4.47k
2405
4.47k
  if (const auto *LI = dyn_cast<LoadInst>(I)) {
2406
2.06k
    Alignment = LI->getAlignment();
2407
2.06k
    IsVolatile = LI->isVolatile();
2408
2.06k
    Flags = MachineMemOperand::MOLoad;
2409
2.06k
    Ptr = LI->getPointerOperand();
2410
2.06k
    ValTy = LI->getType();
2411
2.41k
  } else if (const auto *SI = dyn_cast<StoreInst>(I)) {
2412
2.41k
    Alignment = SI->getAlignment();
2413
2.41k
    IsVolatile = SI->isVolatile();
2414
2.41k
    Flags = MachineMemOperand::MOStore;
2415
2.41k
    Ptr = SI->getPointerOperand();
2416
2.41k
    ValTy = SI->getValueOperand()->getType();
2417
2.41k
  } else
2418
0
    return nullptr;
2419
4.47k
2420
4.47k
  bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr;
2421
4.47k
  bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr;
2422
4.47k
  bool IsDereferenceable =
2423
4.47k
      I->getMetadata(LLVMContext::MD_dereferenceable) != nullptr;
2424
4.47k
  const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range);
2425
4.47k
2426
4.47k
  AAMDNodes AAInfo;
2427
4.47k
  I->getAAMetadata(AAInfo);
2428
4.47k
2429
4.47k
  if (Alignment == 0) // Ensure that codegen never sees alignment 0.
2430
1.14k
    Alignment = DL.getABITypeAlignment(ValTy);
2431
4.47k
2432
4.47k
  unsigned Size = DL.getTypeStoreSize(ValTy);
2433
4.47k
2434
4.47k
  if (IsVolatile)
2435
82
    Flags |= MachineMemOperand::MOVolatile;
2436
4.47k
  if (IsNonTemporal)
2437
273
    Flags |= MachineMemOperand::MONonTemporal;
2438
4.47k
  if (IsDereferenceable)
2439
0
    Flags |= MachineMemOperand::MODereferenceable;
2440
4.47k
  if (IsInvariant)
2441
13
    Flags |= MachineMemOperand::MOInvariant;
2442
4.47k
2443
4.47k
  return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size,
2444
4.47k
                                           Alignment, AAInfo, Ranges);
2445
4.47k
}
2446
2447
1.28k
CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const {
2448
1.28k
  // If both operands are the same, then try to optimize or fold the cmp.
2449
1.28k
  CmpInst::Predicate Predicate = CI->getPredicate();
2450
1.28k
  if (CI->getOperand(0) != CI->getOperand(1))
2451
1.18k
    return Predicate;
2452
101
2453
101
  switch (Predicate) {
2454
101
  
default: 0
llvm_unreachable0
("Invalid predicate!");
2455
101
  
case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break0
;
2456
101
  
case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break4
;
2457
101
  
case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break6
;
2458
101
  
case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break4
;
2459
101
  
case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break4
;
2460
101
  
case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break4
;
2461
101
  
case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break4
;
2462
101
  
case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break4
;
2463
101
  
case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break4
;
2464
101
  
case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break6
;
2465
101
  
case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break4
;
2466
101
  
case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break4
;
2467
101
  
case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break4
;
2468
101
  
case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break4
;
2469
101
  
case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break4
;
2470
101
  
case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break0
;
2471
101
2472
101
  
case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break4
;
2473
101
  
case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break4
;
2474
101
  
case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break4
;
2475
101
  
case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break4
;
2476
101
  
case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break4
;
2477
101
  
case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break4
;
2478
101
  
case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break5
;
2479
101
  
case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break4
;
2480
101
  
case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break4
;
2481
101
  
case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break4
;
2482
101
  }
2483
101
2484
101
  return Predicate;
2485
101
}