Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/TargetPassConfig.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- TargetPassConfig.cpp - Target independent code generation passes ---===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file defines interfaces to access the target independent code
10
// generation passes provided by the LLVM backend.
11
//
12
//===---------------------------------------------------------------------===//
13
14
#include "llvm/CodeGen/TargetPassConfig.h"
15
#include "llvm/ADT/DenseMap.h"
16
#include "llvm/ADT/SmallVector.h"
17
#include "llvm/ADT/StringRef.h"
18
#include "llvm/Analysis/BasicAliasAnalysis.h"
19
#include "llvm/Analysis/CFLAndersAliasAnalysis.h"
20
#include "llvm/Analysis/CFLSteensAliasAnalysis.h"
21
#include "llvm/Analysis/CallGraphSCCPass.h"
22
#include "llvm/Analysis/ScopedNoAliasAA.h"
23
#include "llvm/Analysis/TargetTransformInfo.h"
24
#include "llvm/Analysis/TypeBasedAliasAnalysis.h"
25
#include "llvm/CodeGen/CSEConfigBase.h"
26
#include "llvm/CodeGen/MachineFunctionPass.h"
27
#include "llvm/CodeGen/MachinePassRegistry.h"
28
#include "llvm/CodeGen/Passes.h"
29
#include "llvm/CodeGen/RegAllocRegistry.h"
30
#include "llvm/IR/IRPrintingPasses.h"
31
#include "llvm/IR/LegacyPassManager.h"
32
#include "llvm/IR/Verifier.h"
33
#include "llvm/MC/MCAsmInfo.h"
34
#include "llvm/MC/MCTargetOptions.h"
35
#include "llvm/Pass.h"
36
#include "llvm/Support/CodeGen.h"
37
#include "llvm/Support/CommandLine.h"
38
#include "llvm/Support/Compiler.h"
39
#include "llvm/Support/Debug.h"
40
#include "llvm/Support/ErrorHandling.h"
41
#include "llvm/Support/Threading.h"
42
#include "llvm/Support/SaveAndRestore.h"
43
#include "llvm/Target/TargetMachine.h"
44
#include "llvm/Transforms/Scalar.h"
45
#include "llvm/Transforms/Utils.h"
46
#include "llvm/Transforms/Utils/SymbolRewriter.h"
47
#include <cassert>
48
#include <string>
49
50
using namespace llvm;
51
52
cl::opt<bool> EnableIPRA("enable-ipra", cl::init(false), cl::Hidden,
53
                         cl::desc("Enable interprocedural register allocation "
54
                                  "to reduce load/store at procedure calls."));
55
static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,
56
    cl::desc("Disable Post Regalloc Scheduler"));
57
static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
58
    cl::desc("Disable branch folding"));
59
static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,
60
    cl::desc("Disable tail duplication"));
61
static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,
62
    cl::desc("Disable pre-register allocation tail duplication"));
63
static cl::opt<bool> DisableBlockPlacement("disable-block-placement",
64
    cl::Hidden, cl::desc("Disable probability-driven block placement"));
65
static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",
66
    cl::Hidden, cl::desc("Collect probability-driven block placement stats"));
67
static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
68
    cl::desc("Disable Stack Slot Coloring"));
69
static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
70
    cl::desc("Disable Machine Dead Code Elimination"));
71
static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
72
    cl::desc("Disable Early If-conversion"));
73
static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
74
    cl::desc("Disable Machine LICM"));
75
static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
76
    cl::desc("Disable Machine Common Subexpression Elimination"));
77
static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(
78
    "optimize-regalloc", cl::Hidden,
79
    cl::desc("Enable optimized register allocation compilation path."));
80
static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
81
    cl::Hidden,
82
    cl::desc("Disable Machine LICM"));
83
static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,
84
    cl::desc("Disable Machine Sinking"));
85
static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",
86
    cl::Hidden,
87
    cl::desc("Disable PostRA Machine Sinking"));
88
static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,
89
    cl::desc("Disable Loop Strength Reduction Pass"));
90
static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",
91
    cl::Hidden, cl::desc("Disable ConstantHoisting"));
92
static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,
93
    cl::desc("Disable Codegen Prepare"));
94
static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
95
    cl::desc("Disable Copy Propagation pass"));
96
static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
97
    cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
98
static cl::opt<bool> EnableImplicitNullChecks(
99
    "enable-implicit-null-checks",
100
    cl::desc("Fold null checks into faulting memory operations"),
101
    cl::init(false), cl::Hidden);
102
static cl::opt<bool> DisableMergeICmps("disable-mergeicmps",
103
    cl::desc("Disable MergeICmps Pass"),
104
    cl::init(false), cl::Hidden);
105
static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden,
106
    cl::desc("Print LLVM IR produced by the loop-reduce pass"));
107
static cl::opt<bool> PrintISelInput("print-isel-input", cl::Hidden,
108
    cl::desc("Print LLVM IR input to isel pass"));
109
static cl::opt<bool> PrintGCInfo("print-gc", cl::Hidden,
110
    cl::desc("Dump garbage collector data"));
111
static cl::opt<cl::boolOrDefault>
112
    VerifyMachineCode("verify-machineinstrs", cl::Hidden,
113
                      cl::desc("Verify generated machine code"),
114
                      cl::ZeroOrMore);
115
enum RunOutliner { AlwaysOutline, NeverOutline, TargetDefault };
116
// Enable or disable the MachineOutliner.
117
static cl::opt<RunOutliner> EnableMachineOutliner(
118
    "enable-machine-outliner", cl::desc("Enable the machine outliner"),
119
    cl::Hidden, cl::ValueOptional, cl::init(TargetDefault),
120
    cl::values(clEnumValN(AlwaysOutline, "always",
121
                          "Run on all functions guaranteed to be beneficial"),
122
               clEnumValN(NeverOutline, "never", "Disable all outlining"),
123
               // Sentinel value for unspecified option.
124
               clEnumValN(AlwaysOutline, "", "")));
125
// Enable or disable FastISel. Both options are needed, because
126
// FastISel is enabled by default with -fast, and we wish to be
127
// able to enable or disable fast-isel independently from -O0.
128
static cl::opt<cl::boolOrDefault>
129
EnableFastISelOption("fast-isel", cl::Hidden,
130
  cl::desc("Enable the \"fast\" instruction selector"));
131
132
static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
133
    "global-isel", cl::Hidden,
134
    cl::desc("Enable the \"global\" instruction selector"));
135
136
static cl::opt<std::string> PrintMachineInstrs(
137
    "print-machineinstrs", cl::ValueOptional, cl::desc("Print machine instrs"),
138
    cl::value_desc("pass-name"), cl::init("option-unspecified"), cl::Hidden);
139
140
static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
141
    "global-isel-abort", cl::Hidden,
142
    cl::desc("Enable abort calls when \"global\" instruction selection "
143
             "fails to lower/select an instruction"),
144
    cl::values(
145
        clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),
146
        clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),
147
        clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",
148
                   "Disable the abort but emit a diagnostic on failure")));
149
150
// Temporary option to allow experimenting with MachineScheduler as a post-RA
151
// scheduler. Targets can "properly" enable this with
152
// substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).
153
// Targets can return true in targetSchedulesPostRAScheduling() and
154
// insert a PostRA scheduling pass wherever it wants.
155
cl::opt<bool> MISchedPostRA("misched-postra", cl::Hidden,
156
  cl::desc("Run MachineScheduler post regalloc (independent of preRA sched)"));
157
158
// Experimental option to run live interval analysis early.
159
static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
160
    cl::desc("Run live interval analysis earlier in the pipeline"));
161
162
// Experimental option to use CFL-AA in codegen
163
enum class CFLAAType { None, Steensgaard, Andersen, Both };
164
static cl::opt<CFLAAType> UseCFLAA(
165
    "use-cfl-aa-in-codegen", cl::init(CFLAAType::None), cl::Hidden,
166
    cl::desc("Enable the new, experimental CFL alias analysis in CodeGen"),
167
    cl::values(clEnumValN(CFLAAType::None, "none", "Disable CFL-AA"),
168
               clEnumValN(CFLAAType::Steensgaard, "steens",
169
                          "Enable unification-based CFL-AA"),
170
               clEnumValN(CFLAAType::Andersen, "anders",
171
                          "Enable inclusion-based CFL-AA"),
172
               clEnumValN(CFLAAType::Both, "both",
173
                          "Enable both variants of CFL-AA")));
174
175
/// Option names for limiting the codegen pipeline.
176
/// Those are used in error reporting and we didn't want
177
/// to duplicate their names all over the place.
178
const char *StartAfterOptName = "start-after";
179
const char *StartBeforeOptName = "start-before";
180
const char *StopAfterOptName = "stop-after";
181
const char *StopBeforeOptName = "stop-before";
182
183
static cl::opt<std::string>
184
    StartAfterOpt(StringRef(StartAfterOptName),
185
                  cl::desc("Resume compilation after a specific pass"),
186
                  cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
187
188
static cl::opt<std::string>
189
    StartBeforeOpt(StringRef(StartBeforeOptName),
190
                   cl::desc("Resume compilation before a specific pass"),
191
                   cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
192
193
static cl::opt<std::string>
194
    StopAfterOpt(StringRef(StopAfterOptName),
195
                 cl::desc("Stop compilation after a specific pass"),
196
                 cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
197
198
static cl::opt<std::string>
199
    StopBeforeOpt(StringRef(StopBeforeOptName),
200
                  cl::desc("Stop compilation before a specific pass"),
201
                  cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
202
203
/// Allow standard passes to be disabled by command line options. This supports
204
/// simple binary flags that either suppress the pass or do nothing.
205
/// i.e. -disable-mypass=false has no effect.
206
/// These should be converted to boolOrDefault in order to use applyOverride.
207
static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
208
548k
                                       bool Override) {
209
548k
  if (Override)
210
184
    return IdentifyingPassPtr();
211
547k
  return PassID;
212
547k
}
213
214
/// Allow standard passes to be disabled by the command line, regardless of who
215
/// is adding the pass.
216
///
217
/// StandardID is the pass identified in the standard pass pipeline and provided
218
/// to addPass(). It may be a target-specific ID in the case that the target
219
/// directly adds its own pass, but in that case we harmlessly fall through.
220
///
221
/// TargetID is the pass that the target has configured to override StandardID.
222
///
223
/// StandardID may be a pseudo ID. In that case TargetID is the name of the real
224
/// pass to run. This allows multiple options to control a single pass depending
225
/// on where in the pipeline that pass is added.
226
static IdentifyingPassPtr overridePass(AnalysisID StandardID,
227
1.52M
                                       IdentifyingPassPtr TargetID) {
228
1.52M
  if (StandardID == &PostRASchedulerID)
229
33.5k
    return applyDisable(TargetID, DisablePostRASched);
230
1.48M
231
1.48M
  if (StandardID == &BranchFolderPassID)
232
34.5k
    return applyDisable(TargetID, DisableBranchFold);
233
1.45M
234
1.45M
  if (StandardID == &TailDuplicateID)
235
33.9k
    return applyDisable(TargetID, DisableTailDuplicate);
236
1.41M
237
1.41M
  if (StandardID == &EarlyTailDuplicateID)
238
34.5k
    return applyDisable(TargetID, DisableEarlyTailDup);
239
1.38M
240
1.38M
  if (StandardID == &MachineBlockPlacementID)
241
34.5k
    return applyDisable(TargetID, DisableBlockPlacement);
242
1.35M
243
1.35M
  if (StandardID == &StackSlotColoringID)
244
34.0k
    return applyDisable(TargetID, DisableSSC);
245
1.31M
246
1.31M
  if (StandardID == &DeadMachineInstructionElimID)
247
75.2k
    return applyDisable(TargetID, DisableMachineDCE);
248
1.24M
249
1.24M
  if (StandardID == &EarlyIfConverterID)
250
22.6k
    return applyDisable(TargetID, DisableEarlyIfConversion);
251
1.21M
252
1.21M
  if (StandardID == &EarlyMachineLICMID)
253
36.8k
    return applyDisable(TargetID, DisableMachineLICM);
254
1.18M
255
1.18M
  if (StandardID == &MachineCSEID)
256
36.8k
    return applyDisable(TargetID, DisableMachineCSE);
257
1.14M
258
1.14M
  if (StandardID == &MachineLICMID)
259
33.8k
    return applyDisable(TargetID, DisablePostRAMachineLICM);
260
1.11M
261
1.11M
  if (StandardID == &MachineSinkingID)
262
34.5k
    return applyDisable(TargetID, DisableMachineSink);
263
1.07M
264
1.07M
  if (StandardID == &PostRAMachineSinkingID)
265
34.5k
    return applyDisable(TargetID, DisablePostRAMachineSink);
266
1.04M
267
1.04M
  if (StandardID == &MachineCopyPropagationID)
268
68.3k
    return applyDisable(TargetID, DisableCopyProp);
269
972k
270
972k
  return TargetID;
271
972k
}
272
273
//===---------------------------------------------------------------------===//
274
/// TargetPassConfig
275
//===---------------------------------------------------------------------===//
276
277
INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
278
                "Target Pass Configuration", false, false)
279
char TargetPassConfig::ID = 0;
280
281
namespace {
282
283
struct InsertedPass {
284
  AnalysisID TargetPassID;
285
  IdentifyingPassPtr InsertedPassID;
286
  bool VerifyAfter;
287
  bool PrintAfter;
288
289
  InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID,
290
               bool VerifyAfter, bool PrintAfter)
291
      : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID),
292
14.5k
        VerifyAfter(VerifyAfter), PrintAfter(PrintAfter) {}
293
294
14.4k
  Pass *getInsertedPass() const {
295
14.4k
    assert(InsertedPassID.isValid() && "Illegal Pass ID!");
296
14.4k
    if (InsertedPassID.isInstance())
297
0
      return InsertedPassID.getInstance();
298
14.4k
    Pass *NP = Pass::createPass(InsertedPassID.getID());
299
14.4k
    assert(NP && "Pass ID not registered");
300
14.4k
    return NP;
301
14.4k
  }
302
};
303
304
} // end anonymous namespace
305
306
namespace llvm {
307
308
class PassConfigImpl {
309
public:
310
  // List of passes explicitly substituted by this target. Normally this is
311
  // empty, but it is a convenient way to suppress or replace specific passes
312
  // that are part of a standard pass pipeline without overridding the entire
313
  // pipeline. This mechanism allows target options to inherit a standard pass's
314
  // user interface. For example, a target may disable a standard pass by
315
  // default by substituting a pass ID of zero, and the user may still enable
316
  // that standard pass with an explicit command line option.
317
  DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
318
319
  /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
320
  /// is inserted after each instance of the first one.
321
  SmallVector<InsertedPass, 4> InsertedPasses;
322
};
323
324
} // end namespace llvm
325
326
// Out of line virtual method.
327
42.3k
TargetPassConfig::~TargetPassConfig() {
328
42.3k
  delete Impl;
329
42.3k
}
330
331
170k
static const PassInfo *getPassInfo(StringRef PassName) {
332
170k
  if (PassName.empty())
333
169k
    return nullptr;
334
543
335
543
  const PassRegistry &PR = *PassRegistry::getPassRegistry();
336
543
  const PassInfo *PI = PR.getPassInfo(PassName);
337
543
  if (!PI)
338
5
    report_fatal_error(Twine('\"') + Twine(PassName) +
339
5
                       Twine("\" pass is not registered."));
340
538
  return PI;
341
538
}
342
343
170k
static AnalysisID getPassIDFromName(StringRef PassName) {
344
170k
  const PassInfo *PI = getPassInfo(PassName);
345
170k
  return PI ? 
PI->getTypeInfo()529
:
nullptr169k
;
346
170k
}
347
348
static std::pair<StringRef, unsigned>
349
170k
getPassNameAndInstanceNum(StringRef PassName) {
350
170k
  StringRef Name, InstanceNumStr;
351
170k
  std::tie(Name, InstanceNumStr) = PassName.split(',');
352
170k
353
170k
  unsigned InstanceNum = 0;
354
170k
  if (!InstanceNumStr.empty() && 
InstanceNumStr.getAsInteger(10, InstanceNum)6
)
355
1
    report_fatal_error("invalid pass instance specifier " + PassName);
356
170k
357
170k
  return std::make_pair(Name, InstanceNum);
358
170k
}
359
360
42.5k
void TargetPassConfig::setStartStopPasses() {
361
42.5k
  StringRef StartBeforeName;
362
42.5k
  std::tie(StartBeforeName, StartBeforeInstanceNum) =
363
42.5k
    getPassNameAndInstanceNum(StartBeforeOpt);
364
42.5k
365
42.5k
  StringRef StartAfterName;
366
42.5k
  std::tie(StartAfterName, StartAfterInstanceNum) =
367
42.5k
    getPassNameAndInstanceNum(StartAfterOpt);
368
42.5k
369
42.5k
  StringRef StopBeforeName;
370
42.5k
  std::tie(StopBeforeName, StopBeforeInstanceNum)
371
42.5k
    = getPassNameAndInstanceNum(StopBeforeOpt);
372
42.5k
373
42.5k
  StringRef StopAfterName;
374
42.5k
  std::tie(StopAfterName, StopAfterInstanceNum)
375
42.5k
    = getPassNameAndInstanceNum(StopAfterOpt);
376
42.5k
377
42.5k
  StartBefore = getPassIDFromName(StartBeforeName);
378
42.5k
  StartAfter = getPassIDFromName(StartAfterName);
379
42.5k
  StopBefore = getPassIDFromName(StopBeforeName);
380
42.5k
  StopAfter = getPassIDFromName(StopAfterName);
381
42.5k
  if (StartBefore && 
StartAfter100
)
382
1
    report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") +
383
1
                       Twine(StartAfterOptName) + Twine(" specified!"));
384
42.5k
  if (StopBefore && 
StopAfter59
)
385
1
    report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") +
386
1
                       Twine(StopAfterOptName) + Twine(" specified!"));
387
42.5k
  Started = (StartAfter == nullptr) && 
(StartBefore == nullptr)42.4k
;
388
42.5k
}
389
390
// Out of line constructor provides default values for pass options and
391
// registers all common codegen passes.
392
TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm)
393
42.5k
    : ImmutablePass(ID), PM(&pm), TM(&TM) {
394
42.5k
  Impl = new PassConfigImpl();
395
42.5k
396
42.5k
  // Register all target independent codegen passes to activate their PassIDs,
397
42.5k
  // including this pass itself.
398
42.5k
  initializeCodeGen(*PassRegistry::getPassRegistry());
399
42.5k
400
42.5k
  // Also register alias analysis passes required by codegen passes.
401
42.5k
  initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
402
42.5k
  initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
403
42.5k
404
42.5k
  if (StringRef(PrintMachineInstrs.getValue()).equals(""))
405
15
    TM.Options.PrintMachineCode = true;
406
42.5k
407
42.5k
  if (EnableIPRA.getNumOccurrences())
408
27
    TM.Options.EnableIPRA = EnableIPRA;
409
42.5k
  else {
410
42.5k
    // If not explicitly specified, use target default.
411
42.5k
    TM.Options.EnableIPRA |= TM.useIPRA();
412
42.5k
  }
413
42.5k
414
42.5k
  if (TM.Options.EnableIPRA)
415
3.46k
    setRequiresCodeGenSCCOrder();
416
42.5k
417
42.5k
  if (EnableGlobalISelAbort.getNumOccurrences())
418
6.85k
    TM.Options.GlobalISelAbort = EnableGlobalISelAbort;
419
42.5k
420
42.5k
  setStartStopPasses();
421
42.5k
}
422
423
920k
CodeGenOpt::Level TargetPassConfig::getOptLevel() const {
424
920k
  return TM->getOptLevel();
425
920k
}
426
427
/// Insert InsertedPassID pass after TargetPassID.
428
void TargetPassConfig::insertPass(AnalysisID TargetPassID,
429
                                  IdentifyingPassPtr InsertedPassID,
430
14.5k
                                  bool VerifyAfter, bool PrintAfter) {
431
14.5k
  assert(((!InsertedPassID.isInstance() &&
432
14.5k
           TargetPassID != InsertedPassID.getID()) ||
433
14.5k
          (InsertedPassID.isInstance() &&
434
14.5k
           TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
435
14.5k
         "Insert a pass after itself!");
436
14.5k
  Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID, VerifyAfter,
437
14.5k
                                    PrintAfter);
438
14.5k
}
439
440
/// createPassConfig - Create a pass configuration object to be used by
441
/// addPassToEmitX methods for generating a pipeline of CodeGen passes.
442
///
443
/// Targets may override this to extend TargetPassConfig.
444
0
TargetPassConfig *LLVMTargetMachine::createPassConfig(PassManagerBase &PM) {
445
0
  return new TargetPassConfig(*this, PM);
446
0
}
447
448
TargetPassConfig::TargetPassConfig()
449
1
  : ImmutablePass(ID) {
450
1
  report_fatal_error("Trying to construct TargetPassConfig without a target "
451
1
                     "machine. Scheduling a CodeGen pass without a target "
452
1
                     "triple set?");
453
1
}
454
455
38.0k
bool TargetPassConfig::willCompleteCodeGenPipeline() {
456
38.0k
  return StopBeforeOpt.empty() && 
StopAfterOpt.empty()37.9k
;
457
38.0k
}
458
459
1.99k
bool TargetPassConfig::hasLimitedCodeGenPipeline() {
460
1.99k
  return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||
461
1.99k
         !willCompleteCodeGenPipeline();
462
1.99k
}
463
464
std::string
465
0
TargetPassConfig::getLimitedCodeGenPipelineReason(const char *Separator) const {
466
0
  if (!hasLimitedCodeGenPipeline())
467
0
    return std::string();
468
0
  std::string Res;
469
0
  static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,
470
0
                                              &StopAfterOpt, &StopBeforeOpt};
471
0
  static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
472
0
                                   StopAfterOptName, StopBeforeOptName};
473
0
  bool IsFirst = true;
474
0
  for (int Idx = 0; Idx < 4; ++Idx)
475
0
    if (!PassNames[Idx]->empty()) {
476
0
      if (!IsFirst)
477
0
        Res += Separator;
478
0
      IsFirst = false;
479
0
      Res += OptNames[Idx];
480
0
    }
481
0
  return Res;
482
0
}
483
484
// Helper to verify the analysis is really immutable.
485
45.2k
void TargetPassConfig::setOpt(bool &Opt, bool Val) {
486
45.2k
  assert(!Initialized && "PassConfig is immutable");
487
45.2k
  Opt = Val;
488
45.2k
}
489
490
void TargetPassConfig::substitutePass(AnalysisID StandardID,
491
33.6k
                                      IdentifyingPassPtr TargetID) {
492
33.6k
  Impl->TargetPasses[StandardID] = TargetID;
493
33.6k
}
494
495
1.52M
IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
496
1.52M
  DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
497
1.52M
    I = Impl->TargetPasses.find(ID);
498
1.52M
  if (I == Impl->TargetPasses.end())
499
1.49M
    return ID;
500
25.3k
  return I->second;
501
25.3k
}
502
503
36.3k
bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
504
36.3k
  IdentifyingPassPtr TargetID = getPassSubstitution(ID);
505
36.3k
  IdentifyingPassPtr FinalPtr = overridePass(ID, TargetID);
506
36.3k
  return !FinalPtr.isValid() || 
FinalPtr.isInstance()36.1k
||
507
36.3k
      
FinalPtr.getID() != ID36.1k
;
508
36.3k
}
509
510
/// Add a pass to the PassManager if that pass is supposed to be run.  If the
511
/// Started/Stopped flags indicate either that the compilation should start at
512
/// a later pass or that it should stop after an earlier pass, then do not add
513
/// the pass.  Finally, compare the current pass against the StartAfter
514
/// and StopAfter options and change the Started/Stopped flags accordingly.
515
3.31M
void TargetPassConfig::addPass(Pass *P, bool verifyAfter, bool printAfter) {
516
3.31M
  assert(!Initialized && "PassConfig is immutable");
517
3.31M
518
3.31M
  // Cache the Pass ID here in case the pass manager finds this pass is
519
3.31M
  // redundant with ones already scheduled / available, and deletes it.
520
3.31M
  // Fundamentally, once we add the pass to the manager, we no longer own it
521
3.31M
  // and shouldn't reference it.
522
3.31M
  AnalysisID PassID = P->getPassID();
523
3.31M
524
3.31M
  if (StartBefore == PassID && 
StartBeforeCount++ == StartBeforeInstanceNum101
)
525
98
    Started = true;
526
3.31M
  if (StopBefore == PassID && 
StopBeforeCount++ == StopBeforeInstanceNum58
)
527
57
    Stopped = true;
528
3.31M
  if (Started && 
!Stopped3.29M
) {
529
3.28M
    std::string Banner;
530
3.28M
    // Construct banner message before PM->add() as that may delete the pass.
531
3.28M
    if (AddingMachinePasses && 
(2.11M
printAfter2.11M
||
verifyAfter0
))
532
2.11M
      Banner = std::string("After ") + std::string(P->getPassName());
533
3.28M
    PM->add(P);
534
3.28M
    if (AddingMachinePasses) {
535
2.11M
      if (printAfter)
536
2.11M
        addPrintPass(Banner);
537
2.11M
      if (verifyAfter)
538
1.47M
        addVerifyPass(Banner);
539
2.11M
    }
540
3.28M
541
3.28M
    // Add the passes after the pass P if there is any.
542
3.28M
    for (auto IP : Impl->InsertedPasses) {
543
627k
      if (IP.TargetPassID == PassID)
544
14.4k
        addPass(IP.getInsertedPass(), IP.VerifyAfter, IP.PrintAfter);
545
627k
    }
546
3.28M
  } else {
547
22.5k
    delete P;
548
22.5k
  }
549
3.31M
550
3.31M
  if (StopAfter == PassID && 
StopAfterCount++ == StopAfterInstanceNum254
)
551
248
    Stopped = true;
552
3.31M
553
3.31M
  if (StartAfter == PassID && 
StartAfterCount++ == StartAfterInstanceNum114
)
554
111
    Started = true;
555
3.31M
  if (Stopped && 
!Started11.4k
)
556
0
    report_fatal_error("Cannot stop compilation after pass that is not run");
557
3.31M
}
558
559
/// Add a CodeGen pass at this point in the pipeline after checking for target
560
/// and command line overrides.
561
///
562
/// addPass cannot return a pointer to the pass instance because is internal the
563
/// PassManager and the instance we create here may already be freed.
564
AnalysisID TargetPassConfig::addPass(AnalysisID PassID, bool verifyAfter,
565
1.48M
                                     bool printAfter) {
566
1.48M
  IdentifyingPassPtr TargetID = getPassSubstitution(PassID);
567
1.48M
  IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID);
568
1.48M
  if (!FinalPtr.isValid())
569
14.1k
    return nullptr;
570
1.47M
571
1.47M
  Pass *P;
572
1.47M
  if (FinalPtr.isInstance())
573
0
    P = FinalPtr.getInstance();
574
1.47M
  else {
575
1.47M
    P = Pass::createPass(FinalPtr.getID());
576
1.47M
    if (!P)
577
1.47M
      
llvm_unreachable0
("Pass ID not registered");
578
1.47M
  }
579
1.47M
  AnalysisID FinalID = P->getPassID();
580
1.47M
  addPass(P, verifyAfter, printAfter); // Ends the lifetime of P.
581
1.47M
582
1.47M
  return FinalID;
583
1.47M
}
584
585
41.5k
void TargetPassConfig::printAndVerify(const std::string &Banner) {
586
41.5k
  addPrintPass(Banner);
587
41.5k
  addVerifyPass(Banner);
588
41.5k
}
589
590
2.15M
void TargetPassConfig::addPrintPass(const std::string &Banner) {
591
2.15M
  if (TM->shouldPrintMachineCode())
592
845
    PM->add(createMachineFunctionPrinterPass(dbgs(), Banner));
593
2.15M
}
594
595
1.51M
void TargetPassConfig::addVerifyPass(const std::string &Banner) {
596
1.51M
  bool Verify = VerifyMachineCode == cl::BOU_TRUE;
597
#ifdef EXPENSIVE_CHECKS
598
  if (VerifyMachineCode == cl::BOU_UNSET)
599
    Verify = TM->isMachineVerifierClean();
600
#endif
601
1.51M
  if (Verify)
602
212k
    PM->add(createMachineVerifierPass(Banner));
603
1.51M
}
604
605
/// Add common target configurable passes that perform LLVM IR to IR transforms
606
/// following machine independent optimization.
607
36.3k
void TargetPassConfig::addIRPasses() {
608
36.3k
  switch (UseCFLAA) {
609
36.3k
  case CFLAAType::Steensgaard:
610
0
    addPass(createCFLSteensAAWrapperPass());
611
0
    break;
612
36.3k
  case CFLAAType::Andersen:
613
0
    addPass(createCFLAndersAAWrapperPass());
614
0
    break;
615
36.3k
  case CFLAAType::Both:
616
0
    addPass(createCFLAndersAAWrapperPass());
617
0
    addPass(createCFLSteensAAWrapperPass());
618
0
    break;
619
36.3k
  default:
620
36.3k
    break;
621
36.3k
  }
622
36.3k
623
36.3k
  // Basic AliasAnalysis support.
624
36.3k
  // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
625
36.3k
  // BasicAliasAnalysis wins if they disagree. This is intended to help
626
36.3k
  // support "obvious" type-punning idioms.
627
36.3k
  addPass(createTypeBasedAAWrapperPass());
628
36.3k
  addPass(createScopedNoAliasAAWrapperPass());
629
36.3k
  addPass(createBasicAAWrapperPass());
630
36.3k
631
36.3k
  // Before running any passes, run the verifier to determine if the input
632
36.3k
  // coming from the front-end and/or optimizer is valid.
633
36.3k
  if (!DisableVerify)
634
23.1k
    addPass(createVerifierPass());
635
36.3k
636
36.3k
  // Run loop strength reduction before anything else.
637
36.3k
  if (getOptLevel() != CodeGenOpt::None && 
!DisableLSR34.5k
) {
638
34.4k
    addPass(createLoopStrengthReducePass());
639
34.4k
    if (PrintLSR)
640
3
      addPass(createPrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
641
34.4k
  }
642
36.3k
643
36.3k
  if (getOptLevel() != CodeGenOpt::None) {
644
34.5k
    // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
645
34.5k
    // loads and compares. ExpandMemCmpPass then tries to expand those calls
646
34.5k
    // into optimally-sized loads and compares. The transforms are enabled by a
647
34.5k
    // target lowering hook.
648
34.5k
    if (!DisableMergeICmps)
649
34.5k
      addPass(createMergeICmpsLegacyPass());
650
34.5k
    addPass(createExpandMemCmpPass());
651
34.5k
  }
652
36.3k
653
36.3k
  // Run GC lowering passes for builtin collectors
654
36.3k
  // TODO: add a pass insertion point here
655
36.3k
  addPass(createGCLoweringPass());
656
36.3k
  addPass(createShadowStackGCLoweringPass());
657
36.3k
658
36.3k
  // Make sure that no unreachable blocks are instruction selected.
659
36.3k
  addPass(createUnreachableBlockEliminationPass());
660
36.3k
661
36.3k
  // Prepare expensive constants for SelectionDAG.
662
36.3k
  if (getOptLevel() != CodeGenOpt::None && 
!DisableConstantHoisting34.5k
)
663
34.5k
    addPass(createConstantHoistingPass());
664
36.3k
665
36.3k
  if (getOptLevel() != CodeGenOpt::None && 
!DisablePartialLibcallInlining34.5k
)
666
34.5k
    addPass(createPartiallyInlineLibCallsPass());
667
36.3k
668
36.3k
  // Instrument function entry and exit, e.g. with calls to mcount().
669
36.3k
  addPass(createPostInlineEntryExitInstrumenterPass());
670
36.3k
671
36.3k
  // Add scalarization of target's unsupported masked memory intrinsics pass.
672
36.3k
  // the unsupported intrinsic will be replaced with a chain of basic blocks,
673
36.3k
  // that stores/loads element one-by-one if the appropriate mask bit is set.
674
36.3k
  addPass(createScalarizeMaskedMemIntrinPass());
675
36.3k
676
36.3k
  // Expand reduction intrinsics into shuffle sequences if the target wants to.
677
36.3k
  addPass(createExpandReductionsPass());
678
36.3k
}
679
680
/// Turn exception handling constructs into something the code generators can
681
/// handle.
682
36.3k
void TargetPassConfig::addPassesToHandleExceptions() {
683
36.3k
  const MCAsmInfo *MCAI = TM->getMCAsmInfo();
684
36.3k
  assert(MCAI && "No MCAsmInfo");
685
36.3k
  switch (MCAI->getExceptionHandlingType()) {
686
36.3k
  case ExceptionHandling::SjLj:
687
1.72k
    // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
688
1.72k
    // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
689
1.72k
    // catch info can get misplaced when a selector ends up more than one block
690
1.72k
    // removed from the parent invoke(s). This could happen when a landing
691
1.72k
    // pad is shared by multiple invokes and is also a target of a normal
692
1.72k
    // edge from elsewhere.
693
1.72k
    addPass(createSjLjEHPreparePass());
694
1.72k
    LLVM_FALLTHROUGH;
695
32.0k
  case ExceptionHandling::DwarfCFI:
696
32.0k
  case ExceptionHandling::ARM:
697
32.0k
    addPass(createDwarfEHPass());
698
32.0k
    break;
699
32.0k
  case ExceptionHandling::WinEH:
700
817
    // We support using both GCC-style and MSVC-style exceptions on Windows, so
701
817
    // add both preparation passes. Each pass will only actually run if it
702
817
    // recognizes the personality function.
703
817
    addPass(createWinEHPass());
704
817
    addPass(createDwarfEHPass());
705
817
    break;
706
32.0k
  case ExceptionHandling::Wasm:
707
14
    // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
708
14
    // on catchpads and cleanuppads because it does not outline them into
709
14
    // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
710
14
    // should remove PHIs there.
711
14
    addPass(createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/false));
712
14
    addPass(createWasmEHPass());
713
14
    break;
714
32.0k
  case ExceptionHandling::None:
715
3.48k
    addPass(createLowerInvokePass());
716
3.48k
717
3.48k
    // The lower invoke pass may create unreachable code. Remove it.
718
3.48k
    addPass(createUnreachableBlockEliminationPass());
719
3.48k
    break;
720
36.3k
  }
721
36.3k
}
722
723
/// Add pass to prepare the LLVM IR for code generation. This should be done
724
/// before exception handling preparation passes.
725
36.3k
void TargetPassConfig::addCodeGenPrepare() {
726
36.3k
  if (getOptLevel() != CodeGenOpt::None && 
!DisableCGP34.5k
)
727
34.5k
    addPass(createCodeGenPreparePass());
728
36.3k
  addPass(createRewriteSymbolsPass());
729
36.3k
}
730
731
/// Add common passes that perform LLVM IR to IR transforms in preparation for
732
/// instruction selection.
733
36.3k
void TargetPassConfig::addISelPrepare() {
734
36.3k
  addPreISel();
735
36.3k
736
36.3k
  // Force codegen to run according to the callgraph.
737
36.3k
  if (requiresCodeGenSCCOrder())
738
2.45k
    addPass(new DummyCGSCCPass);
739
36.3k
740
36.3k
  // Add both the safe stack and the stack protection passes: each of them will
741
36.3k
  // only protect functions that have corresponding attributes.
742
36.3k
  addPass(createSafeStackPass());
743
36.3k
  addPass(createStackProtectorPass());
744
36.3k
745
36.3k
  if (PrintISelInput)
746
0
    addPass(createPrintFunctionPass(
747
0
        dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"));
748
36.3k
749
36.3k
  // All passes which modify the LLVM IR are now complete; run the verifier
750
36.3k
  // to ensure that the IR is valid.
751
36.3k
  if (!DisableVerify)
752
23.1k
    addPass(createVerifierPass());
753
36.3k
}
754
755
36.3k
bool TargetPassConfig::addCoreISelPasses() {
756
36.3k
  // Enable FastISel with -fast-isel, but allow that to be overridden.
757
36.3k
  TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE);
758
36.3k
759
36.3k
  // Determine an instruction selector.
760
36.3k
  enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
761
36.3k
  SelectorType Selector;
762
36.3k
763
36.3k
  if (EnableFastISelOption == cl::BOU_TRUE)
764
421
    Selector = SelectorType::FastISel;
765
35.9k
  else if (EnableGlobalISelOption == cl::BOU_TRUE ||
766
35.9k
           
(28.9k
TM->Options.EnableGlobalISel28.9k
&&
767
28.9k
            
EnableGlobalISelOption != cl::BOU_FALSE46
))
768
7.00k
    Selector = SelectorType::GlobalISel;
769
28.9k
  else if (TM->getOptLevel() == CodeGenOpt::None && 
TM->getO0WantsFastISel()1.57k
)
770
1.53k
    Selector = SelectorType::FastISel;
771
27.4k
  else
772
27.4k
    Selector = SelectorType::SelectionDAG;
773
36.3k
774
36.3k
  // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
775
36.3k
  if (Selector == SelectorType::FastISel) {
776
1.95k
    TM->setFastISel(true);
777
1.95k
    TM->setGlobalISel(false);
778
34.4k
  } else if (Selector == SelectorType::GlobalISel) {
779
7.00k
    TM->setFastISel(false);
780
7.00k
    TM->setGlobalISel(true);
781
7.00k
  }
782
36.3k
783
36.3k
  // Add instruction selector passes.
784
36.3k
  if (Selector == SelectorType::GlobalISel) {
785
7.00k
    SaveAndRestore<bool> SavedAddingMachinePasses(AddingMachinePasses, true);
786
7.00k
    if (addIRTranslator())
787
0
      return true;
788
7.00k
789
7.00k
    addPreLegalizeMachineIR();
790
7.00k
791
7.00k
    if (addLegalizeMachineIR())
792
0
      return true;
793
7.00k
794
7.00k
    // Before running the register bank selector, ask the target if it
795
7.00k
    // wants to run some passes.
796
7.00k
    addPreRegBankSelect();
797
7.00k
798
7.00k
    if (addRegBankSelect())
799
0
      return true;
800
7.00k
801
7.00k
    addPreGlobalInstructionSelect();
802
7.00k
803
7.00k
    if (addGlobalInstructionSelect())
804
0
      return true;
805
7.00k
806
7.00k
    // Pass to reset the MachineFunction if the ISel failed.
807
7.00k
    addPass(createResetMachineFunctionPass(
808
7.00k
        reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled()));
809
7.00k
810
7.00k
    // Provide a fallback path when we do not want to abort on
811
7.00k
    // not-yet-supported input.
812
7.00k
    if (!isGlobalISelAbortEnabled() && 
addInstSelector()6.76k
)
813
0
      return true;
814
29.3k
815
29.3k
  } else if (addInstSelector())
816
0
    return true;
817
36.3k
818
36.3k
  // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
819
36.3k
  // FinalizeISel.
820
36.3k
  addPass(&FinalizeISelID);
821
36.3k
822
36.3k
  // Print the instruction selected machine code...
823
36.3k
  printAndVerify("After Instruction Selection");
824
36.3k
825
36.3k
  return false;
826
36.3k
}
827
828
36.3k
bool TargetPassConfig::addISelPasses() {
829
36.3k
  if (TM->useEmulatedTLS())
830
469
    addPass(createLowerEmuTLSPass());
831
36.3k
832
36.3k
  addPass(createPreISelIntrinsicLoweringPass());
833
36.3k
  addPass(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
834
36.3k
  addIRPasses();
835
36.3k
  addCodeGenPrepare();
836
36.3k
  addPassesToHandleExceptions();
837
36.3k
  addISelPrepare();
838
36.3k
839
36.3k
  return addCoreISelPasses();
840
36.3k
}
841
842
/// -regalloc=... command line option.
843
0
static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
844
static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
845
               RegisterPassParser<RegisterRegAlloc>>
846
    RegAlloc("regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),
847
             cl::desc("Register allocator to use"));
848
849
/// Add the complete set of target-independent postISel code generator passes.
850
///
851
/// This can be read as the standard order of major LLVM CodeGen stages. Stages
852
/// with nontrivial configuration or multiple passes are broken out below in
853
/// add%Stage routines.
854
///
855
/// Any TargetPassConfig::addXX routine may be overriden by the Target. The
856
/// addPre/Post methods with empty header implementations allow injecting
857
/// target-specific fixups just before or after major stages. Additionally,
858
/// targets have the flexibility to change pass order within a stage by
859
/// overriding default implementation of add%Stage routines below. Each
860
/// technique has maintainability tradeoffs because alternate pass orders are
861
/// not well supported. addPre/Post works better if the target pass is easily
862
/// tied to a common pass. But if it has subtle dependencies on multiple passes,
863
/// the target should override the stage instead.
864
///
865
/// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
866
/// before/after any target-independent pass. But it's currently overkill.
867
36.3k
void TargetPassConfig::addMachinePasses() {
868
36.3k
  AddingMachinePasses = true;
869
36.3k
870
36.3k
  // Insert a machine instr printer pass after the specified pass.
871
36.3k
  StringRef PrintMachineInstrsPassName = PrintMachineInstrs.getValue();
872
36.3k
  if (!PrintMachineInstrsPassName.equals("") &&
873
36.3k
      
!PrintMachineInstrsPassName.equals("option-unspecified")36.3k
) {
874
11
    if (const PassInfo *TPI = getPassInfo(PrintMachineInstrsPassName)) {
875
10
      const PassRegistry *PR = PassRegistry::getPassRegistry();
876
10
      const PassInfo *IPI = PR->getPassInfo(StringRef("machineinstr-printer"));
877
10
      assert(IPI && "failed to get \"machineinstr-printer\" PassInfo!");
878
10
      const char *TID = (const char *)(TPI->getTypeInfo());
879
10
      const char *IID = (const char *)(IPI->getTypeInfo());
880
10
      insertPass(TID, IID);
881
10
    }
882
11
  }
883
36.3k
884
36.3k
  // Add passes that optimize machine instructions in SSA form.
885
36.3k
  if (getOptLevel() != CodeGenOpt::None) {
886
34.5k
    addMachineSSAOptimization();
887
34.5k
  } else {
888
1.85k
    // If the target requests it, assign local variables to stack slots relative
889
1.85k
    // to one another and simplify frame index references where possible.
890
1.85k
    addPass(&LocalStackSlotAllocationID, false);
891
1.85k
  }
892
36.3k
893
36.3k
  if (TM->Options.EnableIPRA)
894
2.44k
    addPass(createRegUsageInfoPropPass());
895
36.3k
896
36.3k
  // Run pre-ra passes.
897
36.3k
  addPreRegAlloc();
898
36.3k
899
36.3k
  // Run register allocation and passes that are tightly coupled with it,
900
36.3k
  // including phi elimination and scheduling.
901
36.3k
  if (getOptimizeRegAlloc())
902
34.4k
    addOptimizedRegAlloc();
903
1.86k
  else
904
1.86k
    addFastRegAlloc();
905
36.3k
906
36.3k
  // Run post-ra passes.
907
36.3k
  addPostRegAlloc();
908
36.3k
909
36.3k
  // Insert prolog/epilog code.  Eliminate abstract frame index references...
910
36.3k
  if (getOptLevel() != CodeGenOpt::None) {
911
34.5k
    addPass(&PostRAMachineSinkingID);
912
34.5k
    addPass(&ShrinkWrapID);
913
34.5k
  }
914
36.3k
915
36.3k
  // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
916
36.3k
  // do so if it hasn't been disabled, substituted, or overridden.
917
36.3k
  if (!isPassSubstitutedOrOverridden(&PrologEpilogCodeInserterID))
918
36.1k
      addPass(createPrologEpilogInserterPass());
919
36.3k
920
36.3k
  /// Add passes that optimize machine instructions after register allocation.
921
36.3k
  if (getOptLevel() != CodeGenOpt::None)
922
34.5k
    addMachineLateOptimization();
923
36.3k
924
36.3k
  // Expand pseudo instructions before second scheduling pass.
925
36.3k
  addPass(&ExpandPostRAPseudosID);
926
36.3k
927
36.3k
  // Run pre-sched2 passes.
928
36.3k
  addPreSched2();
929
36.3k
930
36.3k
  if (EnableImplicitNullChecks)
931
5
    addPass(&ImplicitNullChecksID);
932
36.3k
933
36.3k
  // Second pass scheduler.
934
36.3k
  // Let Target optionally insert this pass by itself at some other
935
36.3k
  // point.
936
36.3k
  if (getOptLevel() != CodeGenOpt::None &&
937
36.3k
      
!TM->targetSchedulesPostRAScheduling()34.5k
) {
938
33.5k
    if (MISchedPostRA)
939
5
      addPass(&PostMachineSchedulerID);
940
33.4k
    else
941
33.4k
      addPass(&PostRASchedulerID);
942
33.5k
  }
943
36.3k
944
36.3k
  // GC
945
36.3k
  if (addGCPasses()) {
946
33.2k
    if (PrintGCInfo)
947
0
      addPass(createGCInfoPrinter(dbgs()), false, false);
948
33.2k
  }
949
36.3k
950
36.3k
  // Basic block placement.
951
36.3k
  if (getOptLevel() != CodeGenOpt::None)
952
34.5k
    addBlockPlacement();
953
36.3k
954
36.3k
  addPreEmitPass();
955
36.3k
956
36.3k
  if (TM->Options.EnableIPRA)
957
2.44k
    // Collect register usage information and produce a register mask of
958
2.44k
    // clobbered registers, to be used to optimize call sites.
959
2.44k
    addPass(createRegUsageInfoCollector());
960
36.3k
961
36.3k
  addPass(&FuncletLayoutID, false);
962
36.3k
963
36.3k
  addPass(&StackMapLivenessID, false);
964
36.3k
  addPass(&LiveDebugValuesID, false);
965
36.3k
966
36.3k
  // Insert before XRay Instrumentation.
967
36.3k
  addPass(&FEntryInserterID, false);
968
36.3k
969
36.3k
  addPass(&XRayInstrumentationID, false);
970
36.3k
  addPass(&PatchableFunctionID, false);
971
36.3k
972
36.3k
  if (TM->Options.EnableMachineOutliner && 
getOptLevel() != CodeGenOpt::None18.2k
&&
973
36.3k
      
EnableMachineOutliner != NeverOutline17.3k
) {
974
17.3k
    bool RunOnAllFunctions = (EnableMachineOutliner == AlwaysOutline);
975
17.3k
    bool AddOutliner = RunOnAllFunctions ||
976
17.3k
                       
TM->Options.SupportsDefaultOutlining17.3k
;
977
17.3k
    if (AddOutliner)
978
8.62k
      addPass(createMachineOutlinerPass(RunOnAllFunctions));
979
17.3k
  }
980
36.3k
981
36.3k
  // Add passes that directly emit MI after all other MI passes.
982
36.3k
  addPreEmitPass2();
983
36.3k
984
36.3k
  AddingMachinePasses = false;
985
36.3k
}
986
987
/// Add passes that optimize machine instructions in SSA form.
988
34.2k
void TargetPassConfig::addMachineSSAOptimization() {
989
34.2k
  // Pre-ra tail duplication.
990
34.2k
  addPass(&EarlyTailDuplicateID);
991
34.2k
992
34.2k
  // Optimize PHIs before DCE: removing dead PHI cycles may make more
993
34.2k
  // instructions dead.
994
34.2k
  addPass(&OptimizePHIsID, false);
995
34.2k
996
34.2k
  // This pass merges large allocas. StackSlotColoring is a different pass
997
34.2k
  // which merges spill slots.
998
34.2k
  addPass(&StackColoringID, false);
999
34.2k
1000
34.2k
  // If the target requests it, assign local variables to stack slots relative
1001
34.2k
  // to one another and simplify frame index references where possible.
1002
34.2k
  addPass(&LocalStackSlotAllocationID, false);
1003
34.2k
1004
34.2k
  // With optimization, dead code should already be eliminated. However
1005
34.2k
  // there is one known exception: lowered code for arguments that are only
1006
34.2k
  // used by tail calls, where the tail calls reuse the incoming stack
1007
34.2k
  // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1008
34.2k
  addPass(&DeadMachineInstructionElimID);
1009
34.2k
1010
34.2k
  // Allow targets to insert passes that improve instruction level parallelism,
1011
34.2k
  // like if-conversion. Such passes will typically need dominator trees and
1012
34.2k
  // loop info, just like LICM and CSE below.
1013
34.2k
  addILPOpts();
1014
34.2k
1015
34.2k
  addPass(&EarlyMachineLICMID, false);
1016
34.2k
  addPass(&MachineCSEID, false);
1017
34.2k
1018
34.2k
  addPass(&MachineSinkingID);
1019
34.2k
1020
34.2k
  addPass(&PeepholeOptimizerID);
1021
34.2k
  // Clean-up the dead code that may have been generated by peephole
1022
34.2k
  // rewriting.
1023
34.2k
  addPass(&DeadMachineInstructionElimID);
1024
34.2k
}
1025
1026
//===---------------------------------------------------------------------===//
1027
/// Register Allocation Pass Configuration
1028
//===---------------------------------------------------------------------===//
1029
1030
36.3k
bool TargetPassConfig::getOptimizeRegAlloc() const {
1031
36.3k
  switch (OptimizeRegAlloc) {
1032
36.3k
  
case cl::BOU_UNSET: return getOptLevel() != CodeGenOpt::None36.3k
;
1033
36.3k
  
case cl::BOU_TRUE: return true12
;
1034
36.3k
  
case cl::BOU_FALSE: return false26
;
1035
0
  }
1036
0
  llvm_unreachable("Invalid optimize-regalloc state");
1037
0
}
1038
1039
/// A dummy default pass factory indicates whether the register allocator is
1040
/// overridden on the command line.
1041
static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1042
1043
static RegisterRegAlloc
1044
defaultRegAlloc("default",
1045
                "pick register allocator based on -O option",
1046
                useDefaultRegisterAllocator);
1047
1048
35.3k
static void initializeDefaultRegisterAllocatorOnce() {
1049
35.3k
  if (!RegisterRegAlloc::getDefault())
1050
35.3k
    RegisterRegAlloc::setDefault(RegAlloc);
1051
35.3k
}
1052
1053
/// Instantiate the default register allocator pass for this target for either
1054
/// the optimized or unoptimized allocation path. This will be added to the pass
1055
/// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1056
/// in the optimized case.
1057
///
1058
/// A target that uses the standard regalloc pass order for fast or optimized
1059
/// allocation may still override this for per-target regalloc
1060
/// selection. But -regalloc=... always takes precedence.
1061
35.5k
FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1062
35.5k
  if (Optimized)
1063
33.7k
    return createGreedyRegisterAllocator();
1064
1.78k
  else
1065
1.78k
    return createFastRegisterAllocator();
1066
35.5k
}
1067
1068
/// Find and instantiate the register allocation pass requested by this target
1069
/// at the current optimization level.  Different register allocators are
1070
/// defined as separate passes because they may require different analysis.
1071
///
1072
/// This helper ensures that the regalloc= option is always available,
1073
/// even for targets that override the default allocator.
1074
///
1075
/// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1076
/// this can be folded into addPass.
1077
35.6k
FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1078
35.6k
  // Initialize the global default.
1079
35.6k
  llvm::call_once(InitializeDefaultRegisterAllocatorFlag,
1080
35.6k
                  initializeDefaultRegisterAllocatorOnce);
1081
35.6k
1082
35.6k
  RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1083
35.6k
  if (Ctor != useDefaultRegisterAllocator)
1084
105
    return Ctor();
1085
35.5k
1086
35.5k
  // With no -regalloc= override, ask the target for a regalloc pass.
1087
35.5k
  return createTargetRegisterAllocator(Optimized);
1088
35.5k
}
1089
1090
1.82k
bool TargetPassConfig::addRegAssignmentFast() {
1091
1.82k
  if (RegAlloc != &useDefaultRegisterAllocator &&
1092
1.82k
      
RegAlloc != &createFastRegisterAllocator40
)
1093
0
    report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc.");
1094
1.82k
1095
1.82k
  addPass(createRegAllocPass(false));
1096
1.82k
  return true;
1097
1.82k
}
1098
1099
33.8k
bool TargetPassConfig::addRegAssignmentOptimized() {
1100
33.8k
  // Add the selected register allocation pass.
1101
33.8k
  addPass(createRegAllocPass(true));
1102
33.8k
1103
33.8k
  // Allow targets to change the register assignments before rewriting.
1104
33.8k
  addPreRewrite();
1105
33.8k
1106
33.8k
  // Finally rewrite virtual registers.
1107
33.8k
  addPass(&VirtRegRewriterID);
1108
33.8k
  // Perform stack slot coloring and post-ra machine LICM.
1109
33.8k
  //
1110
33.8k
  // FIXME: Re-enable coloring with register when it's capable of adding
1111
33.8k
  // kill markers.
1112
33.8k
  addPass(&StackSlotColoringID);
1113
33.8k
1114
33.8k
  return true;
1115
33.8k
}
1116
1117
/// Return true if the default global register allocator is in use and
1118
/// has not be overriden on the command line with '-regalloc=...'
1119
8.62k
bool TargetPassConfig::usingDefaultRegAlloc() const {
1120
8.62k
  return RegAlloc.getNumOccurrences() == 0;
1121
8.62k
}
1122
1123
/// Add the minimum set of target-independent passes that are required for
1124
/// register allocation. No coalescing or scheduling.
1125
1.84k
void TargetPassConfig::addFastRegAlloc() {
1126
1.84k
  addPass(&PHIEliminationID, false);
1127
1.84k
  addPass(&TwoAddressInstructionPassID, false);
1128
1.84k
1129
1.84k
  addRegAssignmentFast();
1130
1.84k
}
1131
1132
/// Add standard target-independent passes that are tightly coupled with
1133
/// optimized register allocation, including coalescing, machine instruction
1134
/// scheduling, and register allocation itself.
1135
34.2k
void TargetPassConfig::addOptimizedRegAlloc() {
1136
34.2k
  addPass(&DetectDeadLanesID, false);
1137
34.2k
1138
34.2k
  addPass(&ProcessImplicitDefsID, false);
1139
34.2k
1140
34.2k
  // LiveVariables currently requires pure SSA form.
1141
34.2k
  //
1142
34.2k
  // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
1143
34.2k
  // LiveVariables can be removed completely, and LiveIntervals can be directly
1144
34.2k
  // computed. (We still either need to regenerate kill flags after regalloc, or
1145
34.2k
  // preferably fix the scavenger to not depend on them).
1146
34.2k
  addPass(&LiveVariablesID, false);
1147
34.2k
1148
34.2k
  // Edge splitting is smarter with machine loop info.
1149
34.2k
  addPass(&MachineLoopInfoID, false);
1150
34.2k
  addPass(&PHIEliminationID, false);
1151
34.2k
1152
34.2k
  // Eventually, we want to run LiveIntervals before PHI elimination.
1153
34.2k
  if (EarlyLiveIntervals)
1154
0
    addPass(&LiveIntervalsID, false);
1155
34.2k
1156
34.2k
  addPass(&TwoAddressInstructionPassID, false);
1157
34.2k
  addPass(&RegisterCoalescerID);
1158
34.2k
1159
34.2k
  // The machine scheduler may accidentally create disconnected components
1160
34.2k
  // when moving subregister definitions around, avoid this by splitting them to
1161
34.2k
  // separate vregs before. Splitting can also improve reg. allocation quality.
1162
34.2k
  addPass(&RenameIndependentSubregsID);
1163
34.2k
1164
34.2k
  // PreRA instruction scheduling.
1165
34.2k
  addPass(&MachineSchedulerID);
1166
34.2k
1167
34.2k
  if (addRegAssignmentOptimized()) {
1168
33.8k
    // Allow targets to expand pseudo instructions depending on the choice of
1169
33.8k
    // registers before MachineCopyPropagation.
1170
33.8k
    addPostRewrite();
1171
33.8k
1172
33.8k
    // Copy propagate to forward register uses and try to eliminate COPYs that
1173
33.8k
    // were not coalesced.
1174
33.8k
    addPass(&MachineCopyPropagationID);
1175
33.8k
1176
33.8k
    // Run post-ra machine LICM to hoist reloads / remats.
1177
33.8k
    //
1178
33.8k
    // FIXME: can this move into MachineLateOptimization?
1179
33.8k
    addPass(&MachineLICMID);
1180
33.8k
  }
1181
34.2k
}
1182
1183
//===---------------------------------------------------------------------===//
1184
/// Post RegAlloc Pass Configuration
1185
//===---------------------------------------------------------------------===//
1186
1187
/// Add passes that optimize machine instructions after register allocation.
1188
34.5k
void TargetPassConfig::addMachineLateOptimization() {
1189
34.5k
  // Branch folding must be run after regalloc and prolog/epilog insertion.
1190
34.5k
  addPass(&BranchFolderPassID);
1191
34.5k
1192
34.5k
  // Tail duplication.
1193
34.5k
  // Note that duplicating tail just increases code size and degrades
1194
34.5k
  // performance for targets that require Structured Control Flow.
1195
34.5k
  // In addition it can also make CFG irreducible. Thus we disable it.
1196
34.5k
  if (!TM->requiresStructuredCFG())
1197
33.9k
    addPass(&TailDuplicateID);
1198
34.5k
1199
34.5k
  // Copy propagation.
1200
34.5k
  addPass(&MachineCopyPropagationID);
1201
34.5k
}
1202
1203
/// Add standard GC passes.
1204
33.2k
bool TargetPassConfig::addGCPasses() {
1205
33.2k
  addPass(&GCMachineCodeAnalysisID, false);
1206
33.2k
  return true;
1207
33.2k
}
1208
1209
/// Add standard basic block placement passes.
1210
34.5k
void TargetPassConfig::addBlockPlacement() {
1211
34.5k
  if (addPass(&MachineBlockPlacementID)) {
1212
34.0k
    // Run a separate pass to collect block placement statistics.
1213
34.0k
    if (EnableBlockPlacementStats)
1214
0
      addPass(&MachineBlockPlacementStatsID);
1215
34.0k
  }
1216
34.5k
}
1217
1218
//===---------------------------------------------------------------------===//
1219
/// GlobalISel Configuration
1220
//===---------------------------------------------------------------------===//
1221
47.0k
bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1222
47.0k
  return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1223
47.0k
}
1224
1225
7.00k
bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1226
7.00k
  return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1227
7.00k
}
1228
1229
477k
bool TargetPassConfig::isGISelCSEEnabled() const {
1230
477k
  return true;
1231
477k
}
1232
1233
0
std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1234
0
  return make_unique<CSEConfigBase>();
1235
0
}