Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp
Line
Count
Source
1
//===- SpeculativeExecution.cpp ---------------------------------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This pass hoists instructions to enable speculative execution on
10
// targets where branches are expensive. This is aimed at GPUs. It
11
// currently works on simple if-then and if-then-else
12
// patterns.
13
//
14
// Removing branches is not the only motivation for this
15
// pass. E.g. consider this code and assume that there is no
16
// addressing mode for multiplying by sizeof(*a):
17
//
18
//   if (b > 0)
19
//     c = a[i + 1]
20
//   if (d > 0)
21
//     e = a[i + 2]
22
//
23
// turns into
24
//
25
//   p = &a[i + 1];
26
//   if (b > 0)
27
//     c = *p;
28
//   q = &a[i + 2];
29
//   if (d > 0)
30
//     e = *q;
31
//
32
// which could later be optimized to
33
//
34
//   r = &a[i];
35
//   if (b > 0)
36
//     c = r[1];
37
//   if (d > 0)
38
//     e = r[2];
39
//
40
// Later passes sink back much of the speculated code that did not enable
41
// further optimization.
42
//
43
// This pass is more aggressive than the function SpeculativeyExecuteBB in
44
// SimplifyCFG. SimplifyCFG will not speculate if no selects are introduced and
45
// it will speculate at most one instruction. It also will not speculate if
46
// there is a value defined in the if-block that is only used in the then-block.
47
// These restrictions make sense since the speculation in SimplifyCFG seems
48
// aimed at introducing cheap selects, while this pass is intended to do more
49
// aggressive speculation while counting on later passes to either capitalize on
50
// that or clean it up.
51
//
52
// If the pass was created by calling
53
// createSpeculativeExecutionIfHasBranchDivergencePass or the
54
// -spec-exec-only-if-divergent-target option is present, this pass only has an
55
// effect on targets where TargetTransformInfo::hasBranchDivergence() is true;
56
// on other targets, it is a nop.
57
//
58
// This lets you include this pass unconditionally in the IR pass pipeline, but
59
// only enable it for relevant targets.
60
//
61
//===----------------------------------------------------------------------===//
62
63
#include "llvm/Transforms/Scalar/SpeculativeExecution.h"
64
#include "llvm/ADT/SmallPtrSet.h"
65
#include "llvm/Analysis/GlobalsModRef.h"
66
#include "llvm/Analysis/ValueTracking.h"
67
#include "llvm/IR/Instructions.h"
68
#include "llvm/IR/Module.h"
69
#include "llvm/IR/Operator.h"
70
#include "llvm/Support/CommandLine.h"
71
#include "llvm/Support/Debug.h"
72
73
using namespace llvm;
74
75
#define DEBUG_TYPE "speculative-execution"
76
77
// The risk that speculation will not pay off increases with the
78
// number of instructions speculated, so we put a limit on that.
79
static cl::opt<unsigned> SpecExecMaxSpeculationCost(
80
    "spec-exec-max-speculation-cost", cl::init(7), cl::Hidden,
81
    cl::desc("Speculative execution is not applied to basic blocks where "
82
             "the cost of the instructions to speculatively execute "
83
             "exceeds this limit."));
84
85
// Speculating just a few instructions from a larger block tends not
86
// to be profitable and this limit prevents that. A reason for that is
87
// that small basic blocks are more likely to be candidates for
88
// further optimization.
89
static cl::opt<unsigned> SpecExecMaxNotHoisted(
90
    "spec-exec-max-not-hoisted", cl::init(5), cl::Hidden,
91
    cl::desc("Speculative execution is not applied to basic blocks where the "
92
             "number of instructions that would not be speculatively executed "
93
             "exceeds this limit."));
94
95
static cl::opt<bool> SpecExecOnlyIfDivergentTarget(
96
    "spec-exec-only-if-divergent-target", cl::init(false), cl::Hidden,
97
    cl::desc("Speculative execution is applied only to targets with divergent "
98
             "branches, even if the pass was configured to apply only to all "
99
             "targets."));
100
101
namespace {
102
103
class SpeculativeExecutionLegacyPass : public FunctionPass {
104
public:
105
  static char ID;
106
  explicit SpeculativeExecutionLegacyPass(bool OnlyIfDivergentTarget = false)
107
      : FunctionPass(ID), OnlyIfDivergentTarget(OnlyIfDivergentTarget ||
108
                                                SpecExecOnlyIfDivergentTarget),
109
15.8k
        Impl(OnlyIfDivergentTarget) {}
110
111
  void getAnalysisUsage(AnalysisUsage &AU) const override;
112
  bool runOnFunction(Function &F) override;
113
114
493k
  StringRef getPassName() const override {
115
493k
    if (OnlyIfDivergentTarget)
116
465k
      return "Speculatively execute instructions if target has divergent "
117
465k
             "branches";
118
28.2k
    return "Speculatively execute instructions";
119
28.2k
  }
120
121
private:
122
  // Variable preserved purely for correct name printing.
123
  const bool OnlyIfDivergentTarget;
124
125
  SpeculativeExecutionPass Impl;
126
};
127
} // namespace
128
129
char SpeculativeExecutionLegacyPass::ID = 0;
130
36.0k
INITIALIZE_PASS_BEGIN(SpeculativeExecutionLegacyPass, "speculative-execution",
131
36.0k
                      "Speculatively execute instructions", false, false)
132
36.0k
INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
133
36.0k
INITIALIZE_PASS_END(SpeculativeExecutionLegacyPass, "speculative-execution",
134
                    "Speculatively execute instructions", false, false)
135
136
15.8k
void SpeculativeExecutionLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const {
137
15.8k
  AU.addRequired<TargetTransformInfoWrapperPass>();
138
15.8k
  AU.addPreserved<GlobalsAAWrapperPass>();
139
15.8k
  AU.setPreservesCFG();
140
15.8k
}
141
142
493k
bool SpeculativeExecutionLegacyPass::runOnFunction(Function &F) {
143
493k
  if (skipFunction(F))
144
52
    return false;
145
493k
146
493k
  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
147
493k
  return Impl.runImpl(F, TTI);
148
493k
}
149
150
namespace llvm {
151
152
494k
bool SpeculativeExecutionPass::runImpl(Function &F, TargetTransformInfo *TTI) {
153
494k
  if (OnlyIfDivergentTarget && 
!TTI->hasBranchDivergence()465k
) {
154
464k
    LLVM_DEBUG(dbgs() << "Not running SpeculativeExecution because "
155
464k
                         "TTI->hasBranchDivergence() is false.\n");
156
464k
    return false;
157
464k
  }
158
30.1k
159
30.1k
  this->TTI = TTI;
160
30.1k
  bool Changed = false;
161
33.6k
  for (auto& B : F) {
162
33.6k
    Changed |= runOnBasicBlock(B);
163
33.6k
  }
164
30.1k
  return Changed;
165
30.1k
}
166
167
33.6k
bool SpeculativeExecutionPass::runOnBasicBlock(BasicBlock &B) {
168
33.6k
  BranchInst *BI = dyn_cast<BranchInst>(B.getTerminator());
169
33.6k
  if (BI == nullptr)
170
30.2k
    return false;
171
3.37k
172
3.37k
  if (BI->getNumSuccessors() != 2)
173
1.80k
    return false;
174
1.56k
  BasicBlock &Succ0 = *BI->getSuccessor(0);
175
1.56k
  BasicBlock &Succ1 = *BI->getSuccessor(1);
176
1.56k
177
1.56k
  if (&B == &Succ0 || 
&B == &Succ11.53k
||
&Succ0 == &Succ11.36k
) {
178
202
    return false;
179
202
  }
180
1.36k
181
1.36k
  // Hoist from if-then (triangle).
182
1.36k
  if (Succ0.getSinglePredecessor() != nullptr &&
183
1.36k
      
Succ0.getSingleSuccessor() == &Succ1850
) {
184
345
    return considerHoistingFromTo(Succ0, B);
185
345
  }
186
1.02k
187
1.02k
  // Hoist from if-else (triangle).
188
1.02k
  if (Succ1.getSinglePredecessor() != nullptr &&
189
1.02k
      
Succ1.getSingleSuccessor() == &Succ0836
) {
190
347
    return considerHoistingFromTo(Succ1, B);
191
347
  }
192
673
193
673
  // Hoist from if-then-else (diamond), but only if it is equivalent to
194
673
  // an if-else or if-then due to one of the branches doing nothing.
195
673
  if (Succ0.getSinglePredecessor() != nullptr &&
196
673
      
Succ1.getSinglePredecessor() != nullptr505
&&
197
673
      
Succ1.getSingleSuccessor() != nullptr373
&&
198
673
      
Succ1.getSingleSuccessor() != &B212
&&
199
673
      
Succ1.getSingleSuccessor() == Succ0.getSingleSuccessor()185
) {
200
166
    // If a block has only one instruction, then that is a terminator
201
166
    // instruction so that the block does nothing. This does happen.
202
166
    if (Succ1.size() == 1) // equivalent to if-then
203
28
      return considerHoistingFromTo(Succ0, B);
204
138
    if (Succ0.size() == 1) // equivalent to if-else
205
5
      return considerHoistingFromTo(Succ1, B);
206
640
  }
207
640
208
640
  return false;
209
640
}
210
211
static unsigned ComputeSpeculationCost(const Instruction *I,
212
1.94k
                                       const TargetTransformInfo &TTI) {
213
1.94k
  switch (Operator::getOpcode(I)) {
214
1.94k
    case Instruction::GetElementPtr:
215
832
    case Instruction::Add:
216
832
    case Instruction::Mul:
217
832
    case Instruction::And:
218
832
    case Instruction::Or:
219
832
    case Instruction::Select:
220
832
    case Instruction::Shl:
221
832
    case Instruction::Sub:
222
832
    case Instruction::LShr:
223
832
    case Instruction::AShr:
224
832
    case Instruction::Xor:
225
832
    case Instruction::ZExt:
226
832
    case Instruction::SExt:
227
832
    case Instruction::Call:
228
832
    case Instruction::BitCast:
229
832
    case Instruction::PtrToInt:
230
832
    case Instruction::IntToPtr:
231
832
    case Instruction::AddrSpaceCast:
232
832
    case Instruction::FPToUI:
233
832
    case Instruction::FPToSI:
234
832
    case Instruction::UIToFP:
235
832
    case Instruction::SIToFP:
236
832
    case Instruction::FPExt:
237
832
    case Instruction::FPTrunc:
238
832
    case Instruction::FAdd:
239
832
    case Instruction::FSub:
240
832
    case Instruction::FMul:
241
832
    case Instruction::FDiv:
242
832
    case Instruction::FRem:
243
832
    case Instruction::FNeg:
244
832
    case Instruction::ICmp:
245
832
    case Instruction::FCmp:
246
832
      return TTI.getUserCost(I);
247
832
248
1.11k
    default:
249
1.11k
      return UINT_MAX; // Disallow anything not whitelisted.
250
1.94k
  }
251
1.94k
}
252
253
bool SpeculativeExecutionPass::considerHoistingFromTo(
254
725
    BasicBlock &FromBlock, BasicBlock &ToBlock) {
255
725
  SmallPtrSet<const Instruction *, 8> NotHoisted;
256
725
  const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](User *U) {
257
1.09k
    for (Value* V : U->operand_values()) {
258
1.09k
      if (Instruction *I = dyn_cast<Instruction>(V)) {
259
639
        if (NotHoisted.count(I) > 0)
260
149
          return false;
261
639
      }
262
1.09k
    }
263
624
    
return true475
;
264
624
  };
265
725
266
725
  unsigned TotalSpeculationCost = 0;
267
1.94k
  for (auto& I : FromBlock) {
268
1.94k
    const unsigned Cost = ComputeSpeculationCost(&I, *TTI);
269
1.94k
    if (Cost != UINT_MAX && 
isSafeToSpeculativelyExecute(&I)832
&&
270
1.94k
        
AllPrecedingUsesFromBlockHoisted(&I)624
) {
271
475
      TotalSpeculationCost += Cost;
272
475
      if (TotalSpeculationCost > SpecExecMaxSpeculationCost)
273
21
        return false;  // too much to hoist
274
1.46k
    } else {
275
1.46k
      NotHoisted.insert(&I);
276
1.46k
      if (NotHoisted.size() > SpecExecMaxNotHoisted)
277
25
        return false; // too much left behind
278
1.46k
    }
279
1.94k
  }
280
725
281
725
  
if (679
TotalSpeculationCost == 0679
)
282
508
    return false; // nothing to hoist
283
171
284
667
  
for (auto I = FromBlock.begin(); 171
I != FromBlock.end();) {
285
496
    // We have to increment I before moving Current as moving Current
286
496
    // changes the list that I is iterating through.
287
496
    auto Current = I;
288
496
    ++I;
289
496
    if (!NotHoisted.count(&*Current)) {
290
279
      Current->moveBefore(ToBlock.getTerminator());
291
279
    }
292
496
  }
293
171
  return true;
294
171
}
295
296
2.90k
FunctionPass *createSpeculativeExecutionPass() {
297
2.90k
  return new SpeculativeExecutionLegacyPass();
298
2.90k
}
299
300
12.9k
FunctionPass *createSpeculativeExecutionIfHasBranchDivergencePass() {
301
12.9k
  return new SpeculativeExecutionLegacyPass(/* OnlyIfDivergentTarget = */ true);
302
12.9k
}
303
304
SpeculativeExecutionPass::SpeculativeExecutionPass(bool OnlyIfDivergentTarget)
305
    : OnlyIfDivergentTarget(OnlyIfDivergentTarget ||
306
16.0k
                            SpecExecOnlyIfDivergentTarget) {}
307
308
PreservedAnalyses SpeculativeExecutionPass::run(Function &F,
309
1.16k
                                                FunctionAnalysisManager &AM) {
310
1.16k
  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
311
1.16k
312
1.16k
  bool Changed = runImpl(F, TTI);
313
1.16k
314
1.16k
  if (!Changed)
315
1.14k
    return PreservedAnalyses::all();
316
18
  PreservedAnalyses PA;
317
18
  PA.preserve<GlobalsAA>();
318
18
  PA.preserveSet<CFGAnalyses>();
319
18
  return PA;
320
18
}
321
}  // namespace llvm