Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// This pass lowers the 'expect' intrinsic to LLVM metadata.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
15
#include "llvm/ADT/SmallVector.h"
16
#include "llvm/ADT/Statistic.h"
17
#include "llvm/ADT/iterator_range.h"
18
#include "llvm/IR/BasicBlock.h"
19
#include "llvm/IR/Constants.h"
20
#include "llvm/IR/Function.h"
21
#include "llvm/IR/Instructions.h"
22
#include "llvm/IR/Intrinsics.h"
23
#include "llvm/IR/LLVMContext.h"
24
#include "llvm/IR/MDBuilder.h"
25
#include "llvm/IR/Metadata.h"
26
#include "llvm/Pass.h"
27
#include "llvm/Support/CommandLine.h"
28
#include "llvm/Support/Debug.h"
29
#include "llvm/Transforms/Scalar.h"
30
31
using namespace llvm;
32
33
#define DEBUG_TYPE "lower-expect-intrinsic"
34
35
STATISTIC(ExpectIntrinsicsHandled,
36
          "Number of 'expect' intrinsic instructions handled");
37
38
// These default values are chosen to represent an extremely skewed outcome for
39
// a condition, but they leave some room for interpretation by later passes.
40
//
41
// If the documentation for __builtin_expect() was made explicit that it should
42
// only be used in extreme cases, we could make this ratio higher. As it stands,
43
// programmers may be using __builtin_expect() / llvm.expect to annotate that a
44
// branch is likely or unlikely to be taken.
45
//
46
// There is a known dependency on this ratio in CodeGenPrepare when transforming
47
// 'select' instructions. It may be worthwhile to hoist these values to some
48
// shared space, so they can be used directly by other passes.
49
50
static cl::opt<uint32_t> LikelyBranchWeight(
51
    "likely-branch-weight", cl::Hidden, cl::init(2000),
52
    cl::desc("Weight of the branch likely to be taken (default = 2000)"));
53
static cl::opt<uint32_t> UnlikelyBranchWeight(
54
    "unlikely-branch-weight", cl::Hidden, cl::init(1),
55
    cl::desc("Weight of the branch unlikely to be taken (default = 1)"));
56
57
24.2k
static bool handleSwitchExpect(SwitchInst &SI) {
58
24.2k
  CallInst *CI = dyn_cast<CallInst>(SI.getCondition());
59
24.2k
  if (!CI)
60
23.8k
    return false;
61
360
62
360
  Function *Fn = CI->getCalledFunction();
63
360
  if (
!Fn || 360
Fn->getIntrinsicID() != Intrinsic::expect326
)
64
356
    return false;
65
4
66
4
  Value *ArgValue = CI->getArgOperand(0);
67
4
  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
68
4
  if (!ExpectedValue)
69
0
    return false;
70
4
71
4
  SwitchInst::CaseHandle Case = *SI.findCaseValue(ExpectedValue);
72
4
  unsigned n = SI.getNumCases(); // +1 for default case.
73
4
  SmallVector<uint32_t, 16> Weights(n + 1, UnlikelyBranchWeight);
74
4
75
4
  if (Case == *SI.case_default())
76
2
    Weights[0] = LikelyBranchWeight;
77
4
  else
78
2
    Weights[Case.getCaseIndex() + 1] = LikelyBranchWeight;
79
24.2k
80
24.2k
  SI.setMetadata(LLVMContext::MD_prof,
81
24.2k
                 MDBuilder(CI->getContext()).createBranchWeights(Weights));
82
24.2k
83
24.2k
  SI.setCondition(ArgValue);
84
24.2k
  return true;
85
24.2k
}
86
87
/// Handler for PHINodes that define the value argument to an
88
/// @llvm.expect call.
89
///
90
/// If the operand of the phi has a constant value and it 'contradicts'
91
/// with the expected value of phi def, then the corresponding incoming
92
/// edge of the phi is unlikely to be taken. Using that information,
93
/// the branch probability info for the originating branch can be inferred.
94
5.07k
static void handlePhiDef(CallInst *Expect) {
95
5.07k
  Value &Arg = *Expect->getArgOperand(0);
96
5.07k
  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(Expect->getArgOperand(1));
97
5.07k
  if (!ExpectedValue)
98
2
    return;
99
5.07k
  const APInt &ExpectedPhiValue = ExpectedValue->getValue();
100
5.07k
101
5.07k
  // Walk up in backward a list of instructions that
102
5.07k
  // have 'copy' semantics by 'stripping' the copies
103
5.07k
  // until a PHI node or an instruction of unknown kind
104
5.07k
  // is reached. Negation via xor is also handled.
105
5.07k
  //
106
5.07k
  //       C = PHI(...);
107
5.07k
  //       B = C;
108
5.07k
  //       A = B;
109
5.07k
  //       D = __builtin_expect(A, 0);
110
5.07k
  //
111
5.07k
  Value *V = &Arg;
112
5.07k
  SmallVector<Instruction *, 4> Operations;
113
13.6k
  while (
!isa<PHINode>(V)13.6k
) {
114
13.5k
    if (ZExtInst *
ZExt13.5k
= dyn_cast<ZExtInst>(V)) {
115
4.52k
      V = ZExt->getOperand(0);
116
4.52k
      Operations.push_back(ZExt);
117
4.52k
      continue;
118
4.52k
    }
119
9.01k
120
9.01k
    
if (SExtInst *9.01k
SExt9.01k
= dyn_cast<SExtInst>(V)) {
121
48
      V = SExt->getOperand(0);
122
48
      Operations.push_back(SExt);
123
48
      continue;
124
48
    }
125
8.97k
126
8.97k
    BinaryOperator *BinOp = dyn_cast<BinaryOperator>(V);
127
8.97k
    if (
!BinOp || 8.97k
BinOp->getOpcode() != Instruction::Xor4.01k
)
128
4.95k
      return;
129
4.01k
130
4.01k
    ConstantInt *CInt = dyn_cast<ConstantInt>(BinOp->getOperand(1));
131
4.01k
    if (!CInt)
132
0
      return;
133
4.01k
134
4.01k
    V = BinOp->getOperand(0);
135
4.01k
    Operations.push_back(BinOp);
136
4.01k
  }
137
5.07k
138
5.07k
  // Executes the recorded operations on input 'Value'.
139
114
  
auto ApplyOperations = [&](const APInt &Value) 114
{
140
130
    APInt Result = Value;
141
238
    for (auto Op : llvm::reverse(Operations)) {
142
238
      switch (Op->getOpcode()) {
143
96
      case Instruction::Xor:
144
96
        Result ^= cast<ConstantInt>(Op->getOperand(1))->getValue();
145
96
        break;
146
102
      case Instruction::ZExt:
147
102
        Result = Result.zext(Op->getType()->getIntegerBitWidth());
148
102
        break;
149
40
      case Instruction::SExt:
150
40
        Result = Result.sext(Op->getType()->getIntegerBitWidth());
151
40
        break;
152
0
      default:
153
0
        llvm_unreachable("Unexpected operation");
154
130
      }
155
130
    }
156
130
    return Result;
157
130
  };
158
114
159
114
  auto *PhiDef = dyn_cast<PHINode>(V);
160
114
161
114
  // Get the first dominating conditional branch of the operand
162
114
  // i's incoming block.
163
40
  auto GetDomConditional = [&](unsigned i) -> BranchInst * {
164
40
    BasicBlock *BB = PhiDef->getIncomingBlock(i);
165
40
    BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator());
166
40
    if (
BI && 40
BI->isConditional()40
)
167
38
      return BI;
168
2
    BB = BB->getSinglePredecessor();
169
2
    if (!BB)
170
0
      return nullptr;
171
2
    BI = dyn_cast<BranchInst>(BB->getTerminator());
172
2
    if (
!BI || 2
BI->isUnconditional()2
)
173
0
      return nullptr;
174
2
    return BI;
175
2
  };
176
114
177
114
  // Now walk through all Phi operands to find phi oprerands with values
178
114
  // conflicting with the expected phi output value. Any such operand
179
114
  // indicates the incoming edge to that operand is unlikely.
180
358
  for (unsigned i = 0, e = PhiDef->getNumIncomingValues(); 
i != e358
;
++i244
) {
181
244
182
244
    Value *PhiOpnd = PhiDef->getIncomingValue(i);
183
244
    ConstantInt *CI = dyn_cast<ConstantInt>(PhiOpnd);
184
244
    if (!CI)
185
114
      continue;
186
130
187
130
    // Not an interesting case when IsUnlikely is false -- we can not infer
188
130
    // anything useful when the operand value matches the expected phi
189
130
    // output.
190
130
    
if (130
ExpectedPhiValue == ApplyOperations(CI->getValue())130
)
191
90
      continue;
192
40
193
40
    BranchInst *BI = GetDomConditional(i);
194
40
    if (!BI)
195
0
      continue;
196
40
197
40
    MDBuilder MDB(PhiDef->getContext());
198
40
199
40
    // There are two situations in which an operand of the PhiDef comes
200
40
    // from a given successor of a branch instruction BI.
201
40
    // 1) When the incoming block of the operand is the successor block;
202
40
    // 2) When the incoming block is BI's enclosing block and the
203
40
    // successor is the PhiDef's enclosing block.
204
40
    //
205
40
    // Returns true if the operand which comes from OpndIncomingBB
206
40
    // comes from outgoing edge of BI that leads to Succ block.
207
40
    auto *OpndIncomingBB = PhiDef->getIncomingBlock(i);
208
46
    auto IsOpndComingFromSuccessor = [&](BasicBlock *Succ) {
209
46
      if (OpndIncomingBB == Succ)
210
46
        // If this successor is the incoming block for this
211
46
        // Phi operand, then this successor does lead to the Phi.
212
2
        return true;
213
44
      
if (44
OpndIncomingBB == BI->getParent() && 44
Succ == PhiDef->getParent()42
)
214
44
        // Otherwise, if the edge is directly from the branch
215
44
        // to the Phi, this successor is the one feeding this
216
44
        // Phi operand.
217
38
        return true;
218
6
      return false;
219
6
    };
220
40
221
40
    if (IsOpndComingFromSuccessor(BI->getSuccessor(1)))
222
34
      BI->setMetadata(
223
34
          LLVMContext::MD_prof,
224
34
          MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight));
225
6
    else 
if (6
IsOpndComingFromSuccessor(BI->getSuccessor(0))6
)
226
6
      BI->setMetadata(
227
6
          LLVMContext::MD_prof,
228
6
          MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight));
229
244
  }
230
5.07k
}
231
232
// Handle both BranchInst and SelectInst.
233
1.70M
template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
234
1.70M
235
1.70M
  // Handle non-optimized IR code like:
236
1.70M
  //   %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
237
1.70M
  //   %tobool = icmp ne i64 %expval, 0
238
1.70M
  //   br i1 %tobool, label %if.then, label %if.end
239
1.70M
  //
240
1.70M
  // Or the following simpler case:
241
1.70M
  //   %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
242
1.70M
  //   br i1 %expval, label %if.then, label %if.end
243
1.70M
244
1.70M
  CallInst *CI;
245
1.70M
246
1.70M
  ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition());
247
1.70M
  CmpInst::Predicate Predicate;
248
1.70M
  ConstantInt *CmpConstOperand = nullptr;
249
1.70M
  if (
!CmpI1.70M
) {
250
205k
    CI = dyn_cast<CallInst>(BSI.getCondition());
251
205k
    Predicate = CmpInst::ICMP_NE;
252
1.70M
  } else {
253
1.50M
    Predicate = CmpI->getPredicate();
254
1.50M
    if (
Predicate != CmpInst::ICMP_NE && 1.50M
Predicate != CmpInst::ICMP_EQ1.23M
)
255
463k
      return false;
256
1.03M
257
1.03M
    CmpConstOperand = dyn_cast<ConstantInt>(CmpI->getOperand(1));
258
1.03M
    if (!CmpConstOperand)
259
544k
      return false;
260
494k
    CI = dyn_cast<CallInst>(CmpI->getOperand(0));
261
494k
  }
262
1.70M
263
699k
  
if (699k
!CI699k
)
264
557k
    return false;
265
141k
266
141k
  uint64_t ValueComparedTo = 0;
267
141k
  if (
CmpConstOperand141k
) {
268
104k
    if (CmpConstOperand->getBitWidth() > 64)
269
0
      return false;
270
104k
    ValueComparedTo = CmpConstOperand->getZExtValue();
271
104k
  }
272
141k
273
141k
  Function *Fn = CI->getCalledFunction();
274
141k
  if (
!Fn || 141k
Fn->getIntrinsicID() != Intrinsic::expect118k
)
275
136k
    return false;
276
5.06k
277
5.06k
  Value *ArgValue = CI->getArgOperand(0);
278
5.06k
  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
279
5.06k
  if (!ExpectedValue)
280
0
    return false;
281
5.06k
282
5.06k
  MDBuilder MDB(CI->getContext());
283
5.06k
  MDNode *Node;
284
5.06k
285
5.06k
  if ((ExpectedValue->getZExtValue() == ValueComparedTo) ==
286
5.06k
      (Predicate == CmpInst::ICMP_EQ))
287
52
    Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight);
288
5.06k
  else
289
5.00k
    Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight);
290
5.06k
291
5.06k
  BSI.setMetadata(LLVMContext::MD_prof, Node);
292
5.06k
293
5.06k
  if (CmpI)
294
5.05k
    CmpI->setOperand(0, ArgValue);
295
5.06k
  else
296
2
    BSI.setCondition(ArgValue);
297
1.70M
  return true;
298
1.70M
}
LowerExpectIntrinsic.cpp:bool handleBrSelExpect<llvm::BranchInst>(llvm::BranchInst&)
Line
Count
Source
233
1.59M
template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
234
1.59M
235
1.59M
  // Handle non-optimized IR code like:
236
1.59M
  //   %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
237
1.59M
  //   %tobool = icmp ne i64 %expval, 0
238
1.59M
  //   br i1 %tobool, label %if.then, label %if.end
239
1.59M
  //
240
1.59M
  // Or the following simpler case:
241
1.59M
  //   %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
242
1.59M
  //   br i1 %expval, label %if.then, label %if.end
243
1.59M
244
1.59M
  CallInst *CI;
245
1.59M
246
1.59M
  ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition());
247
1.59M
  CmpInst::Predicate Predicate;
248
1.59M
  ConstantInt *CmpConstOperand = nullptr;
249
1.59M
  if (
!CmpI1.59M
) {
250
194k
    CI = dyn_cast<CallInst>(BSI.getCondition());
251
194k
    Predicate = CmpInst::ICMP_NE;
252
1.59M
  } else {
253
1.39M
    Predicate = CmpI->getPredicate();
254
1.39M
    if (
Predicate != CmpInst::ICMP_NE && 1.39M
Predicate != CmpInst::ICMP_EQ1.13M
)
255
393k
      return false;
256
1.00M
257
1.00M
    CmpConstOperand = dyn_cast<ConstantInt>(CmpI->getOperand(1));
258
1.00M
    if (!CmpConstOperand)
259
540k
      return false;
260
463k
    CI = dyn_cast<CallInst>(CmpI->getOperand(0));
261
463k
  }
262
1.59M
263
658k
  
if (658k
!CI658k
)
264
530k
    return false;
265
127k
266
127k
  uint64_t ValueComparedTo = 0;
267
127k
  if (
CmpConstOperand127k
) {
268
90.4k
    if (CmpConstOperand->getBitWidth() > 64)
269
0
      return false;
270
90.4k
    ValueComparedTo = CmpConstOperand->getZExtValue();
271
90.4k
  }
272
127k
273
127k
  Function *Fn = CI->getCalledFunction();
274
127k
  if (
!Fn || 127k
Fn->getIntrinsicID() != Intrinsic::expect117k
)
275
122k
    return false;
276
5.05k
277
5.05k
  Value *ArgValue = CI->getArgOperand(0);
278
5.05k
  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
279
5.05k
  if (!ExpectedValue)
280
0
    return false;
281
5.05k
282
5.05k
  MDBuilder MDB(CI->getContext());
283
5.05k
  MDNode *Node;
284
5.05k
285
5.05k
  if ((ExpectedValue->getZExtValue() == ValueComparedTo) ==
286
5.05k
      (Predicate == CmpInst::ICMP_EQ))
287
52
    Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight);
288
5.05k
  else
289
5.00k
    Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight);
290
5.05k
291
5.05k
  BSI.setMetadata(LLVMContext::MD_prof, Node);
292
5.05k
293
5.05k
  if (CmpI)
294
5.05k
    CmpI->setOperand(0, ArgValue);
295
5.05k
  else
296
2
    BSI.setCondition(ArgValue);
297
1.59M
  return true;
298
1.59M
}
LowerExpectIntrinsic.cpp:bool handleBrSelExpect<llvm::SelectInst>(llvm::SelectInst&)
Line
Count
Source
233
115k
template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) {
234
115k
235
115k
  // Handle non-optimized IR code like:
236
115k
  //   %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1)
237
115k
  //   %tobool = icmp ne i64 %expval, 0
238
115k
  //   br i1 %tobool, label %if.then, label %if.end
239
115k
  //
240
115k
  // Or the following simpler case:
241
115k
  //   %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1)
242
115k
  //   br i1 %expval, label %if.then, label %if.end
243
115k
244
115k
  CallInst *CI;
245
115k
246
115k
  ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition());
247
115k
  CmpInst::Predicate Predicate;
248
115k
  ConstantInt *CmpConstOperand = nullptr;
249
115k
  if (
!CmpI115k
) {
250
10.3k
    CI = dyn_cast<CallInst>(BSI.getCondition());
251
10.3k
    Predicate = CmpInst::ICMP_NE;
252
115k
  } else {
253
105k
    Predicate = CmpI->getPredicate();
254
105k
    if (
Predicate != CmpInst::ICMP_NE && 105k
Predicate != CmpInst::ICMP_EQ102k
)
255
70.1k
      return false;
256
34.9k
257
34.9k
    CmpConstOperand = dyn_cast<ConstantInt>(CmpI->getOperand(1));
258
34.9k
    if (!CmpConstOperand)
259
3.71k
      return false;
260
31.2k
    CI = dyn_cast<CallInst>(CmpI->getOperand(0));
261
31.2k
  }
262
115k
263
41.5k
  
if (41.5k
!CI41.5k
)
264
27.2k
    return false;
265
14.3k
266
14.3k
  uint64_t ValueComparedTo = 0;
267
14.3k
  if (
CmpConstOperand14.3k
) {
268
14.3k
    if (CmpConstOperand->getBitWidth() > 64)
269
0
      return false;
270
14.3k
    ValueComparedTo = CmpConstOperand->getZExtValue();
271
14.3k
  }
272
14.3k
273
14.3k
  Function *Fn = CI->getCalledFunction();
274
14.3k
  if (
!Fn || 14.3k
Fn->getIntrinsicID() != Intrinsic::expect450
)
275
14.3k
    return false;
276
2
277
2
  Value *ArgValue = CI->getArgOperand(0);
278
2
  ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1));
279
2
  if (!ExpectedValue)
280
0
    return false;
281
2
282
2
  MDBuilder MDB(CI->getContext());
283
2
  MDNode *Node;
284
2
285
2
  if ((ExpectedValue->getZExtValue() == ValueComparedTo) ==
286
2
      (Predicate == CmpInst::ICMP_EQ))
287
0
    Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight);
288
2
  else
289
2
    Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight);
290
2
291
2
  BSI.setMetadata(LLVMContext::MD_prof, Node);
292
2
293
2
  if (CmpI)
294
2
    CmpI->setOperand(0, ArgValue);
295
2
  else
296
0
    BSI.setCondition(ArgValue);
297
115k
  return true;
298
115k
}
299
300
2.90M
static bool handleBranchExpect(BranchInst &BI) {
301
2.90M
  if (BI.isUnconditional())
302
1.31M
    return false;
303
1.59M
304
1.59M
  return handleBrSelExpect<BranchInst>(BI);
305
1.59M
}
306
307
763k
static bool lowerExpectIntrinsic(Function &F) {
308
763k
  bool Changed = false;
309
763k
310
3.73M
  for (BasicBlock &BB : F) {
311
3.73M
    // Create "block_weights" metadata.
312
3.73M
    if (BranchInst *
BI3.73M
= dyn_cast<BranchInst>(BB.getTerminator())) {
313
2.90M
      if (handleBranchExpect(*BI))
314
5.05k
        ExpectIntrinsicsHandled++;
315
3.73M
    } else 
if (SwitchInst *830k
SI830k
= dyn_cast<SwitchInst>(BB.getTerminator())) {
316
24.2k
      if (handleSwitchExpect(*SI))
317
4
        ExpectIntrinsicsHandled++;
318
830k
    }
319
3.73M
320
3.73M
    // Remove llvm.expect intrinsics. Iterate backwards in order
321
3.73M
    // to process select instructions before the intrinsic gets
322
3.73M
    // removed.
323
24.1M
    for (auto BI = BB.rbegin(), BE = BB.rend(); 
BI != BE24.1M
;) {
324
20.3M
      Instruction *Inst = &*BI++;
325
20.3M
      CallInst *CI = dyn_cast<CallInst>(Inst);
326
20.3M
      if (
!CI20.3M
) {
327
17.5M
        if (SelectInst *
SI17.5M
= dyn_cast<SelectInst>(Inst)) {
328
115k
          if (handleBrSelExpect(*SI))
329
2
            ExpectIntrinsicsHandled++;
330
115k
        }
331
17.5M
        continue;
332
17.5M
      }
333
2.89M
334
2.89M
      Function *Fn = CI->getCalledFunction();
335
2.89M
      if (
Fn && 2.89M
Fn->getIntrinsicID() == Intrinsic::expect2.78M
) {
336
5.07k
        // Before erasing the llvm.expect, walk backward to find
337
5.07k
        // phi that define llvm.expect's first arg, and
338
5.07k
        // infer branch probability:
339
5.07k
        handlePhiDef(CI);
340
5.07k
        Value *Exp = CI->getArgOperand(0);
341
5.07k
        CI->replaceAllUsesWith(Exp);
342
5.07k
        CI->eraseFromParent();
343
5.07k
        Changed = true;
344
5.07k
      }
345
20.3M
    }
346
3.73M
  }
347
763k
348
763k
  return Changed;
349
763k
}
350
351
PreservedAnalyses LowerExpectIntrinsicPass::run(Function &F,
352
132
                                                FunctionAnalysisManager &) {
353
132
  if (lowerExpectIntrinsic(F))
354
28
    return PreservedAnalyses::none();
355
104
356
104
  return PreservedAnalyses::all();
357
104
}
358
359
namespace {
360
/// \brief Legacy pass for lowering expect intrinsics out of the IR.
361
///
362
/// When this pass is run over a function it uses expect intrinsics which feed
363
/// branches and switches to provide branch weight metadata for those
364
/// terminators. It then removes the expect intrinsics from the IR so the rest
365
/// of the optimizer can ignore them.
366
class LowerExpectIntrinsic : public FunctionPass {
367
public:
368
  static char ID;
369
17.1k
  LowerExpectIntrinsic() : FunctionPass(ID) {
370
17.1k
    initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry());
371
17.1k
  }
372
373
763k
  bool runOnFunction(Function &F) override { return lowerExpectIntrinsic(F); }
374
};
375
}
376
377
char LowerExpectIntrinsic::ID = 0;
378
INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect",
379
                "Lower 'expect' Intrinsics", false, false)
380
381
17.0k
FunctionPass *llvm::createLowerExpectIntrinsicPass() {
382
17.0k
  return new LowerExpectIntrinsic();
383
17.0k
}