/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | // |
10 | | // This pass lowers the 'expect' intrinsic to LLVM metadata. |
11 | | // |
12 | | //===----------------------------------------------------------------------===// |
13 | | |
14 | | #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h" |
15 | | #include "llvm/ADT/SmallVector.h" |
16 | | #include "llvm/ADT/Statistic.h" |
17 | | #include "llvm/ADT/iterator_range.h" |
18 | | #include "llvm/IR/BasicBlock.h" |
19 | | #include "llvm/IR/Constants.h" |
20 | | #include "llvm/IR/Function.h" |
21 | | #include "llvm/IR/Instructions.h" |
22 | | #include "llvm/IR/Intrinsics.h" |
23 | | #include "llvm/IR/LLVMContext.h" |
24 | | #include "llvm/IR/MDBuilder.h" |
25 | | #include "llvm/IR/Metadata.h" |
26 | | #include "llvm/Pass.h" |
27 | | #include "llvm/Support/CommandLine.h" |
28 | | #include "llvm/Support/Debug.h" |
29 | | #include "llvm/Transforms/Scalar.h" |
30 | | |
31 | | using namespace llvm; |
32 | | |
33 | | #define DEBUG_TYPE "lower-expect-intrinsic" |
34 | | |
35 | | STATISTIC(ExpectIntrinsicsHandled, |
36 | | "Number of 'expect' intrinsic instructions handled"); |
37 | | |
38 | | // These default values are chosen to represent an extremely skewed outcome for |
39 | | // a condition, but they leave some room for interpretation by later passes. |
40 | | // |
41 | | // If the documentation for __builtin_expect() was made explicit that it should |
42 | | // only be used in extreme cases, we could make this ratio higher. As it stands, |
43 | | // programmers may be using __builtin_expect() / llvm.expect to annotate that a |
44 | | // branch is likely or unlikely to be taken. |
45 | | // |
46 | | // There is a known dependency on this ratio in CodeGenPrepare when transforming |
47 | | // 'select' instructions. It may be worthwhile to hoist these values to some |
48 | | // shared space, so they can be used directly by other passes. |
49 | | |
50 | | static cl::opt<uint32_t> LikelyBranchWeight( |
51 | | "likely-branch-weight", cl::Hidden, cl::init(2000), |
52 | | cl::desc("Weight of the branch likely to be taken (default = 2000)")); |
53 | | static cl::opt<uint32_t> UnlikelyBranchWeight( |
54 | | "unlikely-branch-weight", cl::Hidden, cl::init(1), |
55 | | cl::desc("Weight of the branch unlikely to be taken (default = 1)")); |
56 | | |
57 | 24.2k | static bool handleSwitchExpect(SwitchInst &SI) { |
58 | 24.2k | CallInst *CI = dyn_cast<CallInst>(SI.getCondition()); |
59 | 24.2k | if (!CI) |
60 | 23.8k | return false; |
61 | 360 | |
62 | 360 | Function *Fn = CI->getCalledFunction(); |
63 | 360 | if (!Fn || 360 Fn->getIntrinsicID() != Intrinsic::expect326 ) |
64 | 356 | return false; |
65 | 4 | |
66 | 4 | Value *ArgValue = CI->getArgOperand(0); |
67 | 4 | ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); |
68 | 4 | if (!ExpectedValue) |
69 | 0 | return false; |
70 | 4 | |
71 | 4 | SwitchInst::CaseHandle Case = *SI.findCaseValue(ExpectedValue); |
72 | 4 | unsigned n = SI.getNumCases(); // +1 for default case. |
73 | 4 | SmallVector<uint32_t, 16> Weights(n + 1, UnlikelyBranchWeight); |
74 | 4 | |
75 | 4 | if (Case == *SI.case_default()) |
76 | 2 | Weights[0] = LikelyBranchWeight; |
77 | 4 | else |
78 | 2 | Weights[Case.getCaseIndex() + 1] = LikelyBranchWeight; |
79 | 24.2k | |
80 | 24.2k | SI.setMetadata(LLVMContext::MD_prof, |
81 | 24.2k | MDBuilder(CI->getContext()).createBranchWeights(Weights)); |
82 | 24.2k | |
83 | 24.2k | SI.setCondition(ArgValue); |
84 | 24.2k | return true; |
85 | 24.2k | } |
86 | | |
87 | | /// Handler for PHINodes that define the value argument to an |
88 | | /// @llvm.expect call. |
89 | | /// |
90 | | /// If the operand of the phi has a constant value and it 'contradicts' |
91 | | /// with the expected value of phi def, then the corresponding incoming |
92 | | /// edge of the phi is unlikely to be taken. Using that information, |
93 | | /// the branch probability info for the originating branch can be inferred. |
94 | 5.07k | static void handlePhiDef(CallInst *Expect) { |
95 | 5.07k | Value &Arg = *Expect->getArgOperand(0); |
96 | 5.07k | ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(Expect->getArgOperand(1)); |
97 | 5.07k | if (!ExpectedValue) |
98 | 2 | return; |
99 | 5.07k | const APInt &ExpectedPhiValue = ExpectedValue->getValue(); |
100 | 5.07k | |
101 | 5.07k | // Walk up in backward a list of instructions that |
102 | 5.07k | // have 'copy' semantics by 'stripping' the copies |
103 | 5.07k | // until a PHI node or an instruction of unknown kind |
104 | 5.07k | // is reached. Negation via xor is also handled. |
105 | 5.07k | // |
106 | 5.07k | // C = PHI(...); |
107 | 5.07k | // B = C; |
108 | 5.07k | // A = B; |
109 | 5.07k | // D = __builtin_expect(A, 0); |
110 | 5.07k | // |
111 | 5.07k | Value *V = &Arg; |
112 | 5.07k | SmallVector<Instruction *, 4> Operations; |
113 | 13.6k | while (!isa<PHINode>(V)13.6k ) { |
114 | 13.5k | if (ZExtInst *ZExt13.5k = dyn_cast<ZExtInst>(V)) { |
115 | 4.52k | V = ZExt->getOperand(0); |
116 | 4.52k | Operations.push_back(ZExt); |
117 | 4.52k | continue; |
118 | 4.52k | } |
119 | 9.01k | |
120 | 9.01k | if (SExtInst *9.01k SExt9.01k = dyn_cast<SExtInst>(V)) { |
121 | 48 | V = SExt->getOperand(0); |
122 | 48 | Operations.push_back(SExt); |
123 | 48 | continue; |
124 | 48 | } |
125 | 8.97k | |
126 | 8.97k | BinaryOperator *BinOp = dyn_cast<BinaryOperator>(V); |
127 | 8.97k | if (!BinOp || 8.97k BinOp->getOpcode() != Instruction::Xor4.01k ) |
128 | 4.95k | return; |
129 | 4.01k | |
130 | 4.01k | ConstantInt *CInt = dyn_cast<ConstantInt>(BinOp->getOperand(1)); |
131 | 4.01k | if (!CInt) |
132 | 0 | return; |
133 | 4.01k | |
134 | 4.01k | V = BinOp->getOperand(0); |
135 | 4.01k | Operations.push_back(BinOp); |
136 | 4.01k | } |
137 | 5.07k | |
138 | 5.07k | // Executes the recorded operations on input 'Value'. |
139 | 114 | auto ApplyOperations = [&](const APInt &Value) 114 { |
140 | 130 | APInt Result = Value; |
141 | 238 | for (auto Op : llvm::reverse(Operations)) { |
142 | 238 | switch (Op->getOpcode()) { |
143 | 96 | case Instruction::Xor: |
144 | 96 | Result ^= cast<ConstantInt>(Op->getOperand(1))->getValue(); |
145 | 96 | break; |
146 | 102 | case Instruction::ZExt: |
147 | 102 | Result = Result.zext(Op->getType()->getIntegerBitWidth()); |
148 | 102 | break; |
149 | 40 | case Instruction::SExt: |
150 | 40 | Result = Result.sext(Op->getType()->getIntegerBitWidth()); |
151 | 40 | break; |
152 | 0 | default: |
153 | 0 | llvm_unreachable("Unexpected operation"); |
154 | 130 | } |
155 | 130 | } |
156 | 130 | return Result; |
157 | 130 | }; |
158 | 114 | |
159 | 114 | auto *PhiDef = dyn_cast<PHINode>(V); |
160 | 114 | |
161 | 114 | // Get the first dominating conditional branch of the operand |
162 | 114 | // i's incoming block. |
163 | 40 | auto GetDomConditional = [&](unsigned i) -> BranchInst * { |
164 | 40 | BasicBlock *BB = PhiDef->getIncomingBlock(i); |
165 | 40 | BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator()); |
166 | 40 | if (BI && 40 BI->isConditional()40 ) |
167 | 38 | return BI; |
168 | 2 | BB = BB->getSinglePredecessor(); |
169 | 2 | if (!BB) |
170 | 0 | return nullptr; |
171 | 2 | BI = dyn_cast<BranchInst>(BB->getTerminator()); |
172 | 2 | if (!BI || 2 BI->isUnconditional()2 ) |
173 | 0 | return nullptr; |
174 | 2 | return BI; |
175 | 2 | }; |
176 | 114 | |
177 | 114 | // Now walk through all Phi operands to find phi oprerands with values |
178 | 114 | // conflicting with the expected phi output value. Any such operand |
179 | 114 | // indicates the incoming edge to that operand is unlikely. |
180 | 358 | for (unsigned i = 0, e = PhiDef->getNumIncomingValues(); i != e358 ; ++i244 ) { |
181 | 244 | |
182 | 244 | Value *PhiOpnd = PhiDef->getIncomingValue(i); |
183 | 244 | ConstantInt *CI = dyn_cast<ConstantInt>(PhiOpnd); |
184 | 244 | if (!CI) |
185 | 114 | continue; |
186 | 130 | |
187 | 130 | // Not an interesting case when IsUnlikely is false -- we can not infer |
188 | 130 | // anything useful when the operand value matches the expected phi |
189 | 130 | // output. |
190 | 130 | if (130 ExpectedPhiValue == ApplyOperations(CI->getValue())130 ) |
191 | 90 | continue; |
192 | 40 | |
193 | 40 | BranchInst *BI = GetDomConditional(i); |
194 | 40 | if (!BI) |
195 | 0 | continue; |
196 | 40 | |
197 | 40 | MDBuilder MDB(PhiDef->getContext()); |
198 | 40 | |
199 | 40 | // There are two situations in which an operand of the PhiDef comes |
200 | 40 | // from a given successor of a branch instruction BI. |
201 | 40 | // 1) When the incoming block of the operand is the successor block; |
202 | 40 | // 2) When the incoming block is BI's enclosing block and the |
203 | 40 | // successor is the PhiDef's enclosing block. |
204 | 40 | // |
205 | 40 | // Returns true if the operand which comes from OpndIncomingBB |
206 | 40 | // comes from outgoing edge of BI that leads to Succ block. |
207 | 40 | auto *OpndIncomingBB = PhiDef->getIncomingBlock(i); |
208 | 46 | auto IsOpndComingFromSuccessor = [&](BasicBlock *Succ) { |
209 | 46 | if (OpndIncomingBB == Succ) |
210 | 46 | // If this successor is the incoming block for this |
211 | 46 | // Phi operand, then this successor does lead to the Phi. |
212 | 2 | return true; |
213 | 44 | if (44 OpndIncomingBB == BI->getParent() && 44 Succ == PhiDef->getParent()42 ) |
214 | 44 | // Otherwise, if the edge is directly from the branch |
215 | 44 | // to the Phi, this successor is the one feeding this |
216 | 44 | // Phi operand. |
217 | 38 | return true; |
218 | 6 | return false; |
219 | 6 | }; |
220 | 40 | |
221 | 40 | if (IsOpndComingFromSuccessor(BI->getSuccessor(1))) |
222 | 34 | BI->setMetadata( |
223 | 34 | LLVMContext::MD_prof, |
224 | 34 | MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight)); |
225 | 6 | else if (6 IsOpndComingFromSuccessor(BI->getSuccessor(0))6 ) |
226 | 6 | BI->setMetadata( |
227 | 6 | LLVMContext::MD_prof, |
228 | 6 | MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight)); |
229 | 244 | } |
230 | 5.07k | } |
231 | | |
232 | | // Handle both BranchInst and SelectInst. |
233 | 1.70M | template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) { |
234 | 1.70M | |
235 | 1.70M | // Handle non-optimized IR code like: |
236 | 1.70M | // %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1) |
237 | 1.70M | // %tobool = icmp ne i64 %expval, 0 |
238 | 1.70M | // br i1 %tobool, label %if.then, label %if.end |
239 | 1.70M | // |
240 | 1.70M | // Or the following simpler case: |
241 | 1.70M | // %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1) |
242 | 1.70M | // br i1 %expval, label %if.then, label %if.end |
243 | 1.70M | |
244 | 1.70M | CallInst *CI; |
245 | 1.70M | |
246 | 1.70M | ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition()); |
247 | 1.70M | CmpInst::Predicate Predicate; |
248 | 1.70M | ConstantInt *CmpConstOperand = nullptr; |
249 | 1.70M | if (!CmpI1.70M ) { |
250 | 205k | CI = dyn_cast<CallInst>(BSI.getCondition()); |
251 | 205k | Predicate = CmpInst::ICMP_NE; |
252 | 1.70M | } else { |
253 | 1.50M | Predicate = CmpI->getPredicate(); |
254 | 1.50M | if (Predicate != CmpInst::ICMP_NE && 1.50M Predicate != CmpInst::ICMP_EQ1.23M ) |
255 | 463k | return false; |
256 | 1.03M | |
257 | 1.03M | CmpConstOperand = dyn_cast<ConstantInt>(CmpI->getOperand(1)); |
258 | 1.03M | if (!CmpConstOperand) |
259 | 544k | return false; |
260 | 494k | CI = dyn_cast<CallInst>(CmpI->getOperand(0)); |
261 | 494k | } |
262 | 1.70M | |
263 | 699k | if (699k !CI699k ) |
264 | 557k | return false; |
265 | 141k | |
266 | 141k | uint64_t ValueComparedTo = 0; |
267 | 141k | if (CmpConstOperand141k ) { |
268 | 104k | if (CmpConstOperand->getBitWidth() > 64) |
269 | 0 | return false; |
270 | 104k | ValueComparedTo = CmpConstOperand->getZExtValue(); |
271 | 104k | } |
272 | 141k | |
273 | 141k | Function *Fn = CI->getCalledFunction(); |
274 | 141k | if (!Fn || 141k Fn->getIntrinsicID() != Intrinsic::expect118k ) |
275 | 136k | return false; |
276 | 5.06k | |
277 | 5.06k | Value *ArgValue = CI->getArgOperand(0); |
278 | 5.06k | ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); |
279 | 5.06k | if (!ExpectedValue) |
280 | 0 | return false; |
281 | 5.06k | |
282 | 5.06k | MDBuilder MDB(CI->getContext()); |
283 | 5.06k | MDNode *Node; |
284 | 5.06k | |
285 | 5.06k | if ((ExpectedValue->getZExtValue() == ValueComparedTo) == |
286 | 5.06k | (Predicate == CmpInst::ICMP_EQ)) |
287 | 52 | Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); |
288 | 5.06k | else |
289 | 5.00k | Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); |
290 | 5.06k | |
291 | 5.06k | BSI.setMetadata(LLVMContext::MD_prof, Node); |
292 | 5.06k | |
293 | 5.06k | if (CmpI) |
294 | 5.05k | CmpI->setOperand(0, ArgValue); |
295 | 5.06k | else |
296 | 2 | BSI.setCondition(ArgValue); |
297 | 1.70M | return true; |
298 | 1.70M | } LowerExpectIntrinsic.cpp:bool handleBrSelExpect<llvm::BranchInst>(llvm::BranchInst&) Line | Count | Source | 233 | 1.59M | template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) { | 234 | 1.59M | | 235 | 1.59M | // Handle non-optimized IR code like: | 236 | 1.59M | // %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1) | 237 | 1.59M | // %tobool = icmp ne i64 %expval, 0 | 238 | 1.59M | // br i1 %tobool, label %if.then, label %if.end | 239 | 1.59M | // | 240 | 1.59M | // Or the following simpler case: | 241 | 1.59M | // %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1) | 242 | 1.59M | // br i1 %expval, label %if.then, label %if.end | 243 | 1.59M | | 244 | 1.59M | CallInst *CI; | 245 | 1.59M | | 246 | 1.59M | ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition()); | 247 | 1.59M | CmpInst::Predicate Predicate; | 248 | 1.59M | ConstantInt *CmpConstOperand = nullptr; | 249 | 1.59M | if (!CmpI1.59M ) { | 250 | 194k | CI = dyn_cast<CallInst>(BSI.getCondition()); | 251 | 194k | Predicate = CmpInst::ICMP_NE; | 252 | 1.59M | } else { | 253 | 1.39M | Predicate = CmpI->getPredicate(); | 254 | 1.39M | if (Predicate != CmpInst::ICMP_NE && 1.39M Predicate != CmpInst::ICMP_EQ1.13M ) | 255 | 393k | return false; | 256 | 1.00M | | 257 | 1.00M | CmpConstOperand = dyn_cast<ConstantInt>(CmpI->getOperand(1)); | 258 | 1.00M | if (!CmpConstOperand) | 259 | 540k | return false; | 260 | 463k | CI = dyn_cast<CallInst>(CmpI->getOperand(0)); | 261 | 463k | } | 262 | 1.59M | | 263 | 658k | if (658k !CI658k ) | 264 | 530k | return false; | 265 | 127k | | 266 | 127k | uint64_t ValueComparedTo = 0; | 267 | 127k | if (CmpConstOperand127k ) { | 268 | 90.4k | if (CmpConstOperand->getBitWidth() > 64) | 269 | 0 | return false; | 270 | 90.4k | ValueComparedTo = CmpConstOperand->getZExtValue(); | 271 | 90.4k | } | 272 | 127k | | 273 | 127k | Function *Fn = CI->getCalledFunction(); | 274 | 127k | if (!Fn || 127k Fn->getIntrinsicID() != Intrinsic::expect117k ) | 275 | 122k | return false; | 276 | 5.05k | | 277 | 5.05k | Value *ArgValue = CI->getArgOperand(0); | 278 | 5.05k | ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); | 279 | 5.05k | if (!ExpectedValue) | 280 | 0 | return false; | 281 | 5.05k | | 282 | 5.05k | MDBuilder MDB(CI->getContext()); | 283 | 5.05k | MDNode *Node; | 284 | 5.05k | | 285 | 5.05k | if ((ExpectedValue->getZExtValue() == ValueComparedTo) == | 286 | 5.05k | (Predicate == CmpInst::ICMP_EQ)) | 287 | 52 | Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); | 288 | 5.05k | else | 289 | 5.00k | Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); | 290 | 5.05k | | 291 | 5.05k | BSI.setMetadata(LLVMContext::MD_prof, Node); | 292 | 5.05k | | 293 | 5.05k | if (CmpI) | 294 | 5.05k | CmpI->setOperand(0, ArgValue); | 295 | 5.05k | else | 296 | 2 | BSI.setCondition(ArgValue); | 297 | 1.59M | return true; | 298 | 1.59M | } |
LowerExpectIntrinsic.cpp:bool handleBrSelExpect<llvm::SelectInst>(llvm::SelectInst&) Line | Count | Source | 233 | 115k | template <class BrSelInst> static bool handleBrSelExpect(BrSelInst &BSI) { | 234 | 115k | | 235 | 115k | // Handle non-optimized IR code like: | 236 | 115k | // %expval = call i64 @llvm.expect.i64(i64 %conv1, i64 1) | 237 | 115k | // %tobool = icmp ne i64 %expval, 0 | 238 | 115k | // br i1 %tobool, label %if.then, label %if.end | 239 | 115k | // | 240 | 115k | // Or the following simpler case: | 241 | 115k | // %expval = call i1 @llvm.expect.i1(i1 %cmp, i1 1) | 242 | 115k | // br i1 %expval, label %if.then, label %if.end | 243 | 115k | | 244 | 115k | CallInst *CI; | 245 | 115k | | 246 | 115k | ICmpInst *CmpI = dyn_cast<ICmpInst>(BSI.getCondition()); | 247 | 115k | CmpInst::Predicate Predicate; | 248 | 115k | ConstantInt *CmpConstOperand = nullptr; | 249 | 115k | if (!CmpI115k ) { | 250 | 10.3k | CI = dyn_cast<CallInst>(BSI.getCondition()); | 251 | 10.3k | Predicate = CmpInst::ICMP_NE; | 252 | 115k | } else { | 253 | 105k | Predicate = CmpI->getPredicate(); | 254 | 105k | if (Predicate != CmpInst::ICMP_NE && 105k Predicate != CmpInst::ICMP_EQ102k ) | 255 | 70.1k | return false; | 256 | 34.9k | | 257 | 34.9k | CmpConstOperand = dyn_cast<ConstantInt>(CmpI->getOperand(1)); | 258 | 34.9k | if (!CmpConstOperand) | 259 | 3.71k | return false; | 260 | 31.2k | CI = dyn_cast<CallInst>(CmpI->getOperand(0)); | 261 | 31.2k | } | 262 | 115k | | 263 | 41.5k | if (41.5k !CI41.5k ) | 264 | 27.2k | return false; | 265 | 14.3k | | 266 | 14.3k | uint64_t ValueComparedTo = 0; | 267 | 14.3k | if (CmpConstOperand14.3k ) { | 268 | 14.3k | if (CmpConstOperand->getBitWidth() > 64) | 269 | 0 | return false; | 270 | 14.3k | ValueComparedTo = CmpConstOperand->getZExtValue(); | 271 | 14.3k | } | 272 | 14.3k | | 273 | 14.3k | Function *Fn = CI->getCalledFunction(); | 274 | 14.3k | if (!Fn || 14.3k Fn->getIntrinsicID() != Intrinsic::expect450 ) | 275 | 14.3k | return false; | 276 | 2 | | 277 | 2 | Value *ArgValue = CI->getArgOperand(0); | 278 | 2 | ConstantInt *ExpectedValue = dyn_cast<ConstantInt>(CI->getArgOperand(1)); | 279 | 2 | if (!ExpectedValue) | 280 | 0 | return false; | 281 | 2 | | 282 | 2 | MDBuilder MDB(CI->getContext()); | 283 | 2 | MDNode *Node; | 284 | 2 | | 285 | 2 | if ((ExpectedValue->getZExtValue() == ValueComparedTo) == | 286 | 2 | (Predicate == CmpInst::ICMP_EQ)) | 287 | 0 | Node = MDB.createBranchWeights(LikelyBranchWeight, UnlikelyBranchWeight); | 288 | 2 | else | 289 | 2 | Node = MDB.createBranchWeights(UnlikelyBranchWeight, LikelyBranchWeight); | 290 | 2 | | 291 | 2 | BSI.setMetadata(LLVMContext::MD_prof, Node); | 292 | 2 | | 293 | 2 | if (CmpI) | 294 | 2 | CmpI->setOperand(0, ArgValue); | 295 | 2 | else | 296 | 0 | BSI.setCondition(ArgValue); | 297 | 115k | return true; | 298 | 115k | } |
|
299 | | |
300 | 2.90M | static bool handleBranchExpect(BranchInst &BI) { |
301 | 2.90M | if (BI.isUnconditional()) |
302 | 1.31M | return false; |
303 | 1.59M | |
304 | 1.59M | return handleBrSelExpect<BranchInst>(BI); |
305 | 1.59M | } |
306 | | |
307 | 763k | static bool lowerExpectIntrinsic(Function &F) { |
308 | 763k | bool Changed = false; |
309 | 763k | |
310 | 3.73M | for (BasicBlock &BB : F) { |
311 | 3.73M | // Create "block_weights" metadata. |
312 | 3.73M | if (BranchInst *BI3.73M = dyn_cast<BranchInst>(BB.getTerminator())) { |
313 | 2.90M | if (handleBranchExpect(*BI)) |
314 | 5.05k | ExpectIntrinsicsHandled++; |
315 | 3.73M | } else if (SwitchInst *830k SI830k = dyn_cast<SwitchInst>(BB.getTerminator())) { |
316 | 24.2k | if (handleSwitchExpect(*SI)) |
317 | 4 | ExpectIntrinsicsHandled++; |
318 | 830k | } |
319 | 3.73M | |
320 | 3.73M | // Remove llvm.expect intrinsics. Iterate backwards in order |
321 | 3.73M | // to process select instructions before the intrinsic gets |
322 | 3.73M | // removed. |
323 | 24.1M | for (auto BI = BB.rbegin(), BE = BB.rend(); BI != BE24.1M ;) { |
324 | 20.3M | Instruction *Inst = &*BI++; |
325 | 20.3M | CallInst *CI = dyn_cast<CallInst>(Inst); |
326 | 20.3M | if (!CI20.3M ) { |
327 | 17.5M | if (SelectInst *SI17.5M = dyn_cast<SelectInst>(Inst)) { |
328 | 115k | if (handleBrSelExpect(*SI)) |
329 | 2 | ExpectIntrinsicsHandled++; |
330 | 115k | } |
331 | 17.5M | continue; |
332 | 17.5M | } |
333 | 2.89M | |
334 | 2.89M | Function *Fn = CI->getCalledFunction(); |
335 | 2.89M | if (Fn && 2.89M Fn->getIntrinsicID() == Intrinsic::expect2.78M ) { |
336 | 5.07k | // Before erasing the llvm.expect, walk backward to find |
337 | 5.07k | // phi that define llvm.expect's first arg, and |
338 | 5.07k | // infer branch probability: |
339 | 5.07k | handlePhiDef(CI); |
340 | 5.07k | Value *Exp = CI->getArgOperand(0); |
341 | 5.07k | CI->replaceAllUsesWith(Exp); |
342 | 5.07k | CI->eraseFromParent(); |
343 | 5.07k | Changed = true; |
344 | 5.07k | } |
345 | 20.3M | } |
346 | 3.73M | } |
347 | 763k | |
348 | 763k | return Changed; |
349 | 763k | } |
350 | | |
351 | | PreservedAnalyses LowerExpectIntrinsicPass::run(Function &F, |
352 | 132 | FunctionAnalysisManager &) { |
353 | 132 | if (lowerExpectIntrinsic(F)) |
354 | 28 | return PreservedAnalyses::none(); |
355 | 104 | |
356 | 104 | return PreservedAnalyses::all(); |
357 | 104 | } |
358 | | |
359 | | namespace { |
360 | | /// \brief Legacy pass for lowering expect intrinsics out of the IR. |
361 | | /// |
362 | | /// When this pass is run over a function it uses expect intrinsics which feed |
363 | | /// branches and switches to provide branch weight metadata for those |
364 | | /// terminators. It then removes the expect intrinsics from the IR so the rest |
365 | | /// of the optimizer can ignore them. |
366 | | class LowerExpectIntrinsic : public FunctionPass { |
367 | | public: |
368 | | static char ID; |
369 | 17.1k | LowerExpectIntrinsic() : FunctionPass(ID) { |
370 | 17.1k | initializeLowerExpectIntrinsicPass(*PassRegistry::getPassRegistry()); |
371 | 17.1k | } |
372 | | |
373 | 763k | bool runOnFunction(Function &F) override { return lowerExpectIntrinsic(F); } |
374 | | }; |
375 | | } |
376 | | |
377 | | char LowerExpectIntrinsic::ID = 0; |
378 | | INITIALIZE_PASS(LowerExpectIntrinsic, "lower-expect", |
379 | | "Lower 'expect' Intrinsics", false, false) |
380 | | |
381 | 17.0k | FunctionPass *llvm::createLowerExpectIntrinsicPass() { |
382 | 17.0k | return new LowerExpectIntrinsic(); |
383 | 17.0k | } |