/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | |
9 | | #include "CGLoopInfo.h" |
10 | | #include "clang/AST/ASTContext.h" |
11 | | #include "clang/AST/Attr.h" |
12 | | #include "clang/AST/Expr.h" |
13 | | #include "clang/Basic/CodeGenOptions.h" |
14 | | #include "llvm/IR/BasicBlock.h" |
15 | | #include "llvm/IR/CFG.h" |
16 | | #include "llvm/IR/Constants.h" |
17 | | #include "llvm/IR/InstrTypes.h" |
18 | | #include "llvm/IR/Instructions.h" |
19 | | #include "llvm/IR/Metadata.h" |
20 | | using namespace clang::CodeGen; |
21 | | using namespace llvm; |
22 | | |
23 | | MDNode * |
24 | 21.2k | LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) { |
25 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
26 | 21.2k | SmallVector<Metadata *, 4> NewLoopProperties; |
27 | 21.2k | NewLoopProperties.push_back(nullptr); |
28 | 21.2k | NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
29 | | |
30 | 21.2k | MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties); |
31 | 21.2k | LoopID->replaceOperandWith(0, LoopID); |
32 | 21.2k | return LoopID; |
33 | 21.2k | } |
34 | | |
35 | | MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs, |
36 | | ArrayRef<Metadata *> LoopProperties, |
37 | 21.2k | bool &HasUserTransforms) { |
38 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
39 | | |
40 | 21.2k | Optional<bool> Enabled; |
41 | 21.2k | if (Attrs.PipelineDisabled) |
42 | 2 | Enabled = false; |
43 | 21.2k | else if (Attrs.PipelineInitiationInterval != 0) |
44 | 3 | Enabled = true; |
45 | | |
46 | 21.2k | if (Enabled != true) { |
47 | 21.2k | SmallVector<Metadata *, 4> NewLoopProperties; |
48 | 21.2k | if (Enabled == false) { |
49 | 2 | NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
50 | 2 | NewLoopProperties.push_back( |
51 | 2 | MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"), |
52 | 2 | ConstantAsMetadata::get(ConstantInt::get( |
53 | 2 | llvm::Type::getInt1Ty(Ctx), 1))})); |
54 | 2 | LoopProperties = NewLoopProperties; |
55 | 2 | } |
56 | 21.2k | return createLoopPropertiesMetadata(LoopProperties); |
57 | 21.2k | } |
58 | | |
59 | 3 | SmallVector<Metadata *, 4> Args; |
60 | 3 | Args.push_back(nullptr); |
61 | 3 | Args.append(LoopProperties.begin(), LoopProperties.end()); |
62 | | |
63 | 3 | if (Attrs.PipelineInitiationInterval > 0) { |
64 | 3 | Metadata *Vals[] = { |
65 | 3 | MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"), |
66 | 3 | ConstantAsMetadata::get(ConstantInt::get( |
67 | 3 | llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))}; |
68 | 3 | Args.push_back(MDNode::get(Ctx, Vals)); |
69 | 3 | } |
70 | | |
71 | | // No follow-up: This is the last transformation. |
72 | | |
73 | 3 | MDNode *LoopID = MDNode::getDistinct(Ctx, Args); |
74 | 3 | LoopID->replaceOperandWith(0, LoopID); |
75 | 3 | HasUserTransforms = true; |
76 | 3 | return LoopID; |
77 | 3 | } |
78 | | |
79 | | MDNode * |
80 | | LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs, |
81 | | ArrayRef<Metadata *> LoopProperties, |
82 | 21.2k | bool &HasUserTransforms) { |
83 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
84 | | |
85 | 21.2k | Optional<bool> Enabled; |
86 | 21.2k | if (Attrs.UnrollEnable == LoopAttributes::Disable) |
87 | 180 | Enabled = false; |
88 | 21.0k | else if (Attrs.UnrollEnable == LoopAttributes::Full) |
89 | 0 | Enabled = None; |
90 | 21.0k | else if (Attrs.UnrollEnable != LoopAttributes::Unspecified || |
91 | 21.0k | Attrs.UnrollCount != 0) |
92 | 38 | Enabled = true; |
93 | | |
94 | 21.2k | if (Enabled != true) { |
95 | | // createFullUnrollMetadata will already have added llvm.loop.unroll.disable |
96 | | // if unrolling is disabled. |
97 | 21.1k | return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms); |
98 | 21.1k | } |
99 | | |
100 | 38 | SmallVector<Metadata *, 4> FollowupLoopProperties; |
101 | | |
102 | | // Apply all loop properties to the unrolled loop. |
103 | 38 | FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
104 | | |
105 | | // Don't unroll an already unrolled loop. |
106 | 38 | FollowupLoopProperties.push_back( |
107 | 38 | MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable"))); |
108 | | |
109 | 38 | bool FollowupHasTransforms = false; |
110 | 38 | MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties, |
111 | 38 | FollowupHasTransforms); |
112 | | |
113 | 38 | SmallVector<Metadata *, 4> Args; |
114 | 38 | Args.push_back(nullptr); |
115 | 38 | Args.append(LoopProperties.begin(), LoopProperties.end()); |
116 | | |
117 | | // Setting unroll.count |
118 | 38 | if (Attrs.UnrollCount > 0) { |
119 | 27 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), |
120 | 27 | ConstantAsMetadata::get(ConstantInt::get( |
121 | 27 | llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; |
122 | 27 | Args.push_back(MDNode::get(Ctx, Vals)); |
123 | 27 | } |
124 | | |
125 | | // Setting unroll.full or unroll.disable |
126 | 38 | if (Attrs.UnrollEnable == LoopAttributes::Enable) { |
127 | 11 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")}; |
128 | 11 | Args.push_back(MDNode::get(Ctx, Vals)); |
129 | 11 | } |
130 | | |
131 | 38 | if (FollowupHasTransforms) |
132 | 2 | Args.push_back(MDNode::get( |
133 | 2 | Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup})); |
134 | | |
135 | 38 | MDNode *LoopID = MDNode::getDistinct(Ctx, Args); |
136 | 38 | LoopID->replaceOperandWith(0, LoopID); |
137 | 38 | HasUserTransforms = true; |
138 | 38 | return LoopID; |
139 | 38 | } |
140 | | |
141 | | MDNode * |
142 | | LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs, |
143 | | ArrayRef<Metadata *> LoopProperties, |
144 | 21.2k | bool &HasUserTransforms) { |
145 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
146 | | |
147 | 21.2k | Optional<bool> Enabled; |
148 | 21.2k | if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable) |
149 | 2 | Enabled = false; |
150 | 21.2k | else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable || |
151 | 21.2k | Attrs.UnrollAndJamCount != 0) |
152 | 4 | Enabled = true; |
153 | | |
154 | 21.2k | if (Enabled != true) { |
155 | 21.2k | SmallVector<Metadata *, 4> NewLoopProperties; |
156 | 21.2k | if (Enabled == false) { |
157 | 2 | NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
158 | 2 | NewLoopProperties.push_back(MDNode::get( |
159 | 2 | Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable"))); |
160 | 2 | LoopProperties = NewLoopProperties; |
161 | 2 | } |
162 | 21.2k | return createPartialUnrollMetadata(Attrs, LoopProperties, |
163 | 21.2k | HasUserTransforms); |
164 | 21.2k | } |
165 | | |
166 | 4 | SmallVector<Metadata *, 4> FollowupLoopProperties; |
167 | 4 | FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
168 | 4 | FollowupLoopProperties.push_back( |
169 | 4 | MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable"))); |
170 | | |
171 | 4 | bool FollowupHasTransforms = false; |
172 | 4 | MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties, |
173 | 4 | FollowupHasTransforms); |
174 | | |
175 | 4 | SmallVector<Metadata *, 4> Args; |
176 | 4 | Args.push_back(nullptr); |
177 | 4 | Args.append(LoopProperties.begin(), LoopProperties.end()); |
178 | | |
179 | | // Setting unroll_and_jam.count |
180 | 4 | if (Attrs.UnrollAndJamCount > 0) { |
181 | 2 | Metadata *Vals[] = { |
182 | 2 | MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"), |
183 | 2 | ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), |
184 | 2 | Attrs.UnrollAndJamCount))}; |
185 | 2 | Args.push_back(MDNode::get(Ctx, Vals)); |
186 | 2 | } |
187 | | |
188 | 4 | if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) { |
189 | 2 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")}; |
190 | 2 | Args.push_back(MDNode::get(Ctx, Vals)); |
191 | 2 | } |
192 | | |
193 | 4 | if (FollowupHasTransforms) |
194 | 1 | Args.push_back(MDNode::get( |
195 | 1 | Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"), |
196 | 1 | Followup})); |
197 | | |
198 | 4 | if (UnrollAndJamInnerFollowup) |
199 | 1 | Args.push_back(MDNode::get( |
200 | 1 | Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"), |
201 | 1 | UnrollAndJamInnerFollowup})); |
202 | | |
203 | 4 | MDNode *LoopID = MDNode::getDistinct(Ctx, Args); |
204 | 4 | LoopID->replaceOperandWith(0, LoopID); |
205 | 4 | HasUserTransforms = true; |
206 | 4 | return LoopID; |
207 | 4 | } |
208 | | |
209 | | MDNode * |
210 | | LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, |
211 | | ArrayRef<Metadata *> LoopProperties, |
212 | 21.2k | bool &HasUserTransforms) { |
213 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
214 | | |
215 | 21.2k | Optional<bool> Enabled; |
216 | 21.2k | if (Attrs.VectorizeEnable == LoopAttributes::Disable) |
217 | 343 | Enabled = false; |
218 | 20.8k | else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || |
219 | 12.2k | Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified || |
220 | 12.2k | Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 012.2k || |
221 | 12.2k | Attrs.VectorizeScalable != LoopAttributes::Unspecified) |
222 | 8.65k | Enabled = true; |
223 | | |
224 | 21.2k | if (Enabled != true) { |
225 | 12.5k | SmallVector<Metadata *, 4> NewLoopProperties; |
226 | 12.5k | if (Enabled == false) { |
227 | 343 | NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
228 | 343 | NewLoopProperties.push_back( |
229 | 343 | MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), |
230 | 343 | ConstantAsMetadata::get(ConstantInt::get( |
231 | 343 | llvm::Type::getInt1Ty(Ctx), 0))})); |
232 | 343 | LoopProperties = NewLoopProperties; |
233 | 343 | } |
234 | 12.5k | return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms); |
235 | 12.5k | } |
236 | | |
237 | | // Apply all loop properties to the vectorized loop. |
238 | 8.65k | SmallVector<Metadata *, 4> FollowupLoopProperties; |
239 | 8.65k | FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
240 | | |
241 | | // Don't vectorize an already vectorized loop. |
242 | 8.65k | FollowupLoopProperties.push_back( |
243 | 8.65k | MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); |
244 | | |
245 | 8.65k | bool FollowupHasTransforms = false; |
246 | 8.65k | MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties, |
247 | 8.65k | FollowupHasTransforms); |
248 | | |
249 | 8.65k | SmallVector<Metadata *, 4> Args; |
250 | 8.65k | Args.push_back(nullptr); |
251 | 8.65k | Args.append(LoopProperties.begin(), LoopProperties.end()); |
252 | | |
253 | | // Setting vectorize.predicate |
254 | 8.65k | bool IsVectorPredicateEnabled = false; |
255 | 8.65k | if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified && |
256 | 4 | Attrs.VectorizeEnable != LoopAttributes::Disable && |
257 | 4 | Attrs.VectorizeWidth < 1) { |
258 | | |
259 | 3 | IsVectorPredicateEnabled = |
260 | 3 | (Attrs.VectorizePredicateEnable == LoopAttributes::Enable); |
261 | | |
262 | 3 | Metadata *Vals[] = { |
263 | 3 | MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"), |
264 | 3 | ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx), |
265 | 3 | IsVectorPredicateEnabled))}; |
266 | 3 | Args.push_back(MDNode::get(Ctx, Vals)); |
267 | 3 | } |
268 | | |
269 | | // Setting vectorize.width |
270 | 8.65k | if (Attrs.VectorizeWidth > 0) { |
271 | 438 | Metadata *Vals[] = { |
272 | 438 | MDString::get(Ctx, "llvm.loop.vectorize.width"), |
273 | 438 | ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), |
274 | 438 | Attrs.VectorizeWidth))}; |
275 | | |
276 | 438 | Args.push_back(MDNode::get(Ctx, Vals)); |
277 | 438 | } |
278 | | |
279 | 8.65k | if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) { |
280 | 20 | bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable; |
281 | 20 | Metadata *Vals[] = { |
282 | 20 | MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"), |
283 | 20 | ConstantAsMetadata::get( |
284 | 20 | ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))}; |
285 | 20 | Args.push_back(MDNode::get(Ctx, Vals)); |
286 | 20 | } |
287 | | |
288 | | // Setting interleave.count |
289 | 8.65k | if (Attrs.InterleaveCount > 0) { |
290 | 20 | Metadata *Vals[] = { |
291 | 20 | MDString::get(Ctx, "llvm.loop.interleave.count"), |
292 | 20 | ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), |
293 | 20 | Attrs.InterleaveCount))}; |
294 | 20 | Args.push_back(MDNode::get(Ctx, Vals)); |
295 | 20 | } |
296 | | |
297 | | // vectorize.enable is set if: |
298 | | // 1) loop hint vectorize.enable is set, or |
299 | | // 2) it is implied when vectorize.predicate is set, or |
300 | | // 3) it is implied when vectorize.width is set to a value > 1 |
301 | | // 4) it is implied when vectorize.scalable.enable is true |
302 | | // 5) it is implied when vectorize.width is unset (0) and the user |
303 | | // explicitly requested fixed-width vectorization, i.e. |
304 | | // vectorize.scalable.enable is false. |
305 | 8.65k | if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || |
306 | 24 | IsVectorPredicateEnabled || Attrs.VectorizeWidth > 123 || |
307 | 9 | Attrs.VectorizeScalable == LoopAttributes::Enable || |
308 | 7 | (Attrs.VectorizeScalable == LoopAttributes::Disable && |
309 | 8.65k | Attrs.VectorizeWidth != 12 )) { |
310 | 8.65k | bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable; |
311 | 8.65k | Args.push_back( |
312 | 8.65k | MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), |
313 | 8.65k | ConstantAsMetadata::get(ConstantInt::get( |
314 | 8.65k | llvm::Type::getInt1Ty(Ctx), AttrVal))})); |
315 | 8.65k | } |
316 | | |
317 | 8.65k | if (FollowupHasTransforms) |
318 | 8 | Args.push_back(MDNode::get( |
319 | 8 | Ctx, |
320 | 8 | {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup})); |
321 | | |
322 | 8.65k | MDNode *LoopID = MDNode::getDistinct(Ctx, Args); |
323 | 8.65k | LoopID->replaceOperandWith(0, LoopID); |
324 | 8.65k | HasUserTransforms = true; |
325 | 8.65k | return LoopID; |
326 | 8.65k | } |
327 | | |
328 | | MDNode * |
329 | | LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs, |
330 | | ArrayRef<Metadata *> LoopProperties, |
331 | 21.2k | bool &HasUserTransforms) { |
332 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
333 | | |
334 | 21.2k | Optional<bool> Enabled; |
335 | 21.2k | if (Attrs.DistributeEnable == LoopAttributes::Disable) |
336 | 7 | Enabled = false; |
337 | 21.2k | if (Attrs.DistributeEnable == LoopAttributes::Enable) |
338 | 3 | Enabled = true; |
339 | | |
340 | 21.2k | if (Enabled != true) { |
341 | 21.2k | SmallVector<Metadata *, 4> NewLoopProperties; |
342 | 21.2k | if (Enabled == false) { |
343 | 7 | NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
344 | 7 | NewLoopProperties.push_back( |
345 | 7 | MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"), |
346 | 7 | ConstantAsMetadata::get(ConstantInt::get( |
347 | 7 | llvm::Type::getInt1Ty(Ctx), 0))})); |
348 | 7 | LoopProperties = NewLoopProperties; |
349 | 7 | } |
350 | 21.2k | return createLoopVectorizeMetadata(Attrs, LoopProperties, |
351 | 21.2k | HasUserTransforms); |
352 | 21.2k | } |
353 | | |
354 | 3 | bool FollowupHasTransforms = false; |
355 | 3 | MDNode *Followup = |
356 | 3 | createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms); |
357 | | |
358 | 3 | SmallVector<Metadata *, 4> Args; |
359 | 3 | Args.push_back(nullptr); |
360 | 3 | Args.append(LoopProperties.begin(), LoopProperties.end()); |
361 | | |
362 | 3 | Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), |
363 | 3 | ConstantAsMetadata::get(ConstantInt::get( |
364 | 3 | llvm::Type::getInt1Ty(Ctx), |
365 | 3 | (Attrs.DistributeEnable == LoopAttributes::Enable)))}; |
366 | 3 | Args.push_back(MDNode::get(Ctx, Vals)); |
367 | | |
368 | 3 | if (FollowupHasTransforms) |
369 | 2 | Args.push_back(MDNode::get( |
370 | 2 | Ctx, |
371 | 2 | {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup})); |
372 | | |
373 | 3 | MDNode *LoopID = MDNode::getDistinct(Ctx, Args); |
374 | 3 | LoopID->replaceOperandWith(0, LoopID); |
375 | 3 | HasUserTransforms = true; |
376 | 3 | return LoopID; |
377 | 3 | } |
378 | | |
379 | | MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs, |
380 | | ArrayRef<Metadata *> LoopProperties, |
381 | 21.2k | bool &HasUserTransforms) { |
382 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
383 | | |
384 | 21.2k | Optional<bool> Enabled; |
385 | 21.2k | if (Attrs.UnrollEnable == LoopAttributes::Disable) |
386 | 180 | Enabled = false; |
387 | 21.0k | else if (Attrs.UnrollEnable == LoopAttributes::Full) |
388 | 4 | Enabled = true; |
389 | | |
390 | 21.2k | if (Enabled != true) { |
391 | 21.2k | SmallVector<Metadata *, 4> NewLoopProperties; |
392 | 21.2k | if (Enabled == false) { |
393 | 180 | NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end()); |
394 | 180 | NewLoopProperties.push_back( |
395 | 180 | MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable"))); |
396 | 180 | LoopProperties = NewLoopProperties; |
397 | 180 | } |
398 | 21.2k | return createLoopDistributeMetadata(Attrs, LoopProperties, |
399 | 21.2k | HasUserTransforms); |
400 | 21.2k | } |
401 | | |
402 | 4 | SmallVector<Metadata *, 4> Args; |
403 | 4 | Args.push_back(nullptr); |
404 | 4 | Args.append(LoopProperties.begin(), LoopProperties.end()); |
405 | 4 | Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full"))); |
406 | | |
407 | | // No follow-up: there is no loop after full unrolling. |
408 | | // TODO: Warn if there are transformations after full unrolling. |
409 | | |
410 | 4 | MDNode *LoopID = MDNode::getDistinct(Ctx, Args); |
411 | 4 | LoopID->replaceOperandWith(0, LoopID); |
412 | 4 | HasUserTransforms = true; |
413 | 4 | return LoopID; |
414 | 4 | } |
415 | | |
416 | | MDNode *LoopInfo::createMetadata( |
417 | | const LoopAttributes &Attrs, |
418 | | llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties, |
419 | 21.2k | bool &HasUserTransforms) { |
420 | 21.2k | SmallVector<Metadata *, 3> LoopProperties; |
421 | | |
422 | | // If we have a valid start debug location for the loop, add it. |
423 | 21.2k | if (StartLoc) { |
424 | 8.53k | LoopProperties.push_back(StartLoc.getAsMDNode()); |
425 | | |
426 | | // If we also have a valid end debug location for the loop, add it. |
427 | 8.53k | if (EndLoc) |
428 | 8.53k | LoopProperties.push_back(EndLoc.getAsMDNode()); |
429 | 8.53k | } |
430 | | |
431 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
432 | 21.2k | if (Attrs.MustProgress) |
433 | 10.9k | LoopProperties.push_back( |
434 | 10.9k | MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress"))); |
435 | | |
436 | 21.2k | assert(!!AccGroup == Attrs.IsParallel && |
437 | 21.2k | "There must be an access group iff the loop is parallel"); |
438 | 21.2k | if (Attrs.IsParallel) { |
439 | 4.83k | LoopProperties.push_back(MDNode::get( |
440 | 4.83k | Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup})); |
441 | 4.83k | } |
442 | | |
443 | 21.2k | LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(), |
444 | 21.2k | AdditionalLoopProperties.end()); |
445 | 21.2k | return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms); |
446 | 21.2k | } |
447 | | |
448 | | LoopAttributes::LoopAttributes(bool IsParallel) |
449 | | : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), |
450 | | UnrollEnable(LoopAttributes::Unspecified), |
451 | | UnrollAndJamEnable(LoopAttributes::Unspecified), |
452 | | VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0), |
453 | | VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0), |
454 | | UnrollCount(0), UnrollAndJamCount(0), |
455 | | DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false), |
456 | 296k | PipelineInitiationInterval(0), MustProgress(false) {} |
457 | | |
458 | 26.9k | void LoopAttributes::clear() { |
459 | 26.9k | IsParallel = false; |
460 | 26.9k | VectorizeWidth = 0; |
461 | 26.9k | VectorizeScalable = LoopAttributes::Unspecified; |
462 | 26.9k | InterleaveCount = 0; |
463 | 26.9k | UnrollCount = 0; |
464 | 26.9k | UnrollAndJamCount = 0; |
465 | 26.9k | VectorizeEnable = LoopAttributes::Unspecified; |
466 | 26.9k | UnrollEnable = LoopAttributes::Unspecified; |
467 | 26.9k | UnrollAndJamEnable = LoopAttributes::Unspecified; |
468 | 26.9k | VectorizePredicateEnable = LoopAttributes::Unspecified; |
469 | 26.9k | DistributeEnable = LoopAttributes::Unspecified; |
470 | 26.9k | PipelineDisabled = false; |
471 | 26.9k | PipelineInitiationInterval = 0; |
472 | 26.9k | MustProgress = false; |
473 | 26.9k | } |
474 | | |
475 | | LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, |
476 | | const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, |
477 | | LoopInfo *Parent) |
478 | | : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc), |
479 | 26.9k | Parent(Parent) { |
480 | | |
481 | 26.9k | if (Attrs.IsParallel) { |
482 | | // Create an access group for this loop. |
483 | 4.83k | LLVMContext &Ctx = Header->getContext(); |
484 | 4.83k | AccGroup = MDNode::getDistinct(Ctx, {}); |
485 | 4.83k | } |
486 | | |
487 | 26.9k | if (!Attrs.IsParallel && Attrs.VectorizeWidth == 022.1k && |
488 | 21.8k | Attrs.VectorizeScalable == LoopAttributes::Unspecified && |
489 | 21.8k | Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 021.8k && |
490 | 21.8k | Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled21.7k && |
491 | 21.7k | Attrs.PipelineInitiationInterval == 0 && |
492 | 21.7k | Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified && |
493 | 21.7k | Attrs.VectorizeEnable == LoopAttributes::Unspecified && |
494 | 17.5k | Attrs.UnrollEnable == LoopAttributes::Unspecified && |
495 | 17.3k | Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && |
496 | 17.3k | Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc17.3k && |
497 | 9.22k | !EndLoc && !Attrs.MustProgress) |
498 | 5.77k | return; |
499 | | |
500 | 21.2k | TempLoopID = MDNode::getTemporary(Header->getContext(), None); |
501 | 21.2k | } |
502 | | |
503 | 26.9k | void LoopInfo::finish() { |
504 | | // We did not annotate the loop body instructions because there are no |
505 | | // attributes for this loop. |
506 | 26.9k | if (!TempLoopID) |
507 | 5.77k | return; |
508 | | |
509 | 21.2k | MDNode *LoopID; |
510 | 21.2k | LoopAttributes CurLoopAttr = Attrs; |
511 | 21.2k | LLVMContext &Ctx = Header->getContext(); |
512 | | |
513 | 21.2k | if (Parent && (3.88k Parent->Attrs.UnrollAndJamEnable3.88k || |
514 | 3.87k | Parent->Attrs.UnrollAndJamCount != 0)) { |
515 | | // Parent unroll-and-jams this loop. |
516 | | // Split the transformations in those that happens before the unroll-and-jam |
517 | | // and those after. |
518 | | |
519 | 5 | LoopAttributes BeforeJam, AfterJam; |
520 | | |
521 | 5 | BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel; |
522 | | |
523 | 5 | BeforeJam.VectorizeWidth = Attrs.VectorizeWidth; |
524 | 5 | BeforeJam.VectorizeScalable = Attrs.VectorizeScalable; |
525 | 5 | BeforeJam.InterleaveCount = Attrs.InterleaveCount; |
526 | 5 | BeforeJam.VectorizeEnable = Attrs.VectorizeEnable; |
527 | 5 | BeforeJam.DistributeEnable = Attrs.DistributeEnable; |
528 | 5 | BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable; |
529 | | |
530 | 5 | switch (Attrs.UnrollEnable) { |
531 | 5 | case LoopAttributes::Unspecified: |
532 | 5 | case LoopAttributes::Disable: |
533 | 5 | BeforeJam.UnrollEnable = Attrs.UnrollEnable; |
534 | 5 | AfterJam.UnrollEnable = Attrs.UnrollEnable; |
535 | 5 | break; |
536 | 0 | case LoopAttributes::Full: |
537 | 0 | BeforeJam.UnrollEnable = LoopAttributes::Full; |
538 | 0 | break; |
539 | 0 | case LoopAttributes::Enable: |
540 | 0 | AfterJam.UnrollEnable = LoopAttributes::Enable; |
541 | 0 | break; |
542 | 5 | } |
543 | | |
544 | 5 | AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable; |
545 | 5 | AfterJam.UnrollCount = Attrs.UnrollCount; |
546 | 5 | AfterJam.PipelineDisabled = Attrs.PipelineDisabled; |
547 | 5 | AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval; |
548 | | |
549 | | // If this loop is subject of an unroll-and-jam by the parent loop, and has |
550 | | // an unroll-and-jam annotation itself, we have to decide whether to first |
551 | | // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The |
552 | | // UnrollAndJam pass processes loops from inner to outer, so we apply the |
553 | | // inner first. |
554 | 5 | BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount; |
555 | 5 | BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable; |
556 | | |
557 | | // Set the inner followup metadata to process by the outer loop. Only |
558 | | // consider the first inner loop. |
559 | 5 | if (!Parent->UnrollAndJamInnerFollowup) { |
560 | | // Splitting the attributes into a BeforeJam and an AfterJam part will |
561 | | // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam) |
562 | | // to be forwarded to the AfterJam part. We detect the situation here and |
563 | | // add it manually. |
564 | 5 | SmallVector<Metadata *, 1> BeforeLoopProperties; |
565 | 5 | if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified || |
566 | 4 | BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified || |
567 | 4 | BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 || |
568 | 4 | BeforeJam.VectorizeScalable == LoopAttributes::Enable) |
569 | 1 | BeforeLoopProperties.push_back( |
570 | 1 | MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); |
571 | | |
572 | 5 | bool InnerFollowupHasTransform = false; |
573 | 5 | MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties, |
574 | 5 | InnerFollowupHasTransform); |
575 | 5 | if (InnerFollowupHasTransform) |
576 | 1 | Parent->UnrollAndJamInnerFollowup = InnerFollowup; |
577 | 5 | } |
578 | | |
579 | 5 | CurLoopAttr = BeforeJam; |
580 | 5 | } |
581 | | |
582 | 21.2k | bool HasUserTransforms = false; |
583 | 21.2k | LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms); |
584 | 21.2k | TempLoopID->replaceAllUsesWith(LoopID); |
585 | 21.2k | } |
586 | | |
587 | | void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc, |
588 | 26.9k | const llvm::DebugLoc &EndLoc) { |
589 | 26.9k | Active.emplace_back( |
590 | 26.9k | new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc, |
591 | 22.8k | Active.empty() ? nullptr : Active.back().get()4.12k )); |
592 | | // Clear the attributes so nested loops do not inherit them. |
593 | 26.9k | StagedAttrs.clear(); |
594 | 26.9k | } |
595 | | |
596 | | void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, |
597 | | const clang::CodeGenOptions &CGOpts, |
598 | | ArrayRef<const clang::Attr *> Attrs, |
599 | | const llvm::DebugLoc &StartLoc, |
600 | 12.0k | const llvm::DebugLoc &EndLoc, bool MustProgress) { |
601 | | // Identify loop hint attributes from Attrs. |
602 | 174 | for (const auto *Attr : Attrs) { |
603 | 174 | const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); |
604 | 174 | const OpenCLUnrollHintAttr *OpenCLHint = |
605 | 174 | dyn_cast<OpenCLUnrollHintAttr>(Attr); |
606 | | |
607 | | // Skip non loop hint attributes |
608 | 174 | if (!LH && !OpenCLHint25 ) { |
609 | 7 | continue; |
610 | 7 | } |
611 | | |
612 | 167 | LoopHintAttr::OptionType Option = LoopHintAttr::Unroll; |
613 | 167 | LoopHintAttr::LoopHintState State = LoopHintAttr::Disable; |
614 | 167 | unsigned ValueInt = 1; |
615 | | // Translate opencl_unroll_hint attribute argument to |
616 | | // equivalent LoopHintAttr enums. |
617 | | // OpenCL v2.0 s6.11.5: |
618 | | // 0 - enable unroll (no argument). |
619 | | // 1 - disable unroll. |
620 | | // other positive integer n - unroll by n. |
621 | 167 | if (OpenCLHint) { |
622 | 18 | ValueInt = OpenCLHint->getUnrollHint(); |
623 | 18 | if (ValueInt == 0) { |
624 | 6 | State = LoopHintAttr::Enable; |
625 | 12 | } else if (ValueInt != 1) { |
626 | 6 | Option = LoopHintAttr::UnrollCount; |
627 | 6 | State = LoopHintAttr::Numeric; |
628 | 6 | } |
629 | 149 | } else if (LH) { |
630 | 149 | auto *ValueExpr = LH->getValue(); |
631 | 149 | if (ValueExpr) { |
632 | 60 | llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); |
633 | 60 | ValueInt = ValueAPS.getSExtValue(); |
634 | 60 | } |
635 | | |
636 | 149 | Option = LH->getOption(); |
637 | 149 | State = LH->getState(); |
638 | 149 | } |
639 | 167 | switch (State) { |
640 | 49 | case LoopHintAttr::Disable: |
641 | 49 | switch (Option) { |
642 | 6 | case LoopHintAttr::Vectorize: |
643 | | // Disable vectorization by specifying a width of 1. |
644 | 6 | setVectorizeWidth(1); |
645 | 6 | setVectorizeScalable(LoopAttributes::Unspecified); |
646 | 6 | break; |
647 | 6 | case LoopHintAttr::Interleave: |
648 | | // Disable interleaving by speciyfing a count of 1. |
649 | 6 | setInterleaveCount(1); |
650 | 6 | break; |
651 | 25 | case LoopHintAttr::Unroll: |
652 | 25 | setUnrollState(LoopAttributes::Disable); |
653 | 25 | break; |
654 | 2 | case LoopHintAttr::UnrollAndJam: |
655 | 2 | setUnrollAndJamState(LoopAttributes::Disable); |
656 | 2 | break; |
657 | 1 | case LoopHintAttr::VectorizePredicate: |
658 | 1 | setVectorizePredicateState(LoopAttributes::Disable); |
659 | 1 | break; |
660 | 7 | case LoopHintAttr::Distribute: |
661 | 7 | setDistributeState(false); |
662 | 7 | break; |
663 | 2 | case LoopHintAttr::PipelineDisabled: |
664 | 2 | setPipelineDisabled(true); |
665 | 2 | break; |
666 | 0 | case LoopHintAttr::UnrollCount: |
667 | 0 | case LoopHintAttr::UnrollAndJamCount: |
668 | 0 | case LoopHintAttr::VectorizeWidth: |
669 | 0 | case LoopHintAttr::InterleaveCount: |
670 | 0 | case LoopHintAttr::PipelineInitiationInterval: |
671 | 0 | llvm_unreachable("Options cannot be disabled."); |
672 | 0 | break; |
673 | 49 | } |
674 | 49 | break; |
675 | 31 | case LoopHintAttr::Enable: |
676 | 31 | switch (Option) { |
677 | 10 | case LoopHintAttr::Vectorize: |
678 | 11 | case LoopHintAttr::Interleave: |
679 | 11 | setVectorizeEnable(true); |
680 | 11 | break; |
681 | 11 | case LoopHintAttr::Unroll: |
682 | 11 | setUnrollState(LoopAttributes::Enable); |
683 | 11 | break; |
684 | 2 | case LoopHintAttr::UnrollAndJam: |
685 | 2 | setUnrollAndJamState(LoopAttributes::Enable); |
686 | 2 | break; |
687 | 3 | case LoopHintAttr::VectorizePredicate: |
688 | 3 | setVectorizePredicateState(LoopAttributes::Enable); |
689 | 3 | break; |
690 | 4 | case LoopHintAttr::Distribute: |
691 | 4 | setDistributeState(true); |
692 | 4 | break; |
693 | 0 | case LoopHintAttr::UnrollCount: |
694 | 0 | case LoopHintAttr::UnrollAndJamCount: |
695 | 0 | case LoopHintAttr::VectorizeWidth: |
696 | 0 | case LoopHintAttr::InterleaveCount: |
697 | 0 | case LoopHintAttr::PipelineDisabled: |
698 | 0 | case LoopHintAttr::PipelineInitiationInterval: |
699 | 0 | llvm_unreachable("Options cannot enabled."); |
700 | 0 | break; |
701 | 31 | } |
702 | 31 | break; |
703 | 15 | case LoopHintAttr::AssumeSafety: |
704 | 15 | switch (Option) { |
705 | 14 | case LoopHintAttr::Vectorize: |
706 | 15 | case LoopHintAttr::Interleave: |
707 | | // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. |
708 | 15 | setParallel(true); |
709 | 15 | setVectorizeEnable(true); |
710 | 15 | break; |
711 | 0 | case LoopHintAttr::Unroll: |
712 | 0 | case LoopHintAttr::UnrollAndJam: |
713 | 0 | case LoopHintAttr::VectorizePredicate: |
714 | 0 | case LoopHintAttr::UnrollCount: |
715 | 0 | case LoopHintAttr::UnrollAndJamCount: |
716 | 0 | case LoopHintAttr::VectorizeWidth: |
717 | 0 | case LoopHintAttr::InterleaveCount: |
718 | 0 | case LoopHintAttr::Distribute: |
719 | 0 | case LoopHintAttr::PipelineDisabled: |
720 | 0 | case LoopHintAttr::PipelineInitiationInterval: |
721 | 0 | llvm_unreachable("Options cannot be used to assume mem safety."); |
722 | 0 | break; |
723 | 15 | } |
724 | 15 | break; |
725 | 4 | case LoopHintAttr::Full: |
726 | 4 | switch (Option) { |
727 | 4 | case LoopHintAttr::Unroll: |
728 | 4 | setUnrollState(LoopAttributes::Full); |
729 | 4 | break; |
730 | 0 | case LoopHintAttr::UnrollAndJam: |
731 | 0 | setUnrollAndJamState(LoopAttributes::Full); |
732 | 0 | break; |
733 | 0 | case LoopHintAttr::Vectorize: |
734 | 0 | case LoopHintAttr::Interleave: |
735 | 0 | case LoopHintAttr::UnrollCount: |
736 | 0 | case LoopHintAttr::UnrollAndJamCount: |
737 | 0 | case LoopHintAttr::VectorizeWidth: |
738 | 0 | case LoopHintAttr::InterleaveCount: |
739 | 0 | case LoopHintAttr::Distribute: |
740 | 0 | case LoopHintAttr::PipelineDisabled: |
741 | 0 | case LoopHintAttr::PipelineInitiationInterval: |
742 | 0 | case LoopHintAttr::VectorizePredicate: |
743 | 0 | llvm_unreachable("Options cannot be used with 'full' hint."); |
744 | 0 | break; |
745 | 4 | } |
746 | 4 | break; |
747 | 18 | case LoopHintAttr::FixedWidth: |
748 | 21 | case LoopHintAttr::ScalableWidth: |
749 | 21 | switch (Option) { |
750 | 21 | case LoopHintAttr::VectorizeWidth: |
751 | 21 | setVectorizeScalable(State == LoopHintAttr::ScalableWidth |
752 | 3 | ? LoopAttributes::Enable |
753 | 18 | : LoopAttributes::Disable); |
754 | 21 | if (LH->getValue()) |
755 | 19 | setVectorizeWidth(ValueInt); |
756 | 21 | break; |
757 | 0 | default: |
758 | 0 | llvm_unreachable("Options cannot be used with 'scalable' hint."); |
759 | 0 | break; |
760 | 21 | } |
761 | 21 | break; |
762 | 47 | case LoopHintAttr::Numeric: |
763 | 47 | switch (Option) { |
764 | 15 | case LoopHintAttr::InterleaveCount: |
765 | 15 | setInterleaveCount(ValueInt); |
766 | 15 | break; |
767 | 27 | case LoopHintAttr::UnrollCount: |
768 | 27 | setUnrollCount(ValueInt); |
769 | 27 | break; |
770 | 2 | case LoopHintAttr::UnrollAndJamCount: |
771 | 2 | setUnrollAndJamCount(ValueInt); |
772 | 2 | break; |
773 | 3 | case LoopHintAttr::PipelineInitiationInterval: |
774 | 3 | setPipelineInitiationInterval(ValueInt); |
775 | 3 | break; |
776 | 0 | case LoopHintAttr::Unroll: |
777 | 0 | case LoopHintAttr::UnrollAndJam: |
778 | 0 | case LoopHintAttr::VectorizePredicate: |
779 | 0 | case LoopHintAttr::Vectorize: |
780 | 0 | case LoopHintAttr::VectorizeWidth: |
781 | 0 | case LoopHintAttr::Interleave: |
782 | 0 | case LoopHintAttr::Distribute: |
783 | 0 | case LoopHintAttr::PipelineDisabled: |
784 | 0 | llvm_unreachable("Options cannot be assigned a value."); |
785 | 0 | break; |
786 | 47 | } |
787 | 47 | break; |
788 | 167 | } |
789 | 167 | } |
790 | | |
791 | 12.0k | setMustProgress(MustProgress); |
792 | | |
793 | 12.0k | if (CGOpts.OptimizationLevel > 0) |
794 | | // Disable unrolling for the loop, if unrolling is disabled (via |
795 | | // -fno-unroll-loops) and no pragmas override the decision. |
796 | 239 | if (!CGOpts.UnrollLoops && |
797 | 166 | (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified && |
798 | 160 | StagedAttrs.UnrollCount == 0)) |
799 | 155 | setUnrollState(LoopAttributes::Disable); |
800 | | |
801 | | /// Stage the attributes. |
802 | 12.0k | push(Header, StartLoc, EndLoc); |
803 | 12.0k | } |
804 | | |
805 | 26.9k | void LoopInfoStack::pop() { |
806 | 26.9k | assert(!Active.empty() && "No active loops to pop"); |
807 | 26.9k | Active.back()->finish(); |
808 | 26.9k | Active.pop_back(); |
809 | 26.9k | } |
810 | | |
811 | 5.03M | void LoopInfoStack::InsertHelper(Instruction *I) const { |
812 | 5.03M | if (I->mayReadOrWriteMemory()) { |
813 | 2.87M | SmallVector<Metadata *, 4> AccessGroups; |
814 | 405k | for (const auto &AL : Active) { |
815 | | // Here we assume that every loop that has an access group is parallel. |
816 | 405k | if (MDNode *Group = AL->getAccessGroup()) |
817 | 47.3k | AccessGroups.push_back(Group); |
818 | 405k | } |
819 | 2.87M | MDNode *UnionMD = nullptr; |
820 | 2.87M | if (AccessGroups.size() == 1) |
821 | 47.2k | UnionMD = cast<MDNode>(AccessGroups[0]); |
822 | 2.82M | else if (AccessGroups.size() >= 2) |
823 | 50 | UnionMD = MDNode::get(I->getContext(), AccessGroups); |
824 | 2.87M | I->setMetadata("llvm.access.group", UnionMD); |
825 | 2.87M | } |
826 | | |
827 | 5.03M | if (!hasInfo()) |
828 | 4.29M | return; |
829 | | |
830 | 742k | const LoopInfo &L = getInfo(); |
831 | 742k | if (!L.getLoopID()) |
832 | 113k | return; |
833 | | |
834 | 628k | if (I->isTerminator()) { |
835 | 127k | for (BasicBlock *Succ : successors(I)) |
836 | 175k | if (Succ == L.getHeader()) { |
837 | 20.7k | I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID()); |
838 | 20.7k | break; |
839 | 20.7k | } |
840 | 127k | return; |
841 | 127k | } |
842 | 628k | } |