Coverage Report

Created: 2021-09-21 08:58

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "CGLoopInfo.h"
10
#include "clang/AST/ASTContext.h"
11
#include "clang/AST/Attr.h"
12
#include "clang/AST/Expr.h"
13
#include "clang/Basic/CodeGenOptions.h"
14
#include "llvm/IR/BasicBlock.h"
15
#include "llvm/IR/CFG.h"
16
#include "llvm/IR/Constants.h"
17
#include "llvm/IR/InstrTypes.h"
18
#include "llvm/IR/Instructions.h"
19
#include "llvm/IR/Metadata.h"
20
using namespace clang::CodeGen;
21
using namespace llvm;
22
23
MDNode *
24
29.0k
LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25
29.0k
  LLVMContext &Ctx = Header->getContext();
26
29.0k
  SmallVector<Metadata *, 4> NewLoopProperties;
27
29.0k
  NewLoopProperties.push_back(nullptr);
28
29.0k
  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29
30
29.0k
  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
31
29.0k
  LoopID->replaceOperandWith(0, LoopID);
32
29.0k
  return LoopID;
33
29.0k
}
34
35
MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
36
                                           ArrayRef<Metadata *> LoopProperties,
37
29.0k
                                           bool &HasUserTransforms) {
38
29.0k
  LLVMContext &Ctx = Header->getContext();
39
40
29.0k
  Optional<bool> Enabled;
41
29.0k
  if (Attrs.PipelineDisabled)
42
2
    Enabled = false;
43
29.0k
  else if (Attrs.PipelineInitiationInterval != 0)
44
3
    Enabled = true;
45
46
29.0k
  if (Enabled != true) {
47
29.0k
    SmallVector<Metadata *, 4> NewLoopProperties;
48
29.0k
    if (Enabled == false) {
49
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
50
2
      NewLoopProperties.push_back(
51
2
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
52
2
                            ConstantAsMetadata::get(ConstantInt::get(
53
2
                                llvm::Type::getInt1Ty(Ctx), 1))}));
54
2
      LoopProperties = NewLoopProperties;
55
2
    }
56
29.0k
    return createLoopPropertiesMetadata(LoopProperties);
57
29.0k
  }
58
59
3
  SmallVector<Metadata *, 4> Args;
60
3
  Args.push_back(nullptr);
61
3
  Args.append(LoopProperties.begin(), LoopProperties.end());
62
63
3
  if (Attrs.PipelineInitiationInterval > 0) {
64
3
    Metadata *Vals[] = {
65
3
        MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66
3
        ConstantAsMetadata::get(ConstantInt::get(
67
3
            llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68
3
    Args.push_back(MDNode::get(Ctx, Vals));
69
3
  }
70
71
  // No follow-up: This is the last transformation.
72
73
3
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74
3
  LoopID->replaceOperandWith(0, LoopID);
75
3
  HasUserTransforms = true;
76
3
  return LoopID;
77
29.0k
}
78
79
MDNode *
80
LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81
                                      ArrayRef<Metadata *> LoopProperties,
82
29.0k
                                      bool &HasUserTransforms) {
83
29.0k
  LLVMContext &Ctx = Header->getContext();
84
85
29.0k
  Optional<bool> Enabled;
86
29.0k
  if (Attrs.UnrollEnable == LoopAttributes::Disable)
87
173
    Enabled = false;
88
28.8k
  else if (Attrs.UnrollEnable == LoopAttributes::Full)
89
0
    Enabled = None;
90
28.8k
  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91
28.8k
           
Attrs.UnrollCount != 028.8k
)
92
74
    Enabled = true;
93
94
29.0k
  if (Enabled != true) {
95
    // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96
    // if unrolling is disabled.
97
28.9k
    return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98
28.9k
  }
99
100
74
  SmallVector<Metadata *, 4> FollowupLoopProperties;
101
102
  // Apply all loop properties to the unrolled loop.
103
74
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104
105
  // Don't unroll an already unrolled loop.
106
74
  FollowupLoopProperties.push_back(
107
74
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108
109
74
  bool FollowupHasTransforms = false;
110
74
  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111
74
                                              FollowupHasTransforms);
112
113
74
  SmallVector<Metadata *, 4> Args;
114
74
  Args.push_back(nullptr);
115
74
  Args.append(LoopProperties.begin(), LoopProperties.end());
116
117
  // Setting unroll.count
118
74
  if (Attrs.UnrollCount > 0) {
119
57
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
120
57
                        ConstantAsMetadata::get(ConstantInt::get(
121
57
                            llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
122
57
    Args.push_back(MDNode::get(Ctx, Vals));
123
57
  }
124
125
  // Setting unroll.full or unroll.disable
126
74
  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
127
19
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
128
19
    Args.push_back(MDNode::get(Ctx, Vals));
129
19
  }
130
131
74
  if (FollowupHasTransforms)
132
2
    Args.push_back(MDNode::get(
133
2
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
134
135
74
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
136
74
  LoopID->replaceOperandWith(0, LoopID);
137
74
  HasUserTransforms = true;
138
74
  return LoopID;
139
29.0k
}
140
141
MDNode *
142
LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
143
                                     ArrayRef<Metadata *> LoopProperties,
144
29.0k
                                     bool &HasUserTransforms) {
145
29.0k
  LLVMContext &Ctx = Header->getContext();
146
147
29.0k
  Optional<bool> Enabled;
148
29.0k
  if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
149
2
    Enabled = false;
150
29.0k
  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
151
29.0k
           
Attrs.UnrollAndJamCount != 029.0k
)
152
4
    Enabled = true;
153
154
29.0k
  if (Enabled != true) {
155
29.0k
    SmallVector<Metadata *, 4> NewLoopProperties;
156
29.0k
    if (Enabled == false) {
157
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
158
2
      NewLoopProperties.push_back(MDNode::get(
159
2
          Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
160
2
      LoopProperties = NewLoopProperties;
161
2
    }
162
29.0k
    return createPartialUnrollMetadata(Attrs, LoopProperties,
163
29.0k
                                       HasUserTransforms);
164
29.0k
  }
165
166
4
  SmallVector<Metadata *, 4> FollowupLoopProperties;
167
4
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
168
4
  FollowupLoopProperties.push_back(
169
4
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
170
171
4
  bool FollowupHasTransforms = false;
172
4
  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
173
4
                                                 FollowupHasTransforms);
174
175
4
  SmallVector<Metadata *, 4> Args;
176
4
  Args.push_back(nullptr);
177
4
  Args.append(LoopProperties.begin(), LoopProperties.end());
178
179
  // Setting unroll_and_jam.count
180
4
  if (Attrs.UnrollAndJamCount > 0) {
181
2
    Metadata *Vals[] = {
182
2
        MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
183
2
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
184
2
                                                 Attrs.UnrollAndJamCount))};
185
2
    Args.push_back(MDNode::get(Ctx, Vals));
186
2
  }
187
188
4
  if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
189
2
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
190
2
    Args.push_back(MDNode::get(Ctx, Vals));
191
2
  }
192
193
4
  if (FollowupHasTransforms)
194
1
    Args.push_back(MDNode::get(
195
1
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
196
1
              Followup}));
197
198
4
  if (UnrollAndJamInnerFollowup)
199
1
    Args.push_back(MDNode::get(
200
1
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
201
1
              UnrollAndJamInnerFollowup}));
202
203
4
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
204
4
  LoopID->replaceOperandWith(0, LoopID);
205
4
  HasUserTransforms = true;
206
4
  return LoopID;
207
29.0k
}
208
209
MDNode *
210
LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
211
                                      ArrayRef<Metadata *> LoopProperties,
212
29.0k
                                      bool &HasUserTransforms) {
213
29.0k
  LLVMContext &Ctx = Header->getContext();
214
215
29.0k
  Optional<bool> Enabled;
216
29.0k
  if (Attrs.VectorizeEnable == LoopAttributes::Disable)
217
343
    Enabled = false;
218
28.6k
  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
219
28.6k
           
Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified20.0k
||
220
28.6k
           
Attrs.InterleaveCount != 020.0k
||
Attrs.VectorizeWidth != 020.0k
||
221
28.6k
           
Attrs.VectorizeScalable != LoopAttributes::Unspecified19.9k
)
222
8.68k
    Enabled = true;
223
224
29.0k
  if (Enabled != true) {
225
20.3k
    SmallVector<Metadata *, 4> NewLoopProperties;
226
20.3k
    if (Enabled == false) {
227
343
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
228
343
      NewLoopProperties.push_back(
229
343
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
230
343
                            ConstantAsMetadata::get(ConstantInt::get(
231
343
                                llvm::Type::getInt1Ty(Ctx), 0))}));
232
343
      LoopProperties = NewLoopProperties;
233
343
    }
234
20.3k
    return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
235
20.3k
  }
236
237
  // Apply all loop properties to the vectorized loop.
238
8.68k
  SmallVector<Metadata *, 4> FollowupLoopProperties;
239
8.68k
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
240
241
  // Don't vectorize an already vectorized loop.
242
8.68k
  FollowupLoopProperties.push_back(
243
8.68k
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
244
245
8.68k
  bool FollowupHasTransforms = false;
246
8.68k
  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
247
8.68k
                                                FollowupHasTransforms);
248
249
8.68k
  SmallVector<Metadata *, 4> Args;
250
8.68k
  Args.push_back(nullptr);
251
8.68k
  Args.append(LoopProperties.begin(), LoopProperties.end());
252
253
  // Setting vectorize.predicate when it has been specified and vectorization
254
  // has not been disabled.
255
8.68k
  bool IsVectorPredicateEnabled = false;
256
8.68k
  if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
257
8
    IsVectorPredicateEnabled =
258
8
        (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
259
260
8
    Metadata *Vals[] = {
261
8
        MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
262
8
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
263
8
                                                 IsVectorPredicateEnabled))};
264
8
    Args.push_back(MDNode::get(Ctx, Vals));
265
8
  }
266
267
  // Setting vectorize.width
268
8.68k
  if (Attrs.VectorizeWidth > 0) {
269
443
    Metadata *Vals[] = {
270
443
        MDString::get(Ctx, "llvm.loop.vectorize.width"),
271
443
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
272
443
                                                 Attrs.VectorizeWidth))};
273
274
443
    Args.push_back(MDNode::get(Ctx, Vals));
275
443
  }
276
277
8.68k
  if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
278
24
    bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
279
24
    Metadata *Vals[] = {
280
24
        MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
281
24
        ConstantAsMetadata::get(
282
24
            ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
283
24
    Args.push_back(MDNode::get(Ctx, Vals));
284
24
  }
285
286
  // Setting interleave.count
287
8.68k
  if (Attrs.InterleaveCount > 0) {
288
20
    Metadata *Vals[] = {
289
20
        MDString::get(Ctx, "llvm.loop.interleave.count"),
290
20
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
291
20
                                                 Attrs.InterleaveCount))};
292
20
    Args.push_back(MDNode::get(Ctx, Vals));
293
20
  }
294
295
  // vectorize.enable is set if:
296
  // 1) loop hint vectorize.enable is set, or
297
  // 2) it is implied when vectorize.predicate is set, or
298
  // 3) it is implied when vectorize.width is set to a value > 1
299
  // 4) it is implied when vectorize.scalable.enable is true
300
  // 5) it is implied when vectorize.width is unset (0) and the user
301
  //    explicitly requested fixed-width vectorization, i.e.
302
  //    vectorize.scalable.enable is false.
303
8.68k
  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
304
8.68k
      
(29
IsVectorPredicateEnabled29
&&
Attrs.VectorizeWidth != 14
) ||
305
8.68k
      
Attrs.VectorizeWidth > 127
||
306
8.68k
      
Attrs.VectorizeScalable == LoopAttributes::Enable12
||
307
8.68k
      
(10
Attrs.VectorizeScalable == LoopAttributes::Disable10
&&
308
8.67k
       
Attrs.VectorizeWidth != 14
)) {
309
8.67k
    bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
310
8.67k
    Args.push_back(
311
8.67k
        MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
312
8.67k
                          ConstantAsMetadata::get(ConstantInt::get(
313
8.67k
                              llvm::Type::getInt1Ty(Ctx), AttrVal))}));
314
8.67k
  }
315
316
8.68k
  if (FollowupHasTransforms)
317
8
    Args.push_back(MDNode::get(
318
8
        Ctx,
319
8
        {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
320
321
8.68k
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
322
8.68k
  LoopID->replaceOperandWith(0, LoopID);
323
8.68k
  HasUserTransforms = true;
324
8.68k
  return LoopID;
325
29.0k
}
326
327
MDNode *
328
LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
329
                                       ArrayRef<Metadata *> LoopProperties,
330
29.0k
                                       bool &HasUserTransforms) {
331
29.0k
  LLVMContext &Ctx = Header->getContext();
332
333
29.0k
  Optional<bool> Enabled;
334
29.0k
  if (Attrs.DistributeEnable == LoopAttributes::Disable)
335
7
    Enabled = false;
336
29.0k
  if (Attrs.DistributeEnable == LoopAttributes::Enable)
337
3
    Enabled = true;
338
339
29.0k
  if (Enabled != true) {
340
29.0k
    SmallVector<Metadata *, 4> NewLoopProperties;
341
29.0k
    if (Enabled == false) {
342
7
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
343
7
      NewLoopProperties.push_back(
344
7
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
345
7
                            ConstantAsMetadata::get(ConstantInt::get(
346
7
                                llvm::Type::getInt1Ty(Ctx), 0))}));
347
7
      LoopProperties = NewLoopProperties;
348
7
    }
349
29.0k
    return createLoopVectorizeMetadata(Attrs, LoopProperties,
350
29.0k
                                       HasUserTransforms);
351
29.0k
  }
352
353
3
  bool FollowupHasTransforms = false;
354
3
  MDNode *Followup =
355
3
      createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
356
357
3
  SmallVector<Metadata *, 4> Args;
358
3
  Args.push_back(nullptr);
359
3
  Args.append(LoopProperties.begin(), LoopProperties.end());
360
361
3
  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
362
3
                      ConstantAsMetadata::get(ConstantInt::get(
363
3
                          llvm::Type::getInt1Ty(Ctx),
364
3
                          (Attrs.DistributeEnable == LoopAttributes::Enable)))};
365
3
  Args.push_back(MDNode::get(Ctx, Vals));
366
367
3
  if (FollowupHasTransforms)
368
2
    Args.push_back(MDNode::get(
369
2
        Ctx,
370
2
        {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
371
372
3
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
373
3
  LoopID->replaceOperandWith(0, LoopID);
374
3
  HasUserTransforms = true;
375
3
  return LoopID;
376
29.0k
}
377
378
MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
379
                                           ArrayRef<Metadata *> LoopProperties,
380
29.0k
                                           bool &HasUserTransforms) {
381
29.0k
  LLVMContext &Ctx = Header->getContext();
382
383
29.0k
  Optional<bool> Enabled;
384
29.0k
  if (Attrs.UnrollEnable == LoopAttributes::Disable)
385
173
    Enabled = false;
386
28.8k
  else if (Attrs.UnrollEnable == LoopAttributes::Full)
387
6
    Enabled = true;
388
389
29.0k
  if (Enabled != true) {
390
29.0k
    SmallVector<Metadata *, 4> NewLoopProperties;
391
29.0k
    if (Enabled == false) {
392
173
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
393
173
      NewLoopProperties.push_back(
394
173
          MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
395
173
      LoopProperties = NewLoopProperties;
396
173
    }
397
29.0k
    return createLoopDistributeMetadata(Attrs, LoopProperties,
398
29.0k
                                        HasUserTransforms);
399
29.0k
  }
400
401
6
  SmallVector<Metadata *, 4> Args;
402
6
  Args.push_back(nullptr);
403
6
  Args.append(LoopProperties.begin(), LoopProperties.end());
404
6
  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
405
406
  // No follow-up: there is no loop after full unrolling.
407
  // TODO: Warn if there are transformations after full unrolling.
408
409
6
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
410
6
  LoopID->replaceOperandWith(0, LoopID);
411
6
  HasUserTransforms = true;
412
6
  return LoopID;
413
29.0k
}
414
415
MDNode *LoopInfo::createMetadata(
416
    const LoopAttributes &Attrs,
417
    llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
418
29.0k
    bool &HasUserTransforms) {
419
29.0k
  SmallVector<Metadata *, 3> LoopProperties;
420
421
  // If we have a valid start debug location for the loop, add it.
422
29.0k
  if (StartLoc) {
423
15.6k
    LoopProperties.push_back(StartLoc.getAsMDNode());
424
425
    // If we also have a valid end debug location for the loop, add it.
426
15.6k
    if (EndLoc)
427
15.6k
      LoopProperties.push_back(EndLoc.getAsMDNode());
428
15.6k
  }
429
430
29.0k
  LLVMContext &Ctx = Header->getContext();
431
29.0k
  if (Attrs.MustProgress)
432
19.2k
    LoopProperties.push_back(
433
19.2k
        MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
434
435
29.0k
  assert(!!AccGroup == Attrs.IsParallel &&
436
29.0k
         "There must be an access group iff the loop is parallel");
437
29.0k
  if (Attrs.IsParallel) {
438
8.86k
    LoopProperties.push_back(MDNode::get(
439
8.86k
        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
440
8.86k
  }
441
442
29.0k
  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
443
29.0k
                        AdditionalLoopProperties.end());
444
29.0k
  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
445
29.0k
}
446
447
LoopAttributes::LoopAttributes(bool IsParallel)
448
    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
449
      UnrollEnable(LoopAttributes::Unspecified),
450
      UnrollAndJamEnable(LoopAttributes::Unspecified),
451
      VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
452
      VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
453
      UnrollCount(0), UnrollAndJamCount(0),
454
      DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
455
378k
      PipelineInitiationInterval(0), MustProgress(false) {}
456
457
35.0k
void LoopAttributes::clear() {
458
35.0k
  IsParallel = false;
459
35.0k
  VectorizeWidth = 0;
460
35.0k
  VectorizeScalable = LoopAttributes::Unspecified;
461
35.0k
  InterleaveCount = 0;
462
35.0k
  UnrollCount = 0;
463
35.0k
  UnrollAndJamCount = 0;
464
35.0k
  VectorizeEnable = LoopAttributes::Unspecified;
465
35.0k
  UnrollEnable = LoopAttributes::Unspecified;
466
35.0k
  UnrollAndJamEnable = LoopAttributes::Unspecified;
467
35.0k
  VectorizePredicateEnable = LoopAttributes::Unspecified;
468
35.0k
  DistributeEnable = LoopAttributes::Unspecified;
469
35.0k
  PipelineDisabled = false;
470
35.0k
  PipelineInitiationInterval = 0;
471
35.0k
  MustProgress = false;
472
35.0k
}
473
474
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
475
                   const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
476
                   LoopInfo *Parent)
477
    : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
478
35.0k
      Parent(Parent) {
479
480
35.0k
  if (Attrs.IsParallel) {
481
    // Create an access group for this loop.
482
8.86k
    LLVMContext &Ctx = Header->getContext();
483
8.86k
    AccGroup = MDNode::getDistinct(Ctx, {});
484
8.86k
  }
485
486
35.0k
  if (!Attrs.IsParallel && 
Attrs.VectorizeWidth == 026.1k
&&
487
35.0k
      
Attrs.VectorizeScalable == LoopAttributes::Unspecified25.9k
&&
488
35.0k
      
Attrs.InterleaveCount == 025.9k
&&
Attrs.UnrollCount == 025.9k
&&
489
35.0k
      
Attrs.UnrollAndJamCount == 025.8k
&&
!Attrs.PipelineDisabled25.8k
&&
490
35.0k
      
Attrs.PipelineInitiationInterval == 025.8k
&&
491
35.0k
      
Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified25.8k
&&
492
35.0k
      
Attrs.VectorizeEnable == LoopAttributes::Unspecified25.8k
&&
493
35.0k
      
Attrs.UnrollEnable == LoopAttributes::Unspecified25.5k
&&
494
35.0k
      
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified25.3k
&&
495
35.0k
      
Attrs.DistributeEnable == LoopAttributes::Unspecified25.3k
&&
!StartLoc25.3k
&&
496
35.0k
      
!EndLoc10.0k
&&
!Attrs.MustProgress10.0k
)
497
6.03k
    return;
498
499
29.0k
  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
500
29.0k
}
501
502
35.0k
void LoopInfo::finish() {
503
  // We did not annotate the loop body instructions because there are no
504
  // attributes for this loop.
505
35.0k
  if (!TempLoopID)
506
6.03k
    return;
507
508
29.0k
  MDNode *LoopID;
509
29.0k
  LoopAttributes CurLoopAttr = Attrs;
510
29.0k
  LLVMContext &Ctx = Header->getContext();
511
512
29.0k
  if (Parent && 
(7.50k
Parent->Attrs.UnrollAndJamEnable7.50k
||
513
7.50k
                 
Parent->Attrs.UnrollAndJamCount != 07.50k
)) {
514
    // Parent unroll-and-jams this loop.
515
    // Split the transformations in those that happens before the unroll-and-jam
516
    // and those after.
517
518
5
    LoopAttributes BeforeJam, AfterJam;
519
520
5
    BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
521
522
5
    BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
523
5
    BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
524
5
    BeforeJam.InterleaveCount = Attrs.InterleaveCount;
525
5
    BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
526
5
    BeforeJam.DistributeEnable = Attrs.DistributeEnable;
527
5
    BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
528
529
5
    switch (Attrs.UnrollEnable) {
530
5
    case LoopAttributes::Unspecified:
531
5
    case LoopAttributes::Disable:
532
5
      BeforeJam.UnrollEnable = Attrs.UnrollEnable;
533
5
      AfterJam.UnrollEnable = Attrs.UnrollEnable;
534
5
      break;
535
0
    case LoopAttributes::Full:
536
0
      BeforeJam.UnrollEnable = LoopAttributes::Full;
537
0
      break;
538
0
    case LoopAttributes::Enable:
539
0
      AfterJam.UnrollEnable = LoopAttributes::Enable;
540
0
      break;
541
5
    }
542
543
5
    AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
544
5
    AfterJam.UnrollCount = Attrs.UnrollCount;
545
5
    AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
546
5
    AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
547
548
    // If this loop is subject of an unroll-and-jam by the parent loop, and has
549
    // an unroll-and-jam annotation itself, we have to decide whether to first
550
    // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
551
    // UnrollAndJam pass processes loops from inner to outer, so we apply the
552
    // inner first.
553
5
    BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
554
5
    BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
555
556
    // Set the inner followup metadata to process by the outer loop. Only
557
    // consider the first inner loop.
558
5
    if (!Parent->UnrollAndJamInnerFollowup) {
559
      // Splitting the attributes into a BeforeJam and an AfterJam part will
560
      // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
561
      // to be forwarded to the AfterJam part. We detect the situation here and
562
      // add it manually.
563
5
      SmallVector<Metadata *, 1> BeforeLoopProperties;
564
5
      if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
565
5
          
BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified4
||
566
5
          
BeforeJam.InterleaveCount != 04
||
BeforeJam.VectorizeWidth != 04
||
567
5
          
BeforeJam.VectorizeScalable == LoopAttributes::Enable4
)
568
1
        BeforeLoopProperties.push_back(
569
1
            MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
570
571
5
      bool InnerFollowupHasTransform = false;
572
5
      MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
573
5
                                             InnerFollowupHasTransform);
574
5
      if (InnerFollowupHasTransform)
575
1
        Parent->UnrollAndJamInnerFollowup = InnerFollowup;
576
5
    }
577
578
5
    CurLoopAttr = BeforeJam;
579
5
  }
580
581
29.0k
  bool HasUserTransforms = false;
582
29.0k
  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
583
29.0k
  TempLoopID->replaceAllUsesWith(LoopID);
584
29.0k
}
585
586
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
587
35.0k
                         const llvm::DebugLoc &EndLoc) {
588
35.0k
  Active.emplace_back(
589
35.0k
      new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
590
35.0k
                   Active.empty() ? 
nullptr27.3k
:
Active.back().get()7.75k
));
591
  // Clear the attributes so nested loops do not inherit them.
592
35.0k
  StagedAttrs.clear();
593
35.0k
}
594
595
void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
596
                         const clang::CodeGenOptions &CGOpts,
597
                         ArrayRef<const clang::Attr *> Attrs,
598
                         const llvm::DebugLoc &StartLoc,
599
19.9k
                         const llvm::DebugLoc &EndLoc, bool MustProgress) {
600
  // Identify loop hint attributes from Attrs.
601
19.9k
  for (const auto *Attr : Attrs) {
602
219
    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
603
219
    const OpenCLUnrollHintAttr *OpenCLHint =
604
219
        dyn_cast<OpenCLUnrollHintAttr>(Attr);
605
606
    // Skip non loop hint attributes
607
219
    if (!LH && 
!OpenCLHint25
) {
608
7
      continue;
609
7
    }
610
611
212
    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
612
212
    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
613
212
    unsigned ValueInt = 1;
614
    // Translate opencl_unroll_hint attribute argument to
615
    // equivalent LoopHintAttr enums.
616
    // OpenCL v2.0 s6.11.5:
617
    // 0 - enable unroll (no argument).
618
    // 1 - disable unroll.
619
    // other positive integer n - unroll by n.
620
212
    if (OpenCLHint) {
621
18
      ValueInt = OpenCLHint->getUnrollHint();
622
18
      if (ValueInt == 0) {
623
6
        State = LoopHintAttr::Enable;
624
12
      } else if (ValueInt != 1) {
625
6
        Option = LoopHintAttr::UnrollCount;
626
6
        State = LoopHintAttr::Numeric;
627
6
      }
628
194
    } else if (LH) {
629
194
      auto *ValueExpr = LH->getValue();
630
194
      if (ValueExpr) {
631
92
        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
632
92
        ValueInt = ValueAPS.getSExtValue();
633
92
      }
634
635
194
      Option = LH->getOption();
636
194
      State = LH->getState();
637
194
    }
638
212
    switch (State) {
639
54
    case LoopHintAttr::Disable:
640
54
      switch (Option) {
641
7
      case LoopHintAttr::Vectorize:
642
        // Disable vectorization by specifying a width of 1.
643
7
        setVectorizeWidth(1);
644
7
        setVectorizeScalable(LoopAttributes::Unspecified);
645
7
        break;
646
6
      case LoopHintAttr::Interleave:
647
        // Disable interleaving by speciyfing a count of 1.
648
6
        setInterleaveCount(1);
649
6
        break;
650
27
      case LoopHintAttr::Unroll:
651
27
        setUnrollState(LoopAttributes::Disable);
652
27
        break;
653
2
      case LoopHintAttr::UnrollAndJam:
654
2
        setUnrollAndJamState(LoopAttributes::Disable);
655
2
        break;
656
3
      case LoopHintAttr::VectorizePredicate:
657
3
        setVectorizePredicateState(LoopAttributes::Disable);
658
3
        break;
659
7
      case LoopHintAttr::Distribute:
660
7
        setDistributeState(false);
661
7
        break;
662
2
      case LoopHintAttr::PipelineDisabled:
663
2
        setPipelineDisabled(true);
664
2
        break;
665
0
      case LoopHintAttr::UnrollCount:
666
0
      case LoopHintAttr::UnrollAndJamCount:
667
0
      case LoopHintAttr::VectorizeWidth:
668
0
      case LoopHintAttr::InterleaveCount:
669
0
      case LoopHintAttr::PipelineInitiationInterval:
670
0
        llvm_unreachable("Options cannot be disabled.");
671
0
        break;
672
54
      }
673
54
      break;
674
54
    case LoopHintAttr::Enable:
675
39
      switch (Option) {
676
14
      case LoopHintAttr::Vectorize:
677
15
      case LoopHintAttr::Interleave:
678
15
        setVectorizeEnable(true);
679
15
        break;
680
13
      case LoopHintAttr::Unroll:
681
13
        setUnrollState(LoopAttributes::Enable);
682
13
        break;
683
2
      case LoopHintAttr::UnrollAndJam:
684
2
        setUnrollAndJamState(LoopAttributes::Enable);
685
2
        break;
686
5
      case LoopHintAttr::VectorizePredicate:
687
5
        setVectorizePredicateState(LoopAttributes::Enable);
688
5
        break;
689
4
      case LoopHintAttr::Distribute:
690
4
        setDistributeState(true);
691
4
        break;
692
0
      case LoopHintAttr::UnrollCount:
693
0
      case LoopHintAttr::UnrollAndJamCount:
694
0
      case LoopHintAttr::VectorizeWidth:
695
0
      case LoopHintAttr::InterleaveCount:
696
0
      case LoopHintAttr::PipelineDisabled:
697
0
      case LoopHintAttr::PipelineInitiationInterval:
698
0
        llvm_unreachable("Options cannot enabled.");
699
0
        break;
700
39
      }
701
39
      break;
702
39
    case LoopHintAttr::AssumeSafety:
703
15
      switch (Option) {
704
14
      case LoopHintAttr::Vectorize:
705
15
      case LoopHintAttr::Interleave:
706
        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
707
15
        setParallel(true);
708
15
        setVectorizeEnable(true);
709
15
        break;
710
0
      case LoopHintAttr::Unroll:
711
0
      case LoopHintAttr::UnrollAndJam:
712
0
      case LoopHintAttr::VectorizePredicate:
713
0
      case LoopHintAttr::UnrollCount:
714
0
      case LoopHintAttr::UnrollAndJamCount:
715
0
      case LoopHintAttr::VectorizeWidth:
716
0
      case LoopHintAttr::InterleaveCount:
717
0
      case LoopHintAttr::Distribute:
718
0
      case LoopHintAttr::PipelineDisabled:
719
0
      case LoopHintAttr::PipelineInitiationInterval:
720
0
        llvm_unreachable("Options cannot be used to assume mem safety.");
721
0
        break;
722
15
      }
723
15
      break;
724
15
    case LoopHintAttr::Full:
725
4
      switch (Option) {
726
4
      case LoopHintAttr::Unroll:
727
4
        setUnrollState(LoopAttributes::Full);
728
4
        break;
729
0
      case LoopHintAttr::UnrollAndJam:
730
0
        setUnrollAndJamState(LoopAttributes::Full);
731
0
        break;
732
0
      case LoopHintAttr::Vectorize:
733
0
      case LoopHintAttr::Interleave:
734
0
      case LoopHintAttr::UnrollCount:
735
0
      case LoopHintAttr::UnrollAndJamCount:
736
0
      case LoopHintAttr::VectorizeWidth:
737
0
      case LoopHintAttr::InterleaveCount:
738
0
      case LoopHintAttr::Distribute:
739
0
      case LoopHintAttr::PipelineDisabled:
740
0
      case LoopHintAttr::PipelineInitiationInterval:
741
0
      case LoopHintAttr::VectorizePredicate:
742
0
        llvm_unreachable("Options cannot be used with 'full' hint.");
743
0
        break;
744
4
      }
745
4
      break;
746
22
    case LoopHintAttr::FixedWidth:
747
25
    case LoopHintAttr::ScalableWidth:
748
25
      switch (Option) {
749
25
      case LoopHintAttr::VectorizeWidth:
750
25
        setVectorizeScalable(State == LoopHintAttr::ScalableWidth
751
25
                                 ? 
LoopAttributes::Enable3
752
25
                                 : 
LoopAttributes::Disable22
);
753
25
        if (LH->getValue())
754
23
          setVectorizeWidth(ValueInt);
755
25
        break;
756
0
      default:
757
0
        llvm_unreachable("Options cannot be used with 'scalable' hint.");
758
0
        break;
759
25
      }
760
25
      break;
761
75
    case LoopHintAttr::Numeric:
762
75
      switch (Option) {
763
15
      case LoopHintAttr::InterleaveCount:
764
15
        setInterleaveCount(ValueInt);
765
15
        break;
766
55
      case LoopHintAttr::UnrollCount:
767
55
        setUnrollCount(ValueInt);
768
55
        break;
769
2
      case LoopHintAttr::UnrollAndJamCount:
770
2
        setUnrollAndJamCount(ValueInt);
771
2
        break;
772
3
      case LoopHintAttr::PipelineInitiationInterval:
773
3
        setPipelineInitiationInterval(ValueInt);
774
3
        break;
775
0
      case LoopHintAttr::Unroll:
776
0
      case LoopHintAttr::UnrollAndJam:
777
0
      case LoopHintAttr::VectorizePredicate:
778
0
      case LoopHintAttr::Vectorize:
779
0
      case LoopHintAttr::VectorizeWidth:
780
0
      case LoopHintAttr::Interleave:
781
0
      case LoopHintAttr::Distribute:
782
0
      case LoopHintAttr::PipelineDisabled:
783
0
        llvm_unreachable("Options cannot be assigned a value.");
784
0
        break;
785
75
      }
786
75
      break;
787
212
    }
788
212
  }
789
790
19.9k
  setMustProgress(MustProgress);
791
792
19.9k
  if (CGOpts.OptimizationLevel > 0)
793
    // Disable unrolling for the loop, if unrolling is disabled (via
794
    // -fno-unroll-loops) and no pragmas override the decision.
795
238
    if (!CGOpts.UnrollLoops &&
796
238
        
(164
StagedAttrs.UnrollEnable == LoopAttributes::Unspecified164
&&
797
164
         
StagedAttrs.UnrollCount == 0156
))
798
146
      setUnrollState(LoopAttributes::Disable);
799
800
  /// Stage the attributes.
801
19.9k
  push(Header, StartLoc, EndLoc);
802
19.9k
}
803
804
35.0k
void LoopInfoStack::pop() {
805
35.0k
  assert(!Active.empty() && "No active loops to pop");
806
0
  Active.back()->finish();
807
35.0k
  Active.pop_back();
808
35.0k
}
809
810
6.92M
void LoopInfoStack::InsertHelper(Instruction *I) const {
811
6.92M
  if (I->mayReadOrWriteMemory()) {
812
4.10M
    SmallVector<Metadata *, 4> AccessGroups;
813
4.10M
    for (const auto &AL : Active) {
814
      // Here we assume that every loop that has an access group is parallel.
815
660k
      if (MDNode *Group = AL->getAccessGroup())
816
89.1k
        AccessGroups.push_back(Group);
817
660k
    }
818
4.10M
    MDNode *UnionMD = nullptr;
819
4.10M
    if (AccessGroups.size() == 1)
820
89.0k
      UnionMD = cast<MDNode>(AccessGroups[0]);
821
4.01M
    else if (AccessGroups.size() >= 2)
822
50
      UnionMD = MDNode::get(I->getContext(), AccessGroups);
823
4.10M
    I->setMetadata("llvm.access.group", UnionMD);
824
4.10M
  }
825
826
6.92M
  if (!hasInfo())
827
5.75M
    return;
828
829
1.17M
  const LoopInfo &L = getInfo();
830
1.17M
  if (!L.getLoopID())
831
115k
    return;
832
833
1.06M
  if (I->isTerminator()) {
834
210k
    for (BasicBlock *Succ : successors(I))
835
293k
      if (Succ == L.getHeader()) {
836
28.5k
        I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
837
28.5k
        break;
838
28.5k
      }
839
210k
    return;
840
210k
  }
841
1.06M
}