Coverage Report

Created: 2020-02-15 09:57

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "CGLoopInfo.h"
10
#include "clang/AST/ASTContext.h"
11
#include "clang/AST/Attr.h"
12
#include "llvm/IR/BasicBlock.h"
13
#include "llvm/IR/CFG.h"
14
#include "llvm/IR/Constants.h"
15
#include "llvm/IR/InstrTypes.h"
16
#include "llvm/IR/Instructions.h"
17
#include "llvm/IR/Metadata.h"
18
using namespace clang::CodeGen;
19
using namespace llvm;
20
21
MDNode *
22
14.8k
LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
23
14.8k
  LLVMContext &Ctx = Header->getContext();
24
14.8k
  SmallVector<Metadata *, 4> NewLoopProperties;
25
14.8k
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26
14.8k
  NewLoopProperties.push_back(TempNode.get());
27
14.8k
  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
28
14.8k
29
14.8k
  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30
14.8k
  LoopID->replaceOperandWith(0, LoopID);
31
14.8k
  return LoopID;
32
14.8k
}
33
34
MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
35
                                           ArrayRef<Metadata *> LoopProperties,
36
14.8k
                                           bool &HasUserTransforms) {
37
14.8k
  LLVMContext &Ctx = Header->getContext();
38
14.8k
39
14.8k
  Optional<bool> Enabled;
40
14.8k
  if (Attrs.PipelineDisabled)
41
2
    Enabled = false;
42
14.8k
  else if (Attrs.PipelineInitiationInterval != 0)
43
3
    Enabled = true;
44
14.8k
45
14.8k
  if (Enabled != true) {
46
14.8k
    SmallVector<Metadata *, 4> NewLoopProperties;
47
14.8k
    if (Enabled == false) {
48
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49
2
      NewLoopProperties.push_back(
50
2
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
51
2
                            ConstantAsMetadata::get(ConstantInt::get(
52
2
                                llvm::Type::getInt1Ty(Ctx), 1))}));
53
2
      LoopProperties = NewLoopProperties;
54
2
    }
55
14.8k
    return createLoopPropertiesMetadata(LoopProperties);
56
14.8k
  }
57
3
58
3
  SmallVector<Metadata *, 4> Args;
59
3
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
60
3
  Args.push_back(TempNode.get());
61
3
  Args.append(LoopProperties.begin(), LoopProperties.end());
62
3
63
3
  if (Attrs.PipelineInitiationInterval > 0) {
64
3
    Metadata *Vals[] = {
65
3
        MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66
3
        ConstantAsMetadata::get(ConstantInt::get(
67
3
            llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68
3
    Args.push_back(MDNode::get(Ctx, Vals));
69
3
  }
70
3
71
3
  // No follow-up: This is the last transformation.
72
3
73
3
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74
3
  LoopID->replaceOperandWith(0, LoopID);
75
3
  HasUserTransforms = true;
76
3
  return LoopID;
77
3
}
78
79
MDNode *
80
LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81
                                      ArrayRef<Metadata *> LoopProperties,
82
14.8k
                                      bool &HasUserTransforms) {
83
14.8k
  LLVMContext &Ctx = Header->getContext();
84
14.8k
85
14.8k
  Optional<bool> Enabled;
86
14.8k
  if (Attrs.UnrollEnable == LoopAttributes::Disable)
87
19
    Enabled = false;
88
14.8k
  else if (Attrs.UnrollEnable == LoopAttributes::Full)
89
0
    Enabled = None;
90
14.8k
  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91
14.8k
           
Attrs.UnrollCount != 014.8k
)
92
32
    Enabled = true;
93
14.8k
94
14.8k
  if (Enabled != true) {
95
14.8k
    // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96
14.8k
    // if unrolling is disabled.
97
14.8k
    return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98
14.8k
  }
99
32
100
32
  SmallVector<Metadata *, 4> FollowupLoopProperties;
101
32
102
32
  // Apply all loop properties to the unrolled loop.
103
32
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104
32
105
32
  // Don't unroll an already unrolled loop.
106
32
  FollowupLoopProperties.push_back(
107
32
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108
32
109
32
  bool FollowupHasTransforms = false;
110
32
  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111
32
                                              FollowupHasTransforms);
112
32
113
32
  SmallVector<Metadata *, 4> Args;
114
32
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115
32
  Args.push_back(TempNode.get());
116
32
  Args.append(LoopProperties.begin(), LoopProperties.end());
117
32
118
32
  // Setting unroll.count
119
32
  if (Attrs.UnrollCount > 0) {
120
22
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121
22
                        ConstantAsMetadata::get(ConstantInt::get(
122
22
                            llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123
22
    Args.push_back(MDNode::get(Ctx, Vals));
124
22
  }
125
32
126
32
  // Setting unroll.full or unroll.disable
127
32
  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128
10
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129
10
    Args.push_back(MDNode::get(Ctx, Vals));
130
10
  }
131
32
132
32
  if (FollowupHasTransforms)
133
2
    Args.push_back(MDNode::get(
134
2
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135
32
136
32
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137
32
  LoopID->replaceOperandWith(0, LoopID);
138
32
  HasUserTransforms = true;
139
32
  return LoopID;
140
32
}
141
142
MDNode *
143
LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144
                                     ArrayRef<Metadata *> LoopProperties,
145
14.8k
                                     bool &HasUserTransforms) {
146
14.8k
  LLVMContext &Ctx = Header->getContext();
147
14.8k
148
14.8k
  Optional<bool> Enabled;
149
14.8k
  if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
150
2
    Enabled = false;
151
14.8k
  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152
14.8k
           
Attrs.UnrollAndJamCount != 014.8k
)
153
4
    Enabled = true;
154
14.8k
155
14.8k
  if (Enabled != true) {
156
14.8k
    SmallVector<Metadata *, 4> NewLoopProperties;
157
14.8k
    if (Enabled == false) {
158
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159
2
      NewLoopProperties.push_back(MDNode::get(
160
2
          Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161
2
      LoopProperties = NewLoopProperties;
162
2
    }
163
14.8k
    return createPartialUnrollMetadata(Attrs, LoopProperties,
164
14.8k
                                       HasUserTransforms);
165
14.8k
  }
166
4
167
4
  SmallVector<Metadata *, 4> FollowupLoopProperties;
168
4
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169
4
  FollowupLoopProperties.push_back(
170
4
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171
4
172
4
  bool FollowupHasTransforms = false;
173
4
  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174
4
                                                 FollowupHasTransforms);
175
4
176
4
  SmallVector<Metadata *, 4> Args;
177
4
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178
4
  Args.push_back(TempNode.get());
179
4
  Args.append(LoopProperties.begin(), LoopProperties.end());
180
4
181
4
  // Setting unroll_and_jam.count
182
4
  if (Attrs.UnrollAndJamCount > 0) {
183
2
    Metadata *Vals[] = {
184
2
        MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
185
2
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186
2
                                                 Attrs.UnrollAndJamCount))};
187
2
    Args.push_back(MDNode::get(Ctx, Vals));
188
2
  }
189
4
190
4
  if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
191
2
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
192
2
    Args.push_back(MDNode::get(Ctx, Vals));
193
2
  }
194
4
195
4
  if (FollowupHasTransforms)
196
1
    Args.push_back(MDNode::get(
197
1
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
198
1
              Followup}));
199
4
200
4
  if (UnrollAndJamInnerFollowup)
201
1
    Args.push_back(MDNode::get(
202
1
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
203
1
              UnrollAndJamInnerFollowup}));
204
4
205
4
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206
4
  LoopID->replaceOperandWith(0, LoopID);
207
4
  HasUserTransforms = true;
208
4
  return LoopID;
209
4
}
210
211
MDNode *
212
LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
213
                                      ArrayRef<Metadata *> LoopProperties,
214
14.8k
                                      bool &HasUserTransforms) {
215
14.8k
  LLVMContext &Ctx = Header->getContext();
216
14.8k
217
14.8k
  Optional<bool> Enabled;
218
14.8k
  if (Attrs.VectorizeEnable == LoopAttributes::Disable)
219
184
    Enabled = false;
220
14.6k
  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
221
14.6k
           
Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified8.10k
||
222
14.6k
           
Attrs.InterleaveCount != 08.09k
||
Attrs.VectorizeWidth != 08.09k
)
223
6.59k
    Enabled = true;
224
14.8k
225
14.8k
  if (Enabled != true) {
226
8.26k
    SmallVector<Metadata *, 4> NewLoopProperties;
227
8.26k
    if (Enabled == false) {
228
184
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229
184
      NewLoopProperties.push_back(
230
184
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
231
184
                            ConstantAsMetadata::get(ConstantInt::get(
232
184
                                llvm::Type::getInt1Ty(Ctx), 0))}));
233
184
      LoopProperties = NewLoopProperties;
234
184
    }
235
8.26k
    return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
236
8.26k
  }
237
6.59k
238
6.59k
  // Apply all loop properties to the vectorized loop.
239
6.59k
  SmallVector<Metadata *, 4> FollowupLoopProperties;
240
6.59k
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
241
6.59k
242
6.59k
  // Don't vectorize an already vectorized loop.
243
6.59k
  FollowupLoopProperties.push_back(
244
6.59k
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
245
6.59k
246
6.59k
  bool FollowupHasTransforms = false;
247
6.59k
  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248
6.59k
                                                FollowupHasTransforms);
249
6.59k
250
6.59k
  SmallVector<Metadata *, 4> Args;
251
6.59k
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
252
6.59k
  Args.push_back(TempNode.get());
253
6.59k
  Args.append(LoopProperties.begin(), LoopProperties.end());
254
6.59k
255
6.59k
  // Setting vectorize.predicate
256
6.59k
  bool IsVectorPredicateEnabled = false;
257
6.59k
  if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
258
6.59k
      
Attrs.VectorizeEnable != LoopAttributes::Disable4
&&
259
6.59k
      
Attrs.VectorizeWidth < 14
) {
260
3
261
3
    IsVectorPredicateEnabled =
262
3
        (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
263
3
264
3
    Metadata *Vals[] = {
265
3
        MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
266
3
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
267
3
                                                 IsVectorPredicateEnabled))};
268
3
    Args.push_back(MDNode::get(Ctx, Vals));
269
3
  }
270
6.59k
271
6.59k
  // Setting vectorize.width
272
6.59k
  if (Attrs.VectorizeWidth > 0) {
273
431
    Metadata *Vals[] = {
274
431
        MDString::get(Ctx, "llvm.loop.vectorize.width"),
275
431
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
276
431
                                                 Attrs.VectorizeWidth))};
277
431
    Args.push_back(MDNode::get(Ctx, Vals));
278
431
  }
279
6.59k
280
6.59k
  // Setting interleave.count
281
6.59k
  if (Attrs.InterleaveCount > 0) {
282
15
    Metadata *Vals[] = {
283
15
        MDString::get(Ctx, "llvm.loop.interleave.count"),
284
15
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
285
15
                                                 Attrs.InterleaveCount))};
286
15
    Args.push_back(MDNode::get(Ctx, Vals));
287
15
  }
288
6.59k
289
6.59k
  // vectorize.enable is set if:
290
6.59k
  // 1) loop hint vectorize.enable is set, or
291
6.59k
  // 2) it is implied when vectorize.predicate is set, or
292
6.59k
  // 3) it is implied when vectorize.width is set.
293
6.59k
  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
294
6.59k
      
IsVectorPredicateEnabled17
||
295
6.59k
      
Attrs.VectorizeWidth > 116
) {
296
6.59k
    bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
297
6.59k
    Args.push_back(
298
6.59k
        MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
299
6.59k
                          ConstantAsMetadata::get(ConstantInt::get(
300
6.59k
                              llvm::Type::getInt1Ty(Ctx), AttrVal))}));
301
6.59k
  }
302
6.59k
303
6.59k
  if (FollowupHasTransforms)
304
8
    Args.push_back(MDNode::get(
305
8
        Ctx,
306
8
        {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
307
6.59k
308
6.59k
  MDNode *LoopID = MDNode::get(Ctx, Args);
309
6.59k
  LoopID->replaceOperandWith(0, LoopID);
310
6.59k
  HasUserTransforms = true;
311
6.59k
  return LoopID;
312
6.59k
}
313
314
MDNode *
315
LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
316
                                       ArrayRef<Metadata *> LoopProperties,
317
14.8k
                                       bool &HasUserTransforms) {
318
14.8k
  LLVMContext &Ctx = Header->getContext();
319
14.8k
320
14.8k
  Optional<bool> Enabled;
321
14.8k
  if (Attrs.DistributeEnable == LoopAttributes::Disable)
322
2
    Enabled = false;
323
14.8k
  if (Attrs.DistributeEnable == LoopAttributes::Enable)
324
3
    Enabled = true;
325
14.8k
326
14.8k
  if (Enabled != true) {
327
14.8k
    SmallVector<Metadata *, 4> NewLoopProperties;
328
14.8k
    if (Enabled == false) {
329
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
330
2
      NewLoopProperties.push_back(
331
2
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
332
2
                            ConstantAsMetadata::get(ConstantInt::get(
333
2
                                llvm::Type::getInt1Ty(Ctx), 0))}));
334
2
      LoopProperties = NewLoopProperties;
335
2
    }
336
14.8k
    return createLoopVectorizeMetadata(Attrs, LoopProperties,
337
14.8k
                                       HasUserTransforms);
338
14.8k
  }
339
3
340
3
  bool FollowupHasTransforms = false;
341
3
  MDNode *Followup =
342
3
      createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
343
3
344
3
  SmallVector<Metadata *, 4> Args;
345
3
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
346
3
  Args.push_back(TempNode.get());
347
3
  Args.append(LoopProperties.begin(), LoopProperties.end());
348
3
349
3
  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
350
3
                      ConstantAsMetadata::get(ConstantInt::get(
351
3
                          llvm::Type::getInt1Ty(Ctx),
352
3
                          (Attrs.DistributeEnable == LoopAttributes::Enable)))};
353
3
  Args.push_back(MDNode::get(Ctx, Vals));
354
3
355
3
  if (FollowupHasTransforms)
356
2
    Args.push_back(MDNode::get(
357
2
        Ctx,
358
2
        {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
359
3
360
3
  MDNode *LoopID = MDNode::get(Ctx, Args);
361
3
  LoopID->replaceOperandWith(0, LoopID);
362
3
  HasUserTransforms = true;
363
3
  return LoopID;
364
3
}
365
366
MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
367
                                           ArrayRef<Metadata *> LoopProperties,
368
14.8k
                                           bool &HasUserTransforms) {
369
14.8k
  LLVMContext &Ctx = Header->getContext();
370
14.8k
371
14.8k
  Optional<bool> Enabled;
372
14.8k
  if (Attrs.UnrollEnable == LoopAttributes::Disable)
373
19
    Enabled = false;
374
14.8k
  else if (Attrs.UnrollEnable == LoopAttributes::Full)
375
2
    Enabled = true;
376
14.8k
377
14.8k
  if (Enabled != true) {
378
14.8k
    SmallVector<Metadata *, 4> NewLoopProperties;
379
14.8k
    if (Enabled == false) {
380
19
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
381
19
      NewLoopProperties.push_back(
382
19
          MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
383
19
      LoopProperties = NewLoopProperties;
384
19
    }
385
14.8k
    return createLoopDistributeMetadata(Attrs, LoopProperties,
386
14.8k
                                        HasUserTransforms);
387
14.8k
  }
388
2
389
2
  SmallVector<Metadata *, 4> Args;
390
2
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
391
2
  Args.push_back(TempNode.get());
392
2
  Args.append(LoopProperties.begin(), LoopProperties.end());
393
2
  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
394
2
395
2
  // No follow-up: there is no loop after full unrolling.
396
2
  // TODO: Warn if there are transformations after full unrolling.
397
2
398
2
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
399
2
  LoopID->replaceOperandWith(0, LoopID);
400
2
  HasUserTransforms = true;
401
2
  return LoopID;
402
2
}
403
404
MDNode *LoopInfo::createMetadata(
405
    const LoopAttributes &Attrs,
406
    llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
407
14.8k
    bool &HasUserTransforms) {
408
14.8k
  SmallVector<Metadata *, 3> LoopProperties;
409
14.8k
410
14.8k
  // If we have a valid start debug location for the loop, add it.
411
14.8k
  if (StartLoc) {
412
8.01k
    LoopProperties.push_back(StartLoc.getAsMDNode());
413
8.01k
414
8.01k
    // If we also have a valid end debug location for the loop, add it.
415
8.01k
    if (EndLoc)
416
8.01k
      LoopProperties.push_back(EndLoc.getAsMDNode());
417
8.01k
  }
418
14.8k
419
14.8k
  assert(!!AccGroup == Attrs.IsParallel &&
420
14.8k
         "There must be an access group iff the loop is parallel");
421
14.8k
  if (Attrs.IsParallel) {
422
3.86k
    LLVMContext &Ctx = Header->getContext();
423
3.86k
    LoopProperties.push_back(MDNode::get(
424
3.86k
        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
425
3.86k
  }
426
14.8k
427
14.8k
  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
428
14.8k
                        AdditionalLoopProperties.end());
429
14.8k
  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
430
14.8k
}
431
432
LoopAttributes::LoopAttributes(bool IsParallel)
433
    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
434
      UnrollEnable(LoopAttributes::Unspecified),
435
      UnrollAndJamEnable(LoopAttributes::Unspecified),
436
      VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
437
      InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
438
      DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
439
243k
      PipelineInitiationInterval(0) {}
440
441
21.5k
void LoopAttributes::clear() {
442
21.5k
  IsParallel = false;
443
21.5k
  VectorizeWidth = 0;
444
21.5k
  InterleaveCount = 0;
445
21.5k
  UnrollCount = 0;
446
21.5k
  UnrollAndJamCount = 0;
447
21.5k
  VectorizeEnable = LoopAttributes::Unspecified;
448
21.5k
  UnrollEnable = LoopAttributes::Unspecified;
449
21.5k
  UnrollAndJamEnable = LoopAttributes::Unspecified;
450
21.5k
  VectorizePredicateEnable = LoopAttributes::Unspecified;
451
21.5k
  DistributeEnable = LoopAttributes::Unspecified;
452
21.5k
  PipelineDisabled = false;
453
21.5k
  PipelineInitiationInterval = 0;
454
21.5k
}
455
456
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
457
                   const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
458
                   LoopInfo *Parent)
459
    : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
460
21.5k
      Parent(Parent) {
461
21.5k
462
21.5k
  if (Attrs.IsParallel) {
463
3.86k
    // Create an access group for this loop.
464
3.86k
    LLVMContext &Ctx = Header->getContext();
465
3.86k
    AccGroup = MDNode::getDistinct(Ctx, {});
466
3.86k
  }
467
21.5k
468
21.5k
  if (!Attrs.IsParallel && 
Attrs.VectorizeWidth == 017.7k
&&
469
21.5k
      
Attrs.InterleaveCount == 017.3k
&&
Attrs.UnrollCount == 017.3k
&&
470
21.5k
      
Attrs.UnrollAndJamCount == 017.3k
&&
!Attrs.PipelineDisabled17.3k
&&
471
21.5k
      
Attrs.PipelineInitiationInterval == 017.3k
&&
472
21.5k
      
Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified17.3k
&&
473
21.5k
      
Attrs.VectorizeEnable == LoopAttributes::Unspecified17.3k
&&
474
21.5k
      
Attrs.UnrollEnable == LoopAttributes::Unspecified14.4k
&&
475
21.5k
      
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified14.3k
&&
476
21.5k
      
Attrs.DistributeEnable == LoopAttributes::Unspecified14.3k
&&
!StartLoc14.3k
&&
477
21.5k
      
!EndLoc6.72k
)
478
6.72k
    return;
479
14.8k
480
14.8k
  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
481
14.8k
}
482
483
21.5k
void LoopInfo::finish() {
484
21.5k
  // We did not annotate the loop body instructions because there are no
485
21.5k
  // attributes for this loop.
486
21.5k
  if (!TempLoopID)
487
6.72k
    return;
488
14.8k
489
14.8k
  MDNode *LoopID;
490
14.8k
  LoopAttributes CurLoopAttr = Attrs;
491
14.8k
  LLVMContext &Ctx = Header->getContext();
492
14.8k
493
14.8k
  if (Parent && 
(3.33k
Parent->Attrs.UnrollAndJamEnable3.33k
||
494
3.33k
                 Parent->Attrs.UnrollAndJamCount != 0)) {
495
1
    // Parent unroll-and-jams this loop.
496
1
    // Split the transformations in those that happens before the unroll-and-jam
497
1
    // and those after.
498
1
499
1
    LoopAttributes BeforeJam, AfterJam;
500
1
501
1
    BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
502
1
503
1
    BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
504
1
    BeforeJam.InterleaveCount = Attrs.InterleaveCount;
505
1
    BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
506
1
    BeforeJam.DistributeEnable = Attrs.DistributeEnable;
507
1
    BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
508
1
509
1
    switch (Attrs.UnrollEnable) {
510
1
    case LoopAttributes::Unspecified:
511
1
    case LoopAttributes::Disable:
512
1
      BeforeJam.UnrollEnable = Attrs.UnrollEnable;
513
1
      AfterJam.UnrollEnable = Attrs.UnrollEnable;
514
1
      break;
515
1
    case LoopAttributes::Full:
516
0
      BeforeJam.UnrollEnable = LoopAttributes::Full;
517
0
      break;
518
1
    case LoopAttributes::Enable:
519
0
      AfterJam.UnrollEnable = LoopAttributes::Enable;
520
0
      break;
521
1
    }
522
1
523
1
    AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
524
1
    AfterJam.UnrollCount = Attrs.UnrollCount;
525
1
    AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
526
1
    AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
527
1
528
1
    // If this loop is subject of an unroll-and-jam by the parent loop, and has
529
1
    // an unroll-and-jam annotation itself, we have to decide whether to first
530
1
    // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
531
1
    // UnrollAndJam pass processes loops from inner to outer, so we apply the
532
1
    // inner first.
533
1
    BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
534
1
    BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
535
1
536
1
    // Set the inner followup metadata to process by the outer loop. Only
537
1
    // consider the first inner loop.
538
1
    if (!Parent->UnrollAndJamInnerFollowup) {
539
1
      // Splitting the attributes into a BeforeJam and an AfterJam part will
540
1
      // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
541
1
      // to be forwarded to the AfterJam part. We detect the situation here and
542
1
      // add it manually.
543
1
      SmallVector<Metadata *, 1> BeforeLoopProperties;
544
1
      if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
545
1
          
BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified0
||
546
1
          
BeforeJam.InterleaveCount != 00
||
BeforeJam.VectorizeWidth != 00
)
547
1
        BeforeLoopProperties.push_back(
548
1
            MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
549
1
550
1
      bool InnerFollowupHasTransform = false;
551
1
      MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
552
1
                                             InnerFollowupHasTransform);
553
1
      if (InnerFollowupHasTransform)
554
1
        Parent->UnrollAndJamInnerFollowup = InnerFollowup;
555
1
    }
556
1
557
1
    CurLoopAttr = BeforeJam;
558
1
  }
559
14.8k
560
14.8k
  bool HasUserTransforms = false;
561
14.8k
  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
562
14.8k
  TempLoopID->replaceAllUsesWith(LoopID);
563
14.8k
}
564
565
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
566
21.5k
                         const llvm::DebugLoc &EndLoc) {
567
21.5k
  Active.emplace_back(
568
21.5k
      new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
569
21.5k
                   Active.empty() ? 
nullptr17.8k
:
Active.back().get()3.70k
));
570
21.5k
  // Clear the attributes so nested loops do not inherit them.
571
21.5k
  StagedAttrs.clear();
572
21.5k
}
573
574
void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
575
                         ArrayRef<const clang::Attr *> Attrs,
576
                         const llvm::DebugLoc &StartLoc,
577
10.5k
                         const llvm::DebugLoc &EndLoc) {
578
10.5k
579
10.5k
  // Identify loop hint attributes from Attrs.
580
10.5k
  for (const auto *Attr : Attrs) {
581
136
    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
582
136
    const OpenCLUnrollHintAttr *OpenCLHint =
583
136
        dyn_cast<OpenCLUnrollHintAttr>(Attr);
584
136
585
136
    // Skip non loop hint attributes
586
136
    if (!LH && 
!OpenCLHint18
) {
587
0
      continue;
588
0
    }
589
136
590
136
    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
591
136
    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
592
136
    unsigned ValueInt = 1;
593
136
    // Translate opencl_unroll_hint attribute argument to
594
136
    // equivalent LoopHintAttr enums.
595
136
    // OpenCL v2.0 s6.11.5:
596
136
    // 0 - enable unroll (no argument).
597
136
    // 1 - disable unroll.
598
136
    // other positive integer n - unroll by n.
599
136
    if (OpenCLHint) {
600
18
      ValueInt = OpenCLHint->getUnrollHint();
601
18
      if (ValueInt == 0) {
602
6
        State = LoopHintAttr::Enable;
603
12
      } else if (ValueInt != 1) {
604
6
        Option = LoopHintAttr::UnrollCount;
605
6
        State = LoopHintAttr::Numeric;
606
6
      }
607
118
    } else if (LH) {
608
118
      auto *ValueExpr = LH->getValue();
609
118
      if (ValueExpr) {
610
47
        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
611
47
        ValueInt = ValueAPS.getSExtValue();
612
47
      }
613
118
614
118
      Option = LH->getOption();
615
118
      State = LH->getState();
616
118
    }
617
136
    switch (State) {
618
36
    case LoopHintAttr::Disable:
619
36
      switch (Option) {
620
4
      case LoopHintAttr::Vectorize:
621
4
        // Disable vectorization by specifying a width of 1.
622
4
        setVectorizeWidth(1);
623
4
        break;
624
6
      case LoopHintAttr::Interleave:
625
6
        // Disable interleaving by speciyfing a count of 1.
626
6
        setInterleaveCount(1);
627
6
        break;
628
19
      case LoopHintAttr::Unroll:
629
19
        setUnrollState(LoopAttributes::Disable);
630
19
        break;
631
2
      case LoopHintAttr::UnrollAndJam:
632
2
        setUnrollAndJamState(LoopAttributes::Disable);
633
2
        break;
634
1
      case LoopHintAttr::VectorizePredicate:
635
1
        setVectorizePredicateState(LoopAttributes::Disable);
636
1
        break;
637
2
      case LoopHintAttr::Distribute:
638
2
        setDistributeState(false);
639
2
        break;
640
2
      case LoopHintAttr::PipelineDisabled:
641
2
        setPipelineDisabled(true);
642
2
        break;
643
0
      case LoopHintAttr::UnrollCount:
644
0
      case LoopHintAttr::UnrollAndJamCount:
645
0
      case LoopHintAttr::VectorizeWidth:
646
0
      case LoopHintAttr::InterleaveCount:
647
0
      case LoopHintAttr::PipelineInitiationInterval:
648
0
        llvm_unreachable("Options cannot be disabled.");
649
0
        break;
650
36
      }
651
36
      break;
652
36
    case LoopHintAttr::Enable:
653
30
      switch (Option) {
654
11
      case LoopHintAttr::Vectorize:
655
11
      case LoopHintAttr::Interleave:
656
11
        setVectorizeEnable(true);
657
11
        break;
658
11
      case LoopHintAttr::Unroll:
659
10
        setUnrollState(LoopAttributes::Enable);
660
10
        break;
661
11
      case LoopHintAttr::UnrollAndJam:
662
2
        setUnrollAndJamState(LoopAttributes::Enable);
663
2
        break;
664
11
      case LoopHintAttr::VectorizePredicate:
665
3
        setVectorizePredicateState(LoopAttributes::Enable);
666
3
        break;
667
11
      case LoopHintAttr::Distribute:
668
4
        setDistributeState(true);
669
4
        break;
670
11
      case LoopHintAttr::UnrollCount:
671
0
      case LoopHintAttr::UnrollAndJamCount:
672
0
      case LoopHintAttr::VectorizeWidth:
673
0
      case LoopHintAttr::InterleaveCount:
674
0
      case LoopHintAttr::PipelineDisabled:
675
0
      case LoopHintAttr::PipelineInitiationInterval:
676
0
        llvm_unreachable("Options cannot enabled.");
677
0
        break;
678
30
      }
679
30
      break;
680
30
    case LoopHintAttr::AssumeSafety:
681
15
      switch (Option) {
682
15
      case LoopHintAttr::Vectorize:
683
15
      case LoopHintAttr::Interleave:
684
15
        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
685
15
        setParallel(true);
686
15
        setVectorizeEnable(true);
687
15
        break;
688
15
      case LoopHintAttr::Unroll:
689
0
      case LoopHintAttr::UnrollAndJam:
690
0
      case LoopHintAttr::VectorizePredicate:
691
0
      case LoopHintAttr::UnrollCount:
692
0
      case LoopHintAttr::UnrollAndJamCount:
693
0
      case LoopHintAttr::VectorizeWidth:
694
0
      case LoopHintAttr::InterleaveCount:
695
0
      case LoopHintAttr::Distribute:
696
0
      case LoopHintAttr::PipelineDisabled:
697
0
      case LoopHintAttr::PipelineInitiationInterval:
698
0
        llvm_unreachable("Options cannot be used to assume mem safety.");
699
0
        break;
700
15
      }
701
15
      break;
702
15
    case LoopHintAttr::Full:
703
2
      switch (Option) {
704
2
      case LoopHintAttr::Unroll:
705
2
        setUnrollState(LoopAttributes::Full);
706
2
        break;
707
0
      case LoopHintAttr::UnrollAndJam:
708
0
        setUnrollAndJamState(LoopAttributes::Full);
709
0
        break;
710
0
      case LoopHintAttr::Vectorize:
711
0
      case LoopHintAttr::Interleave:
712
0
      case LoopHintAttr::UnrollCount:
713
0
      case LoopHintAttr::UnrollAndJamCount:
714
0
      case LoopHintAttr::VectorizeWidth:
715
0
      case LoopHintAttr::InterleaveCount:
716
0
      case LoopHintAttr::Distribute:
717
0
      case LoopHintAttr::PipelineDisabled:
718
0
      case LoopHintAttr::PipelineInitiationInterval:
719
0
      case LoopHintAttr::VectorizePredicate:
720
0
        llvm_unreachable("Options cannot be used with 'full' hint.");
721
0
        break;
722
2
      }
723
2
      break;
724
53
    case LoopHintAttr::Numeric:
725
53
      switch (Option) {
726
16
      case LoopHintAttr::VectorizeWidth:
727
16
        setVectorizeWidth(ValueInt);
728
16
        break;
729
10
      case LoopHintAttr::InterleaveCount:
730
10
        setInterleaveCount(ValueInt);
731
10
        break;
732
22
      case LoopHintAttr::UnrollCount:
733
22
        setUnrollCount(ValueInt);
734
22
        break;
735
2
      case LoopHintAttr::UnrollAndJamCount:
736
2
        setUnrollAndJamCount(ValueInt);
737
2
        break;
738
3
      case LoopHintAttr::PipelineInitiationInterval:
739
3
        setPipelineInitiationInterval(ValueInt);
740
3
        break;
741
0
      case LoopHintAttr::Unroll:
742
0
      case LoopHintAttr::UnrollAndJam:
743
0
      case LoopHintAttr::VectorizePredicate:
744
0
      case LoopHintAttr::Vectorize:
745
0
      case LoopHintAttr::Interleave:
746
0
      case LoopHintAttr::Distribute:
747
0
      case LoopHintAttr::PipelineDisabled:
748
0
        llvm_unreachable("Options cannot be assigned a value.");
749
0
        break;
750
53
      }
751
53
      break;
752
136
    }
753
136
  }
754
10.5k
755
10.5k
  /// Stage the attributes.
756
10.5k
  push(Header, StartLoc, EndLoc);
757
10.5k
}
758
759
21.5k
void LoopInfoStack::pop() {
760
21.5k
  assert(!Active.empty() && "No active loops to pop");
761
21.5k
  Active.back()->finish();
762
21.5k
  Active.pop_back();
763
21.5k
}
764
765
4.16M
void LoopInfoStack::InsertHelper(Instruction *I) const {
766
4.16M
  if (I->mayReadOrWriteMemory()) {
767
2.36M
    SmallVector<Metadata *, 4> AccessGroups;
768
2.36M
    for (const auto &AL : Active) {
769
348k
      // Here we assume that every loop that has an access group is parallel.
770
348k
      if (MDNode *Group = AL->getAccessGroup())
771
39.5k
        AccessGroups.push_back(Group);
772
348k
    }
773
2.36M
    MDNode *UnionMD = nullptr;
774
2.36M
    if (AccessGroups.size() == 1)
775
39.4k
      UnionMD = cast<MDNode>(AccessGroups[0]);
776
2.32M
    else if (AccessGroups.size() >= 2)
777
50
      UnionMD = MDNode::get(I->getContext(), AccessGroups);
778
2.36M
    I->setMetadata("llvm.access.group", UnionMD);
779
2.36M
  }
780
4.16M
781
4.16M
  if (!hasInfo())
782
3.53M
    return;
783
635k
784
635k
  const LoopInfo &L = getInfo();
785
635k
  if (!L.getLoopID())
786
122k
    return;
787
513k
788
513k
  if (I->isTerminator()) {
789
102k
    for (BasicBlock *Succ : successors(I))
790
142k
      if (Succ == L.getHeader()) {
791
14.4k
        I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
792
14.4k
        break;
793
14.4k
      }
794
102k
    return;
795
102k
  }
796
513k
}