Coverage Report

Created: 2020-09-22 08:39

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "CGLoopInfo.h"
10
#include "clang/AST/ASTContext.h"
11
#include "clang/AST/Attr.h"
12
#include "clang/AST/Expr.h"
13
#include "clang/Basic/CodeGenOptions.h"
14
#include "llvm/IR/BasicBlock.h"
15
#include "llvm/IR/CFG.h"
16
#include "llvm/IR/Constants.h"
17
#include "llvm/IR/InstrTypes.h"
18
#include "llvm/IR/Instructions.h"
19
#include "llvm/IR/Metadata.h"
20
using namespace clang::CodeGen;
21
using namespace llvm;
22
23
MDNode *
24
17.1k
LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25
17.1k
  LLVMContext &Ctx = Header->getContext();
26
17.1k
  SmallVector<Metadata *, 4> NewLoopProperties;
27
17.1k
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
28
17.1k
  NewLoopProperties.push_back(TempNode.get());
29
17.1k
  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
30
31
17.1k
  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
32
17.1k
  LoopID->replaceOperandWith(0, LoopID);
33
17.1k
  return LoopID;
34
17.1k
}
35
36
MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37
                                           ArrayRef<Metadata *> LoopProperties,
38
17.1k
                                           bool &HasUserTransforms) {
39
17.1k
  LLVMContext &Ctx = Header->getContext();
40
41
17.1k
  Optional<bool> Enabled;
42
17.1k
  if (Attrs.PipelineDisabled)
43
2
    Enabled = false;
44
17.1k
  else if (Attrs.PipelineInitiationInterval != 0)
45
3
    Enabled = true;
46
47
17.1k
  if (Enabled != true) {
48
17.1k
    SmallVector<Metadata *, 4> NewLoopProperties;
49
17.1k
    if (Enabled == false) {
50
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
51
2
      NewLoopProperties.push_back(
52
2
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
53
2
                            ConstantAsMetadata::get(ConstantInt::get(
54
2
                                llvm::Type::getInt1Ty(Ctx), 1))}));
55
2
      LoopProperties = NewLoopProperties;
56
2
    }
57
17.1k
    return createLoopPropertiesMetadata(LoopProperties);
58
17.1k
  }
59
60
3
  SmallVector<Metadata *, 4> Args;
61
3
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
62
3
  Args.push_back(TempNode.get());
63
3
  Args.append(LoopProperties.begin(), LoopProperties.end());
64
65
3
  if (Attrs.PipelineInitiationInterval > 0) {
66
3
    Metadata *Vals[] = {
67
3
        MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
68
3
        ConstantAsMetadata::get(ConstantInt::get(
69
3
            llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
70
3
    Args.push_back(MDNode::get(Ctx, Vals));
71
3
  }
72
73
  // No follow-up: This is the last transformation.
74
75
3
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
76
3
  LoopID->replaceOperandWith(0, LoopID);
77
3
  HasUserTransforms = true;
78
3
  return LoopID;
79
3
}
80
81
MDNode *
82
LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
83
                                      ArrayRef<Metadata *> LoopProperties,
84
17.1k
                                      bool &HasUserTransforms) {
85
17.1k
  LLVMContext &Ctx = Header->getContext();
86
87
17.1k
  Optional<bool> Enabled;
88
17.1k
  if (Attrs.UnrollEnable == LoopAttributes::Disable)
89
158
    Enabled = false;
90
16.9k
  else if (Attrs.UnrollEnable == LoopAttributes::Full)
91
0
    Enabled = None;
92
16.9k
  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
93
16.9k
           Attrs.UnrollCount != 0)
94
38
    Enabled = true;
95
96
17.1k
  if (Enabled != true) {
97
    // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
98
    // if unrolling is disabled.
99
17.0k
    return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
100
17.0k
  }
101
102
38
  SmallVector<Metadata *, 4> FollowupLoopProperties;
103
104
  // Apply all loop properties to the unrolled loop.
105
38
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
106
107
  // Don't unroll an already unrolled loop.
108
38
  FollowupLoopProperties.push_back(
109
38
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
110
111
38
  bool FollowupHasTransforms = false;
112
38
  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
113
38
                                              FollowupHasTransforms);
114
115
38
  SmallVector<Metadata *, 4> Args;
116
38
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
117
38
  Args.push_back(TempNode.get());
118
38
  Args.append(LoopProperties.begin(), LoopProperties.end());
119
120
  // Setting unroll.count
121
38
  if (Attrs.UnrollCount > 0) {
122
27
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
123
27
                        ConstantAsMetadata::get(ConstantInt::get(
124
27
                            llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
125
27
    Args.push_back(MDNode::get(Ctx, Vals));
126
27
  }
127
128
  // Setting unroll.full or unroll.disable
129
38
  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
130
11
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
131
11
    Args.push_back(MDNode::get(Ctx, Vals));
132
11
  }
133
134
38
  if (FollowupHasTransforms)
135
2
    Args.push_back(MDNode::get(
136
2
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
137
138
38
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
139
38
  LoopID->replaceOperandWith(0, LoopID);
140
38
  HasUserTransforms = true;
141
38
  return LoopID;
142
38
}
143
144
MDNode *
145
LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
146
                                     ArrayRef<Metadata *> LoopProperties,
147
17.1k
                                     bool &HasUserTransforms) {
148
17.1k
  LLVMContext &Ctx = Header->getContext();
149
150
17.1k
  Optional<bool> Enabled;
151
17.1k
  if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
152
2
    Enabled = false;
153
17.1k
  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
154
17.1k
           Attrs.UnrollAndJamCount != 0)
155
4
    Enabled = true;
156
157
17.1k
  if (Enabled != true) {
158
17.1k
    SmallVector<Metadata *, 4> NewLoopProperties;
159
17.1k
    if (Enabled == false) {
160
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
161
2
      NewLoopProperties.push_back(MDNode::get(
162
2
          Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
163
2
      LoopProperties = NewLoopProperties;
164
2
    }
165
17.1k
    return createPartialUnrollMetadata(Attrs, LoopProperties,
166
17.1k
                                       HasUserTransforms);
167
17.1k
  }
168
169
4
  SmallVector<Metadata *, 4> FollowupLoopProperties;
170
4
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
171
4
  FollowupLoopProperties.push_back(
172
4
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
173
174
4
  bool FollowupHasTransforms = false;
175
4
  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
176
4
                                                 FollowupHasTransforms);
177
178
4
  SmallVector<Metadata *, 4> Args;
179
4
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
180
4
  Args.push_back(TempNode.get());
181
4
  Args.append(LoopProperties.begin(), LoopProperties.end());
182
183
  // Setting unroll_and_jam.count
184
4
  if (Attrs.UnrollAndJamCount > 0) {
185
2
    Metadata *Vals[] = {
186
2
        MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
187
2
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
188
2
                                                 Attrs.UnrollAndJamCount))};
189
2
    Args.push_back(MDNode::get(Ctx, Vals));
190
2
  }
191
192
4
  if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
193
2
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
194
2
    Args.push_back(MDNode::get(Ctx, Vals));
195
2
  }
196
197
4
  if (FollowupHasTransforms)
198
1
    Args.push_back(MDNode::get(
199
1
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
200
1
              Followup}));
201
202
4
  if (UnrollAndJamInnerFollowup)
203
1
    Args.push_back(MDNode::get(
204
1
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
205
1
              UnrollAndJamInnerFollowup}));
206
207
4
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
208
4
  LoopID->replaceOperandWith(0, LoopID);
209
4
  HasUserTransforms = true;
210
4
  return LoopID;
211
4
}
212
213
MDNode *
214
LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
215
                                      ArrayRef<Metadata *> LoopProperties,
216
17.1k
                                      bool &HasUserTransforms) {
217
17.1k
  LLVMContext &Ctx = Header->getContext();
218
219
17.1k
  Optional<bool> Enabled;
220
17.1k
  if (Attrs.VectorizeEnable == LoopAttributes::Disable)
221
283
    Enabled = false;
222
16.8k
  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
223
8.52k
           Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
224
8.51k
           Attrs.InterleaveCount != 0 || 
Attrs.VectorizeWidth != 08.51k
)
225
8.33k
    Enabled = true;
226
227
17.1k
  if (Enabled != true) {
228
8.78k
    SmallVector<Metadata *, 4> NewLoopProperties;
229
8.78k
    if (Enabled == false) {
230
283
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
231
283
      NewLoopProperties.push_back(
232
283
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
233
283
                            ConstantAsMetadata::get(ConstantInt::get(
234
283
                                llvm::Type::getInt1Ty(Ctx), 0))}));
235
283
      LoopProperties = NewLoopProperties;
236
283
    }
237
8.78k
    return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
238
8.78k
  }
239
240
  // Apply all loop properties to the vectorized loop.
241
8.33k
  SmallVector<Metadata *, 4> FollowupLoopProperties;
242
8.33k
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
243
244
  // Don't vectorize an already vectorized loop.
245
8.33k
  FollowupLoopProperties.push_back(
246
8.33k
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
247
248
8.33k
  bool FollowupHasTransforms = false;
249
8.33k
  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
250
8.33k
                                                FollowupHasTransforms);
251
252
8.33k
  SmallVector<Metadata *, 4> Args;
253
8.33k
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
254
8.33k
  Args.push_back(TempNode.get());
255
8.33k
  Args.append(LoopProperties.begin(), LoopProperties.end());
256
257
  // Setting vectorize.predicate
258
8.33k
  bool IsVectorPredicateEnabled = false;
259
8.33k
  if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
260
4
      Attrs.VectorizeEnable != LoopAttributes::Disable &&
261
4
      Attrs.VectorizeWidth < 1) {
262
263
3
    IsVectorPredicateEnabled =
264
3
        (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
265
266
3
    Metadata *Vals[] = {
267
3
        MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
268
3
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
269
3
                                                 IsVectorPredicateEnabled))};
270
3
    Args.push_back(MDNode::get(Ctx, Vals));
271
3
  }
272
273
  // Setting vectorize.width
274
8.33k
  if (Attrs.VectorizeWidth > 0) {
275
441
    Metadata *Vals[] = {
276
441
        MDString::get(Ctx, "llvm.loop.vectorize.width"),
277
441
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
278
441
                                                 Attrs.VectorizeWidth))};
279
441
    Args.push_back(MDNode::get(Ctx, Vals));
280
441
  }
281
282
  // Setting interleave.count
283
8.33k
  if (Attrs.InterleaveCount > 0) {
284
15
    Metadata *Vals[] = {
285
15
        MDString::get(Ctx, "llvm.loop.interleave.count"),
286
15
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
287
15
                                                 Attrs.InterleaveCount))};
288
15
    Args.push_back(MDNode::get(Ctx, Vals));
289
15
  }
290
291
  // vectorize.enable is set if:
292
  // 1) loop hint vectorize.enable is set, or
293
  // 2) it is implied when vectorize.predicate is set, or
294
  // 3) it is implied when vectorize.width is set.
295
8.33k
  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
296
19
      IsVectorPredicateEnabled ||
297
8.32k
      
Attrs.VectorizeWidth > 118
) {
298
8.32k
    bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
299
8.32k
    Args.push_back(
300
8.32k
        MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
301
8.32k
                          ConstantAsMetadata::get(ConstantInt::get(
302
8.32k
                              llvm::Type::getInt1Ty(Ctx), AttrVal))}));
303
8.32k
  }
304
305
8.33k
  if (FollowupHasTransforms)
306
8
    Args.push_back(MDNode::get(
307
8
        Ctx,
308
8
        {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
309
310
8.33k
  MDNode *LoopID = MDNode::get(Ctx, Args);
311
8.33k
  LoopID->replaceOperandWith(0, LoopID);
312
8.33k
  HasUserTransforms = true;
313
8.33k
  return LoopID;
314
8.33k
}
315
316
MDNode *
317
LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
318
                                       ArrayRef<Metadata *> LoopProperties,
319
17.1k
                                       bool &HasUserTransforms) {
320
17.1k
  LLVMContext &Ctx = Header->getContext();
321
322
17.1k
  Optional<bool> Enabled;
323
17.1k
  if (Attrs.DistributeEnable == LoopAttributes::Disable)
324
2
    Enabled = false;
325
17.1k
  if (Attrs.DistributeEnable == LoopAttributes::Enable)
326
3
    Enabled = true;
327
328
17.1k
  if (Enabled != true) {
329
17.1k
    SmallVector<Metadata *, 4> NewLoopProperties;
330
17.1k
    if (Enabled == false) {
331
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
332
2
      NewLoopProperties.push_back(
333
2
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
334
2
                            ConstantAsMetadata::get(ConstantInt::get(
335
2
                                llvm::Type::getInt1Ty(Ctx), 0))}));
336
2
      LoopProperties = NewLoopProperties;
337
2
    }
338
17.1k
    return createLoopVectorizeMetadata(Attrs, LoopProperties,
339
17.1k
                                       HasUserTransforms);
340
17.1k
  }
341
342
3
  bool FollowupHasTransforms = false;
343
3
  MDNode *Followup =
344
3
      createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
345
346
3
  SmallVector<Metadata *, 4> Args;
347
3
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
348
3
  Args.push_back(TempNode.get());
349
3
  Args.append(LoopProperties.begin(), LoopProperties.end());
350
351
3
  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
352
3
                      ConstantAsMetadata::get(ConstantInt::get(
353
3
                          llvm::Type::getInt1Ty(Ctx),
354
3
                          (Attrs.DistributeEnable == LoopAttributes::Enable)))};
355
3
  Args.push_back(MDNode::get(Ctx, Vals));
356
357
3
  if (FollowupHasTransforms)
358
2
    Args.push_back(MDNode::get(
359
2
        Ctx,
360
2
        {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
361
362
3
  MDNode *LoopID = MDNode::get(Ctx, Args);
363
3
  LoopID->replaceOperandWith(0, LoopID);
364
3
  HasUserTransforms = true;
365
3
  return LoopID;
366
3
}
367
368
MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
369
                                           ArrayRef<Metadata *> LoopProperties,
370
17.1k
                                           bool &HasUserTransforms) {
371
17.1k
  LLVMContext &Ctx = Header->getContext();
372
373
17.1k
  Optional<bool> Enabled;
374
17.1k
  if (Attrs.UnrollEnable == LoopAttributes::Disable)
375
158
    Enabled = false;
376
16.9k
  else if (Attrs.UnrollEnable == LoopAttributes::Full)
377
4
    Enabled = true;
378
379
17.1k
  if (Enabled != true) {
380
17.1k
    SmallVector<Metadata *, 4> NewLoopProperties;
381
17.1k
    if (Enabled == false) {
382
158
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
383
158
      NewLoopProperties.push_back(
384
158
          MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
385
158
      LoopProperties = NewLoopProperties;
386
158
    }
387
17.1k
    return createLoopDistributeMetadata(Attrs, LoopProperties,
388
17.1k
                                        HasUserTransforms);
389
17.1k
  }
390
391
4
  SmallVector<Metadata *, 4> Args;
392
4
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
393
4
  Args.push_back(TempNode.get());
394
4
  Args.append(LoopProperties.begin(), LoopProperties.end());
395
4
  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
396
397
  // No follow-up: there is no loop after full unrolling.
398
  // TODO: Warn if there are transformations after full unrolling.
399
400
4
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
401
4
  LoopID->replaceOperandWith(0, LoopID);
402
4
  HasUserTransforms = true;
403
4
  return LoopID;
404
4
}
405
406
MDNode *LoopInfo::createMetadata(
407
    const LoopAttributes &Attrs,
408
    llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
409
17.1k
    bool &HasUserTransforms) {
410
17.1k
  SmallVector<Metadata *, 3> LoopProperties;
411
412
  // If we have a valid start debug location for the loop, add it.
413
17.1k
  if (StartLoc) {
414
8.28k
    LoopProperties.push_back(StartLoc.getAsMDNode());
415
416
    // If we also have a valid end debug location for the loop, add it.
417
8.28k
    if (EndLoc)
418
8.28k
      LoopProperties.push_back(EndLoc.getAsMDNode());
419
8.28k
  }
420
421
17.1k
  assert(!!AccGroup == Attrs.IsParallel &&
422
17.1k
         "There must be an access group iff the loop is parallel");
423
17.1k
  if (Attrs.IsParallel) {
424
4.73k
    LLVMContext &Ctx = Header->getContext();
425
4.73k
    LoopProperties.push_back(MDNode::get(
426
4.73k
        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
427
4.73k
  }
428
429
17.1k
  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
430
17.1k
                        AdditionalLoopProperties.end());
431
17.1k
  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
432
17.1k
}
433
434
LoopAttributes::LoopAttributes(bool IsParallel)
435
    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
436
      UnrollEnable(LoopAttributes::Unspecified),
437
      UnrollAndJamEnable(LoopAttributes::Unspecified),
438
      VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
439
      InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
440
      DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
441
288k
      PipelineInitiationInterval(0) {}
442
443
25.7k
void LoopAttributes::clear() {
444
25.7k
  IsParallel = false;
445
25.7k
  VectorizeWidth = 0;
446
25.7k
  InterleaveCount = 0;
447
25.7k
  UnrollCount = 0;
448
25.7k
  UnrollAndJamCount = 0;
449
25.7k
  VectorizeEnable = LoopAttributes::Unspecified;
450
25.7k
  UnrollEnable = LoopAttributes::Unspecified;
451
25.7k
  UnrollAndJamEnable = LoopAttributes::Unspecified;
452
25.7k
  VectorizePredicateEnable = LoopAttributes::Unspecified;
453
25.7k
  DistributeEnable = LoopAttributes::Unspecified;
454
25.7k
  PipelineDisabled = false;
455
25.7k
  PipelineInitiationInterval = 0;
456
25.7k
}
457
458
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
459
                   const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
460
                   LoopInfo *Parent)
461
    : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
462
25.7k
      Parent(Parent) {
463
464
25.7k
  if (Attrs.IsParallel) {
465
    // Create an access group for this loop.
466
4.73k
    LLVMContext &Ctx = Header->getContext();
467
4.73k
    AccGroup = MDNode::getDistinct(Ctx, {});
468
4.73k
  }
469
470
25.7k
  if (!Attrs.IsParallel && 
Attrs.VectorizeWidth == 021.0k
&&
471
20.7k
      Attrs.InterleaveCount == 0 && 
Attrs.UnrollCount == 020.7k
&&
472
20.7k
      Attrs.UnrollAndJamCount == 0 && 
!Attrs.PipelineDisabled20.6k
&&
473
20.6k
      Attrs.PipelineInitiationInterval == 0 &&
474
20.6k
      Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
475
20.6k
      Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
476
16.6k
      Attrs.UnrollEnable == LoopAttributes::Unspecified &&
477
16.5k
      Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
478
16.5k
      Attrs.DistributeEnable == LoopAttributes::Unspecified && 
!StartLoc16.5k
&&
479
8.66k
      !EndLoc)
480
8.66k
    return;
481
482
17.1k
  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
483
17.1k
}
484
485
25.7k
void LoopInfo::finish() {
486
  // We did not annotate the loop body instructions because there are no
487
  // attributes for this loop.
488
25.7k
  if (!TempLoopID)
489
8.66k
    return;
490
491
17.1k
  MDNode *LoopID;
492
17.1k
  LoopAttributes CurLoopAttr = Attrs;
493
17.1k
  LLVMContext &Ctx = Header->getContext();
494
495
17.1k
  if (Parent && 
(3.61k
Parent->Attrs.UnrollAndJamEnable3.61k
||
496
3.61k
                 Parent->Attrs.UnrollAndJamCount != 0)) {
497
    // Parent unroll-and-jams this loop.
498
    // Split the transformations in those that happens before the unroll-and-jam
499
    // and those after.
500
501
1
    LoopAttributes BeforeJam, AfterJam;
502
503
1
    BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
504
505
1
    BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
506
1
    BeforeJam.InterleaveCount = Attrs.InterleaveCount;
507
1
    BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
508
1
    BeforeJam.DistributeEnable = Attrs.DistributeEnable;
509
1
    BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
510
511
1
    switch (Attrs.UnrollEnable) {
512
1
    case LoopAttributes::Unspecified:
513
1
    case LoopAttributes::Disable:
514
1
      BeforeJam.UnrollEnable = Attrs.UnrollEnable;
515
1
      AfterJam.UnrollEnable = Attrs.UnrollEnable;
516
1
      break;
517
0
    case LoopAttributes::Full:
518
0
      BeforeJam.UnrollEnable = LoopAttributes::Full;
519
0
      break;
520
0
    case LoopAttributes::Enable:
521
0
      AfterJam.UnrollEnable = LoopAttributes::Enable;
522
0
      break;
523
1
    }
524
525
1
    AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
526
1
    AfterJam.UnrollCount = Attrs.UnrollCount;
527
1
    AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
528
1
    AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
529
530
    // If this loop is subject of an unroll-and-jam by the parent loop, and has
531
    // an unroll-and-jam annotation itself, we have to decide whether to first
532
    // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
533
    // UnrollAndJam pass processes loops from inner to outer, so we apply the
534
    // inner first.
535
1
    BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
536
1
    BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
537
538
    // Set the inner followup metadata to process by the outer loop. Only
539
    // consider the first inner loop.
540
1
    if (!Parent->UnrollAndJamInnerFollowup) {
541
      // Splitting the attributes into a BeforeJam and an AfterJam part will
542
      // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
543
      // to be forwarded to the AfterJam part. We detect the situation here and
544
      // add it manually.
545
1
      SmallVector<Metadata *, 1> BeforeLoopProperties;
546
1
      if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
547
0
          BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
548
0
          BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
549
1
        BeforeLoopProperties.push_back(
550
1
            MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
551
552
1
      bool InnerFollowupHasTransform = false;
553
1
      MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
554
1
                                             InnerFollowupHasTransform);
555
1
      if (InnerFollowupHasTransform)
556
1
        Parent->UnrollAndJamInnerFollowup = InnerFollowup;
557
1
    }
558
559
1
    CurLoopAttr = BeforeJam;
560
1
  }
561
562
17.1k
  bool HasUserTransforms = false;
563
17.1k
  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
564
17.1k
  TempLoopID->replaceAllUsesWith(LoopID);
565
17.1k
}
566
567
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
568
25.7k
                         const llvm::DebugLoc &EndLoc) {
569
25.7k
  Active.emplace_back(
570
25.7k
      new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
571
21.7k
                   Active.empty() ? nullptr : 
Active.back().get()4.05k
));
572
  // Clear the attributes so nested loops do not inherit them.
573
25.7k
  StagedAttrs.clear();
574
25.7k
}
575
576
void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
577
                         const clang::CodeGenOptions &CGOpts,
578
                         ArrayRef<const clang::Attr *> Attrs,
579
                         const llvm::DebugLoc &StartLoc,
580
11.4k
                         const llvm::DebugLoc &EndLoc) {
581
582
  // Identify loop hint attributes from Attrs.
583
154
  for (const auto *Attr : Attrs) {
584
154
    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
585
154
    const OpenCLUnrollHintAttr *OpenCLHint =
586
154
        dyn_cast<OpenCLUnrollHintAttr>(Attr);
587
588
    // Skip non loop hint attributes
589
154
    if (!LH && 
!OpenCLHint25
) {
590
7
      continue;
591
7
    }
592
593
147
    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
594
147
    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
595
147
    unsigned ValueInt = 1;
596
    // Translate opencl_unroll_hint attribute argument to
597
    // equivalent LoopHintAttr enums.
598
    // OpenCL v2.0 s6.11.5:
599
    // 0 - enable unroll (no argument).
600
    // 1 - disable unroll.
601
    // other positive integer n - unroll by n.
602
147
    if (OpenCLHint) {
603
18
      ValueInt = OpenCLHint->getUnrollHint();
604
18
      if (ValueInt == 0) {
605
6
        State = LoopHintAttr::Enable;
606
12
      } else if (ValueInt != 1) {
607
6
        Option = LoopHintAttr::UnrollCount;
608
6
        State = LoopHintAttr::Numeric;
609
6
      }
610
129
    } else if (LH) {
611
129
      auto *ValueExpr = LH->getValue();
612
129
      if (ValueExpr) {
613
52
        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
614
52
        ValueInt = ValueAPS.getSExtValue();
615
52
      }
616
617
129
      Option = LH->getOption();
618
129
      State = LH->getState();
619
129
    }
620
147
    switch (State) {
621
39
    case LoopHintAttr::Disable:
622
39
      switch (Option) {
623
6
      case LoopHintAttr::Vectorize:
624
        // Disable vectorization by specifying a width of 1.
625
6
        setVectorizeWidth(1);
626
6
        break;
627
6
      case LoopHintAttr::Interleave:
628
        // Disable interleaving by speciyfing a count of 1.
629
6
        setInterleaveCount(1);
630
6
        break;
631
20
      case LoopHintAttr::Unroll:
632
20
        setUnrollState(LoopAttributes::Disable);
633
20
        break;
634
2
      case LoopHintAttr::UnrollAndJam:
635
2
        setUnrollAndJamState(LoopAttributes::Disable);
636
2
        break;
637
1
      case LoopHintAttr::VectorizePredicate:
638
1
        setVectorizePredicateState(LoopAttributes::Disable);
639
1
        break;
640
2
      case LoopHintAttr::Distribute:
641
2
        setDistributeState(false);
642
2
        break;
643
2
      case LoopHintAttr::PipelineDisabled:
644
2
        setPipelineDisabled(true);
645
2
        break;
646
0
      case LoopHintAttr::UnrollCount:
647
0
      case LoopHintAttr::UnrollAndJamCount:
648
0
      case LoopHintAttr::VectorizeWidth:
649
0
      case LoopHintAttr::InterleaveCount:
650
0
      case LoopHintAttr::PipelineInitiationInterval:
651
0
        llvm_unreachable("Options cannot be disabled.");
652
0
        break;
653
39
      }
654
39
      break;
655
31
    case LoopHintAttr::Enable:
656
31
      switch (Option) {
657
11
      case LoopHintAttr::Vectorize:
658
11
      case LoopHintAttr::Interleave:
659
11
        setVectorizeEnable(true);
660
11
        break;
661
11
      case LoopHintAttr::Unroll:
662
11
        setUnrollState(LoopAttributes::Enable);
663
11
        break;
664
2
      case LoopHintAttr::UnrollAndJam:
665
2
        setUnrollAndJamState(LoopAttributes::Enable);
666
2
        break;
667
3
      case LoopHintAttr::VectorizePredicate:
668
3
        setVectorizePredicateState(LoopAttributes::Enable);
669
3
        break;
670
4
      case LoopHintAttr::Distribute:
671
4
        setDistributeState(true);
672
4
        break;
673
0
      case LoopHintAttr::UnrollCount:
674
0
      case LoopHintAttr::UnrollAndJamCount:
675
0
      case LoopHintAttr::VectorizeWidth:
676
0
      case LoopHintAttr::InterleaveCount:
677
0
      case LoopHintAttr::PipelineDisabled:
678
0
      case LoopHintAttr::PipelineInitiationInterval:
679
0
        llvm_unreachable("Options cannot enabled.");
680
0
        break;
681
31
      }
682
31
      break;
683
15
    case LoopHintAttr::AssumeSafety:
684
15
      switch (Option) {
685
15
      case LoopHintAttr::Vectorize:
686
15
      case LoopHintAttr::Interleave:
687
        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
688
15
        setParallel(true);
689
15
        setVectorizeEnable(true);
690
15
        break;
691
0
      case LoopHintAttr::Unroll:
692
0
      case LoopHintAttr::UnrollAndJam:
693
0
      case LoopHintAttr::VectorizePredicate:
694
0
      case LoopHintAttr::UnrollCount:
695
0
      case LoopHintAttr::UnrollAndJamCount:
696
0
      case LoopHintAttr::VectorizeWidth:
697
0
      case LoopHintAttr::InterleaveCount:
698
0
      case LoopHintAttr::Distribute:
699
0
      case LoopHintAttr::PipelineDisabled:
700
0
      case LoopHintAttr::PipelineInitiationInterval:
701
0
        llvm_unreachable("Options cannot be used to assume mem safety.");
702
0
        break;
703
15
      }
704
15
      break;
705
4
    case LoopHintAttr::Full:
706
4
      switch (Option) {
707
4
      case LoopHintAttr::Unroll:
708
4
        setUnrollState(LoopAttributes::Full);
709
4
        break;
710
0
      case LoopHintAttr::UnrollAndJam:
711
0
        setUnrollAndJamState(LoopAttributes::Full);
712
0
        break;
713
0
      case LoopHintAttr::Vectorize:
714
0
      case LoopHintAttr::Interleave:
715
0
      case LoopHintAttr::UnrollCount:
716
0
      case LoopHintAttr::UnrollAndJamCount:
717
0
      case LoopHintAttr::VectorizeWidth:
718
0
      case LoopHintAttr::InterleaveCount:
719
0
      case LoopHintAttr::Distribute:
720
0
      case LoopHintAttr::PipelineDisabled:
721
0
      case LoopHintAttr::PipelineInitiationInterval:
722
0
      case LoopHintAttr::VectorizePredicate:
723
0
        llvm_unreachable("Options cannot be used with 'full' hint.");
724
0
        break;
725
4
      }
726
4
      break;
727
58
    case LoopHintAttr::Numeric:
728
58
      switch (Option) {
729
16
      case LoopHintAttr::VectorizeWidth:
730
16
        setVectorizeWidth(ValueInt);
731
16
        break;
732
10
      case LoopHintAttr::InterleaveCount:
733
10
        setInterleaveCount(ValueInt);
734
10
        break;
735
27
      case LoopHintAttr::UnrollCount:
736
27
        setUnrollCount(ValueInt);
737
27
        break;
738
2
      case LoopHintAttr::UnrollAndJamCount:
739
2
        setUnrollAndJamCount(ValueInt);
740
2
        break;
741
3
      case LoopHintAttr::PipelineInitiationInterval:
742
3
        setPipelineInitiationInterval(ValueInt);
743
3
        break;
744
0
      case LoopHintAttr::Unroll:
745
0
      case LoopHintAttr::UnrollAndJam:
746
0
      case LoopHintAttr::VectorizePredicate:
747
0
      case LoopHintAttr::Vectorize:
748
0
      case LoopHintAttr::Interleave:
749
0
      case LoopHintAttr::Distribute:
750
0
      case LoopHintAttr::PipelineDisabled:
751
0
        llvm_unreachable("Options cannot be assigned a value.");
752
0
        break;
753
58
      }
754
58
      break;
755
147
    }
756
147
  }
757
758
11.4k
  if (CGOpts.OptimizationLevel > 0)
759
    // Disable unrolling for the loop, if unrolling is disabled (via
760
    // -fno-unroll-loops) and no pragmas override the decision.
761
229
    if (!CGOpts.UnrollLoops &&
762
149
        (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
763
143
         StagedAttrs.UnrollCount == 0))
764
138
      setUnrollState(LoopAttributes::Disable);
765
766
  /// Stage the attributes.
767
11.4k
  push(Header, StartLoc, EndLoc);
768
11.4k
}
769
770
25.7k
void LoopInfoStack::pop() {
771
25.7k
  assert(!Active.empty() && "No active loops to pop");
772
25.7k
  Active.back()->finish();
773
25.7k
  Active.pop_back();
774
25.7k
}
775
776
4.89M
void LoopInfoStack::InsertHelper(Instruction *I) const {
777
4.89M
  if (I->mayReadOrWriteMemory()) {
778
2.79M
    SmallVector<Metadata *, 4> AccessGroups;
779
396k
    for (const auto &AL : Active) {
780
      // Here we assume that every loop that has an access group is parallel.
781
396k
      if (MDNode *Group = AL->getAccessGroup())
782
46.7k
        AccessGroups.push_back(Group);
783
396k
    }
784
2.79M
    MDNode *UnionMD = nullptr;
785
2.79M
    if (AccessGroups.size() == 1)
786
46.6k
      UnionMD = cast<MDNode>(AccessGroups[0]);
787
2.74M
    else if (AccessGroups.size() >= 2)
788
50
      UnionMD = MDNode::get(I->getContext(), AccessGroups);
789
2.79M
    I->setMetadata("llvm.access.group", UnionMD);
790
2.79M
  }
791
792
4.89M
  if (!hasInfo())
793
4.17M
    return;
794
795
725k
  const LoopInfo &L = getInfo();
796
725k
  if (!L.getLoopID())
797
160k
    return;
798
799
564k
  if (I->isTerminator()) {
800
113k
    for (BasicBlock *Succ : successors(I))
801
156k
      if (Succ == L.getHeader()) {
802
16.7k
        I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
803
16.7k
        break;
804
16.7k
      }
805
113k
    return;
806
113k
  }
807
564k
}