Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/CodeGen/CGLoopInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "CGLoopInfo.h"
10
#include "clang/AST/ASTContext.h"
11
#include "clang/AST/Attr.h"
12
#include "llvm/IR/BasicBlock.h"
13
#include "llvm/IR/CFG.h"
14
#include "llvm/IR/Constants.h"
15
#include "llvm/IR/InstrTypes.h"
16
#include "llvm/IR/Instructions.h"
17
#include "llvm/IR/Metadata.h"
18
using namespace clang::CodeGen;
19
using namespace llvm;
20
21
MDNode *
22
41.8k
LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
23
41.8k
  LLVMContext &Ctx = Header->getContext();
24
41.8k
  SmallVector<Metadata *, 4> NewLoopProperties;
25
41.8k
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26
41.8k
  NewLoopProperties.push_back(TempNode.get());
27
41.8k
  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
28
41.8k
29
41.8k
  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30
41.8k
  LoopID->replaceOperandWith(0, LoopID);
31
41.8k
  return LoopID;
32
41.8k
}
33
34
MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
35
                                           ArrayRef<Metadata *> LoopProperties,
36
41.8k
                                           bool &HasUserTransforms) {
37
41.8k
  LLVMContext &Ctx = Header->getContext();
38
41.8k
39
41.8k
  Optional<bool> Enabled;
40
41.8k
  if (Attrs.PipelineDisabled)
41
2
    Enabled = false;
42
41.8k
  else if (Attrs.PipelineInitiationInterval != 0)
43
3
    Enabled = true;
44
41.8k
45
41.8k
  if (Enabled != true) {
46
41.8k
    SmallVector<Metadata *, 4> NewLoopProperties;
47
41.8k
    if (Enabled == false) {
48
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49
2
      NewLoopProperties.push_back(
50
2
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
51
2
                            ConstantAsMetadata::get(ConstantInt::get(
52
2
                                llvm::Type::getInt1Ty(Ctx), 1))}));
53
2
      LoopProperties = NewLoopProperties;
54
2
    }
55
41.8k
    return createLoopPropertiesMetadata(LoopProperties);
56
41.8k
  }
57
3
58
3
  SmallVector<Metadata *, 4> Args;
59
3
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
60
3
  Args.push_back(TempNode.get());
61
3
  Args.append(LoopProperties.begin(), LoopProperties.end());
62
3
63
3
  if (Attrs.PipelineInitiationInterval > 0) {
64
3
    Metadata *Vals[] = {
65
3
        MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66
3
        ConstantAsMetadata::get(ConstantInt::get(
67
3
            llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68
3
    Args.push_back(MDNode::get(Ctx, Vals));
69
3
  }
70
3
71
3
  // No follow-up: This is the last transformation.
72
3
73
3
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74
3
  LoopID->replaceOperandWith(0, LoopID);
75
3
  HasUserTransforms = true;
76
3
  return LoopID;
77
3
}
78
79
MDNode *
80
LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81
                                      ArrayRef<Metadata *> LoopProperties,
82
41.8k
                                      bool &HasUserTransforms) {
83
41.8k
  LLVMContext &Ctx = Header->getContext();
84
41.8k
85
41.8k
  Optional<bool> Enabled;
86
41.8k
  if (Attrs.UnrollEnable == LoopAttributes::Disable)
87
19
    Enabled = false;
88
41.8k
  else if (Attrs.UnrollEnable == LoopAttributes::Full)
89
0
    Enabled = None;
90
41.8k
  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91
41.8k
           
Attrs.UnrollCount != 041.8k
)
92
30
    Enabled = true;
93
41.8k
94
41.8k
  if (Enabled != true) {
95
41.8k
    // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96
41.8k
    // if unrolling is disabled.
97
41.8k
    return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98
41.8k
  }
99
30
100
30
  SmallVector<Metadata *, 4> FollowupLoopProperties;
101
30
102
30
  // Apply all loop properties to the unrolled loop.
103
30
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104
30
105
30
  // Don't unroll an already unrolled loop.
106
30
  FollowupLoopProperties.push_back(
107
30
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108
30
109
30
  bool FollowupHasTransforms = false;
110
30
  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111
30
                                              FollowupHasTransforms);
112
30
113
30
  SmallVector<Metadata *, 4> Args;
114
30
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115
30
  Args.push_back(TempNode.get());
116
30
  Args.append(LoopProperties.begin(), LoopProperties.end());
117
30
118
30
  // Setting unroll.count
119
30
  if (Attrs.UnrollCount > 0) {
120
22
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121
22
                        ConstantAsMetadata::get(ConstantInt::get(
122
22
                            llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123
22
    Args.push_back(MDNode::get(Ctx, Vals));
124
22
  }
125
30
126
30
  // Setting unroll.full or unroll.disable
127
30
  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128
8
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129
8
    Args.push_back(MDNode::get(Ctx, Vals));
130
8
  }
131
30
132
30
  if (FollowupHasTransforms)
133
2
    Args.push_back(MDNode::get(
134
2
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135
30
136
30
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137
30
  LoopID->replaceOperandWith(0, LoopID);
138
30
  HasUserTransforms = true;
139
30
  return LoopID;
140
30
}
141
142
MDNode *
143
LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144
                                     ArrayRef<Metadata *> LoopProperties,
145
41.8k
                                     bool &HasUserTransforms) {
146
41.8k
  LLVMContext &Ctx = Header->getContext();
147
41.8k
148
41.8k
  Optional<bool> Enabled;
149
41.8k
  if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
150
2
    Enabled = false;
151
41.8k
  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152
41.8k
           
Attrs.UnrollAndJamCount != 041.8k
)
153
4
    Enabled = true;
154
41.8k
155
41.8k
  if (Enabled != true) {
156
41.8k
    SmallVector<Metadata *, 4> NewLoopProperties;
157
41.8k
    if (Enabled == false) {
158
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159
2
      NewLoopProperties.push_back(MDNode::get(
160
2
          Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161
2
      LoopProperties = NewLoopProperties;
162
2
    }
163
41.8k
    return createPartialUnrollMetadata(Attrs, LoopProperties,
164
41.8k
                                       HasUserTransforms);
165
41.8k
  }
166
4
167
4
  SmallVector<Metadata *, 4> FollowupLoopProperties;
168
4
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169
4
  FollowupLoopProperties.push_back(
170
4
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171
4
172
4
  bool FollowupHasTransforms = false;
173
4
  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174
4
                                                 FollowupHasTransforms);
175
4
176
4
  SmallVector<Metadata *, 4> Args;
177
4
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178
4
  Args.push_back(TempNode.get());
179
4
  Args.append(LoopProperties.begin(), LoopProperties.end());
180
4
181
4
  // Setting unroll_and_jam.count
182
4
  if (Attrs.UnrollAndJamCount > 0) {
183
2
    Metadata *Vals[] = {
184
2
        MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
185
2
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186
2
                                                 Attrs.UnrollAndJamCount))};
187
2
    Args.push_back(MDNode::get(Ctx, Vals));
188
2
  }
189
4
190
4
  if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
191
2
    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
192
2
    Args.push_back(MDNode::get(Ctx, Vals));
193
2
  }
194
4
195
4
  if (FollowupHasTransforms)
196
1
    Args.push_back(MDNode::get(
197
1
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
198
1
              Followup}));
199
4
200
4
  if (UnrollAndJamInnerFollowup)
201
1
    Args.push_back(MDNode::get(
202
1
        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
203
1
              UnrollAndJamInnerFollowup}));
204
4
205
4
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206
4
  LoopID->replaceOperandWith(0, LoopID);
207
4
  HasUserTransforms = true;
208
4
  return LoopID;
209
4
}
210
211
MDNode *
212
LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
213
                                      ArrayRef<Metadata *> LoopProperties,
214
41.8k
                                      bool &HasUserTransforms) {
215
41.8k
  LLVMContext &Ctx = Header->getContext();
216
41.8k
217
41.8k
  Optional<bool> Enabled;
218
41.8k
  if (Attrs.VectorizeEnable == LoopAttributes::Disable)
219
0
    Enabled = false;
220
41.8k
  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
221
41.8k
           
Attrs.InterleaveCount != 036.3k
||
Attrs.VectorizeWidth != 036.3k
)
222
5.49k
    Enabled = true;
223
41.8k
224
41.8k
  if (Enabled != true) {
225
36.3k
    SmallVector<Metadata *, 4> NewLoopProperties;
226
36.3k
    if (Enabled == false) {
227
0
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
228
0
      NewLoopProperties.push_back(
229
0
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
230
0
                            ConstantAsMetadata::get(ConstantInt::get(
231
0
                                llvm::Type::getInt1Ty(Ctx), 0))}));
232
0
      LoopProperties = NewLoopProperties;
233
0
    }
234
36.3k
    return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
235
36.3k
  }
236
5.49k
237
5.49k
  // Apply all loop properties to the vectorized loop.
238
5.49k
  SmallVector<Metadata *, 4> FollowupLoopProperties;
239
5.49k
  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
240
5.49k
241
5.49k
  // Don't vectorize an already vectorized loop.
242
5.49k
  FollowupLoopProperties.push_back(
243
5.49k
      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
244
5.49k
245
5.49k
  bool FollowupHasTransforms = false;
246
5.49k
  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
247
5.49k
                                                FollowupHasTransforms);
248
5.49k
249
5.49k
  SmallVector<Metadata *, 4> Args;
250
5.49k
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
251
5.49k
  Args.push_back(TempNode.get());
252
5.49k
  Args.append(LoopProperties.begin(), LoopProperties.end());
253
5.49k
254
5.49k
  // Setting vectorize.width
255
5.49k
  if (Attrs.VectorizeWidth > 0) {
256
297
    Metadata *Vals[] = {
257
297
        MDString::get(Ctx, "llvm.loop.vectorize.width"),
258
297
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
259
297
                                                 Attrs.VectorizeWidth))};
260
297
    Args.push_back(MDNode::get(Ctx, Vals));
261
297
  }
262
5.49k
263
5.49k
  // Setting interleave.count
264
5.49k
  if (Attrs.InterleaveCount > 0) {
265
15
    Metadata *Vals[] = {
266
15
        MDString::get(Ctx, "llvm.loop.interleave.count"),
267
15
        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
268
15
                                                 Attrs.InterleaveCount))};
269
15
    Args.push_back(MDNode::get(Ctx, Vals));
270
15
  }
271
5.49k
272
5.49k
  // Setting vectorize.enable
273
5.49k
  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) {
274
5.48k
    Metadata *Vals[] = {
275
5.48k
        MDString::get(Ctx, "llvm.loop.vectorize.enable"),
276
5.48k
        ConstantAsMetadata::get(ConstantInt::get(
277
5.48k
            llvm::Type::getInt1Ty(Ctx),
278
5.48k
            (Attrs.VectorizeEnable == LoopAttributes::Enable)))};
279
5.48k
    Args.push_back(MDNode::get(Ctx, Vals));
280
5.48k
  }
281
5.49k
282
5.49k
  if (FollowupHasTransforms)
283
8
    Args.push_back(MDNode::get(
284
8
        Ctx,
285
8
        {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
286
5.49k
287
5.49k
  MDNode *LoopID = MDNode::get(Ctx, Args);
288
5.49k
  LoopID->replaceOperandWith(0, LoopID);
289
5.49k
  HasUserTransforms = true;
290
5.49k
  return LoopID;
291
5.49k
}
292
293
MDNode *
294
LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
295
                                       ArrayRef<Metadata *> LoopProperties,
296
41.8k
                                       bool &HasUserTransforms) {
297
41.8k
  LLVMContext &Ctx = Header->getContext();
298
41.8k
299
41.8k
  Optional<bool> Enabled;
300
41.8k
  if (Attrs.DistributeEnable == LoopAttributes::Disable)
301
2
    Enabled = false;
302
41.8k
  if (Attrs.DistributeEnable == LoopAttributes::Enable)
303
3
    Enabled = true;
304
41.8k
305
41.8k
  if (Enabled != true) {
306
41.8k
    SmallVector<Metadata *, 4> NewLoopProperties;
307
41.8k
    if (Enabled == false) {
308
2
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
309
2
      NewLoopProperties.push_back(
310
2
          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
311
2
                            ConstantAsMetadata::get(ConstantInt::get(
312
2
                                llvm::Type::getInt1Ty(Ctx), 0))}));
313
2
      LoopProperties = NewLoopProperties;
314
2
    }
315
41.8k
    return createLoopVectorizeMetadata(Attrs, LoopProperties,
316
41.8k
                                       HasUserTransforms);
317
41.8k
  }
318
3
319
3
  bool FollowupHasTransforms = false;
320
3
  MDNode *Followup =
321
3
      createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
322
3
323
3
  SmallVector<Metadata *, 4> Args;
324
3
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
325
3
  Args.push_back(TempNode.get());
326
3
  Args.append(LoopProperties.begin(), LoopProperties.end());
327
3
328
3
  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
329
3
                      ConstantAsMetadata::get(ConstantInt::get(
330
3
                          llvm::Type::getInt1Ty(Ctx),
331
3
                          (Attrs.DistributeEnable == LoopAttributes::Enable)))};
332
3
  Args.push_back(MDNode::get(Ctx, Vals));
333
3
334
3
  if (FollowupHasTransforms)
335
2
    Args.push_back(MDNode::get(
336
2
        Ctx,
337
2
        {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
338
3
339
3
  MDNode *LoopID = MDNode::get(Ctx, Args);
340
3
  LoopID->replaceOperandWith(0, LoopID);
341
3
  HasUserTransforms = true;
342
3
  return LoopID;
343
3
}
344
345
MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
346
                                           ArrayRef<Metadata *> LoopProperties,
347
41.8k
                                           bool &HasUserTransforms) {
348
41.8k
  LLVMContext &Ctx = Header->getContext();
349
41.8k
350
41.8k
  Optional<bool> Enabled;
351
41.8k
  if (Attrs.UnrollEnable == LoopAttributes::Disable)
352
19
    Enabled = false;
353
41.8k
  else if (Attrs.UnrollEnable == LoopAttributes::Full)
354
2
    Enabled = true;
355
41.8k
356
41.8k
  if (Enabled != true) {
357
41.8k
    SmallVector<Metadata *, 4> NewLoopProperties;
358
41.8k
    if (Enabled == false) {
359
19
      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
360
19
      NewLoopProperties.push_back(
361
19
          MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
362
19
      LoopProperties = NewLoopProperties;
363
19
    }
364
41.8k
    return createLoopDistributeMetadata(Attrs, LoopProperties,
365
41.8k
                                        HasUserTransforms);
366
41.8k
  }
367
2
368
2
  SmallVector<Metadata *, 4> Args;
369
2
  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
370
2
  Args.push_back(TempNode.get());
371
2
  Args.append(LoopProperties.begin(), LoopProperties.end());
372
2
  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
373
2
374
2
  // No follow-up: there is no loop after full unrolling.
375
2
  // TODO: Warn if there are transformations after full unrolling.
376
2
377
2
  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
378
2
  LoopID->replaceOperandWith(0, LoopID);
379
2
  HasUserTransforms = true;
380
2
  return LoopID;
381
2
}
382
383
MDNode *LoopInfo::createMetadata(
384
    const LoopAttributes &Attrs,
385
    llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
386
41.8k
    bool &HasUserTransforms) {
387
41.8k
  SmallVector<Metadata *, 3> LoopProperties;
388
41.8k
389
41.8k
  // If we have a valid start debug location for the loop, add it.
390
41.8k
  if (StartLoc) {
391
36.1k
    LoopProperties.push_back(StartLoc.getAsMDNode());
392
36.1k
393
36.1k
    // If we also have a valid end debug location for the loop, add it.
394
36.1k
    if (EndLoc)
395
36.1k
      LoopProperties.push_back(EndLoc.getAsMDNode());
396
36.1k
  }
397
41.8k
398
41.8k
  assert(!!AccGroup == Attrs.IsParallel &&
399
41.8k
         "There must be an access group iff the loop is parallel");
400
41.8k
  if (Attrs.IsParallel) {
401
3.14k
    LLVMContext &Ctx = Header->getContext();
402
3.14k
    LoopProperties.push_back(MDNode::get(
403
3.14k
        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
404
3.14k
  }
405
41.8k
406
41.8k
  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
407
41.8k
                        AdditionalLoopProperties.end());
408
41.8k
  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
409
41.8k
}
410
411
LoopAttributes::LoopAttributes(bool IsParallel)
412
    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
413
      UnrollEnable(LoopAttributes::Unspecified),
414
      UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
415
      InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
416
      DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
417
388k
      PipelineInitiationInterval(0) {}
418
419
83.3k
void LoopAttributes::clear() {
420
83.3k
  IsParallel = false;
421
83.3k
  VectorizeWidth = 0;
422
83.3k
  InterleaveCount = 0;
423
83.3k
  UnrollCount = 0;
424
83.3k
  UnrollAndJamCount = 0;
425
83.3k
  VectorizeEnable = LoopAttributes::Unspecified;
426
83.3k
  UnrollEnable = LoopAttributes::Unspecified;
427
83.3k
  UnrollAndJamEnable = LoopAttributes::Unspecified;
428
83.3k
  DistributeEnable = LoopAttributes::Unspecified;
429
83.3k
  PipelineDisabled = false;
430
83.3k
  PipelineInitiationInterval = 0;
431
83.3k
}
432
433
LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
434
                   const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
435
                   LoopInfo *Parent)
436
    : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
437
83.3k
      Parent(Parent) {
438
83.3k
439
83.3k
  if (Attrs.IsParallel) {
440
3.14k
    // Create an access group for this loop.
441
3.14k
    LLVMContext &Ctx = Header->getContext();
442
3.14k
    AccGroup = MDNode::getDistinct(Ctx, {});
443
3.14k
  }
444
83.3k
445
83.3k
  if (!Attrs.IsParallel && 
Attrs.VectorizeWidth == 080.1k
&&
446
83.3k
      
Attrs.InterleaveCount == 079.9k
&&
Attrs.UnrollCount == 079.9k
&&
447
83.3k
      
Attrs.UnrollAndJamCount == 079.9k
&&
!Attrs.PipelineDisabled79.9k
&&
448
83.3k
      
Attrs.PipelineInitiationInterval == 079.9k
&&
449
83.3k
      
Attrs.VectorizeEnable == LoopAttributes::Unspecified79.9k
&&
450
83.3k
      
Attrs.UnrollEnable == LoopAttributes::Unspecified77.4k
&&
451
83.3k
      
Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified77.4k
&&
452
83.3k
      
Attrs.DistributeEnable == LoopAttributes::Unspecified77.4k
&&
!StartLoc77.4k
&&
453
83.3k
      
!EndLoc41.4k
)
454
41.4k
    return;
455
41.8k
456
41.8k
  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
457
41.8k
}
458
459
83.3k
void LoopInfo::finish() {
460
83.3k
  // We did not annotate the loop body instructions because there are no
461
83.3k
  // attributes for this loop.
462
83.3k
  if (!TempLoopID)
463
41.4k
    return;
464
41.8k
465
41.8k
  MDNode *LoopID;
466
41.8k
  LoopAttributes CurLoopAttr = Attrs;
467
41.8k
  LLVMContext &Ctx = Header->getContext();
468
41.8k
469
41.8k
  if (Parent && 
(3.30k
Parent->Attrs.UnrollAndJamEnable3.30k
||
470
3.30k
                 Parent->Attrs.UnrollAndJamCount != 0)) {
471
1
    // Parent unroll-and-jams this loop.
472
1
    // Split the transformations in those that happens before the unroll-and-jam
473
1
    // and those after.
474
1
475
1
    LoopAttributes BeforeJam, AfterJam;
476
1
477
1
    BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
478
1
479
1
    BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
480
1
    BeforeJam.InterleaveCount = Attrs.InterleaveCount;
481
1
    BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
482
1
    BeforeJam.DistributeEnable = Attrs.DistributeEnable;
483
1
484
1
    switch (Attrs.UnrollEnable) {
485
1
    case LoopAttributes::Unspecified:
486
1
    case LoopAttributes::Disable:
487
1
      BeforeJam.UnrollEnable = Attrs.UnrollEnable;
488
1
      AfterJam.UnrollEnable = Attrs.UnrollEnable;
489
1
      break;
490
1
    case LoopAttributes::Full:
491
0
      BeforeJam.UnrollEnable = LoopAttributes::Full;
492
0
      break;
493
1
    case LoopAttributes::Enable:
494
0
      AfterJam.UnrollEnable = LoopAttributes::Enable;
495
0
      break;
496
1
    }
497
1
498
1
    AfterJam.UnrollCount = Attrs.UnrollCount;
499
1
    AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
500
1
    AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
501
1
502
1
    // If this loop is subject of an unroll-and-jam by the parent loop, and has
503
1
    // an unroll-and-jam annotation itself, we have to decide whether to first
504
1
    // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
505
1
    // UnrollAndJam pass processes loops from inner to outer, so we apply the
506
1
    // inner first.
507
1
    BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
508
1
    BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
509
1
510
1
    // Set the inner followup metadata to process by the outer loop. Only
511
1
    // consider the first inner loop.
512
1
    if (!Parent->UnrollAndJamInnerFollowup) {
513
1
      // Splitting the attributes into a BeforeJam and an AfterJam part will
514
1
      // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
515
1
      // to be forwarded to the AfterJam part. We detect the situation here and
516
1
      // add it manually.
517
1
      SmallVector<Metadata *, 1> BeforeLoopProperties;
518
1
      if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
519
1
          
BeforeJam.InterleaveCount != 00
||
BeforeJam.VectorizeWidth != 00
)
520
1
        BeforeLoopProperties.push_back(
521
1
            MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
522
1
523
1
      bool InnerFollowupHasTransform = false;
524
1
      MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
525
1
                                             InnerFollowupHasTransform);
526
1
      if (InnerFollowupHasTransform)
527
1
        Parent->UnrollAndJamInnerFollowup = InnerFollowup;
528
1
    }
529
1
530
1
    CurLoopAttr = BeforeJam;
531
1
  }
532
41.8k
533
41.8k
  bool HasUserTransforms = false;
534
41.8k
  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
535
41.8k
  TempLoopID->replaceAllUsesWith(LoopID);
536
41.8k
}
537
538
void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
539
83.3k
                         const llvm::DebugLoc &EndLoc) {
540
83.3k
  Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
541
83.3k
                            Active.empty() ? 
nullptr68.1k
:
&Active.back()15.1k
));
542
83.3k
  // Clear the attributes so nested loops do not inherit them.
543
83.3k
  StagedAttrs.clear();
544
83.3k
}
545
546
void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
547
                         ArrayRef<const clang::Attr *> Attrs,
548
                         const llvm::DebugLoc &StartLoc,
549
73.8k
                         const llvm::DebugLoc &EndLoc) {
550
73.8k
551
73.8k
  // Identify loop hint attributes from Attrs.
552
73.8k
  for (const auto *Attr : Attrs) {
553
119
    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
554
119
    const OpenCLUnrollHintAttr *OpenCLHint =
555
119
        dyn_cast<OpenCLUnrollHintAttr>(Attr);
556
119
557
119
    // Skip non loop hint attributes
558
119
    if (!LH && 
!OpenCLHint18
) {
559
0
      continue;
560
0
    }
561
119
562
119
    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
563
119
    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
564
119
    unsigned ValueInt = 1;
565
119
    // Translate opencl_unroll_hint attribute argument to
566
119
    // equivalent LoopHintAttr enums.
567
119
    // OpenCL v2.0 s6.11.5:
568
119
    // 0 - enable unroll (no argument).
569
119
    // 1 - disable unroll.
570
119
    // other positive integer n - unroll by n.
571
119
    if (OpenCLHint) {
572
18
      ValueInt = OpenCLHint->getUnrollHint();
573
18
      if (ValueInt == 0) {
574
6
        State = LoopHintAttr::Enable;
575
12
      } else if (ValueInt != 1) {
576
6
        Option = LoopHintAttr::UnrollCount;
577
6
        State = LoopHintAttr::Numeric;
578
6
      }
579
101
    } else if (LH) {
580
101
      auto *ValueExpr = LH->getValue();
581
101
      if (ValueExpr) {
582
43
        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
583
43
        ValueInt = ValueAPS.getSExtValue();
584
43
      }
585
101
586
101
      Option = LH->getOption();
587
101
      State = LH->getState();
588
101
    }
589
119
    switch (State) {
590
119
    case LoopHintAttr::Disable:
591
33
      switch (Option) {
592
33
      case LoopHintAttr::Vectorize:
593
2
        // Disable vectorization by specifying a width of 1.
594
2
        setVectorizeWidth(1);
595
2
        break;
596
33
      case LoopHintAttr::Interleave:
597
6
        // Disable interleaving by speciyfing a count of 1.
598
6
        setInterleaveCount(1);
599
6
        break;
600
33
      case LoopHintAttr::Unroll:
601
19
        setUnrollState(LoopAttributes::Disable);
602
19
        break;
603
33
      case LoopHintAttr::UnrollAndJam:
604
2
        setUnrollAndJamState(LoopAttributes::Disable);
605
2
        break;
606
33
      case LoopHintAttr::Distribute:
607
2
        setDistributeState(false);
608
2
        break;
609
33
      case LoopHintAttr::PipelineDisabled:
610
2
        setPipelineDisabled(true);
611
2
        break;
612
33
      case LoopHintAttr::UnrollCount:
613
0
      case LoopHintAttr::UnrollAndJamCount:
614
0
      case LoopHintAttr::VectorizeWidth:
615
0
      case LoopHintAttr::InterleaveCount:
616
0
      case LoopHintAttr::PipelineInitiationInterval:
617
0
        llvm_unreachable("Options cannot be disabled.");
618
0
        break;
619
33
      }
620
33
      break;
621
33
    case LoopHintAttr::Enable:
622
20
      switch (Option) {
623
20
      case LoopHintAttr::Vectorize:
624
6
      case LoopHintAttr::Interleave:
625
6
        setVectorizeEnable(true);
626
6
        break;
627
8
      case LoopHintAttr::Unroll:
628
8
        setUnrollState(LoopAttributes::Enable);
629
8
        break;
630
6
      case LoopHintAttr::UnrollAndJam:
631
2
        setUnrollAndJamState(LoopAttributes::Enable);
632
2
        break;
633
6
      case LoopHintAttr::Distribute:
634
4
        setDistributeState(true);
635
4
        break;
636
6
      case LoopHintAttr::UnrollCount:
637
0
      case LoopHintAttr::UnrollAndJamCount:
638
0
      case LoopHintAttr::VectorizeWidth:
639
0
      case LoopHintAttr::InterleaveCount:
640
0
      case LoopHintAttr::PipelineDisabled:
641
0
      case LoopHintAttr::PipelineInitiationInterval:
642
0
        llvm_unreachable("Options cannot enabled.");
643
0
        break;
644
20
      }
645
20
      break;
646
20
    case LoopHintAttr::AssumeSafety:
647
15
      switch (Option) {
648
15
      case LoopHintAttr::Vectorize:
649
15
      case LoopHintAttr::Interleave:
650
15
        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
651
15
        setParallel(true);
652
15
        setVectorizeEnable(true);
653
15
        break;
654
15
      case LoopHintAttr::Unroll:
655
0
      case LoopHintAttr::UnrollAndJam:
656
0
      case LoopHintAttr::UnrollCount:
657
0
      case LoopHintAttr::UnrollAndJamCount:
658
0
      case LoopHintAttr::VectorizeWidth:
659
0
      case LoopHintAttr::InterleaveCount:
660
0
      case LoopHintAttr::Distribute:
661
0
      case LoopHintAttr::PipelineDisabled:
662
0
      case LoopHintAttr::PipelineInitiationInterval:
663
0
        llvm_unreachable("Options cannot be used to assume mem safety.");
664
0
        break;
665
15
      }
666
15
      break;
667
15
    case LoopHintAttr::Full:
668
2
      switch (Option) {
669
2
      case LoopHintAttr::Unroll:
670
2
        setUnrollState(LoopAttributes::Full);
671
2
        break;
672
2
      case LoopHintAttr::UnrollAndJam:
673
0
        setUnrollAndJamState(LoopAttributes::Full);
674
0
        break;
675
2
      case LoopHintAttr::Vectorize:
676
0
      case LoopHintAttr::Interleave:
677
0
      case LoopHintAttr::UnrollCount:
678
0
      case LoopHintAttr::UnrollAndJamCount:
679
0
      case LoopHintAttr::VectorizeWidth:
680
0
      case LoopHintAttr::InterleaveCount:
681
0
      case LoopHintAttr::Distribute:
682
0
      case LoopHintAttr::PipelineDisabled:
683
0
      case LoopHintAttr::PipelineInitiationInterval:
684
0
        llvm_unreachable("Options cannot be used with 'full' hint.");
685
0
        break;
686
2
      }
687
2
      break;
688
49
    case LoopHintAttr::Numeric:
689
49
      switch (Option) {
690
49
      case LoopHintAttr::VectorizeWidth:
691
12
        setVectorizeWidth(ValueInt);
692
12
        break;
693
49
      case LoopHintAttr::InterleaveCount:
694
10
        setInterleaveCount(ValueInt);
695
10
        break;
696
49
      case LoopHintAttr::UnrollCount:
697
22
        setUnrollCount(ValueInt);
698
22
        break;
699
49
      case LoopHintAttr::UnrollAndJamCount:
700
2
        setUnrollAndJamCount(ValueInt);
701
2
        break;
702
49
      case LoopHintAttr::PipelineInitiationInterval:
703
3
        setPipelineInitiationInterval(ValueInt);
704
3
        break;
705
49
      case LoopHintAttr::Unroll:
706
0
      case LoopHintAttr::UnrollAndJam:
707
0
      case LoopHintAttr::Vectorize:
708
0
      case LoopHintAttr::Interleave:
709
0
      case LoopHintAttr::Distribute:
710
0
      case LoopHintAttr::PipelineDisabled:
711
0
        llvm_unreachable("Options cannot be assigned a value.");
712
0
        break;
713
49
      }
714
49
      break;
715
119
    }
716
119
  }
717
73.8k
718
73.8k
  /// Stage the attributes.
719
73.8k
  push(Header, StartLoc, EndLoc);
720
73.8k
}
721
722
83.3k
void LoopInfoStack::pop() {
723
83.3k
  assert(!Active.empty() && "No active loops to pop");
724
83.3k
  Active.back().finish();
725
83.3k
  Active.pop_back();
726
83.3k
}
727
728
11.9M
void LoopInfoStack::InsertHelper(Instruction *I) const {
729
11.9M
  if (I->mayReadOrWriteMemory()) {
730
6.47M
    SmallVector<Metadata *, 4> AccessGroups;
731
6.47M
    for (const LoopInfo &AL : Active) {
732
1.59M
      // Here we assume that every loop that has an access group is parallel.
733
1.59M
      if (MDNode *Group = AL.getAccessGroup())
734
30.8k
        AccessGroups.push_back(Group);
735
1.59M
    }
736
6.47M
    MDNode *UnionMD = nullptr;
737
6.47M
    if (AccessGroups.size() == 1)
738
30.8k
      UnionMD = cast<MDNode>(AccessGroups[0]);
739
6.44M
    else if (AccessGroups.size() >= 2)
740
22
      UnionMD = MDNode::get(I->getContext(), AccessGroups);
741
6.47M
    I->setMetadata("llvm.access.group", UnionMD);
742
6.47M
  }
743
11.9M
744
11.9M
  if (!hasInfo())
745
9.58M
    return;
746
2.40M
747
2.40M
  const LoopInfo &L = getInfo();
748
2.40M
  if (!L.getLoopID())
749
1.86M
    return;
750
542k
751
542k
  if (I->isTerminator()) {
752
98.8k
    for (BasicBlock *Succ : successors(I))
753
130k
      if (Succ == L.getHeader()) {
754
16.8k
        I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
755
16.8k
        break;
756
16.8k
      }
757
98.8k
    return;
758
98.8k
  }
759
542k
}