Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/Utils/LoopUnroll.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- UnrollLoop.cpp - Loop unrolling utilities -------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements some loop unrolling utilities. It does not define any
10
// actual pass or policy, but provides a single function to perform loop
11
// unrolling.
12
//
13
// The process of unrolling can produce extraneous basic blocks linked with
14
// unconditional branches.  This will be corrected in the future.
15
//
16
//===----------------------------------------------------------------------===//
17
18
#include "llvm/ADT/SmallPtrSet.h"
19
#include "llvm/ADT/Statistic.h"
20
#include "llvm/Analysis/AssumptionCache.h"
21
#include "llvm/Analysis/InstructionSimplify.h"
22
#include "llvm/Analysis/LoopIterator.h"
23
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
24
#include "llvm/Analysis/ScalarEvolution.h"
25
#include "llvm/Transforms/Utils/Local.h"
26
#include "llvm/IR/BasicBlock.h"
27
#include "llvm/IR/DataLayout.h"
28
#include "llvm/IR/DebugInfoMetadata.h"
29
#include "llvm/IR/Dominators.h"
30
#include "llvm/IR/IntrinsicInst.h"
31
#include "llvm/IR/LLVMContext.h"
32
#include "llvm/Support/Debug.h"
33
#include "llvm/Support/raw_ostream.h"
34
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
35
#include "llvm/Transforms/Utils/Cloning.h"
36
#include "llvm/Transforms/Utils/LoopSimplify.h"
37
#include "llvm/Transforms/Utils/LoopUtils.h"
38
#include "llvm/Transforms/Utils/SimplifyIndVar.h"
39
#include "llvm/Transforms/Utils/UnrollLoop.h"
40
using namespace llvm;
41
42
8
#define DEBUG_TYPE "loop-unroll"
43
44
// TODO: Should these be here or in LoopUnroll?
45
STATISTIC(NumCompletelyUnrolled, "Number of loops completely unrolled");
46
STATISTIC(NumUnrolled, "Number of loops unrolled (completely or otherwise)");
47
STATISTIC(NumUnrolledWithHeader, "Number of loops unrolled without a "
48
                                 "conditional latch (completely or otherwise)");
49
50
static cl::opt<bool>
51
UnrollRuntimeEpilog("unroll-runtime-epilog", cl::init(false), cl::Hidden,
52
                    cl::desc("Allow runtime unrolled loops to be unrolled "
53
                             "with epilog instead of prolog."));
54
55
static cl::opt<bool>
56
UnrollVerifyDomtree("unroll-verify-domtree", cl::Hidden,
57
                    cl::desc("Verify domtree after unrolling"),
58
#ifdef EXPENSIVE_CHECKS
59
    cl::init(true)
60
#else
61
    cl::init(false)
62
#endif
63
                    );
64
65
/// Convert the instruction operands from referencing the current values into
66
/// those specified by VMap.
67
1.82M
void llvm::remapInstruction(Instruction *I, ValueToValueMapTy &VMap) {
68
5.69M
  for (unsigned op = 0, E = I->getNumOperands(); op != E; 
++op3.87M
) {
69
3.87M
    Value *Op = I->getOperand(op);
70
3.87M
71
3.87M
    // Unwrap arguments of dbg.value intrinsics.
72
3.87M
    bool Wrapped = false;
73
3.87M
    if (auto *V = dyn_cast<MetadataAsValue>(Op))
74
111
      if (auto *Unwrapped = dyn_cast<ValueAsMetadata>(V->getMetadata())) {
75
37
        Op = Unwrapped->getValue();
76
37
        Wrapped = true;
77
37
      }
78
3.87M
79
3.87M
    auto wrap = [&](Value *V) {
80
2.22M
      auto &C = I->getContext();
81
2.22M
      return Wrapped ? 
MetadataAsValue::get(C, ValueAsMetadata::get(V))37
:
V2.22M
;
82
2.22M
    };
83
3.87M
84
3.87M
    ValueToValueMapTy::iterator It = VMap.find(Op);
85
3.87M
    if (It != VMap.end())
86
2.22M
      I->setOperand(op, wrap(It->second));
87
3.87M
  }
88
1.82M
89
1.82M
  if (PHINode *PN = dyn_cast<PHINode>(I)) {
90
25.2k
    for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; 
++i17.0k
) {
91
17.0k
      ValueToValueMapTy::iterator It = VMap.find(PN->getIncomingBlock(i));
92
17.0k
      if (It != VMap.end())
93
16.8k
        PN->setIncomingBlock(i, cast<BasicBlock>(It->second));
94
17.0k
    }
95
8.18k
  }
96
1.82M
}
97
98
/// Check if unrolling created a situation where we need to insert phi nodes to
99
/// preserve LCSSA form.
100
/// \param Blocks is a vector of basic blocks representing unrolled loop.
101
/// \param L is the outer loop.
102
/// It's possible that some of the blocks are in L, and some are not. In this
103
/// case, if there is a use is outside L, and definition is inside L, we need to
104
/// insert a phi-node, otherwise LCSSA will be broken.
105
/// The function is just a helper function for llvm::UnrollLoop that returns
106
/// true if this situation occurs, indicating that LCSSA needs to be fixed.
107
static bool needToInsertPhisForLCSSA(Loop *L, std::vector<BasicBlock *> Blocks,
108
3.12k
                                     LoopInfo *LI) {
109
8.03k
  for (BasicBlock *BB : Blocks) {
110
8.03k
    if (LI->getLoopFor(BB) == L)
111
7.71k
      continue;
112
1.31k
    
for (Instruction &I : *BB)316
{
113
2.65k
      for (Use &U : I.operands()) {
114
2.65k
        if (auto Def = dyn_cast<Instruction>(U)) {
115
1.53k
          Loop *DefLoop = LI->getLoopFor(Def->getParent());
116
1.53k
          if (!DefLoop)
117
84
            continue;
118
1.45k
          if (DefLoop->contains(L))
119
153
            return true;
120
1.45k
        }
121
2.65k
      }
122
1.31k
    }
123
316
  }
124
3.12k
  
return false2.97k
;
125
3.12k
}
126
127
/// Adds ClonedBB to LoopInfo, creates a new loop for ClonedBB if necessary
128
/// and adds a mapping from the original loop to the new loop to NewLoops.
129
/// Returns nullptr if no new loop was created and a pointer to the
130
/// original loop OriginalBB was part of otherwise.
131
const Loop* llvm::addClonedBlockToLoopInfo(BasicBlock *OriginalBB,
132
                                           BasicBlock *ClonedBB, LoopInfo *LI,
133
281k
                                           NewLoopsMap &NewLoops) {
134
281k
  // Figure out which loop New is in.
135
281k
  const Loop *OldLoop = LI->getLoopFor(OriginalBB);
136
281k
  assert(OldLoop && "Should (at least) be in the loop being unrolled!");
137
281k
138
281k
  Loop *&NewLoop = NewLoops[OldLoop];
139
281k
  if (!NewLoop) {
140
2.82k
    // Found a new sub-loop.
141
2.82k
    assert(OriginalBB == OldLoop->getHeader() &&
142
2.82k
           "Header should be first in RPO");
143
2.82k
144
2.82k
    NewLoop = LI->AllocateLoop();
145
2.82k
    Loop *NewLoopParent = NewLoops.lookup(OldLoop->getParentLoop());
146
2.82k
147
2.82k
    if (NewLoopParent)
148
1.85k
      NewLoopParent->addChildLoop(NewLoop);
149
964
    else
150
964
      LI->addTopLevelLoop(NewLoop);
151
2.82k
152
2.82k
    NewLoop->addBasicBlockToLoop(ClonedBB, *LI);
153
2.82k
    return OldLoop;
154
278k
  } else {
155
278k
    NewLoop->addBasicBlockToLoop(ClonedBB, *LI);
156
278k
    return nullptr;
157
278k
  }
158
281k
}
159
160
/// The function chooses which type of unroll (epilog or prolog) is more
161
/// profitabale.
162
/// Epilog unroll is more profitable when there is PHI that starts from
163
/// constant.  In this case epilog will leave PHI start from constant,
164
/// but prolog will convert it to non-constant.
165
///
166
/// loop:
167
///   PN = PHI [I, Latch], [CI, PreHeader]
168
///   I = foo(PN)
169
///   ...
170
///
171
/// Epilog unroll case.
172
/// loop:
173
///   PN = PHI [I2, Latch], [CI, PreHeader]
174
///   I1 = foo(PN)
175
///   I2 = foo(I1)
176
///   ...
177
/// Prolog unroll case.
178
///   NewPN = PHI [PrologI, Prolog], [CI, PreHeader]
179
/// loop:
180
///   PN = PHI [I2, Latch], [NewPN, PreHeader]
181
///   I1 = foo(PN)
182
///   I2 = foo(I1)
183
///   ...
184
///
185
13.4k
static bool isEpilogProfitable(Loop *L) {
186
13.4k
  BasicBlock *PreHeader = L->getLoopPreheader();
187
13.4k
  BasicBlock *Header = L->getHeader();
188
13.4k
  assert(PreHeader && Header);
189
15.7k
  for (const PHINode &PN : Header->phis()) {
190
15.7k
    if (isa<ConstantInt>(PN.getIncomingValueForBlock(PreHeader)))
191
10.6k
      return true;
192
15.7k
  }
193
13.4k
  
return false2.83k
;
194
13.4k
}
195
196
/// Perform some cleanup and simplifications on loops after unrolling. It is
197
/// useful to simplify the IV's in the new loop, as well as do a quick
198
/// simplify/dce pass of the instructions.
199
void llvm::simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
200
                                   ScalarEvolution *SE, DominatorTree *DT,
201
11.2k
                                   AssumptionCache *AC) {
202
11.2k
  // Simplify any new induction variables in the partially unrolled loop.
203
11.2k
  if (SE && SimplifyIVs) {
204
1.82k
    SmallVector<WeakTrackingVH, 16> DeadInsts;
205
1.82k
    simplifyLoopIVs(L, SE, DT, LI, DeadInsts);
206
1.82k
207
1.82k
    // Aggressively clean up dead instructions that simplifyLoopIVs already
208
1.82k
    // identified. Any remaining should be cleaned up below.
209
12.6k
    while (!DeadInsts.empty())
210
10.7k
      if (Instruction *Inst =
211
10.7k
              dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()))
212
10.7k
        RecursivelyDeleteTriviallyDeadInstructions(Inst);
213
1.82k
  }
214
11.2k
215
11.2k
  // At this point, the code is well formed.  We now do a quick sweep over the
216
11.2k
  // inserted code, doing constant propagation and dead code elimination as we
217
11.2k
  // go.
218
11.2k
  const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
219
100k
  for (BasicBlock *BB : L->getBlocks()) {
220
1.94M
    for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
221
1.84M
      Instruction *Inst = &*I++;
222
1.84M
223
1.84M
      if (Value *V = SimplifyInstruction(Inst, {DL, nullptr, DT, AC}))
224
526k
        if (LI->replacementPreservesLCSSAForm(Inst, V))
225
525k
          Inst->replaceAllUsesWith(V);
226
1.84M
      if (isInstructionTriviallyDead(Inst))
227
527k
        BB->getInstList().erase(Inst);
228
1.84M
    }
229
100k
  }
230
11.2k
231
11.2k
  // TODO: after peeling or unrolling, previously loop variant conditions are
232
11.2k
  // likely to fold to constants, eagerly propagating those here will require
233
11.2k
  // fewer cleanup passes to be run.  Alternatively, a LoopEarlyCSE might be
234
11.2k
  // appropriate.
235
11.2k
}
236
237
/// Unroll the given loop by Count. The loop must be in LCSSA form.  Unrolling
238
/// can only fail when the loop's latch block is not terminated by a conditional
239
/// branch instruction. However, if the trip count (and multiple) are not known,
240
/// loop unrolling will mostly produce more code that is no faster.
241
///
242
/// TripCount is the upper bound of the iteration on which control exits
243
/// LatchBlock. Control may exit the loop prior to TripCount iterations either
244
/// via an early branch in other loop block or via LatchBlock terminator. This
245
/// is relaxed from the general definition of trip count which is the number of
246
/// times the loop header executes. Note that UnrollLoop assumes that the loop
247
/// counter test is in LatchBlock in order to remove unnecesssary instances of
248
/// the test.  If control can exit the loop from the LatchBlock's terminator
249
/// prior to TripCount iterations, flag PreserveCondBr needs to be set.
250
///
251
/// PreserveCondBr indicates whether the conditional branch of the LatchBlock
252
/// needs to be preserved.  It is needed when we use trip count upper bound to
253
/// fully unroll the loop. If PreserveOnlyFirst is also set then only the first
254
/// conditional branch needs to be preserved.
255
///
256
/// Similarly, TripMultiple divides the number of times that the LatchBlock may
257
/// execute without exiting the loop.
258
///
259
/// If AllowRuntime is true then UnrollLoop will consider unrolling loops that
260
/// have a runtime (i.e. not compile time constant) trip count.  Unrolling these
261
/// loops require a unroll "prologue" that runs "RuntimeTripCount % Count"
262
/// iterations before branching into the unrolled loop.  UnrollLoop will not
263
/// runtime-unroll the loop if computing RuntimeTripCount will be expensive and
264
/// AllowExpensiveTripCount is false.
265
///
266
/// If we want to perform PGO-based loop peeling, PeelCount is set to the
267
/// number of iterations we want to peel off.
268
///
269
/// The LoopInfo Analysis that is passed will be kept consistent.
270
///
271
/// This utility preserves LoopInfo. It will also preserve ScalarEvolution and
272
/// DominatorTree if they are non-null.
273
///
274
/// If RemainderLoop is non-null, it will receive the remainder loop (if
275
/// required and not fully unrolled).
276
LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
277
                                  ScalarEvolution *SE, DominatorTree *DT,
278
                                  AssumptionCache *AC,
279
                                  OptimizationRemarkEmitter *ORE,
280
13.8k
                                  bool PreserveLCSSA, Loop **RemainderLoop) {
281
13.8k
282
13.8k
  BasicBlock *Preheader = L->getLoopPreheader();
283
13.8k
  if (!Preheader) {
284
0
    LLVM_DEBUG(dbgs() << "  Can't unroll; loop preheader-insertion failed.\n");
285
0
    return LoopUnrollResult::Unmodified;
286
0
  }
287
13.8k
288
13.8k
  BasicBlock *LatchBlock = L->getLoopLatch();
289
13.8k
  if (!LatchBlock) {
290
0
    LLVM_DEBUG(dbgs() << "  Can't unroll; loop exit-block-insertion failed.\n");
291
0
    return LoopUnrollResult::Unmodified;
292
0
  }
293
13.8k
294
13.8k
  // Loops with indirectbr cannot be cloned.
295
13.8k
  if (!L->isSafeToClone()) {
296
1
    LLVM_DEBUG(dbgs() << "  Can't unroll; Loop body cannot be cloned.\n");
297
1
    return LoopUnrollResult::Unmodified;
298
1
  }
299
13.8k
300
13.8k
  // The current loop unroll pass can unroll loops with a single latch or header
301
13.8k
  // that's a conditional branch exiting the loop.
302
13.8k
  // FIXME: The implementation can be extended to work with more complicated
303
13.8k
  // cases, e.g. loops with multiple latches.
304
13.8k
  BasicBlock *Header = L->getHeader();
305
13.8k
  BranchInst *HeaderBI = dyn_cast<BranchInst>(Header->getTerminator());
306
13.8k
  BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator());
307
13.8k
308
13.8k
  // FIXME: Support loops without conditional latch and multiple exiting blocks.
309
13.8k
  if (!BI ||
310
13.8k
      (BI->isUnconditional() && 
(339
!HeaderBI339
||
HeaderBI->isUnconditional()143
||
311
339
                                 
L->getExitingBlock() != Header138
))) {
312
321
    LLVM_DEBUG(dbgs() << "  Can't unroll; loop not terminated by a conditional "
313
321
                         "branch in the latch or header.\n");
314
321
    return LoopUnrollResult::Unmodified;
315
321
  }
316
13.4k
317
20.0k
  
auto CheckLatchSuccessors = [&](unsigned S1, unsigned S2) 13.4k
{
318
20.0k
    return BI->isConditional() && BI->getSuccessor(S1) == Header &&
319
20.0k
           
!L->contains(BI->getSuccessor(S2))13.4k
;
320
20.0k
  };
321
13.4k
322
13.4k
  // If we have a conditional latch, it must exit the loop.
323
13.4k
  if (BI && BI->isConditional() && 
!CheckLatchSuccessors(0, 1)13.4k
&&
324
13.4k
      
!CheckLatchSuccessors(1, 0)6.58k
) {
325
0
    LLVM_DEBUG(
326
0
        dbgs() << "Can't unroll; a conditional latch must exit the loop");
327
0
    return LoopUnrollResult::Unmodified;
328
0
  }
329
13.4k
330
13.4k
  auto CheckHeaderSuccessors = [&](unsigned S1, unsigned S2) {
331
31
    return HeaderBI && HeaderBI->isConditional() &&
332
31
           L->contains(HeaderBI->getSuccessor(S1)) &&
333
31
           
!L->contains(HeaderBI->getSuccessor(S2))18
;
334
31
  };
335
13.4k
336
13.4k
  // If we do not have a conditional latch, the header must exit the loop.
337
13.4k
  if (BI && !BI->isConditional() && 
HeaderBI18
&&
HeaderBI->isConditional()18
&&
338
13.4k
      
!CheckHeaderSuccessors(0, 1)18
&&
!CheckHeaderSuccessors(1, 0)13
) {
339
0
    LLVM_DEBUG(dbgs() << "Can't unroll; conditional header must exit the loop");
340
0
    return LoopUnrollResult::Unmodified;
341
0
  }
342
13.4k
343
13.4k
  if (Header->hasAddressTaken()) {
344
2
    // The loop-rotate pass can be helpful to avoid this in many cases.
345
2
    LLVM_DEBUG(
346
2
        dbgs() << "  Won't unroll loop: address of header block is taken.\n");
347
2
    return LoopUnrollResult::Unmodified;
348
2
  }
349
13.4k
350
13.4k
  if (ULO.TripCount != 0)
351
13.4k
    LLVM_DEBUG(dbgs() << "  Trip Count = " << ULO.TripCount << "\n");
352
13.4k
  if (ULO.TripMultiple != 1)
353
13.4k
    LLVM_DEBUG(dbgs() << "  Trip Multiple = " << ULO.TripMultiple << "\n");
354
13.4k
355
13.4k
  // Effectively "DCE" unrolled iterations that are beyond the tripcount
356
13.4k
  // and will never be executed.
357
13.4k
  if (ULO.TripCount != 0 && 
ULO.Count > ULO.TripCount9.57k
)
358
0
    ULO.Count = ULO.TripCount;
359
13.4k
360
13.4k
  // Don't enter the unroll code if there is nothing to do.
361
13.4k
  if (ULO.TripCount == 0 && 
ULO.Count < 23.92k
&&
ULO.PeelCount == 0109
) {
362
1
    LLVM_DEBUG(dbgs() << "Won't unroll; almost nothing to do\n");
363
1
    return LoopUnrollResult::Unmodified;
364
1
  }
365
13.4k
366
13.4k
  assert(ULO.Count > 0);
367
13.4k
  assert(ULO.TripMultiple > 0);
368
13.4k
  assert(ULO.TripCount == 0 || ULO.TripCount % ULO.TripMultiple == 0);
369
13.4k
370
13.4k
  // Are we eliminating the loop control altogether?
371
13.4k
  bool CompletelyUnroll = ULO.Count == ULO.TripCount;
372
13.4k
  SmallVector<BasicBlock *, 4> ExitBlocks;
373
13.4k
  L->getExitBlocks(ExitBlocks);
374
13.4k
  std::vector<BasicBlock*> OriginalLoopBlocks = L->getBlocks();
375
13.4k
376
13.4k
  // Go through all exits of L and see if there are any phi-nodes there. We just
377
13.4k
  // conservatively assume that they're inserted to preserve LCSSA form, which
378
13.4k
  // means that complete unrolling might break this form. We need to either fix
379
13.4k
  // it in-place after the transformation, or entirely rebuild LCSSA. TODO: For
380
13.4k
  // now we just recompute LCSSA for the outer loop, but it should be possible
381
13.4k
  // to fix it in-place.
382
13.4k
  bool NeedToFixLCSSA = PreserveLCSSA && CompletelyUnroll &&
383
13.4k
                        
any_of(ExitBlocks, [](const BasicBlock *BB) 9.37k
{
384
10.0k
                          return isa<PHINode>(BB->begin());
385
10.0k
                        });
386
13.4k
387
13.4k
  // We assume a run-time trip count if the compiler cannot
388
13.4k
  // figure out the loop trip count and the unroll-runtime
389
13.4k
  // flag is specified.
390
13.4k
  bool RuntimeTripCount =
391
13.4k
      (ULO.TripCount == 0 && 
ULO.Count > 03.92k
&&
ULO.AllowRuntime3.92k
);
392
13.4k
393
13.4k
  assert((!RuntimeTripCount || !ULO.PeelCount) &&
394
13.4k
         "Did not expect runtime trip-count unrolling "
395
13.4k
         "and peeling for the same loop");
396
13.4k
397
13.4k
  bool Peeled = false;
398
13.4k
  if (ULO.PeelCount) {
399
119
    Peeled = peelLoop(L, ULO.PeelCount, LI, SE, DT, AC, PreserveLCSSA);
400
119
401
119
    // Successful peeling may result in a change in the loop preheader/trip
402
119
    // counts. If we later unroll the loop, we want these to be updated.
403
119
    if (Peeled) {
404
119
      // According to our guards and profitability checks the only
405
119
      // meaningful exit should be latch block. Other exits go to deopt,
406
119
      // so we do not worry about them.
407
119
      BasicBlock *ExitingBlock = L->getLoopLatch();
408
119
      assert(ExitingBlock && "Loop without exiting block?");
409
119
      assert(L->isLoopExiting(ExitingBlock) && "Latch is not exiting?");
410
119
      Preheader = L->getLoopPreheader();
411
119
      ULO.TripCount = SE->getSmallConstantTripCount(L, ExitingBlock);
412
119
      ULO.TripMultiple = SE->getSmallConstantTripMultiple(L, ExitingBlock);
413
119
    }
414
119
  }
415
13.4k
416
13.4k
  // Loops containing convergent instructions must have a count that divides
417
13.4k
  // their TripMultiple.
418
13.4k
  LLVM_DEBUG(
419
13.4k
      {
420
13.4k
        bool HasConvergent = false;
421
13.4k
        for (auto &BB : L->blocks())
422
13.4k
          for (auto &I : *BB)
423
13.4k
            if (auto CS = CallSite(&I))
424
13.4k
              HasConvergent |= CS.isConvergent();
425
13.4k
        assert((!HasConvergent || ULO.TripMultiple % ULO.Count == 0) &&
426
13.4k
               "Unroll count must divide trip multiple if loop contains a "
427
13.4k
               "convergent operation.");
428
13.4k
      });
429
13.4k
430
13.4k
  bool EpilogProfitability =
431
13.4k
      UnrollRuntimeEpilog.getNumOccurrences() ? 
UnrollRuntimeEpilog47
432
13.4k
                                              : 
isEpilogProfitable(L)13.4k
;
433
13.4k
434
13.4k
  if (RuntimeTripCount && 
ULO.TripMultiple % ULO.Count != 03.80k
&&
435
13.4k
      !UnrollRuntimeLoopRemainder(L, ULO.Count, ULO.AllowExpensiveTripCount,
436
3.80k
                                  EpilogProfitability, ULO.UnrollRemainder,
437
3.80k
                                  ULO.ForgetAllSCEV, LI, SE, DT, AC,
438
3.80k
                                  PreserveLCSSA, RemainderLoop)) {
439
2.35k
    if (ULO.Force)
440
12
      RuntimeTripCount = false;
441
2.34k
    else {
442
2.34k
      LLVM_DEBUG(dbgs() << "Won't unroll; remainder loop could not be "
443
2.34k
                           "generated when assuming runtime trip count\n");
444
2.34k
      return LoopUnrollResult::Unmodified;
445
2.34k
    }
446
11.1k
  }
447
11.1k
448
11.1k
  // If we know the trip count, we know the multiple...
449
11.1k
  unsigned BreakoutTrip = 0;
450
11.1k
  if (ULO.TripCount != 0) {
451
9.57k
    BreakoutTrip = ULO.TripCount % ULO.Count;
452
9.57k
    ULO.TripMultiple = 0;
453
9.57k
  } else {
454
1.57k
    // Figure out what multiple to use.
455
1.57k
    BreakoutTrip = ULO.TripMultiple =
456
1.57k
        (unsigned)GreatestCommonDivisor64(ULO.Count, ULO.TripMultiple);
457
1.57k
  }
458
11.1k
459
11.1k
  using namespace ore;
460
11.1k
  // Report the unrolling decision.
461
11.1k
  if (CompletelyUnroll) {
462
9.37k
    LLVM_DEBUG(dbgs() << "COMPLETELY UNROLLING loop %" << Header->getName()
463
9.37k
                      << " with trip count " << ULO.TripCount << "!\n");
464
9.37k
    if (ORE)
465
9.35k
      ORE->emit([&]() {
466
3
        return OptimizationRemark(DEBUG_TYPE, "FullyUnrolled", L->getStartLoc(),
467
3
                                  L->getHeader())
468
3
               << "completely unrolled loop with "
469
3
               << NV("UnrollCount", ULO.TripCount) << " iterations";
470
3
      });
471
9.37k
  } else 
if (1.77k
ULO.PeelCount1.77k
) {
472
118
    LLVM_DEBUG(dbgs() << "PEELING loop %" << Header->getName()
473
118
                      << " with iteration count " << ULO.PeelCount << "!\n");
474
118
    if (ORE)
475
118
      ORE->emit([&]() {
476
0
        return OptimizationRemark(DEBUG_TYPE, "Peeled", L->getStartLoc(),
477
0
                                  L->getHeader())
478
0
               << " peeled loop by " << NV("PeelCount", ULO.PeelCount)
479
0
               << " iterations";
480
0
      });
481
1.65k
  } else {
482
1.65k
    auto DiagBuilder = [&]() {
483
5
      OptimizationRemark Diag(DEBUG_TYPE, "PartialUnrolled", L->getStartLoc(),
484
5
                              L->getHeader());
485
5
      return Diag << "unrolled loop by a factor of "
486
5
                  << NV("UnrollCount", ULO.Count);
487
5
    };
488
1.65k
489
1.65k
    LLVM_DEBUG(dbgs() << "UNROLLING loop %" << Header->getName() << " by "
490
1.65k
                      << ULO.Count);
491
1.65k
    if (ULO.TripMultiple == 0 || 
BreakoutTrip != ULO.TripMultiple1.46k
) {
492
185
      LLVM_DEBUG(dbgs() << " with a breakout at trip " << BreakoutTrip);
493
185
      if (ORE)
494
185
        ORE->emit([&]() {
495
3
          return DiagBuilder() << " with a breakout at trip "
496
3
                               << NV("BreakoutTrip", BreakoutTrip);
497
3
        });
498
1.46k
    } else if (ULO.TripMultiple != 1) {
499
5
      LLVM_DEBUG(dbgs() << " with " << ULO.TripMultiple << " trips per branch");
500
5
      if (ORE)
501
5
        ORE->emit([&]() {
502
0
          return DiagBuilder()
503
0
                 << " with " << NV("TripMultiple", ULO.TripMultiple)
504
0
                 << " trips per branch";
505
0
        });
506
1.46k
    } else if (RuntimeTripCount) {
507
1.44k
      LLVM_DEBUG(dbgs() << " with run-time trip count");
508
1.44k
      if (ORE)
509
1.44k
        ORE->emit(
510
1.44k
            [&]() 
{ return DiagBuilder() << " with run-time trip count"; }2
);
511
1.44k
    }
512
1.65k
    LLVM_DEBUG(dbgs() << "!\n");
513
1.65k
  }
514
11.1k
515
11.1k
  // We are going to make changes to this loop. SCEV may be keeping cached info
516
11.1k
  // about it, in particular about backedge taken count. The changes we make
517
11.1k
  // are guaranteed to invalidate this information for our loop. It is tempting
518
11.1k
  // to only invalidate the loop being unrolled, but it is incorrect as long as
519
11.1k
  // all exiting branches from all inner loops have impact on the outer loops,
520
11.1k
  // and if something changes inside them then any of outer loops may also
521
11.1k
  // change. When we forget outermost loop, we also forget all contained loops
522
11.1k
  // and this is what we need here.
523
11.1k
  if (SE) {
524
11.1k
    if (ULO.ForgetAllSCEV)
525
0
      SE->forgetAllLoops();
526
11.1k
    else
527
11.1k
      SE->forgetTopmostLoop(L);
528
11.1k
  }
529
11.1k
530
11.1k
  bool ContinueOnTrue;
531
11.1k
  bool LatchIsExiting = BI->isConditional();
532
11.1k
  BasicBlock *LoopExit = nullptr;
533
11.1k
  if (LatchIsExiting) {
534
11.1k
    ContinueOnTrue = L->contains(BI->getSuccessor(0));
535
11.1k
    LoopExit = BI->getSuccessor(ContinueOnTrue);
536
11.1k
  } else {
537
17
    NumUnrolledWithHeader++;
538
17
    ContinueOnTrue = L->contains(HeaderBI->getSuccessor(0));
539
17
    LoopExit = HeaderBI->getSuccessor(ContinueOnTrue);
540
17
  }
541
11.1k
542
11.1k
  // For the first iteration of the loop, we should use the precloned values for
543
11.1k
  // PHI nodes.  Insert associations now.
544
11.1k
  ValueToValueMapTy LastValueMap;
545
11.1k
  std::vector<PHINode*> OrigPHINode;
546
26.7k
  for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); 
++I15.6k
) {
547
15.6k
    OrigPHINode.push_back(cast<PHINode>(I));
548
15.6k
  }
549
11.1k
550
11.1k
  std::vector<BasicBlock *> Headers;
551
11.1k
  std::vector<BasicBlock *> HeaderSucc;
552
11.1k
  std::vector<BasicBlock *> Latches;
553
11.1k
  Headers.push_back(Header);
554
11.1k
  Latches.push_back(LatchBlock);
555
11.1k
556
11.1k
  if (!LatchIsExiting) {
557
17
    auto *Term = cast<BranchInst>(Header->getTerminator());
558
17
    if (Term->isUnconditional() || L->contains(Term->getSuccessor(0))) {
559
5
      assert(L->contains(Term->getSuccessor(0)));
560
5
      HeaderSucc.push_back(Term->getSuccessor(0));
561
12
    } else {
562
12
      assert(L->contains(Term->getSuccessor(1)));
563
12
      HeaderSucc.push_back(Term->getSuccessor(1));
564
12
    }
565
17
  }
566
11.1k
567
11.1k
  // The current on-the-fly SSA update requires blocks to be processed in
568
11.1k
  // reverse postorder so that LastValueMap contains the correct value at each
569
11.1k
  // exit.
570
11.1k
  LoopBlocksDFS DFS(L);
571
11.1k
  DFS.perform(LI);
572
11.1k
573
11.1k
  // Stash the DFS iterators before adding blocks to the loop.
574
11.1k
  LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO();
575
11.1k
  LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO();
576
11.1k
577
11.1k
  std::vector<BasicBlock*> UnrolledLoopBlocks = L->getBlocks();
578
11.1k
579
11.1k
  // Loop Unrolling might create new loops. While we do preserve LoopInfo, we
580
11.1k
  // might break loop-simplified form for these loops (as they, e.g., would
581
11.1k
  // share the same exit blocks). We'll keep track of loops for which we can
582
11.1k
  // break this so that later we can re-simplify them.
583
11.1k
  SmallSetVector<Loop *, 4> LoopsToSimplify;
584
11.1k
  for (Loop *SubLoop : *L)
585
757
    LoopsToSimplify.insert(SubLoop);
586
11.1k
587
11.1k
  if (Header->getParent()->isDebugInfoForProfiling())
588
3
    for (BasicBlock *BB : L->getBlocks())
589
3
      for (Instruction &I : *BB)
590
80
        if (!isa<DbgInfoIntrinsic>(&I))
591
78
          if (const DILocation *DIL = I.getDebugLoc()) {
592
76
            auto NewDIL = DIL->cloneByMultiplyingDuplicationFactor(ULO.Count);
593
76
            if (NewDIL)
594
76
              I.setDebugLoc(NewDIL.getValue());
595
76
            else
596
76
              LLVM_DEBUG(dbgs()
597
76
                         << "Failed to create new discriminator: "
598
76
                         << DIL->getFilename() << " Line: " << DIL->getLine());
599
76
          }
600
11.1k
601
215k
  for (unsigned It = 1; It != ULO.Count; 
++It204k
) {
602
204k
    std::vector<BasicBlock*> NewBlocks;
603
204k
    SmallDenseMap<const Loop *, Loop *, 4> NewLoops;
604
204k
    NewLoops[L] = L;
605
204k
606
483k
    for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; 
++BB279k
) {
607
279k
      ValueToValueMapTy VMap;
608
279k
      BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It));
609
279k
      Header->getParent()->getBasicBlockList().push_back(New);
610
279k
611
279k
      assert((*BB != Header || LI->getLoopFor(*BB) == L) &&
612
279k
             "Header should not be in a sub-loop");
613
279k
      // Tell LI about New.
614
279k
      const Loop *OldLoop = addClonedBlockToLoopInfo(*BB, New, LI, NewLoops);
615
279k
      if (OldLoop)
616
1.65k
        LoopsToSimplify.insert(NewLoops[OldLoop]);
617
279k
618
279k
      if (*BB == Header)
619
204k
        // Loop over all of the PHI nodes in the block, changing them to use
620
204k
        // the incoming values from the previous block.
621
230k
        
for (PHINode *OrigPHI : OrigPHINode)204k
{
622
230k
          PHINode *NewPHI = cast<PHINode>(VMap[OrigPHI]);
623
230k
          Value *InVal = NewPHI->getIncomingValueForBlock(LatchBlock);
624
230k
          if (Instruction *InValI = dyn_cast<Instruction>(InVal))
625
230k
            if (It > 1 && 
L->contains(InValI)215k
)
626
215k
              InVal = LastValueMap[InValI];
627
230k
          VMap[OrigPHI] = InVal;
628
230k
          New->getInstList().erase(NewPHI);
629
230k
        }
630
279k
631
279k
      // Update our running map of newest clones
632
279k
      LastValueMap[*BB] = New;
633
279k
      for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
634
2.33M
           VI != VE; 
++VI2.05M
)
635
2.05M
        LastValueMap[VI->first] = VI->second;
636
279k
637
279k
      // Add phi entries for newly created values to all exit blocks.
638
547k
      for (BasicBlock *Succ : successors(*BB)) {
639
547k
        if (L->contains(Succ))
640
291k
          continue;
641
255k
        for (PHINode &PHI : Succ->phis()) {
642
113k
          Value *Incoming = PHI.getIncomingValueForBlock(*BB);
643
113k
          ValueToValueMapTy::iterator It = LastValueMap.find(Incoming);
644
113k
          if (It != LastValueMap.end())
645
65.6k
            Incoming = It->second;
646
113k
          PHI.addIncoming(Incoming, New);
647
113k
        }
648
255k
      }
649
279k
      // Keep track of new headers and latches as we create them, so that
650
279k
      // we can insert the proper branches later.
651
279k
      if (*BB == Header)
652
204k
        Headers.push_back(New);
653
279k
      if (*BB == LatchBlock)
654
204k
        Latches.push_back(New);
655
279k
656
279k
      // Keep track of the successor of the new header in the current iteration.
657
279k
      for (auto *Pred : predecessors(*BB))
658
613k
        if (Pred == Header) {
659
209k
          HeaderSucc.push_back(New);
660
209k
          break;
661
209k
        }
662
279k
663
279k
      NewBlocks.push_back(New);
664
279k
      UnrolledLoopBlocks.push_back(New);
665
279k
666
279k
      // Update DomTree: since we just copy the loop body, and each copy has a
667
279k
      // dedicated entry block (copy of the header block), this header's copy
668
279k
      // dominates all copied blocks. That means, dominance relations in the
669
279k
      // copied body are the same as in the original body.
670
279k
      if (DT) {
671
279k
        if (*BB == Header)
672
204k
          DT->addNewBlock(New, Latches[It - 1]);
673
75.6k
        else {
674
75.6k
          auto BBDomNode = DT->getNode(*BB);
675
75.6k
          auto BBIDom = BBDomNode->getIDom();
676
75.6k
          BasicBlock *OriginalBBIDom = BBIDom->getBlock();
677
75.6k
          DT->addNewBlock(
678
75.6k
              New, cast<BasicBlock>(LastValueMap[cast<Value>(OriginalBBIDom)]));
679
75.6k
        }
680
279k
      }
681
279k
    }
682
204k
683
204k
    // Remap all instructions in the most recent iteration
684
279k
    for (BasicBlock *NewBlock : NewBlocks) {
685
1.82M
      for (Instruction &I : *NewBlock) {
686
1.82M
        ::remapInstruction(&I, LastValueMap);
687
1.82M
        if (auto *II = dyn_cast<IntrinsicInst>(&I))
688
16.6k
          if (II->getIntrinsicID() == Intrinsic::assume)
689
19
            AC->registerAssumption(II);
690
1.82M
      }
691
279k
    }
692
204k
  }
693
11.1k
694
11.1k
  // Loop over the PHI nodes in the original block, setting incoming values.
695
15.6k
  for (PHINode *PN : OrigPHINode) {
696
15.6k
    if (CompletelyUnroll) {
697
11.9k
      PN->replaceAllUsesWith(PN->getIncomingValueForBlock(Preheader));
698
11.9k
      Header->getInstList().erase(PN);
699
11.9k
    } else 
if (3.68k
ULO.Count > 13.68k
) {
700
3.45k
      Value *InVal = PN->removeIncomingValue(LatchBlock, false);
701
3.45k
      // If this value was defined in the loop, take the value defined by the
702
3.45k
      // last iteration of the loop.
703
3.45k
      if (Instruction *InValI = dyn_cast<Instruction>(InVal)) {
704
3.45k
        if (L->contains(InValI))
705
3.45k
          InVal = LastValueMap[InVal];
706
3.45k
      }
707
3.45k
      assert(Latches.back() == LastValueMap[LatchBlock] && "bad last latch");
708
3.45k
      PN->addIncoming(InVal, Latches.back());
709
3.45k
    }
710
15.6k
  }
711
11.1k
712
11.1k
  auto setDest = [LoopExit, ContinueOnTrue](BasicBlock *Src, BasicBlock *Dest,
713
11.1k
                                            ArrayRef<BasicBlock *> NextBlocks,
714
11.1k
                                            BasicBlock *CurrentHeader,
715
215k
                                            bool NeedConditional) {
716
215k
    auto *Term = cast<BranchInst>(Src->getTerminator());
717
215k
    if (NeedConditional) {
718
2.02k
      // Update the conditional branch's successor for the following
719
2.02k
      // iteration.
720
2.02k
      Term->setSuccessor(!ContinueOnTrue, Dest);
721
213k
    } else {
722
213k
      // Remove phi operands at this loop exit
723
213k
      if (Dest != LoopExit) {
724
203k
        BasicBlock *BB = Src;
725
407k
        for (BasicBlock *Succ : successors(BB)) {
726
407k
          if (Succ == CurrentHeader)
727
203k
            continue;
728
203k
          for (PHINode &Phi : Succ->phis())
729
63.1k
            Phi.removeIncomingValue(BB, false);
730
203k
        }
731
203k
      }
732
213k
      // Replace the conditional branch with an unconditional one.
733
213k
      BranchInst::Create(Dest, Term);
734
213k
      Term->eraseFromParent();
735
213k
    }
736
215k
  };
737
11.1k
738
11.1k
  // Now that all the basic blocks for the unrolled iterations are in place,
739
11.1k
  // set up the branches to connect them.
740
11.1k
  if (LatchIsExiting) {
741
11.1k
    // Set up latches to branch to the new header in the unrolled iterations or
742
11.1k
    // the loop exit for the last latch in a fully unrolled loop.
743
226k
    for (unsigned i = 0, e = Latches.size(); i != e; 
++i215k
) {
744
215k
      // The branch destination.
745
215k
      unsigned j = (i + 1) % e;
746
215k
      BasicBlock *Dest = Headers[j];
747
215k
      bool NeedConditional = true;
748
215k
749
215k
      if (RuntimeTripCount && 
j != 06.78k
) {
750
5.34k
        NeedConditional = false;
751
5.34k
      }
752
215k
753
215k
      // For a complete unroll, make the last iteration end with a branch
754
215k
      // to the exit block.
755
215k
      if (CompletelyUnroll) {
756
206k
        if (j == 0)
757
9.36k
          Dest = LoopExit;
758
206k
        // If using trip count upper bound to completely unroll, we need to keep
759
206k
        // the conditional branch except the last one because the loop may exit
760
206k
        // after any iteration.
761
206k
        assert(NeedConditional &&
762
206k
               "NeedCondition cannot be modified by both complete "
763
206k
               "unrolling and runtime unrolling");
764
206k
        NeedConditional =
765
206k
            (ULO.PreserveCondBr && 
j489
&&
!(362
ULO.PreserveOnlyFirst362
&&
i != 0319
));
766
206k
      } else 
if (8.45k
j != BreakoutTrip8.45k
&&
767
8.45k
                 
(6.80k
ULO.TripMultiple == 06.80k
||
j % ULO.TripMultiple != 05.54k
)) {
768
1.26k
        // If we know the trip count or a multiple of it, we can safely use an
769
1.26k
        // unconditional branch for some iterations.
770
1.26k
        NeedConditional = false;
771
1.26k
      }
772
215k
773
215k
      setDest(Latches[i], Dest, Headers, Headers[i], NeedConditional);
774
215k
    }
775
11.1k
  } else {
776
17
    // Setup headers to branch to their new successors in the unrolled
777
17
    // iterations.
778
58
    for (unsigned i = 0, e = Headers.size(); i != e; 
++i41
) {
779
41
      // The branch destination.
780
41
      unsigned j = (i + 1) % e;
781
41
      BasicBlock *Dest = HeaderSucc[i];
782
41
      bool NeedConditional = true;
783
41
784
41
      if (RuntimeTripCount && 
j != 00
)
785
0
        NeedConditional = false;
786
41
787
41
      if (CompletelyUnroll)
788
31
        // We cannot drop the conditional branch for the last condition, as we
789
31
        // may have to execute the loop body depending on the condition.
790
31
        NeedConditional = j == 0 || 
ULO.PreserveCondBr17
;
791
10
      else if (j != BreakoutTrip &&
792
10
               
(7
ULO.TripMultiple == 07
||
j % ULO.TripMultiple != 04
))
793
3
        // If we know the trip count or a multiple of it, we can safely use an
794
3
        // unconditional branch for some iterations.
795
3
        NeedConditional = false;
796
41
797
41
      setDest(Headers[i], Dest, Headers, Headers[i], NeedConditional);
798
41
    }
799
17
800
17
    // Set up latches to branch to the new header in the unrolled iterations or
801
17
    // the loop exit for the last latch in a fully unrolled loop.
802
17
803
58
    for (unsigned i = 0, e = Latches.size(); i != e; 
++i41
) {
804
41
      // The original branch was replicated in each unrolled iteration.
805
41
      BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator());
806
41
807
41
      // The branch destination.
808
41
      unsigned j = (i + 1) % e;
809
41
      BasicBlock *Dest = Headers[j];
810
41
811
41
      // When completely unrolling, the last latch becomes unreachable.
812
41
      if (CompletelyUnroll && 
j == 031
)
813
14
        new UnreachableInst(Term->getContext(), Term);
814
27
      else
815
27
        // Replace the conditional branch with an unconditional one.
816
27
        BranchInst::Create(Dest, Term);
817
41
818
41
      Term->eraseFromParent();
819
41
    }
820
17
  }
821
11.1k
822
11.1k
  // Update dominators of blocks we might reach through exits.
823
11.1k
  // Immediate dominator of such block might change, because we add more
824
11.1k
  // routes which can lead to the exit: we can now reach it from the copied
825
11.1k
  // iterations too.
826
11.1k
  if (DT && ULO.Count > 1) {
827
20.6k
    for (auto *BB : OriginalLoopBlocks) {
828
20.6k
      auto *BBDomNode = DT->getNode(BB);
829
20.6k
      SmallVector<BasicBlock *, 16> ChildrenToUpdate;
830
32.2k
      for (auto *ChildDomNode : BBDomNode->getChildren()) {
831
32.2k
        auto *ChildBB = ChildDomNode->getBlock();
832
32.2k
        if (!L->contains(ChildBB))
833
11.5k
          ChildrenToUpdate.push_back(ChildBB);
834
32.2k
      }
835
20.6k
      BasicBlock *NewIDom;
836
20.6k
      BasicBlock *&TermBlock = LatchIsExiting ? 
LatchBlock20.6k
:
Header36
;
837
20.6k
      auto &TermBlocks = LatchIsExiting ? 
Latches20.6k
:
Headers36
;
838
20.6k
      if (BB == TermBlock) {
839
10.3k
        // The latch is special because we emit unconditional branches in
840
10.3k
        // some cases where the original loop contained a conditional branch.
841
10.3k
        // Since the latch is always at the bottom of the loop, if the latch
842
10.3k
        // dominated an exit before unrolling, the new dominator of that exit
843
10.3k
        // must also be a latch.  Specifically, the dominator is the first
844
10.3k
        // latch which ends in a conditional branch, or the last latch if
845
10.3k
        // there is no such latch.
846
10.3k
        // For loops exiting from the header, we limit the supported loops
847
10.3k
        // to have a single exiting block.
848
10.3k
        NewIDom = TermBlocks.back();
849
214k
        for (BasicBlock *Iter : TermBlocks) {
850
214k
          Instruction *Term = Iter->getTerminator();
851
214k
          if (isa<BranchInst>(Term) && cast<BranchInst>(Term)->isConditional()) {
852
1.78k
            NewIDom = Iter;
853
1.78k
            break;
854
1.78k
          }
855
214k
        }
856
10.3k
      } else {
857
10.2k
        // The new idom of the block will be the nearest common dominator
858
10.2k
        // of all copies of the previous idom. This is equivalent to the
859
10.2k
        // nearest common dominator of the previous idom and the first latch,
860
10.2k
        // which dominates all copies of the previous idom.
861
10.2k
        NewIDom = DT->findNearestCommonDominator(BB, LatchBlock);
862
10.2k
      }
863
20.6k
      for (auto *ChildBB : ChildrenToUpdate)
864
11.5k
        DT->changeImmediateDominator(ChildBB, NewIDom);
865
20.6k
    }
866
10.3k
  }
867
11.1k
868
11.1k
  assert(!DT || !UnrollVerifyDomtree ||
869
11.1k
         DT->verify(DominatorTree::VerificationLevel::Fast));
870
11.1k
871
11.1k
  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
872
11.1k
  // Merge adjacent basic blocks, if possible.
873
215k
  for (BasicBlock *Latch : Latches) {
874
215k
    BranchInst *Term = dyn_cast<BranchInst>(Latch->getTerminator());
875
215k
    assert((Term ||
876
215k
            (CompletelyUnroll && !LatchIsExiting && Latch == Latches.back())) &&
877
215k
           "Need a branch as terminator, except when fully unrolling with "
878
215k
           "unconditional latch");
879
215k
    if (Term && 
Term->isUnconditional()215k
) {
880
213k
      BasicBlock *Dest = Term->getSuccessor(0);
881
213k
      BasicBlock *Fold = Dest->getUniquePredecessor();
882
213k
      if (MergeBlockIntoPredecessor(Dest, &DTU, LI)) {
883
211k
        // Dest has been folded into Fold. Update our worklists accordingly.
884
211k
        std::replace(Latches.begin(), Latches.end(), Dest, Fold);
885
211k
        UnrolledLoopBlocks.erase(std::remove(UnrolledLoopBlocks.begin(),
886
211k
                                             UnrolledLoopBlocks.end(), Dest),
887
211k
                                 UnrolledLoopBlocks.end());
888
211k
      }
889
213k
    }
890
215k
  }
891
11.1k
892
11.1k
  // At this point, the code is well formed.  We now simplify the unrolled loop,
893
11.1k
  // doing constant propagation and dead code elimination as we go.
894
11.1k
  simplifyLoopAfterUnroll(L, !CompletelyUnroll && 
(1.77k
ULO.Count > 11.77k
||
Peeled119
), LI,
895
11.1k
                          SE, DT, AC);
896
11.1k
897
11.1k
  NumCompletelyUnrolled += CompletelyUnroll;
898
11.1k
  ++NumUnrolled;
899
11.1k
900
11.1k
  Loop *OuterL = L->getParentLoop();
901
11.1k
  // Update LoopInfo if the loop is completely removed.
902
11.1k
  if (CompletelyUnroll)
903
9.37k
    LI->erase(L);
904
11.1k
905
11.1k
  // After complete unrolling most of the blocks should be contained in OuterL.
906
11.1k
  // However, some of them might happen to be out of OuterL (e.g. if they
907
11.1k
  // precede a loop exit). In this case we might need to insert PHI nodes in
908
11.1k
  // order to preserve LCSSA form.
909
11.1k
  // We don't need to check this if we already know that we need to fix LCSSA
910
11.1k
  // form.
911
11.1k
  // TODO: For now we just recompute LCSSA for the outer loop in this case, but
912
11.1k
  // it should be possible to fix it in-place.
913
11.1k
  if (PreserveLCSSA && OuterL && 
CompletelyUnroll4.05k
&&
!NeedToFixLCSSA3.72k
)
914
3.12k
    NeedToFixLCSSA |= ::needToInsertPhisForLCSSA(OuterL, UnrolledLoopBlocks, LI);
915
11.1k
916
11.1k
  // If we have a pass and a DominatorTree we should re-simplify impacted loops
917
11.1k
  // to ensure subsequent analyses can rely on this form. We want to simplify
918
11.1k
  // at least one layer outside of the loop that was unrolled so that any
919
11.1k
  // changes to the parent loop exposed by the unrolling are considered.
920
11.1k
  if (DT) {
921
11.1k
    if (OuterL) {
922
4.05k
      // OuterL includes all loops for which we can break loop-simplify, so
923
4.05k
      // it's sufficient to simplify only it (it'll recursively simplify inner
924
4.05k
      // loops too).
925
4.05k
      if (NeedToFixLCSSA) {
926
756
        // LCSSA must be performed on the outermost affected loop. The unrolled
927
756
        // loop's last loop latch is guaranteed to be in the outermost loop
928
756
        // after LoopInfo's been updated by LoopInfo::erase.
929
756
        Loop *LatchLoop = LI->getLoopFor(Latches.back());
930
756
        Loop *FixLCSSALoop = OuterL;
931
756
        if (!FixLCSSALoop->contains(LatchLoop))
932
16
          
while (12
FixLCSSALoop->getParentLoop() != LatchLoop)
933
4
            FixLCSSALoop = FixLCSSALoop->getParentLoop();
934
756
935
756
        formLCSSARecursively(*FixLCSSALoop, *DT, LI, SE);
936
3.29k
      } else if (PreserveLCSSA) {
937
3.29k
        assert(OuterL->isLCSSAForm(*DT) &&
938
3.29k
               "Loops should be in LCSSA form after loop-unroll.");
939
3.29k
      }
940
4.05k
941
4.05k
      // TODO: That potentially might be compile-time expensive. We should try
942
4.05k
      // to fix the loop-simplified form incrementally.
943
4.05k
      simplifyLoop(OuterL, DT, LI, SE, AC, nullptr, PreserveLCSSA);
944
7.09k
    } else {
945
7.09k
      // Simplify loops for which we might've broken loop-simplify form.
946
7.09k
      for (Loop *SubLoop : LoopsToSimplify)
947
1.75k
        simplifyLoop(SubLoop, DT, LI, SE, AC, nullptr, PreserveLCSSA);
948
7.09k
    }
949
11.1k
  }
950
11.1k
951
11.1k
  return CompletelyUnroll ? 
LoopUnrollResult::FullyUnrolled9.37k
952
11.1k
                          : 
LoopUnrollResult::PartiallyUnrolled1.77k
;
953
11.1k
}
954
955
/// Given an llvm.loop loop id metadata node, returns the loop hint metadata
956
/// node with the given name (for example, "llvm.loop.unroll.count"). If no
957
/// such metadata node exists, then nullptr is returned.
958
246k
MDNode *llvm::GetUnrollMetadata(MDNode *LoopID, StringRef Name) {
959
246k
  // First operand should refer to the loop id itself.
960
246k
  assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
961
246k
  assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
962
246k
963
679k
  for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; 
++i433k
) {
964
436k
    MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
965
436k
    if (!MD)
966
0
      continue;
967
436k
968
436k
    MDString *S = dyn_cast<MDString>(MD->getOperand(0));
969
436k
    if (!S)
970
255k
      continue;
971
180k
972
180k
    if (Name.equals(S->getString()))
973
3.27k
      return MD;
974
180k
  }
975
246k
  
return nullptr243k
;
976
246k
}