Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Analysis/IVUsers.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- IVUsers.cpp - Induction Variable Users -------------------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements bookkeeping for "interesting" users of expressions
10
// computed from induction variables.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "llvm/Analysis/IVUsers.h"
15
#include "llvm/ADT/STLExtras.h"
16
#include "llvm/Analysis/AssumptionCache.h"
17
#include "llvm/Analysis/CodeMetrics.h"
18
#include "llvm/Analysis/LoopAnalysisManager.h"
19
#include "llvm/Analysis/LoopPass.h"
20
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
21
#include "llvm/Analysis/ValueTracking.h"
22
#include "llvm/Config/llvm-config.h"
23
#include "llvm/IR/Constants.h"
24
#include "llvm/IR/DataLayout.h"
25
#include "llvm/IR/DerivedTypes.h"
26
#include "llvm/IR/Dominators.h"
27
#include "llvm/IR/Instructions.h"
28
#include "llvm/IR/Module.h"
29
#include "llvm/IR/Type.h"
30
#include "llvm/Support/Debug.h"
31
#include "llvm/Support/raw_ostream.h"
32
#include <algorithm>
33
using namespace llvm;
34
35
#define DEBUG_TYPE "iv-users"
36
37
AnalysisKey IVUsersAnalysis::Key;
38
39
IVUsers IVUsersAnalysis::run(Loop &L, LoopAnalysisManager &AM,
40
12
                             LoopStandardAnalysisResults &AR) {
41
12
  return IVUsers(&L, &AR.AC, &AR.LI, &AR.DT, &AR.SE);
42
12
}
43
44
char IVUsersWrapperPass::ID = 0;
45
48.6k
INITIALIZE_PASS_BEGIN(IVUsersWrapperPass, "iv-users",
46
48.6k
                      "Induction Variable Users", false, true)
47
48.6k
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
48
48.6k
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
49
48.6k
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
50
48.6k
INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
51
48.6k
INITIALIZE_PASS_END(IVUsersWrapperPass, "iv-users", "Induction Variable Users",
52
                    false, true)
53
54
0
Pass *llvm::createIVUsersPass() { return new IVUsersWrapperPass(); }
55
56
/// isInteresting - Test whether the given expression is "interesting" when
57
/// used by the given expression, within the context of analyzing the
58
/// given loop.
59
static bool isInteresting(const SCEV *S, const Instruction *I, const Loop *L,
60
1.20M
                          ScalarEvolution *SE, LoopInfo *LI) {
61
1.20M
  // An addrec is interesting if it's affine or if it has an interesting start.
62
1.20M
  if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
63
945k
    // Keep things simple. Don't touch loop-variant strides unless they're
64
945k
    // only used outside the loop and we can simplify them.
65
945k
    if (AR->getLoop() == L)
66
925k
      return AR->isAffine() ||
67
925k
             
(302
!L->contains(I)302
&&
68
302
              
SE->getSCEVAtScope(AR, LI->getLoopFor(I->getParent())) != AR7
);
69
19.7k
    // Otherwise recurse to see if the start value is interesting, and that
70
19.7k
    // the step value is not interesting, since we don't yet know how to
71
19.7k
    // do effective SCEV expansions for addrecs with interesting steps.
72
19.7k
    return isInteresting(AR->getStart(), I, L, SE, LI) &&
73
19.7k
          
!isInteresting(AR->getStepRecurrence(*SE), I, L, SE, LI)19.7k
;
74
19.7k
  }
75
262k
76
262k
  // An add is interesting if exactly one of its operands is interesting.
77
262k
  if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
78
61.6k
    bool AnyInterestingYet = false;
79
61.6k
    for (const auto *Op : Add->operands())
80
125k
      if (isInteresting(Op, I, L, SE, LI)) {
81
58.7k
        if (AnyInterestingYet)
82
0
          return false;
83
58.7k
        AnyInterestingYet = true;
84
58.7k
      }
85
61.6k
    return AnyInterestingYet;
86
200k
  }
87
200k
88
200k
  // Nothing else is interesting here.
89
200k
  return false;
90
200k
}
91
92
/// Return true if all loop headers that dominate this block are in simplified
93
/// form.
94
static bool isSimplifiedLoopNest(BasicBlock *BB, const DominatorTree *DT,
95
                                 const LoopInfo *LI,
96
1.37M
                                 SmallPtrSetImpl<Loop*> &SimpleLoopNests) {
97
1.37M
  Loop *NearestLoop = nullptr;
98
1.37M
  for (DomTreeNode *Rung = DT->getNode(BB);
99
4.08M
       Rung; 
Rung = Rung->getIDom()2.71M
) {
100
3.93M
    BasicBlock *DomBB = Rung->getBlock();
101
3.93M
    Loop *DomLoop = LI->getLoopFor(DomBB);
102
3.93M
    if (DomLoop && 
DomLoop->getHeader() == DomBB2.38M
) {
103
1.52M
      // If the domtree walk reaches a loop with no preheader, return false.
104
1.52M
      if (!DomLoop->isLoopSimplifyForm())
105
36
        return false;
106
1.52M
      // If we have already checked this loop nest, stop checking.
107
1.52M
      if (SimpleLoopNests.count(DomLoop))
108
1.22M
        break;
109
298k
      // If we have not already checked this loop nest, remember the loop
110
298k
      // header nearest to BB. The nearest loop may not contain BB.
111
298k
      if (!NearestLoop)
112
158k
        NearestLoop = DomLoop;
113
298k
    }
114
3.93M
  }
115
1.37M
  
if (1.37M
NearestLoop1.37M
)
116
158k
    SimpleLoopNests.insert(NearestLoop);
117
1.37M
  return true;
118
1.37M
}
119
120
/// IVUseShouldUsePostIncValue - We have discovered a "User" of an IV expression
121
/// and now we need to decide whether the user should use the preinc or post-inc
122
/// value.  If this user should use the post-inc version of the IV, return true.
123
///
124
/// Choosing wrong here can break dominance properties (if we choose to use the
125
/// post-inc value when we cannot) or it can end up adding extra live-ranges to
126
/// the loop, resulting in reg-reg copies (if we use the pre-inc value when we
127
/// should use the post-inc value).
128
static bool IVUseShouldUsePostIncValue(Instruction *User, Value *Operand,
129
642k
                                       const Loop *L, DominatorTree *DT) {
130
642k
  // If the user is in the loop, use the preinc value.
131
642k
  if (L->contains(User))
132
590k
    return false;
133
52.1k
134
52.1k
  BasicBlock *LatchBlock = L->getLoopLatch();
135
52.1k
  if (!LatchBlock)
136
0
    return false;
137
52.1k
138
52.1k
  // Ok, the user is outside of the loop.  If it is dominated by the latch
139
52.1k
  // block, use the post-inc value.
140
52.1k
  if (DT->dominates(LatchBlock, User->getParent()))
141
22.5k
    return true;
142
29.6k
143
29.6k
  // There is one case we have to be careful of: PHI nodes.  These little guys
144
29.6k
  // can live in blocks that are not dominated by the latch block, but (since
145
29.6k
  // their uses occur in the predecessor block, not the block the PHI lives in)
146
29.6k
  // should still use the post-inc value.  Check for this case now.
147
29.6k
  PHINode *PN = dyn_cast<PHINode>(User);
148
29.6k
  if (!PN || 
!Operand17.7k
)
149
11.8k
    return false; // not a phi, not dominated by latch block.
150
17.7k
151
17.7k
  // Look at all of the uses of Operand by the PHI node.  If any use corresponds
152
17.7k
  // to a block that is not dominated by the latch block, give up and use the
153
17.7k
  // preincremented value.
154
52.1k
  
for (unsigned i = 0, e = PN->getNumIncomingValues(); 17.7k
i != e;
++i34.4k
)
155
41.0k
    if (PN->getIncomingValue(i) == Operand &&
156
41.0k
        
!DT->dominates(LatchBlock, PN->getIncomingBlock(i))17.8k
)
157
6.59k
      return false;
158
17.7k
159
17.7k
  // Okay, all uses of Operand by PN are in predecessor blocks that really are
160
17.7k
  // dominated by the latch block.  Use the post-incremented value.
161
17.7k
  
return true11.1k
;
162
17.7k
}
163
164
/// AddUsersImpl - Inspect the specified instruction.  If it is a
165
/// reducible SCEV, recursively add its users to the IVUsesByStride set and
166
/// return true.  Otherwise, return false.
167
bool IVUsers::AddUsersImpl(Instruction *I,
168
1.62M
                           SmallPtrSetImpl<Loop*> &SimpleLoopNests) {
169
1.62M
  const DataLayout &DL = I->getModule()->getDataLayout();
170
1.62M
171
1.62M
  // Add this IV user to the Processed set before returning false to ensure that
172
1.62M
  // all IV users are members of the set. See IVUsers::isIVUserOrOperand.
173
1.62M
  if (!Processed.insert(I).second)
174
240
    return true;    // Instruction already handled.
175
1.62M
176
1.62M
  if (!SE->isSCEVable(I->getType()))
177
284k
    return false;   // Void and FP expressions cannot be reduced.
178
1.33M
179
1.33M
  // IVUsers is used by LSR which assumes that all SCEV expressions are safe to
180
1.33M
  // pass to SCEVExpander. Expressions are not safe to expand if they represent
181
1.33M
  // operations that are not safe to speculate, namely integer division.
182
1.33M
  if (!isa<PHINode>(I) && 
!isSafeToSpeculativelyExecute(I)1.08M
)
183
158k
    return false;
184
1.18M
185
1.18M
  // LSR is not APInt clean, do not touch integers bigger than 64-bits.
186
1.18M
  // Also avoid creating IVs of non-native types. For example, we don't want a
187
1.18M
  // 64-bit IV in 32-bit code just because the loop has one 64-bit cast.
188
1.18M
  uint64_t Width = SE->getTypeSizeInBits(I->getType());
189
1.18M
  if (Width > 64 || 
!DL.isLegalInteger(Width)1.18M
)
190
138k
    return false;
191
1.04M
192
1.04M
  // Don't attempt to promote ephemeral values to indvars. They will be removed
193
1.04M
  // later anyway.
194
1.04M
  if (EphValues.count(I))
195
1
    return false;
196
1.04M
197
1.04M
  // Get the symbolic expression for this instruction.
198
1.04M
  const SCEV *ISE = SE->getSCEV(I);
199
1.04M
200
1.04M
  // If we've come to an uninteresting expression, stop the traversal and
201
1.04M
  // call this a user.
202
1.04M
  if (!isInteresting(ISE, I, L, SE, LI))
203
117k
    return false;
204
925k
205
925k
  SmallPtrSet<Instruction *, 4> UniqueUsers;
206
1.52M
  for (Use &U : I->uses()) {
207
1.52M
    Instruction *User = cast<Instruction>(U.getUser());
208
1.52M
    if (!UniqueUsers.insert(User).second)
209
1.00k
      continue;
210
1.52M
211
1.52M
    // Do not infinitely recurse on PHI nodes.
212
1.52M
    if (isa<PHINode>(User) && 
Processed.count(User)172k
)
213
148k
      continue;
214
1.37M
215
1.37M
    // Only consider IVUsers that are dominated by simplified loop
216
1.37M
    // headers. Otherwise, SCEVExpander will crash.
217
1.37M
    BasicBlock *UseBB = User->getParent();
218
1.37M
    // A phi's use is live out of its predecessor block.
219
1.37M
    if (PHINode *PHI = dyn_cast<PHINode>(User)) {
220
23.3k
      unsigned OperandNo = U.getOperandNo();
221
23.3k
      unsigned ValNo = PHINode::getIncomingValueNumForOperand(OperandNo);
222
23.3k
      UseBB = PHI->getIncomingBlock(ValNo);
223
23.3k
    }
224
1.37M
    if (!isSimplifiedLoopNest(UseBB, DT, LI, SimpleLoopNests))
225
36
      return false;
226
1.37M
227
1.37M
    // Descend recursively, but not into PHI nodes outside the current loop.
228
1.37M
    // It's important to see the entire expression outside the loop to get
229
1.37M
    // choices that depend on addressing mode use right, although we won't
230
1.37M
    // consider references outside the loop in all cases.
231
1.37M
    // If User is already in Processed, we don't want to recurse into it again,
232
1.37M
    // but do want to record a second reference in the same instruction.
233
1.37M
    bool AddUserToIVUsers = false;
234
1.37M
    if (LI->getLoopFor(User->getParent()) != L) {
235
105k
      if (isa<PHINode>(User) || 
Processed.count(User)83.3k
||
236
105k
          
!AddUsersImpl(User, SimpleLoopNests)83.0k
) {
237
69.2k
        LLVM_DEBUG(dbgs() << "FOUND USER in other loop: " << *User << '\n'
238
69.2k
                          << "   OF SCEV: " << *ISE << '\n');
239
69.2k
        AddUserToIVUsers = true;
240
69.2k
      }
241
1.26M
    } else if (Processed.count(User) || 
!AddUsersImpl(User, SimpleLoopNests)1.26M
) {
242
529k
      LLVM_DEBUG(dbgs() << "FOUND USER: " << *User << '\n'
243
529k
                        << "   OF SCEV: " << *ISE << '\n');
244
529k
      AddUserToIVUsers = true;
245
529k
    }
246
1.37M
247
1.37M
    if (AddUserToIVUsers) {
248
599k
      // Okay, we found a user that we cannot reduce.
249
599k
      IVStrideUse &NewUse = AddUser(User, I);
250
599k
      // Autodetect the post-inc loop set, populating NewUse.PostIncLoops.
251
599k
      // The regular return value here is discarded; instead of recording
252
599k
      // it, we just recompute it when we need it.
253
599k
      const SCEV *OriginalISE = ISE;
254
599k
255
642k
      auto NormalizePred = [&](const SCEVAddRecExpr *AR) {
256
642k
        auto *L = AR->getLoop();
257
642k
        bool Result = IVUseShouldUsePostIncValue(User, I, L, DT);
258
642k
        if (Result)
259
33.6k
          NewUse.PostIncLoops.insert(L);
260
642k
        return Result;
261
642k
      };
262
599k
263
599k
      ISE = normalizeForPostIncUseIf(ISE, NormalizePred, *SE);
264
599k
265
599k
      // PostIncNormalization effectively simplifies the expression under
266
599k
      // pre-increment assumptions. Those assumptions (no wrapping) might not
267
599k
      // hold for the post-inc value. Catch such cases by making sure the
268
599k
      // transformation is invertible.
269
599k
      if (OriginalISE != ISE) {
270
33.1k
        const SCEV *DenormalizedISE =
271
33.1k
            denormalizeForPostIncUse(ISE, NewUse.PostIncLoops, *SE);
272
33.1k
273
33.1k
        // If we normalized the expression, but denormalization doesn't give the
274
33.1k
        // original one, discard this user.
275
33.1k
        if (OriginalISE != DenormalizedISE) {
276
32
          LLVM_DEBUG(dbgs()
277
32
                     << "   DISCARDING (NORMALIZATION ISN'T INVERTIBLE): "
278
32
                     << *ISE << '\n');
279
32
          IVUses.pop_back();
280
32
          return false;
281
32
        }
282
599k
      }
283
599k
      LLVM_DEBUG(if (SE->getSCEV(I) != ISE) dbgs()
284
599k
                 << "   NORMALIZED TO: " << *ISE << '\n');
285
599k
    }
286
1.37M
  }
287
925k
  
return true925k
;
288
925k
}
289
290
274k
bool IVUsers::AddUsersIfInteresting(Instruction *I) {
291
274k
  // SCEVExpander can only handle users that are dominated by simplified loop
292
274k
  // entries. Keep track of all loops that are only dominated by other simple
293
274k
  // loops so we don't traverse the domtree for each user.
294
274k
  SmallPtrSet<Loop*,16> SimpleLoopNests;
295
274k
296
274k
  return AddUsersImpl(I, SimpleLoopNests);
297
274k
}
298
299
599k
IVStrideUse &IVUsers::AddUser(Instruction *User, Value *Operand) {
300
599k
  IVUses.push_back(new IVStrideUse(this, User, Operand));
301
599k
  return IVUses.back();
302
599k
}
303
304
IVUsers::IVUsers(Loop *L, AssumptionCache *AC, LoopInfo *LI, DominatorTree *DT,
305
                 ScalarEvolution *SE)
306
208k
    : L(L), AC(AC), LI(LI), DT(DT), SE(SE), IVUses() {
307
208k
  // Collect ephemeral values so that AddUsersIfInteresting skips them.
308
208k
  EphValues.clear();
309
208k
  CodeMetrics::collectEphemeralValues(L, AC, EphValues);
310
208k
311
208k
  // Find all uses of induction variables in this loop, and categorize
312
208k
  // them by stride.  Start by finding all of the PHI nodes in the header for
313
208k
  // this loop.  If they are induction variables, inspect their uses.
314
483k
  for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); 
++I274k
)
315
274k
    (void)AddUsersIfInteresting(&*I);
316
208k
}
317
318
23
void IVUsers::print(raw_ostream &OS, const Module *M) const {
319
23
  OS << "IV Users for loop ";
320
23
  L->getHeader()->printAsOperand(OS, false);
321
23
  if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
322
19
    OS << " with backedge-taken count " << *SE->getBackedgeTakenCount(L);
323
19
  }
324
23
  OS << ":\n";
325
23
326
23
  for (const IVStrideUse &IVUse : IVUses) {
327
20
    OS << "  ";
328
20
    IVUse.getOperandValToReplace()->printAsOperand(OS, false);
329
20
    OS << " = " << *getReplacementExpr(IVUse);
330
20
    for (auto PostIncLoop : IVUse.PostIncLoops) {
331
13
      OS << " (post-inc with loop ";
332
13
      PostIncLoop->getHeader()->printAsOperand(OS, false);
333
13
      OS << ")";
334
13
    }
335
20
    OS << " in  ";
336
20
    if (IVUse.getUser())
337
20
      IVUse.getUser()->print(OS);
338
0
    else
339
0
      OS << "Printing <null> User";
340
20
    OS << '\n';
341
20
  }
342
23
}
343
344
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
345
LLVM_DUMP_METHOD void IVUsers::dump() const { print(dbgs()); }
346
#endif
347
348
208k
void IVUsers::releaseMemory() {
349
208k
  Processed.clear();
350
208k
  IVUses.clear();
351
208k
}
352
353
34.4k
IVUsersWrapperPass::IVUsersWrapperPass() : LoopPass(ID) {
354
34.4k
  initializeIVUsersWrapperPassPass(*PassRegistry::getPassRegistry());
355
34.4k
}
356
357
34.4k
void IVUsersWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
358
34.4k
  AU.addRequired<AssumptionCacheTracker>();
359
34.4k
  AU.addRequired<LoopInfoWrapperPass>();
360
34.4k
  AU.addRequired<DominatorTreeWrapperPass>();
361
34.4k
  AU.addRequired<ScalarEvolutionWrapperPass>();
362
34.4k
  AU.setPreservesAll();
363
34.4k
}
364
365
208k
bool IVUsersWrapperPass::runOnLoop(Loop *L, LPPassManager &LPM) {
366
208k
  auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
367
208k
      *L->getHeader()->getParent());
368
208k
  auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
369
208k
  auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
370
208k
  auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
371
208k
372
208k
  IU.reset(new IVUsers(L, AC, LI, DT, SE));
373
208k
  return false;
374
208k
}
375
376
18
void IVUsersWrapperPass::print(raw_ostream &OS, const Module *M) const {
377
18
  IU->print(OS, M);
378
18
}
379
380
208k
void IVUsersWrapperPass::releaseMemory() { IU->releaseMemory(); }
381
382
/// getReplacementExpr - Return a SCEV expression which computes the
383
/// value of the OperandValToReplace.
384
1.00M
const SCEV *IVUsers::getReplacementExpr(const IVStrideUse &IU) const {
385
1.00M
  return SE->getSCEV(IU.getOperandValToReplace());
386
1.00M
}
387
388
/// getExpr - Return the expression for the use.
389
1.00M
const SCEV *IVUsers::getExpr(const IVStrideUse &IU) const {
390
1.00M
  return normalizeForPostIncUse(getReplacementExpr(IU), IU.getPostIncLoops(),
391
1.00M
                                *SE);
392
1.00M
}
393
394
3.75k
static const SCEVAddRecExpr *findAddRecForLoop(const SCEV *S, const Loop *L) {
395
3.75k
  if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
396
3.53k
    if (AR->getLoop() == L)
397
3.52k
      return AR;
398
10
    return findAddRecForLoop(AR->getStart(), L);
399
10
  }
400
217
401
217
  if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(S)) {
402
207
    for (const auto *Op : Add->operands())
403
217
      if (const SCEVAddRecExpr *AR = findAddRecForLoop(Op, L))
404
207
        return AR;
405
207
    
return nullptr0
;
406
10
  }
407
10
408
10
  return nullptr;
409
10
}
410
411
3.52k
const SCEV *IVUsers::getStride(const IVStrideUse &IU, const Loop *L) const {
412
3.52k
  if (const SCEVAddRecExpr *AR = findAddRecForLoop(getExpr(IU), L))
413
3.52k
    return AR->getStepRecurrence(*SE);
414
0
  return nullptr;
415
0
}
416
417
106k
void IVStrideUse::transformToPostInc(const Loop *L) {
418
106k
  PostIncLoops.insert(L);
419
106k
}
420
421
107
void IVStrideUse::deleted() {
422
107
  // Remove this user from the list.
423
107
  Parent->Processed.erase(this->getUser());
424
107
  Parent->IVUses.erase(this);
425
107
  // this now dangles!
426
107
}