Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Analysis/ScalarEvolutionExpander.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- ScalarEvolutionExpander.cpp - Scalar Evolution Analysis ------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file contains the implementation of the scalar evolution expander,
10
// which is used to generate the code corresponding to a given scalar evolution
11
// expression.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#include "llvm/Analysis/ScalarEvolutionExpander.h"
16
#include "llvm/ADT/STLExtras.h"
17
#include "llvm/ADT/SmallSet.h"
18
#include "llvm/Analysis/InstructionSimplify.h"
19
#include "llvm/Analysis/LoopInfo.h"
20
#include "llvm/Analysis/TargetTransformInfo.h"
21
#include "llvm/IR/DataLayout.h"
22
#include "llvm/IR/Dominators.h"
23
#include "llvm/IR/IntrinsicInst.h"
24
#include "llvm/IR/LLVMContext.h"
25
#include "llvm/IR/Module.h"
26
#include "llvm/IR/PatternMatch.h"
27
#include "llvm/Support/Debug.h"
28
#include "llvm/Support/raw_ostream.h"
29
30
using namespace llvm;
31
using namespace PatternMatch;
32
33
/// ReuseOrCreateCast - Arrange for there to be a cast of V to Ty at IP,
34
/// reusing an existing cast if a suitable one exists, moving an existing
35
/// cast if a suitable one exists but isn't in the right place, or
36
/// creating a new one.
37
Value *SCEVExpander::ReuseOrCreateCast(Value *V, Type *Ty,
38
                                       Instruction::CastOps Op,
39
291k
                                       BasicBlock::iterator IP) {
40
291k
  // This function must be called with the builder having a valid insertion
41
291k
  // point. It doesn't need to be the actual IP where the uses of the returned
42
291k
  // cast will be added, but it must dominate such IP.
43
291k
  // We use this precondition to produce a cast that will dominate all its
44
291k
  // uses. In particular, this is crucial for the case where the builder's
45
291k
  // insertion point *is* the point where we were asked to put the cast.
46
291k
  // Since we don't know the builder's insertion point is actually
47
291k
  // where the uses will be added (only that it dominates it), we are
48
291k
  // not allowed to move it.
49
291k
  BasicBlock::iterator BIP = Builder.GetInsertPoint();
50
291k
51
291k
  Instruction *Ret = nullptr;
52
291k
53
291k
  // Check to see if there is already a cast!
54
291k
  for (User *U : V->users())
55
880k
    if (U->getType() == Ty)
56
176k
      if (CastInst *CI = dyn_cast<CastInst>(U))
57
167k
        if (CI->getOpcode() == Op) {
58
167k
          // If the cast isn't where we want it, create a new cast at IP.
59
167k
          // Likewise, do not reuse a cast at BIP because it must dominate
60
167k
          // instructions that might be inserted before BIP.
61
167k
          if (BasicBlock::iterator(CI) != IP || 
BIP == IP164k
) {
62
3.31k
            // Create a new cast, and leave the old cast in place in case
63
3.31k
            // it is being used as an insert point.
64
3.31k
            Ret = CastInst::Create(Op, V, Ty, "", &*IP);
65
3.31k
            Ret->takeName(CI);
66
3.31k
            CI->replaceAllUsesWith(Ret);
67
3.31k
            break;
68
3.31k
          }
69
164k
          Ret = CI;
70
164k
          break;
71
164k
        }
72
291k
73
291k
  // Create a new cast.
74
291k
  if (!Ret)
75
124k
    Ret = CastInst::Create(Op, V, Ty, V->getName(), &*IP);
76
291k
77
291k
  // We assert at the end of the function since IP might point to an
78
291k
  // instruction with different dominance properties than a cast
79
291k
  // (an invoke for example) and not dominate BIP (but the cast does).
80
291k
  assert(SE.DT.dominates(Ret, &*BIP));
81
291k
82
291k
  rememberInstruction(Ret);
83
291k
  return Ret;
84
291k
}
85
86
static BasicBlock::iterator findInsertPointAfter(Instruction *I,
87
287k
                                                 BasicBlock *MustDominate) {
88
287k
  BasicBlock::iterator IP = ++I->getIterator();
89
287k
  if (auto *II = dyn_cast<InvokeInst>(I))
90
0
    IP = II->getNormalDest()->begin();
91
287k
92
682k
  while (isa<PHINode>(IP))
93
394k
    ++IP;
94
287k
95
287k
  if (isa<FuncletPadInst>(IP) || 
isa<LandingPadInst>(IP)287k
) {
96
1
    ++IP;
97
287k
  } else if (isa<CatchSwitchInst>(IP)) {
98
3
    IP = MustDominate->getFirstInsertionPt();
99
287k
  } else {
100
287k
    assert(!IP->isEHPad() && "unexpected eh pad!");
101
287k
  }
102
287k
103
287k
  return IP;
104
287k
}
105
106
/// InsertNoopCastOfTo - Insert a cast of V to the specified type,
107
/// which must be possible with a noop cast, doing what we can to share
108
/// the casts.
109
2.11M
Value *SCEVExpander::InsertNoopCastOfTo(Value *V, Type *Ty) {
110
2.11M
  Instruction::CastOps Op = CastInst::getCastOpcode(V, false, Ty, false);
111
2.11M
  assert((Op == Instruction::BitCast ||
112
2.11M
          Op == Instruction::PtrToInt ||
113
2.11M
          Op == Instruction::IntToPtr) &&
114
2.11M
         "InsertNoopCastOfTo cannot perform non-noop casts!");
115
2.11M
  assert(SE.getTypeSizeInBits(V->getType()) == SE.getTypeSizeInBits(Ty) &&
116
2.11M
         "InsertNoopCastOfTo cannot change sizes!");
117
2.11M
118
2.11M
  // Short-circuit unnecessary bitcasts.
119
2.11M
  if (Op == Instruction::BitCast) {
120
2.11M
    if (V->getType() == Ty)
121
1.80M
      return V;
122
308k
    if (CastInst *CI = dyn_cast<CastInst>(V)) {
123
5.31k
      if (CI->getOperand(0)->getType() == Ty)
124
4.42k
        return CI->getOperand(0);
125
311k
    }
126
308k
  }
127
311k
  // Short-circuit unnecessary inttoptr<->ptrtoint casts.
128
311k
  if ((Op == Instruction::PtrToInt || 
Op == Instruction::IntToPtr307k
) &&
129
311k
      
SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(V->getType())7.42k
) {
130
7.42k
    if (CastInst *CI = dyn_cast<CastInst>(V))
131
446
      if ((CI->getOpcode() == Instruction::PtrToInt ||
132
446
           
CI->getOpcode() == Instruction::IntToPtr445
) &&
133
446
          SE.getTypeSizeInBits(CI->getType()) ==
134
445
          SE.getTypeSizeInBits(CI->getOperand(0)->getType()))
135
445
        return CI->getOperand(0);
136
6.98k
    if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
137
16
      if ((CE->getOpcode() == Instruction::PtrToInt ||
138
16
           CE->getOpcode() == Instruction::IntToPtr) &&
139
16
          SE.getTypeSizeInBits(CE->getType()) ==
140
0
          SE.getTypeSizeInBits(CE->getOperand(0)->getType()))
141
0
        return CE->getOperand(0);
142
310k
  }
143
310k
144
310k
  // Fold a cast of a constant.
145
310k
  if (Constant *C = dyn_cast<Constant>(V))
146
18.8k
    return ConstantExpr::getCast(Op, C, Ty);
147
291k
148
291k
  // Cast the argument at the beginning of the entry block, after
149
291k
  // any bitcasts of other arguments.
150
291k
  if (Argument *A = dyn_cast<Argument>(V)) {
151
4.11k
    BasicBlock::iterator IP = A->getParent()->getEntryBlock().begin();
152
5.50k
    while ((isa<BitCastInst>(IP) &&
153
5.50k
            
isa<Argument>(cast<BitCastInst>(IP)->getOperand(0))3.55k
&&
154
5.50k
            
cast<BitCastInst>(IP)->getOperand(0) != A3.55k
) ||
155
5.50k
           
isa<DbgInfoIntrinsic>(IP)4.11k
)
156
1.39k
      ++IP;
157
4.11k
    return ReuseOrCreateCast(A, Ty, Op, IP);
158
4.11k
  }
159
287k
160
287k
  // Cast the instruction immediately after the instruction.
161
287k
  Instruction *I = cast<Instruction>(V);
162
287k
  BasicBlock::iterator IP = findInsertPointAfter(I, Builder.GetInsertBlock());
163
287k
  return ReuseOrCreateCast(I, Ty, Op, IP);
164
287k
}
165
166
/// InsertBinop - Insert the specified binary operator, doing a small amount
167
/// of work to avoid inserting an obviously redundant operation, and hoisting
168
/// to an outer loop when the opportunity is there and it is safe.
169
Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode,
170
                                 Value *LHS, Value *RHS,
171
82.7k
                                 SCEV::NoWrapFlags Flags, bool IsSafeToHoist) {
172
82.7k
  // Fold a binop with constant operands.
173
82.7k
  if (Constant *CLHS = dyn_cast<Constant>(LHS))
174
5.49k
    if (Constant *CRHS = dyn_cast<Constant>(RHS))
175
258
      return ConstantExpr::get(Opcode, CLHS, CRHS);
176
82.5k
177
82.5k
  // Do a quick scan to see if we have this binop nearby.  If so, reuse it.
178
82.5k
  unsigned ScanLimit = 6;
179
82.5k
  BasicBlock::iterator BlockBegin = Builder.GetInsertBlock()->begin();
180
82.5k
  // Scanning starts from the last instruction before the insertion point.
181
82.5k
  BasicBlock::iterator IP = Builder.GetInsertPoint();
182
82.5k
  if (IP != BlockBegin) {
183
76.9k
    --IP;
184
316k
    for (; ScanLimit; 
--IP, --ScanLimit239k
) {
185
293k
      // Don't count dbg.value against the ScanLimit, to avoid perturbing the
186
293k
      // generated code.
187
293k
      if (isa<DbgInfoIntrinsic>(IP))
188
1
        ScanLimit++;
189
293k
190
293k
      auto canGenerateIncompatiblePoison = [&Flags](Instruction *I) {
191
5.66k
        // Ensure that no-wrap flags match.
192
5.66k
        if (isa<OverflowingBinaryOperator>(I)) {
193
5.63k
          if (I->hasNoSignedWrap() != (Flags & SCEV::FlagNSW))
194
4.29k
            return true;
195
1.34k
          if (I->hasNoUnsignedWrap() != (Flags & SCEV::FlagNUW))
196
119
            return true;
197
1.24k
        }
198
1.24k
        // Conservatively, do not use any instruction which has any of exact
199
1.24k
        // flags installed.
200
1.24k
        if (isa<PossiblyExactOperator>(I) && 
I->isExact()24
)
201
18
          return true;
202
1.22k
        return false;
203
1.22k
      };
204
293k
      if (IP->getOpcode() == (unsigned)Opcode && 
IP->getOperand(0) == LHS39.0k
&&
205
293k
          
IP->getOperand(1) == RHS7.78k
&&
!canGenerateIncompatiblePoison(&*IP)5.66k
)
206
1.22k
        return &*IP;
207
292k
      if (IP == BlockBegin) 
break52.4k
;
208
292k
    }
209
76.9k
  }
210
82.5k
211
82.5k
  // Save the original insertion point so we can restore it when we're done.
212
82.5k
  DebugLoc Loc = Builder.GetInsertPoint()->getDebugLoc();
213
81.2k
  SCEVInsertPointGuard Guard(Builder, this);
214
81.2k
215
81.2k
  if (IsSafeToHoist) {
216
81.1k
    // Move the insertion point out of as many loops as we can.
217
82.4k
    while (const Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock())) {
218
42.6k
      if (!L->isLoopInvariant(LHS) || 
!L->isLoopInvariant(RHS)6.79k
)
break41.3k
;
219
1.32k
      BasicBlock *Preheader = L->getLoopPreheader();
220
1.32k
      if (!Preheader) 
break8
;
221
1.31k
222
1.31k
      // Ok, move up a level.
223
1.31k
      Builder.SetInsertPoint(Preheader->getTerminator());
224
1.31k
    }
225
81.1k
  }
226
81.2k
227
81.2k
  // If we haven't found this binop, insert it.
228
81.2k
  Instruction *BO = cast<Instruction>(Builder.CreateBinOp(Opcode, LHS, RHS));
229
81.2k
  BO->setDebugLoc(Loc);
230
81.2k
  if (Flags & SCEV::FlagNUW)
231
18.7k
    BO->setHasNoUnsignedWrap();
232
81.2k
  if (Flags & SCEV::FlagNSW)
233
25.2k
    BO->setHasNoSignedWrap();
234
81.2k
  rememberInstruction(BO);
235
81.2k
236
81.2k
  return BO;
237
82.5k
}
238
239
/// FactorOutConstant - Test if S is divisible by Factor, using signed
240
/// division. If so, update S with Factor divided out and return true.
241
/// S need not be evenly divisible if a reasonable remainder can be
242
/// computed.
243
/// TODO: When ScalarEvolution gets a SCEVSDivExpr, this can be made
244
/// unnecessary; in its place, just signed-divide Ops[i] by the scale and
245
/// check to see if the divide was folded.
246
static bool FactorOutConstant(const SCEV *&S, const SCEV *&Remainder,
247
                              const SCEV *Factor, ScalarEvolution &SE,
248
532k
                              const DataLayout &DL) {
249
532k
  // Everything is divisible by one.
250
532k
  if (Factor->isOne())
251
124k
    return true;
252
407k
253
407k
  // x/x == 1.
254
407k
  if (S == Factor) {
255
31.4k
    S = SE.getConstant(S->getType(), 1);
256
31.4k
    return true;
257
31.4k
  }
258
375k
259
375k
  // For a Constant, check for a multiple of the given factor.
260
375k
  if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
261
282k
    // 0/x == 0.
262
282k
    if (C->isZero())
263
10.9k
      return true;
264
271k
    // Check for divisibility.
265
271k
    if (const SCEVConstant *FC = dyn_cast<SCEVConstant>(Factor)) {
266
271k
      ConstantInt *CI =
267
271k
          ConstantInt::get(SE.getContext(), C->getAPInt().sdiv(FC->getAPInt()));
268
271k
      // If the quotient is zero and the remainder is non-zero, reject
269
271k
      // the value at this scale. It will be considered for subsequent
270
271k
      // smaller scales.
271
271k
      if (!CI->isZero()) {
272
215k
        const SCEV *Div = SE.getConstant(CI);
273
215k
        S = Div;
274
215k
        Remainder = SE.getAddExpr(
275
215k
            Remainder, SE.getConstant(C->getAPInt().srem(FC->getAPInt())));
276
215k
        return true;
277
215k
      }
278
149k
    }
279
271k
  }
280
149k
281
149k
  // In a Mul, check if there is a constant operand which is a multiple
282
149k
  // of the given factor.
283
149k
  if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
284
54.9k
    // Size is known, check if there is a constant operand which is a multiple
285
54.9k
    // of the given factor. If so, we can factor it.
286
54.9k
    const SCEVConstant *FC = cast<SCEVConstant>(Factor);
287
54.9k
    if (const SCEVConstant *C = dyn_cast<SCEVConstant>(M->getOperand(0)))
288
54.9k
      if (!C->getAPInt().srem(FC->getAPInt())) {
289
48.3k
        SmallVector<const SCEV *, 4> NewMulOps(M->op_begin(), M->op_end());
290
48.3k
        NewMulOps[0] = SE.getConstant(C->getAPInt().sdiv(FC->getAPInt()));
291
48.3k
        S = SE.getMulExpr(NewMulOps);
292
48.3k
        return true;
293
48.3k
      }
294
101k
  }
295
101k
296
101k
  // In an AddRec, check if both start and step are divisible.
297
101k
  if (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(S)) {
298
2.85k
    const SCEV *Step = A->getStepRecurrence(SE);
299
2.85k
    const SCEV *StepRem = SE.getConstant(Step->getType(), 0);
300
2.85k
    if (!FactorOutConstant(Step, StepRem, Factor, SE, DL))
301
1.18k
      return false;
302
1.66k
    if (!StepRem->isZero())
303
4
      return false;
304
1.65k
    const SCEV *Start = A->getStart();
305
1.65k
    if (!FactorOutConstant(Start, Remainder, Factor, SE, DL))
306
0
      return false;
307
1.65k
    S = SE.getAddRecExpr(Start, Step, A->getLoop(),
308
1.65k
                         A->getNoWrapFlags(SCEV::FlagNW));
309
1.65k
    return true;
310
1.65k
  }
311
98.6k
312
98.6k
  return false;
313
98.6k
}
314
315
/// SimplifyAddOperands - Sort and simplify a list of add operands. NumAddRecs
316
/// is the number of SCEVAddRecExprs present, which are kept at the end of
317
/// the list.
318
///
319
static void SimplifyAddOperands(SmallVectorImpl<const SCEV *> &Ops,
320
                                Type *Ty,
321
416k
                                ScalarEvolution &SE) {
322
416k
  unsigned NumAddRecs = 0;
323
418k
  for (unsigned i = Ops.size(); i > 0 && 
isa<SCEVAddRecExpr>(Ops[i-1])7.55k
;
--i1.81k
)
324
1.81k
    ++NumAddRecs;
325
416k
  // Group Ops into non-addrecs and addrecs.
326
416k
  SmallVector<const SCEV *, 8> NoAddRecs(Ops.begin(), Ops.end() - NumAddRecs);
327
416k
  SmallVector<const SCEV *, 8> AddRecs(Ops.end() - NumAddRecs, Ops.end());
328
416k
  // Let ScalarEvolution sort and simplify the non-addrecs list.
329
416k
  const SCEV *Sum = NoAddRecs.empty() ?
330
411k
                    SE.getConstant(Ty, 0) :
331
416k
                    
SE.getAddExpr(NoAddRecs)5.73k
;
332
416k
  // If it returned an add, use the operands. Otherwise it simplified
333
416k
  // the sum into a single value, so just use that.
334
416k
  Ops.clear();
335
416k
  if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Sum))
336
274
    Ops.append(Add->op_begin(), Add->op_end());
337
416k
  else if (!Sum->isZero())
338
5.44k
    Ops.push_back(Sum);
339
416k
  // Then append the addrecs.
340
416k
  Ops.append(AddRecs.begin(), AddRecs.end());
341
416k
}
342
343
/// SplitAddRecs - Flatten a list of add operands, moving addrec start values
344
/// out to the top level. For example, convert {a + b,+,c} to a, b, {0,+,d}.
345
/// This helps expose more opportunities for folding parts of the expressions
346
/// into GEP indices.
347
///
348
static void SplitAddRecs(SmallVectorImpl<const SCEV *> &Ops,
349
                         Type *Ty,
350
440k
                         ScalarEvolution &SE) {
351
440k
  // Find the addrecs.
352
440k
  SmallVector<const SCEV *, 8> AddRecs;
353
897k
  for (unsigned i = 0, e = Ops.size(); i != e; 
++i456k
)
354
458k
    
while (const SCEVAddRecExpr *456k
A = dyn_cast<SCEVAddRecExpr>(Ops[i])) {
355
2.06k
      const SCEV *Start = A->getStart();
356
2.06k
      if (Start->isZero()) 
break331
;
357
1.73k
      const SCEV *Zero = SE.getConstant(Ty, 0);
358
1.73k
      AddRecs.push_back(SE.getAddRecExpr(Zero,
359
1.73k
                                         A->getStepRecurrence(SE),
360
1.73k
                                         A->getLoop(),
361
1.73k
                                         A->getNoWrapFlags(SCEV::FlagNW)));
362
1.73k
      if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Start)) {
363
169
        Ops[i] = Zero;
364
169
        Ops.append(Add->op_begin(), Add->op_end());
365
169
        e += Add->getNumOperands();
366
1.56k
      } else {
367
1.56k
        Ops[i] = Start;
368
1.56k
      }
369
1.73k
    }
370
440k
  if (!AddRecs.empty()) {
371
1.66k
    // Add the addrecs onto the end of the list.
372
1.66k
    Ops.append(AddRecs.begin(), AddRecs.end());
373
1.66k
    // Resort the operand list, moving any constants to the front.
374
1.66k
    SimplifyAddOperands(Ops, Ty, SE);
375
1.66k
  }
376
440k
}
377
378
/// expandAddToGEP - Expand an addition expression with a pointer type into
379
/// a GEP instead of using ptrtoint+arithmetic+inttoptr. This helps
380
/// BasicAliasAnalysis and other passes analyze the result. See the rules
381
/// for getelementptr vs. inttoptr in
382
/// http://llvm.org/docs/LangRef.html#pointeraliasing
383
/// for details.
384
///
385
/// Design note: The correctness of using getelementptr here depends on
386
/// ScalarEvolution not recognizing inttoptr and ptrtoint operators, as
387
/// they may introduce pointer arithmetic which may not be safely converted
388
/// into getelementptr.
389
///
390
/// Design note: It might seem desirable for this function to be more
391
/// loop-aware. If some of the indices are loop-invariant while others
392
/// aren't, it might seem desirable to emit multiple GEPs, keeping the
393
/// loop-invariant portions of the overall computation outside the loop.
394
/// However, there are a few reasons this is not done here. Hoisting simple
395
/// arithmetic is a low-level optimization that often isn't very
396
/// important until late in the optimization process. In fact, passes
397
/// like InstructionCombining will combine GEPs, even if it means
398
/// pushing loop-invariant computation down into loops, so even if the
399
/// GEPs were split here, the work would quickly be undone. The
400
/// LoopStrengthReduction pass, which is usually run quite late (and
401
/// after the last InstructionCombining pass), takes care of hoisting
402
/// loop-invariant portions of expressions, after considering what
403
/// can be folded using target addressing modes.
404
///
405
Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
406
                                    const SCEV *const *op_end,
407
                                    PointerType *PTy,
408
                                    Type *Ty,
409
440k
                                    Value *V) {
410
440k
  Type *OriginalElTy = PTy->getElementType();
411
440k
  Type *ElTy = OriginalElTy;
412
440k
  SmallVector<Value *, 4> GepIndices;
413
440k
  SmallVector<const SCEV *, 8> Ops(op_begin, op_end);
414
440k
  bool AnyNonZeroIndices = false;
415
440k
416
440k
  // Split AddRecs up into parts as either of the parts may be usable
417
440k
  // without the other.
418
440k
  SplitAddRecs(Ops, Ty, SE);
419
440k
420
440k
  Type *IntPtrTy = DL.getIntPtrType(PTy);
421
440k
422
440k
  // Descend down the pointer's type and attempt to convert the other
423
440k
  // operands into GEP indices, at each level. The first index in a GEP
424
440k
  // indexes into the array implied by the pointer operand; the rest of
425
440k
  // the indices index into the element or field type selected by the
426
440k
  // preceding index.
427
508k
  for (;;) {
428
508k
    // If the scale size is not 0, attempt to factor out a scale for
429
508k
    // array indexing.
430
508k
    SmallVector<const SCEV *, 8> ScaledOps;
431
508k
    if (ElTy->isSized()) {
432
508k
      const SCEV *ElSize = SE.getSizeOfExpr(IntPtrTy, ElTy);
433
508k
      if (!ElSize->isZero()) {
434
508k
        SmallVector<const SCEV *, 8> NewOps;
435
527k
        for (const SCEV *Op : Ops) {
436
527k
          const SCEV *Remainder = SE.getConstant(Ty, 0);
437
527k
          if (FactorOutConstant(Op, Remainder, ElSize, SE, DL)) {
438
429k
            // Op now has ElSize factored out.
439
429k
            ScaledOps.push_back(Op);
440
429k
            if (!Remainder->isZero())
441
2.38k
              NewOps.push_back(Remainder);
442
429k
            AnyNonZeroIndices = true;
443
429k
          } else {
444
98.6k
            // The operand was not divisible, so add it to the list of operands
445
98.6k
            // we'll scan next iteration.
446
98.6k
            NewOps.push_back(Op);
447
98.6k
          }
448
527k
        }
449
508k
        // If we made any changes, update Ops.
450
508k
        if (!ScaledOps.empty()) {
451
415k
          Ops = NewOps;
452
415k
          SimplifyAddOperands(Ops, Ty, SE);
453
415k
        }
454
508k
      }
455
508k
    }
456
508k
457
508k
    // Record the scaled array index for this level of the type. If
458
508k
    // we didn't find any operands that could be factored, tentatively
459
508k
    // assume that element zero was selected (since the zero offset
460
508k
    // would obviously be folded away).
461
508k
    Value *Scaled = ScaledOps.empty() ?
462
93.4k
                    Constant::getNullValue(Ty) :
463
508k
                    
expandCodeFor(SE.getAddExpr(ScaledOps), Ty)415k
;
464
508k
    GepIndices.push_back(Scaled);
465
508k
466
508k
    // Collect struct field index operands.
467
564k
    while (StructType *STy = dyn_cast<StructType>(ElTy)) {
468
61.6k
      bool FoundFieldNo = false;
469
61.6k
      // An empty struct has no fields.
470
61.6k
      if (STy->getNumElements() == 0) 
break1
;
471
61.6k
      // Field offsets are known. See if a constant offset falls within any of
472
61.6k
      // the struct fields.
473
61.6k
      if (Ops.empty())
474
6.17k
        break;
475
55.4k
      if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[0]))
476
33.3k
        if (SE.getTypeSizeInBits(C->getType()) <= 64) {
477
33.3k
          const StructLayout &SL = *DL.getStructLayout(STy);
478
33.3k
          uint64_t FullOffset = C->getValue()->getZExtValue();
479
33.3k
          if (FullOffset < SL.getSizeInBytes()) {
480
32.9k
            unsigned ElIdx = SL.getElementContainingOffset(FullOffset);
481
32.9k
            GepIndices.push_back(
482
32.9k
                ConstantInt::get(Type::getInt32Ty(Ty->getContext()), ElIdx));
483
32.9k
            ElTy = STy->getTypeAtIndex(ElIdx);
484
32.9k
            Ops[0] =
485
32.9k
                SE.getConstant(Ty, FullOffset - SL.getElementOffset(ElIdx));
486
32.9k
            AnyNonZeroIndices = true;
487
32.9k
            FoundFieldNo = true;
488
32.9k
          }
489
33.3k
        }
490
55.4k
      // If no struct field offsets were found, tentatively assume that
491
55.4k
      // field zero was selected (since the zero offset would obviously
492
55.4k
      // be folded away).
493
55.4k
      if (!FoundFieldNo) {
494
22.5k
        ElTy = STy->getTypeAtIndex(0u);
495
22.5k
        GepIndices.push_back(
496
22.5k
          Constant::getNullValue(Type::getInt32Ty(Ty->getContext())));
497
22.5k
      }
498
55.4k
    }
499
508k
500
508k
    if (ArrayType *ATy = dyn_cast<ArrayType>(ElTy))
501
67.8k
      ElTy = ATy->getElementType();
502
440k
    else
503
440k
      break;
504
508k
  }
505
440k
506
440k
  // If none of the operands were convertible to proper GEP indices, cast
507
440k
  // the base to i8* and do an ugly getelementptr with that. It's still
508
440k
  // better than ptrtoint+arithmetic+inttoptr at least.
509
440k
  if (!AnyNonZeroIndices) {
510
23.2k
    // Cast the base to i8*.
511
23.2k
    V = InsertNoopCastOfTo(V,
512
23.2k
       Type::getInt8PtrTy(Ty->getContext(), PTy->getAddressSpace()));
513
23.2k
514
23.2k
    assert(!isa<Instruction>(V) ||
515
23.2k
           SE.DT.dominates(cast<Instruction>(V), &*Builder.GetInsertPoint()));
516
23.2k
517
23.2k
    // Expand the operands for a plain byte offset.
518
23.2k
    Value *Idx = expandCodeFor(SE.getAddExpr(Ops), Ty);
519
23.2k
520
23.2k
    // Fold a GEP with constant operands.
521
23.2k
    if (Constant *CLHS = dyn_cast<Constant>(V))
522
3.35k
      if (Constant *CRHS = dyn_cast<Constant>(Idx))
523
19
        return ConstantExpr::getGetElementPtr(Type::getInt8Ty(Ty->getContext()),
524
19
                                              CLHS, CRHS);
525
23.2k
526
23.2k
    // Do a quick scan to see if we have this GEP nearby.  If so, reuse it.
527
23.2k
    unsigned ScanLimit = 6;
528
23.2k
    BasicBlock::iterator BlockBegin = Builder.GetInsertBlock()->begin();
529
23.2k
    // Scanning starts from the last instruction before the insertion point.
530
23.2k
    BasicBlock::iterator IP = Builder.GetInsertPoint();
531
23.2k
    if (IP != BlockBegin) {
532
23.1k
      --IP;
533
128k
      for (; ScanLimit; 
--IP, --ScanLimit104k
) {
534
115k
        // Don't count dbg.value against the ScanLimit, to avoid perturbing the
535
115k
        // generated code.
536
115k
        if (isa<DbgInfoIntrinsic>(IP))
537
0
          ScanLimit++;
538
115k
        if (IP->getOpcode() == Instruction::GetElementPtr &&
539
115k
            
IP->getOperand(0) == V39.0k
&&
IP->getOperand(1) == Idx1.71k
)
540
1.37k
          return &*IP;
541
114k
        if (IP == BlockBegin) 
break9.28k
;
542
114k
      }
543
23.1k
    }
544
23.2k
545
23.2k
    // Save the original insertion point so we can restore it when we're done.
546
23.2k
    SCEVInsertPointGuard Guard(Builder, this);
547
21.8k
548
21.8k
    // Move the insertion point out of as many loops as we can.
549
21.8k
    while (const Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock())) {
550
18.9k
      if (!L->isLoopInvariant(V) || 
!L->isLoopInvariant(Idx)11.6k
)
break18.9k
;
551
16
      BasicBlock *Preheader = L->getLoopPreheader();
552
16
      if (!Preheader) 
break0
;
553
16
554
16
      // Ok, move up a level.
555
16
      Builder.SetInsertPoint(Preheader->getTerminator());
556
16
    }
557
21.8k
558
21.8k
    // Emit a GEP.
559
21.8k
    Value *GEP = Builder.CreateGEP(Builder.getInt8Ty(), V, Idx, "uglygep");
560
21.8k
    rememberInstruction(GEP);
561
21.8k
562
21.8k
    return GEP;
563
417k
  }
564
417k
565
417k
  {
566
417k
    SCEVInsertPointGuard Guard(Builder, this);
567
417k
568
417k
    // Move the insertion point out of as many loops as we can.
569
421k
    while (const Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock())) {
570
352k
      if (!L->isLoopInvariant(V)) 
break287k
;
571
64.8k
572
64.8k
      bool AnyIndexNotLoopInvariant = any_of(
573
70.8k
          GepIndices, [L](Value *Op) { return !L->isLoopInvariant(Op); });
574
64.8k
575
64.8k
      if (AnyIndexNotLoopInvariant)
576
61.3k
        break;
577
3.51k
578
3.51k
      BasicBlock *Preheader = L->getLoopPreheader();
579
3.51k
      if (!Preheader) 
break0
;
580
3.51k
581
3.51k
      // Ok, move up a level.
582
3.51k
      Builder.SetInsertPoint(Preheader->getTerminator());
583
3.51k
    }
584
417k
585
417k
    // Insert a pretty getelementptr. Note that this GEP is not marked inbounds,
586
417k
    // because ScalarEvolution may have changed the address arithmetic to
587
417k
    // compute a value which is beyond the end of the allocated object.
588
417k
    Value *Casted = V;
589
417k
    if (V->getType() != PTy)
590
1.25k
      Casted = InsertNoopCastOfTo(Casted, PTy);
591
417k
    Value *GEP = Builder.CreateGEP(OriginalElTy, Casted, GepIndices, "scevgep");
592
417k
    Ops.push_back(SE.getUnknown(GEP));
593
417k
    rememberInstruction(GEP);
594
417k
  }
595
417k
596
417k
  return expand(SE.getAddExpr(Ops));
597
417k
}
598
599
Value *SCEVExpander::expandAddToGEP(const SCEV *Op, PointerType *PTy, Type *Ty,
600
54.2k
                                    Value *V) {
601
54.2k
  const SCEV *const Ops[1] = {Op};
602
54.2k
  return expandAddToGEP(Ops, Ops + 1, PTy, Ty, V);
603
54.2k
}
604
605
/// PickMostRelevantLoop - Given two loops pick the one that's most relevant for
606
/// SCEV expansion. If they are nested, this is the most nested. If they are
607
/// neighboring, pick the later.
608
static const Loop *PickMostRelevantLoop(const Loop *A, const Loop *B,
609
230k
                                        DominatorTree &DT) {
610
230k
  if (!A) 
return B213k
;
611
17.0k
  if (!B) 
return A8.59k
;
612
8.47k
  if (A->contains(B)) 
return B6.22k
;
613
2.25k
  if (B->contains(A)) 
return A1.60k
;
614
642
  if (DT.dominates(A->getHeader(), B->getHeader())) 
return B294
;
615
348
  if (DT.dominates(B->getHeader(), A->getHeader())) return A;
616
0
  return A; // Arbitrarily break the tie.
617
0
}
618
619
/// getRelevantLoop - Get the most relevant loop associated with the given
620
/// expression, according to PickMostRelevantLoop.
621
1.15M
const Loop *SCEVExpander::getRelevantLoop(const SCEV *S) {
622
1.15M
  // Test whether we've already computed the most relevant loop for this SCEV.
623
1.15M
  auto Pair = RelevantLoops.insert(std::make_pair(S, nullptr));
624
1.15M
  if (!Pair.second)
625
396k
    return Pair.first->second;
626
758k
627
758k
  if (isa<SCEVConstant>(S))
628
128k
    // A constant has no relevant loops.
629
128k
    return nullptr;
630
630k
  if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
631
505k
    if (const Instruction *I = dyn_cast<Instruction>(U->getValue()))
632
257k
      return Pair.first->second = SE.LI.getLoopFor(I->getParent());
633
248k
    // A non-instruction has no relevant loops.
634
248k
    return nullptr;
635
248k
  }
636
124k
  if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(S)) {
637
84.1k
    const Loop *L = nullptr;
638
84.1k
    if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S))
639
2.28k
      L = AR->getLoop();
640
84.1k
    for (const SCEV *Op : N->operands())
641
172k
      L = PickMostRelevantLoop(L, getRelevantLoop(Op), SE.DT);
642
84.1k
    return RelevantLoops[N] = L;
643
84.1k
  }
644
40.7k
  if (const SCEVCastExpr *C = dyn_cast<SCEVCastExpr>(S)) {
645
29.8k
    const Loop *Result = getRelevantLoop(C->getOperand());
646
29.8k
    return RelevantLoops[C] = Result;
647
29.8k
  }
648
10.8k
  if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
649
10.8k
    const Loop *Result = PickMostRelevantLoop(
650
10.8k
        getRelevantLoop(D->getLHS()), getRelevantLoop(D->getRHS()), SE.DT);
651
10.8k
    return RelevantLoops[D] = Result;
652
10.8k
  }
653
0
  llvm_unreachable("Unexpected SCEV type!");
654
0
}
655
656
namespace {
657
658
/// LoopCompare - Compare loops by PickMostRelevantLoop.
659
class LoopCompare {
660
  DominatorTree &DT;
661
public:
662
452k
  explicit LoopCompare(DominatorTree &dt) : DT(dt) {}
663
664
  bool operator()(std::pair<const Loop *, const SCEV *> LHS,
665
486k
                  std::pair<const Loop *, const SCEV *> RHS) const {
666
486k
    // Keep pointer operands sorted at the end.
667
486k
    if (LHS.second->getType()->isPointerTy() !=
668
486k
        RHS.second->getType()->isPointerTy())
669
383k
      return LHS.second->getType()->isPointerTy();
670
103k
671
103k
    // Compare loops with PickMostRelevantLoop.
672
103k
    if (LHS.first != RHS.first)
673
47.1k
      return PickMostRelevantLoop(LHS.first, RHS.first, DT) != LHS.first;
674
56.0k
675
56.0k
    // If one operand is a non-constant negative and the other is not,
676
56.0k
    // put the non-constant negative on the right so that a sub can
677
56.0k
    // be used instead of a negate and add.
678
56.0k
    if (LHS.second->isNonConstantNegative()) {
679
2.62k
      if (!RHS.second->isNonConstantNegative())
680
2.18k
        return false;
681
53.4k
    } else if (RHS.second->isNonConstantNegative())
682
9.04k
      return true;
683
44.8k
684
44.8k
    // Otherwise they are equivalent according to this comparison.
685
44.8k
    return false;
686
44.8k
  }
687
};
688
689
}
690
691
424k
Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) {
692
424k
  Type *Ty = SE.getEffectiveSCEVType(S->getType());
693
424k
694
424k
  // Collect all the add operands in a loop, along with their associated loops.
695
424k
  // Iterate in reverse so that constants are emitted last, all else equal, and
696
424k
  // so that pointer operands are inserted first, which the code below relies on
697
424k
  // to form more involved GEPs.
698
424k
  SmallVector<std::pair<const Loop *, const SCEV *>, 8> OpsAndLoops;
699
424k
  for (std::reverse_iterator<SCEVAddExpr::op_iterator> I(S->op_end()),
700
1.30M
       E(S->op_begin()); I != E; 
++I875k
)
701
875k
    OpsAndLoops.push_back(std::make_pair(getRelevantLoop(*I), *I));
702
424k
703
424k
  // Sort by loop. Use a stable sort so that constants follow non-constants and
704
424k
  // pointer operands precede non-pointer operands.
705
424k
  llvm::stable_sort(OpsAndLoops, LoopCompare(SE.DT));
706
424k
707
424k
  // Emit instructions to add all the operands. Hoist as much as possible
708
424k
  // out of loops, and form meaningful getelementptrs where possible.
709
424k
  Value *Sum = nullptr;
710
1.28M
  for (auto I = OpsAndLoops.begin(), E = OpsAndLoops.end(); I != E;) {
711
860k
    const Loop *CurLoop = I->first;
712
860k
    const SCEV *Op = I->second;
713
860k
    if (!Sum) {
714
424k
      // This is the first operand. Just expand it.
715
424k
      Sum = expand(Op);
716
424k
      ++I;
717
435k
    } else if (PointerType *PTy = dyn_cast<PointerType>(Sum->getType())) {
718
386k
      // The running sum expression is a pointer. Try to form a getelementptr
719
386k
      // at this level with that as the base.
720
386k
      SmallVector<const SCEV *, 4> NewOps;
721
788k
      for (; I != E && 
I->first == CurLoop408k
;
++I401k
) {
722
401k
        // If the operand is SCEVUnknown and not instructions, peek through
723
401k
        // it, to enable more of it to be folded into the GEP.
724
401k
        const SCEV *X = I->second;
725
401k
        if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(X))
726
269k
          if (!isa<Instruction>(U->getValue()))
727
209k
            X = SE.getSCEV(U->getValue());
728
401k
        NewOps.push_back(X);
729
401k
      }
730
386k
      Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, Sum);
731
386k
    } else 
if (PointerType *49.4k
PTy49.4k
= dyn_cast<PointerType>(Op->getType())) {
732
0
      // The running sum is an integer, and there's a pointer at this level.
733
0
      // Try to form a getelementptr. If the running sum is instructions,
734
0
      // use a SCEVUnknown to avoid re-analyzing them.
735
0
      SmallVector<const SCEV *, 4> NewOps;
736
0
      NewOps.push_back(isa<Instruction>(Sum) ? SE.getUnknown(Sum) :
737
0
                                               SE.getSCEV(Sum));
738
0
      for (++I; I != E && I->first == CurLoop; ++I)
739
0
        NewOps.push_back(I->second);
740
0
      Sum = expandAddToGEP(NewOps.begin(), NewOps.end(), PTy, Ty, expand(Op));
741
49.4k
    } else if (Op->isNonConstantNegative()) {
742
9.47k
      // Instead of doing a negate and add, just do a subtract.
743
9.47k
      Value *W = expandCodeFor(SE.getNegativeSCEV(Op), Ty);
744
9.47k
      Sum = InsertNoopCastOfTo(Sum, Ty);
745
9.47k
      Sum = InsertBinop(Instruction::Sub, Sum, W, SCEV::FlagAnyWrap,
746
9.47k
                        /*IsSafeToHoist*/ true);
747
9.47k
      ++I;
748
39.9k
    } else {
749
39.9k
      // A simple add.
750
39.9k
      Value *W = expandCodeFor(Op, Ty);
751
39.9k
      Sum = InsertNoopCastOfTo(Sum, Ty);
752
39.9k
      // Canonicalize a constant to the RHS.
753
39.9k
      if (isa<Constant>(Sum)) 
std::swap(Sum, W)15.1k
;
754
39.9k
      Sum = InsertBinop(Instruction::Add, Sum, W, S->getNoWrapFlags(),
755
39.9k
                        /*IsSafeToHoist*/ true);
756
39.9k
      ++I;
757
39.9k
    }
758
860k
  }
759
424k
760
424k
  return Sum;
761
424k
}
762
763
27.4k
Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) {
764
27.4k
  Type *Ty = SE.getEffectiveSCEVType(S->getType());
765
27.4k
766
27.4k
  // Collect all the mul operands in a loop, along with their associated loops.
767
27.4k
  // Iterate in reverse so that constants are emitted last, all else equal.
768
27.4k
  SmallVector<std::pair<const Loop *, const SCEV *>, 8> OpsAndLoops;
769
27.4k
  for (std::reverse_iterator<SCEVMulExpr::op_iterator> I(S->op_end()),
770
83.1k
       E(S->op_begin()); I != E; 
++I55.7k
)
771
55.7k
    OpsAndLoops.push_back(std::make_pair(getRelevantLoop(*I), *I));
772
27.4k
773
27.4k
  // Sort by loop. Use a stable sort so that constants follow non-constants.
774
27.4k
  llvm::stable_sort(OpsAndLoops, LoopCompare(SE.DT));
775
27.4k
776
27.4k
  // Emit instructions to mul all the operands. Hoist as much as possible
777
27.4k
  // out of loops.
778
27.4k
  Value *Prod = nullptr;
779
27.4k
  auto I = OpsAndLoops.begin();
780
27.4k
781
27.4k
  // Expand the calculation of X pow N in the following manner:
782
27.4k
  // Let N = P1 + P2 + ... + PK, where all P are powers of 2. Then:
783
27.4k
  // X pow N = (X pow P1) * (X pow P2) * ... * (X pow PK).
784
52.5k
  const auto ExpandOpBinPowN = [this, &I, &OpsAndLoops, &Ty]() {
785
52.5k
    auto E = I;
786
52.5k
    // Calculate how many times the same operand from the same loop is included
787
52.5k
    // into this power.
788
52.5k
    uint64_t Exponent = 0;
789
52.5k
    const uint64_t MaxExponent = UINT64_MAX >> 1;
790
52.5k
    // No one sane will ever try to calculate such huge exponents, but if we
791
52.5k
    // need this, we stop on UINT64_MAX / 2 because we need to exit the loop
792
52.5k
    // below when the power of 2 exceeds our Exponent, and we want it to be
793
52.5k
    // 1u << 31 at most to not deal with unsigned overflow.
794
105k
    while (E != OpsAndLoops.end() && 
*I == *E80.7k
&&
Exponent != MaxExponent53.1k
) {
795
53.1k
      ++Exponent;
796
53.1k
      ++E;
797
53.1k
    }
798
52.5k
    assert(Exponent > 0 && "Trying to calculate a zeroth exponent of operand?");
799
52.5k
800
52.5k
    // Calculate powers with exponents 1, 2, 4, 8 etc. and include those of them
801
52.5k
    // that are needed into the result.
802
52.5k
    Value *P = expandCodeFor(I->second, Ty);
803
52.5k
    Value *Result = nullptr;
804
52.5k
    if (Exponent & 1)
805
52.4k
      Result = P;
806
52.6k
    for (uint64_t BinExp = 2; BinExp <= Exponent; 
BinExp <<= 1135
) {
807
135
      P = InsertBinop(Instruction::Mul, P, P, SCEV::FlagAnyWrap,
808
135
                      /*IsSafeToHoist*/ true);
809
135
      if (Exponent & BinExp)
810
59
        Result = Result ? InsertBinop(Instruction::Mul, Result, P,
811
19
                                      SCEV::FlagAnyWrap,
812
19
                                      /*IsSafeToHoist*/ true)
813
59
                        : 
P40
;
814
135
    }
815
52.5k
816
52.5k
    I = E;
817
52.5k
    assert(Result && "Nothing was expanded?");
818
52.5k
    return Result;
819
52.5k
  };
820
27.4k
821
82.5k
  while (I != OpsAndLoops.end()) {
822
55.1k
    if (!Prod) {
823
27.4k
      // This is the first operand. Just expand it.
824
27.4k
      Prod = ExpandOpBinPowN();
825
27.6k
    } else if (I->second->isAllOnesValue()) {
826
2.58k
      // Instead of doing a multiply by negative one, just do a negate.
827
2.58k
      Prod = InsertNoopCastOfTo(Prod, Ty);
828
2.58k
      Prod = InsertBinop(Instruction::Sub, Constant::getNullValue(Ty), Prod,
829
2.58k
                         SCEV::FlagAnyWrap, /*IsSafeToHoist*/ true);
830
2.58k
      ++I;
831
25.0k
    } else {
832
25.0k
      // A simple mul.
833
25.0k
      Value *W = ExpandOpBinPowN();
834
25.0k
      Prod = InsertNoopCastOfTo(Prod, Ty);
835
25.0k
      // Canonicalize a constant to the RHS.
836
25.0k
      if (isa<Constant>(Prod)) 
std::swap(Prod, W)16.1k
;
837
25.0k
      const APInt *RHS;
838
25.0k
      if (match(W, m_Power2(RHS))) {
839
19.8k
        // Canonicalize Prod*(1<<C) to Prod<<C.
840
19.8k
        assert(!Ty->isVectorTy() && "vector types are not SCEVable");
841
19.8k
        auto NWFlags = S->getNoWrapFlags();
842
19.8k
        // clear nsw flag if shl will produce poison value.
843
19.8k
        if (RHS->logBase2() == RHS->getBitWidth() - 1)
844
2
          NWFlags = ScalarEvolution::clearFlags(NWFlags, SCEV::FlagNSW);
845
19.8k
        Prod = InsertBinop(Instruction::Shl, Prod,
846
19.8k
                           ConstantInt::get(Ty, RHS->logBase2()), NWFlags,
847
19.8k
                           /*IsSafeToHoist*/ true);
848
19.8k
      } else {
849
5.18k
        Prod = InsertBinop(Instruction::Mul, Prod, W, S->getNoWrapFlags(),
850
5.18k
                           /*IsSafeToHoist*/ true);
851
5.18k
      }
852
25.0k
    }
853
55.1k
  }
854
27.4k
855
27.4k
  return Prod;
856
27.4k
}
857
858
5.49k
Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) {
859
5.49k
  Type *Ty = SE.getEffectiveSCEVType(S->getType());
860
5.49k
861
5.49k
  Value *LHS = expandCodeFor(S->getLHS(), Ty);
862
5.49k
  if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(S->getRHS())) {
863
5.35k
    const APInt &RHS = SC->getAPInt();
864
5.35k
    if (RHS.isPowerOf2())
865
4.59k
      return InsertBinop(Instruction::LShr, LHS,
866
4.59k
                         ConstantInt::get(Ty, RHS.logBase2()),
867
4.59k
                         SCEV::FlagAnyWrap, /*IsSafeToHoist*/ true);
868
905
  }
869
905
870
905
  Value *RHS = expandCodeFor(S->getRHS(), Ty);
871
905
  return InsertBinop(Instruction::UDiv, LHS, RHS, SCEV::FlagAnyWrap,
872
905
                     /*IsSafeToHoist*/ SE.isKnownNonZero(S->getRHS()));
873
905
}
874
875
/// Move parts of Base into Rest to leave Base with the minimal
876
/// expression that provides a pointer operand suitable for a
877
/// GEP expansion.
878
static void ExposePointerBase(const SCEV *&Base, const SCEV *&Rest,
879
4.65k
                              ScalarEvolution &SE) {
880
4.88k
  while (const SCEVAddRecExpr *A = dyn_cast<SCEVAddRecExpr>(Base)) {
881
224
    Base = A->getStart();
882
224
    Rest = SE.getAddExpr(Rest,
883
224
                         SE.getAddRecExpr(SE.getConstant(A->getType(), 0),
884
224
                                          A->getStepRecurrence(SE),
885
224
                                          A->getLoop(),
886
224
                                          A->getNoWrapFlags(SCEV::FlagNW)));
887
224
  }
888
4.65k
  if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(Base)) {
889
1.90k
    Base = A->getOperand(A->getNumOperands()-1);
890
1.90k
    SmallVector<const SCEV *, 8> NewAddOps(A->op_begin(), A->op_end());
891
1.90k
    NewAddOps.back() = Rest;
892
1.90k
    Rest = SE.getAddExpr(NewAddOps);
893
1.90k
    ExposePointerBase(Base, Rest, SE);
894
1.90k
  }
895
4.65k
}
896
897
/// Determine if this is a well-behaved chain of instructions leading back to
898
/// the PHI. If so, it may be reused by expanded expressions.
899
bool SCEVExpander::isNormalAddRecExprPHI(PHINode *PN, Instruction *IncV,
900
517
                                         const Loop *L) {
901
517
  if (IncV->getNumOperands() == 0 || isa<PHINode>(IncV) ||
902
517
      
(514
isa<CastInst>(IncV)514
&&
!isa<BitCastInst>(IncV)195
))
903
198
    return false;
904
319
  // If any of the operands don't dominate the insert position, bail.
905
319
  // Addrec operands are always loop-invariant, so this can only happen
906
319
  // if there are instructions which haven't been hoisted.
907
319
  if (L == IVIncInsertLoop) {
908
0
    for (User::op_iterator OI = IncV->op_begin()+1,
909
0
           OE = IncV->op_end(); OI != OE; ++OI)
910
0
      if (Instruction *OInst = dyn_cast<Instruction>(OI))
911
0
        if (!SE.DT.dominates(OInst, IVIncInsertPos))
912
0
          return false;
913
0
  }
914
319
  // Advance to the next instruction.
915
319
  IncV = dyn_cast<Instruction>(IncV->getOperand(0));
916
319
  if (!IncV)
917
2
    return false;
918
317
919
317
  if (IncV->mayHaveSideEffects())
920
0
    return false;
921
317
922
317
  if (IncV == PN)
923
311
    return true;
924
6
925
6
  return isNormalAddRecExprPHI(PN, IncV, L);
926
6
}
927
928
/// getIVIncOperand returns an induction variable increment's induction
929
/// variable operand.
930
///
931
/// If allowScale is set, any type of GEP is allowed as long as the nonIV
932
/// operands dominate InsertPos.
933
///
934
/// If allowScale is not set, ensure that a GEP increment conforms to one of the
935
/// simple patterns generated by getAddRecExprPHILiterally and
936
/// expandAddtoGEP. If the pattern isn't recognized, return NULL.
937
Instruction *SCEVExpander::getIVIncOperand(Instruction *IncV,
938
                                           Instruction *InsertPos,
939
170k
                                           bool allowScale) {
940
170k
  if (IncV == InsertPos)
941
13
    return nullptr;
942
170k
943
170k
  switch (IncV->getOpcode()) {
944
170k
  default:
945
249
    return nullptr;
946
170k
  // Check for a simple Add/Sub or GEP of a loop invariant step.
947
170k
  case Instruction::Add:
948
129k
  case Instruction::Sub: {
949
129k
    Instruction *OInst = dyn_cast<Instruction>(IncV->getOperand(1));
950
129k
    if (!OInst || 
SE.DT.dominates(OInst, InsertPos)1.25k
)
951
129k
      return dyn_cast<Instruction>(IncV->getOperand(0));
952
32
    return nullptr;
953
32
  }
954
2.20k
  case Instruction::BitCast:
955
2.20k
    return dyn_cast<Instruction>(IncV->getOperand(0));
956
38.3k
  case Instruction::GetElementPtr:
957
76.9k
    for (auto I = IncV->op_begin() + 1, E = IncV->op_end(); I != E; 
++I38.5k
) {
958
40.9k
      if (isa<Constant>(*I))
959
38.5k
        continue;
960
2.45k
      if (Instruction *OInst = dyn_cast<Instruction>(*I)) {
961
2.44k
        if (!SE.DT.dominates(OInst, InsertPos))
962
43
          return nullptr;
963
2.41k
      }
964
2.41k
      if (allowScale) {
965
70
        // allow any kind of GEP as long as it can be hoisted.
966
70
        continue;
967
70
      }
968
2.34k
      // This must be a pointer addition of constants (pretty), which is already
969
2.34k
      // handled, or some number of address-size elements (ugly). Ugly geps
970
2.34k
      // have 2 operands. i1* is used by the expander to represent an
971
2.34k
      // address-size element.
972
2.34k
      if (IncV->getNumOperands() != 2)
973
0
        return nullptr;
974
2.34k
      unsigned AS = cast<PointerType>(IncV->getType())->getAddressSpace();
975
2.34k
      if (IncV->getType() != Type::getInt1PtrTy(SE.getContext(), AS)
976
2.34k
          && 
IncV->getType() != Type::getInt8PtrTy(SE.getContext(), AS)2.22k
)
977
140
        return nullptr;
978
2.20k
      break;
979
2.20k
    }
980
38.3k
    
return dyn_cast<Instruction>(IncV->getOperand(0))38.2k
;
981
170k
  }
982
170k
}
983
984
/// If the insert point of the current builder or any of the builders on the
985
/// stack of saved builders has 'I' as its insert point, update it to point to
986
/// the instruction after 'I'.  This is intended to be used when the instruction
987
/// 'I' is being moved.  If this fixup is not done and 'I' is moved to a
988
/// different block, the inconsistent insert point (with a mismatched
989
/// Instruction and Block) can lead to an instruction being inserted in a block
990
/// other than its parent.
991
24.6k
void SCEVExpander::fixupInsertPoints(Instruction *I) {
992
24.6k
  BasicBlock::iterator It(*I);
993
24.6k
  BasicBlock::iterator NewInsertPt = std::next(It);
994
24.6k
  if (Builder.GetInsertPoint() == It)
995
1
    Builder.SetInsertPoint(&*NewInsertPt);
996
24.6k
  for (auto *InsertPtGuard : InsertPointGuards)
997
26
    if (InsertPtGuard->GetInsertPoint() == It)
998
1
      InsertPtGuard->SetInsertPoint(NewInsertPt);
999
24.6k
}
1000
1001
/// hoistStep - Attempt to hoist a simple IV increment above InsertPos to make
1002
/// it available to other uses in this loop. Recursively hoist any operands,
1003
/// until we reach a value that dominates InsertPos.
1004
158k
bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos) {
1005
158k
  if (SE.DT.dominates(IncV, InsertPos))
1006
133k
      return true;
1007
24.9k
1008
24.9k
  // InsertPos must itself dominate IncV so that IncV's new position satisfies
1009
24.9k
  // its existing users.
1010
24.9k
  if (isa<PHINode>(InsertPos) ||
1011
24.9k
      
!SE.DT.dominates(InsertPos->getParent(), IncV->getParent())24.9k
)
1012
215
    return false;
1013
24.7k
1014
24.7k
  if (!SE.LI.movementPreservesLCSSAForm(IncV, InsertPos))
1015
0
    return false;
1016
24.7k
1017
24.7k
  // Check that the chain of IV operands leading back to Phi can be hoisted.
1018
24.7k
  SmallVector<Instruction*, 4> IVIncs;
1019
24.8k
  for(;;) {
1020
24.8k
    Instruction *Oper = getIVIncOperand(IncV, InsertPos, /*allowScale*/true);
1021
24.8k
    if (!Oper)
1022
125
      return false;
1023
24.6k
    // IncV is safe to hoist.
1024
24.6k
    IVIncs.push_back(IncV);
1025
24.6k
    IncV = Oper;
1026
24.6k
    if (SE.DT.dominates(IncV, InsertPos))
1027
24.6k
      break;
1028
24.6k
  }
1029
49.2k
  
for (auto I = IVIncs.rbegin(), E = IVIncs.rend(); 24.6k
I != E;
++I24.6k
) {
1030
24.6k
    fixupInsertPoints(*I);
1031
24.6k
    (*I)->moveBefore(InsertPos);
1032
24.6k
  }
1033
24.6k
  return true;
1034
24.7k
}
1035
1036
/// Determine if this cyclic phi is in a form that would have been generated by
1037
/// LSR. We don't care if the phi was actually expanded in this pass, as long
1038
/// as it is in a low-cost form, for example, no implied multiplication. This
1039
/// should match any patterns generated by getAddRecExprPHILiterally and
1040
/// expandAddtoGEP.
1041
bool SCEVExpander::isExpandedAddRecExprPHI(PHINode *PN, Instruction *IncV,
1042
142k
                                           const Loop *L) {
1043
142k
  for(Instruction *IVOper = IncV;
1044
145k
      (IVOper = getIVIncOperand(IVOper, L->getLoopPreheader()->getTerminator(),
1045
145k
                                /*allowScale=*/false));) {
1046
144k
    if (IVOper == PN)
1047
142k
      return true;
1048
144k
  }
1049
142k
  
return false352
;
1050
142k
}
1051
1052
/// expandIVInc - Expand an IV increment at Builder's current InsertPos.
1053
/// Typically this is the LatchBlock terminator or IVIncInsertPos, but we may
1054
/// need to materialize IV increments elsewhere to handle difficult situations.
1055
Value *SCEVExpander::expandIVInc(PHINode *PN, Value *StepV, const Loop *L,
1056
                                 Type *ExpandTy, Type *IntTy,
1057
135k
                                 bool useSubtract) {
1058
135k
  Value *IncV;
1059
135k
  // If the PHI is a pointer, use a GEP, otherwise use an add or sub.
1060
135k
  if (ExpandTy->isPointerTy()) {
1061
52.4k
    PointerType *GEPPtrTy = cast<PointerType>(ExpandTy);
1062
52.4k
    // If the step isn't constant, don't use an implicitly scaled GEP, because
1063
52.4k
    // that would require a multiply inside the loop.
1064
52.4k
    if (!isa<ConstantInt>(StepV))
1065
1.25k
      GEPPtrTy = PointerType::get(Type::getInt1Ty(SE.getContext()),
1066
1.25k
                                  GEPPtrTy->getAddressSpace());
1067
52.4k
    IncV = expandAddToGEP(SE.getSCEV(StepV), GEPPtrTy, IntTy, PN);
1068
52.4k
    if (IncV->getType() != PN->getType()) {
1069
17.9k
      IncV = Builder.CreateBitCast(IncV, PN->getType());
1070
17.9k
      rememberInstruction(IncV);
1071
17.9k
    }
1072
82.9k
  } else {
1073
82.9k
    IncV = useSubtract ?
1074
92
      Builder.CreateSub(PN, StepV, Twine(IVName) + ".iv.next") :
1075
82.9k
      
Builder.CreateAdd(PN, StepV, Twine(IVName) + ".iv.next")82.8k
;
1076
82.9k
    rememberInstruction(IncV);
1077
82.9k
  }
1078
135k
  return IncV;
1079
135k
}
1080
1081
/// Hoist the addrec instruction chain rooted in the loop phi above the
1082
/// position. This routine assumes that this is possible (has been checked).
1083
void SCEVExpander::hoistBeforePos(DominatorTree *DT, Instruction *InstToHoist,
1084
133k
                                  Instruction *Pos, PHINode *LoopPhi) {
1085
133k
  do {
1086
133k
    if (DT->dominates(InstToHoist, Pos))
1087
133k
      break;
1088
0
    // Make sure the increment is where we want it. But don't move it
1089
0
    // down past a potential existing post-inc user.
1090
0
    fixupInsertPoints(InstToHoist);
1091
0
    InstToHoist->moveBefore(Pos);
1092
0
    Pos = InstToHoist;
1093
0
    InstToHoist = cast<Instruction>(InstToHoist->getOperand(0));
1094
0
  } while (InstToHoist != LoopPhi);
1095
133k
}
1096
1097
/// Check whether we can cheaply express the requested SCEV in terms of
1098
/// the available PHI SCEV by truncation and/or inversion of the step.
1099
static bool canBeCheaplyTransformed(ScalarEvolution &SE,
1100
                                    const SCEVAddRecExpr *Phi,
1101
                                    const SCEVAddRecExpr *Requested,
1102
2.58k
                                    bool &InvertStep) {
1103
2.58k
  Type *PhiTy = SE.getEffectiveSCEVType(Phi->getType());
1104
2.58k
  Type *RequestedTy = SE.getEffectiveSCEVType(Requested->getType());
1105
2.58k
1106
2.58k
  if (RequestedTy->getIntegerBitWidth() > PhiTy->getIntegerBitWidth())
1107
82
    return false;
1108
2.50k
1109
2.50k
  // Try truncate it if necessary.
1110
2.50k
  Phi = dyn_cast<SCEVAddRecExpr>(SE.getTruncateOrNoop(Phi, RequestedTy));
1111
2.50k
  if (!Phi)
1112
44
    return false;
1113
2.46k
1114
2.46k
  // Check whether truncation will help.
1115
2.46k
  if (Phi == Requested) {
1116
46
    InvertStep = false;
1117
46
    return true;
1118
46
  }
1119
2.41k
1120
2.41k
  // Check whether inverting will help: {R,+,-1} == R - {0,+,1}.
1121
2.41k
  if (SE.getAddExpr(Requested->getStart(),
1122
2.41k
                    SE.getNegativeSCEV(Requested)) == Phi) {
1123
127
    InvertStep = true;
1124
127
    return true;
1125
127
  }
1126
2.28k
1127
2.28k
  return false;
1128
2.28k
}
1129
1130
135k
static bool IsIncrementNSW(ScalarEvolution &SE, const SCEVAddRecExpr *AR) {
1131
135k
  if (!isa<IntegerType>(AR->getType()))
1132
52.4k
    return false;
1133
82.8k
1134
82.8k
  unsigned BitWidth = cast<IntegerType>(AR->getType())->getBitWidth();
1135
82.8k
  Type *WideTy = IntegerType::get(AR->getType()->getContext(), BitWidth * 2);
1136
82.8k
  const SCEV *Step = AR->getStepRecurrence(SE);
1137
82.8k
  const SCEV *OpAfterExtend = SE.getAddExpr(SE.getSignExtendExpr(Step, WideTy),
1138
82.8k
                                            SE.getSignExtendExpr(AR, WideTy));
1139
82.8k
  const SCEV *ExtendAfterOp =
1140
82.8k
    SE.getSignExtendExpr(SE.getAddExpr(AR, Step), WideTy);
1141
82.8k
  return ExtendAfterOp == OpAfterExtend;
1142
82.8k
}
1143
1144
135k
static bool IsIncrementNUW(ScalarEvolution &SE, const SCEVAddRecExpr *AR) {
1145
135k
  if (!isa<IntegerType>(AR->getType()))
1146
52.4k
    return false;
1147
82.8k
1148
82.8k
  unsigned BitWidth = cast<IntegerType>(AR->getType())->getBitWidth();
1149
82.8k
  Type *WideTy = IntegerType::get(AR->getType()->getContext(), BitWidth * 2);
1150
82.8k
  const SCEV *Step = AR->getStepRecurrence(SE);
1151
82.8k
  const SCEV *OpAfterExtend = SE.getAddExpr(SE.getZeroExtendExpr(Step, WideTy),
1152
82.8k
                                            SE.getZeroExtendExpr(AR, WideTy));
1153
82.8k
  const SCEV *ExtendAfterOp =
1154
82.8k
    SE.getZeroExtendExpr(SE.getAddExpr(AR, Step), WideTy);
1155
82.8k
  return ExtendAfterOp == OpAfterExtend;
1156
82.8k
}
1157
1158
/// getAddRecExprPHILiterally - Helper for expandAddRecExprLiterally. Expand
1159
/// the base addrec, which is the addrec without any non-loop-dominating
1160
/// values, and return the PHI.
1161
PHINode *
1162
SCEVExpander::getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized,
1163
                                        const Loop *L,
1164
                                        Type *ExpandTy,
1165
                                        Type *IntTy,
1166
                                        Type *&TruncTy,
1167
272k
                                        bool &InvertStep) {
1168
272k
  assert((!IVIncInsertLoop||IVIncInsertPos) && "Uninitialized insert position");
1169
272k
1170
272k
  // Reuse a previously-inserted PHI, if present.
1171
272k
  BasicBlock *LatchBlock = L->getLoopLatch();
1172
272k
  if (LatchBlock) {
1173
272k
    PHINode *AddRecPhiMatch = nullptr;
1174
272k
    Instruction *IncV = nullptr;
1175
272k
    TruncTy = nullptr;
1176
272k
    InvertStep = false;
1177
272k
1178
272k
    // Only try partially matching scevs that need truncation and/or
1179
272k
    // step-inversion if we know this loop is outside the current loop.
1180
272k
    bool TryNonMatchingSCEV =
1181
272k
        IVIncInsertLoop &&
1182
272k
        
SE.DT.properlyDominates(LatchBlock, IVIncInsertLoop->getHeader())250k
;
1183
272k
1184
465k
    for (PHINode &PN : L->getHeader()->phis()) {
1185
465k
      if (!SE.isSCEVable(PN.getType()))
1186
29.7k
        continue;
1187
435k
1188
435k
      const SCEVAddRecExpr *PhiSCEV = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(&PN));
1189
435k
      if (!PhiSCEV)
1190
43.9k
        continue;
1191
391k
1192
391k
      bool IsMatchingSCEV = PhiSCEV == Normalized;
1193
391k
      // We only handle truncation and inversion of phi recurrences for the
1194
391k
      // expanded expression if the expanded expression's loop dominates the
1195
391k
      // loop we insert to. Check now, so we can bail out early.
1196
391k
      if (!IsMatchingSCEV && 
!TryNonMatchingSCEV253k
)
1197
251k
          continue;
1198
140k
1199
140k
      // TODO: this possibly can be reworked to avoid this cast at all.
1200
140k
      Instruction *TempIncV =
1201
140k
          dyn_cast<Instruction>(PN.getIncomingValueForBlock(LatchBlock));
1202
140k
      if (!TempIncV)
1203
1
        continue;
1204
140k
1205
140k
      // Check whether we can reuse this PHI node.
1206
140k
      if (LSRMode) {
1207
139k
        if (!isExpandedAddRecExprPHI(&PN, TempIncV, L))
1208
330
          continue;
1209
139k
        if (L == IVIncInsertLoop && 
!hoistIVInc(TempIncV, IVIncInsertPos)133k
)
1210
0
          continue;
1211
511
      } else {
1212
511
        if (!isNormalAddRecExprPHI(&PN, TempIncV, L))
1213
200
          continue;
1214
139k
      }
1215
139k
1216
139k
      // Stop if we have found an exact match SCEV.
1217
139k
      if (IsMatchingSCEV) {
1218
137k
        IncV = TempIncV;
1219
137k
        TruncTy = nullptr;
1220
137k
        InvertStep = false;
1221
137k
        AddRecPhiMatch = &PN;
1222
137k
        break;
1223
137k
      }
1224
2.59k
1225
2.59k
      // Try whether the phi can be translated into the requested form
1226
2.59k
      // (truncated and/or offset by a constant).
1227
2.59k
      if ((!TruncTy || 
InvertStep39
) &&
1228
2.59k
          
canBeCheaplyTransformed(SE, PhiSCEV, Normalized, InvertStep)2.58k
) {
1229
173
        // Record the phi node. But don't stop we might find an exact match
1230
173
        // later.
1231
173
        AddRecPhiMatch = &PN;
1232
173
        IncV = TempIncV;
1233
173
        TruncTy = SE.getEffectiveSCEVType(Normalized->getType());
1234
173
      }
1235
2.59k
    }
1236
272k
1237
272k
    if (AddRecPhiMatch) {
1238
137k
      // Potentially, move the increment. We have made sure in
1239
137k
      // isExpandedAddRecExprPHI or hoistIVInc that this is possible.
1240
137k
      if (L == IVIncInsertLoop)
1241
133k
        hoistBeforePos(&SE.DT, IncV, IVIncInsertPos, AddRecPhiMatch);
1242
137k
1243
137k
      // Ok, the add recurrence looks usable.
1244
137k
      // Remember this PHI, even in post-inc mode.
1245
137k
      InsertedValues.insert(AddRecPhiMatch);
1246
137k
      // Remember the increment.
1247
137k
      rememberInstruction(IncV);
1248
137k
      return AddRecPhiMatch;
1249
137k
    }
1250
135k
  }
1251
135k
1252
135k
  // Save the original insertion point so we can restore it when we're done.
1253
135k
  SCEVInsertPointGuard Guard(Builder, this);
1254
135k
1255
135k
  // Another AddRec may need to be recursively expanded below. For example, if
1256
135k
  // this AddRec is quadratic, the StepV may itself be an AddRec in this
1257
135k
  // loop. Remove this loop from the PostIncLoops set before expanding such
1258
135k
  // AddRecs. Otherwise, we cannot find a valid position for the step
1259
135k
  // (i.e. StepV can never dominate its loop header).  Ideally, we could do
1260
135k
  // SavedIncLoops.swap(PostIncLoops), but we generally have a single element,
1261
135k
  // so it's not worth implementing SmallPtrSet::swap.
1262
135k
  PostIncLoopSet SavedPostIncLoops = PostIncLoops;
1263
135k
  PostIncLoops.clear();
1264
135k
1265
135k
  // Expand code for the start value into the loop preheader.
1266
135k
  assert(L->getLoopPreheader() &&
1267
135k
         "Can't expand add recurrences without a loop preheader!");
1268
135k
  Value *StartV = expandCodeFor(Normalized->getStart(), ExpandTy,
1269
135k
                                L->getLoopPreheader()->getTerminator());
1270
135k
1271
135k
  // StartV must have been be inserted into L's preheader to dominate the new
1272
135k
  // phi.
1273
135k
  assert(!isa<Instruction>(StartV) ||
1274
135k
         SE.DT.properlyDominates(cast<Instruction>(StartV)->getParent(),
1275
135k
                                 L->getHeader()));
1276
135k
1277
135k
  // Expand code for the step value. Do this before creating the PHI so that PHI
1278
135k
  // reuse code doesn't see an incomplete PHI.
1279
135k
  const SCEV *Step = Normalized->getStepRecurrence(SE);
1280
135k
  // If the stride is negative, insert a sub instead of an add for the increment
1281
135k
  // (unless it's a constant, because subtracts of constants are canonicalized
1282
135k
  // to adds).
1283
135k
  bool useSubtract = !ExpandTy->isPointerTy() && 
Step->isNonConstantNegative()82.9k
;
1284
135k
  if (useSubtract)
1285
92
    Step = SE.getNegativeSCEV(Step);
1286
135k
  // Expand the step somewhere that dominates the loop header.
1287
135k
  Value *StepV = expandCodeFor(Step, IntTy, &L->getHeader()->front());
1288
135k
1289
135k
  // The no-wrap behavior proved by IsIncrement(NUW|NSW) is only applicable if
1290
135k
  // we actually do emit an addition.  It does not apply if we emit a
1291
135k
  // subtraction.
1292
135k
  bool IncrementIsNUW = !useSubtract && 
IsIncrementNUW(SE, Normalized)135k
;
1293
135k
  bool IncrementIsNSW = !useSubtract && 
IsIncrementNSW(SE, Normalized)135k
;
1294
135k
1295
135k
  // Create the PHI.
1296
135k
  BasicBlock *Header = L->getHeader();
1297
135k
  Builder.SetInsertPoint(Header, Header->begin());
1298
135k
  pred_iterator HPB = pred_begin(Header), HPE = pred_end(Header);
1299
135k
  PHINode *PN = Builder.CreatePHI(ExpandTy, std::distance(HPB, HPE),
1300
135k
                                  Twine(IVName) + ".iv");
1301
135k
  rememberInstruction(PN);
1302
135k
1303
135k
  // Create the step instructions and populate the PHI.
1304
406k
  for (pred_iterator HPI = HPB; HPI != HPE; 
++HPI270k
) {
1305
270k
    BasicBlock *Pred = *HPI;
1306
270k
1307
270k
    // Add a start value.
1308
270k
    if (!L->contains(Pred)) {
1309
135k
      PN->addIncoming(StartV, Pred);
1310
135k
      continue;
1311
135k
    }
1312
135k
1313
135k
    // Create a step value and add it to the PHI.
1314
135k
    // If IVIncInsertLoop is non-null and equal to the addrec's loop, insert the
1315
135k
    // instructions at IVIncInsertPos.
1316
135k
    Instruction *InsertPos = L == IVIncInsertLoop ?
1317
108k
      IVIncInsertPos : 
Pred->getTerminator()26.9k
;
1318
135k
    Builder.SetInsertPoint(InsertPos);
1319
135k
    Value *IncV = expandIVInc(PN, StepV, L, ExpandTy, IntTy, useSubtract);
1320
135k
1321
135k
    if (isa<OverflowingBinaryOperator>(IncV)) {
1322
82.9k
      if (IncrementIsNUW)
1323
27.9k
        cast<BinaryOperator>(IncV)->setHasNoUnsignedWrap();
1324
82.9k
      if (IncrementIsNSW)
1325
38.9k
        cast<BinaryOperator>(IncV)->setHasNoSignedWrap();
1326
82.9k
    }
1327
135k
    PN->addIncoming(IncV, Pred);
1328
135k
  }
1329
135k
1330
135k
  // After expanding subexpressions, restore the PostIncLoops set so the caller
1331
135k
  // can ensure that IVIncrement dominates the current uses.
1332
135k
  PostIncLoops = SavedPostIncLoops;
1333
135k
1334
135k
  // Remember this PHI, even in post-inc mode.
1335
135k
  InsertedValues.insert(PN);
1336
135k
1337
135k
  return PN;
1338
135k
}
1339
1340
272k
Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
1341
272k
  Type *STy = S->getType();
1342
272k
  Type *IntTy = SE.getEffectiveSCEVType(STy);
1343
272k
  const Loop *L = S->getLoop();
1344
272k
1345
272k
  // Determine a normalized form of this expression, which is the expression
1346
272k
  // before any post-inc adjustment is made.
1347
272k
  const SCEVAddRecExpr *Normalized = S;
1348
272k
  if (PostIncLoops.count(L)) {
1349
106k
    PostIncLoopSet Loops;
1350
106k
    Loops.insert(L);
1351
106k
    Normalized = cast<SCEVAddRecExpr>(normalizeForPostIncUse(S, Loops, SE));
1352
106k
  }
1353
272k
1354
272k
  // Strip off any non-loop-dominating component from the addrec start.
1355
272k
  const SCEV *Start = Normalized->getStart();
1356
272k
  const SCEV *PostLoopOffset = nullptr;
1357
272k
  if (!SE.properlyDominates(Start, L->getHeader())) {
1358
1
    PostLoopOffset = Start;
1359
1
    Start = SE.getConstant(Normalized->getType(), 0);
1360
1
    Normalized = cast<SCEVAddRecExpr>(
1361
1
      SE.getAddRecExpr(Start, Normalized->getStepRecurrence(SE),
1362
1
                       Normalized->getLoop(),
1363
1
                       Normalized->getNoWrapFlags(SCEV::FlagNW)));
1364
1
  }
1365
272k
1366
272k
  // Strip off any non-loop-dominating component from the addrec step.
1367
272k
  const SCEV *Step = Normalized->getStepRecurrence(SE);
1368
272k
  const SCEV *PostLoopScale = nullptr;
1369
272k
  if (!SE.dominates(Step, L->getHeader())) {
1370
0
    PostLoopScale = Step;
1371
0
    Step = SE.getConstant(Normalized->getType(), 1);
1372
0
    if (!Start->isZero()) {
1373
0
        // The normalization below assumes that Start is constant zero, so if
1374
0
        // it isn't re-associate Start to PostLoopOffset.
1375
0
        assert(!PostLoopOffset && "Start not-null but PostLoopOffset set?");
1376
0
        PostLoopOffset = Start;
1377
0
        Start = SE.getConstant(Normalized->getType(), 0);
1378
0
    }
1379
0
    Normalized =
1380
0
      cast<SCEVAddRecExpr>(SE.getAddRecExpr(
1381
0
                             Start, Step, Normalized->getLoop(),
1382
0
                             Normalized->getNoWrapFlags(SCEV::FlagNW)));
1383
0
  }
1384
272k
1385
272k
  // Expand the core addrec. If we need post-loop scaling, force it to
1386
272k
  // expand to an integer type to avoid the need for additional casting.
1387
272k
  Type *ExpandTy = PostLoopScale ? 
IntTy0
: STy;
1388
272k
  // We can't use a pointer type for the addrec if the pointer type is
1389
272k
  // non-integral.
1390
272k
  Type *AddRecPHIExpandTy =
1391
272k
      DL.isNonIntegralPointerType(STy) ? 
Normalized->getType()2
:
ExpandTy272k
;
1392
272k
1393
272k
  // In some cases, we decide to reuse an existing phi node but need to truncate
1394
272k
  // it and/or invert the step.
1395
272k
  Type *TruncTy = nullptr;
1396
272k
  bool InvertStep = false;
1397
272k
  PHINode *PN = getAddRecExprPHILiterally(Normalized, L, AddRecPHIExpandTy,
1398
272k
                                          IntTy, TruncTy, InvertStep);
1399
272k
1400
272k
  // Accommodate post-inc mode, if necessary.
1401
272k
  Value *Result;
1402
272k
  if (!PostIncLoops.count(L))
1403
165k
    Result = PN;
1404
106k
  else {
1405
106k
    // In PostInc mode, use the post-incremented value.
1406
106k
    BasicBlock *LatchBlock = L->getLoopLatch();
1407
106k
    assert(LatchBlock && "PostInc mode requires a unique loop latch!");
1408
106k
    Result = PN->getIncomingValueForBlock(LatchBlock);
1409
106k
1410
106k
    // For an expansion to use the postinc form, the client must call
1411
106k
    // expandCodeFor with an InsertPoint that is either outside the PostIncLoop
1412
106k
    // or dominated by IVIncInsertPos.
1413
106k
    if (isa<Instruction>(Result) &&
1414
106k
        !SE.DT.dominates(cast<Instruction>(Result),
1415
106k
                         &*Builder.GetInsertPoint())) {
1416
1
      // The induction variable's postinc expansion does not dominate this use.
1417
1
      // IVUsers tries to prevent this case, so it is rare. However, it can
1418
1
      // happen when an IVUser outside the loop is not dominated by the latch
1419
1
      // block. Adjusting IVIncInsertPos before expansion begins cannot handle
1420
1
      // all cases. Consider a phi outside whose operand is replaced during
1421
1
      // expansion with the value of the postinc user. Without fundamentally
1422
1
      // changing the way postinc users are tracked, the only remedy is
1423
1
      // inserting an extra IV increment. StepV might fold into PostLoopOffset,
1424
1
      // but hopefully expandCodeFor handles that.
1425
1
      bool useSubtract =
1426
1
        !ExpandTy->isPointerTy() && Step->isNonConstantNegative();
1427
1
      if (useSubtract)
1428
0
        Step = SE.getNegativeSCEV(Step);
1429
1
      Value *StepV;
1430
1
      {
1431
1
        // Expand the step somewhere that dominates the loop header.
1432
1
        SCEVInsertPointGuard Guard(Builder, this);
1433
1
        StepV = expandCodeFor(Step, IntTy, &L->getHeader()->front());
1434
1
      }
1435
1
      Result = expandIVInc(PN, StepV, L, ExpandTy, IntTy, useSubtract);
1436
1
    }
1437
106k
  }
1438
272k
1439
272k
  // We have decided to reuse an induction variable of a dominating loop. Apply
1440
272k
  // truncation and/or inversion of the step.
1441
272k
  if (TruncTy) {
1442
173
    Type *ResTy = Result->getType();
1443
173
    // Normalize the result type.
1444
173
    if (ResTy != SE.getEffectiveSCEVType(ResTy))
1445
0
      Result = InsertNoopCastOfTo(Result, SE.getEffectiveSCEVType(ResTy));
1446
173
    // Truncate the result.
1447
173
    if (TruncTy != Result->getType()) {
1448
69
      Result = Builder.CreateTrunc(Result, TruncTy);
1449
69
      rememberInstruction(Result);
1450
69
    }
1451
173
    // Invert the result.
1452
173
    if (InvertStep) {
1453
127
      Result = Builder.CreateSub(expandCodeFor(Normalized->getStart(), TruncTy),
1454
127
                                 Result);
1455
127
      rememberInstruction(Result);
1456
127
    }
1457
173
  }
1458
272k
1459
272k
  // Re-apply any non-loop-dominating scale.
1460
272k
  if (PostLoopScale) {
1461
0
    assert(S->isAffine() && "Can't linearly scale non-affine recurrences.");
1462
0
    Result = InsertNoopCastOfTo(Result, IntTy);
1463
0
    Result = Builder.CreateMul(Result,
1464
0
                               expandCodeFor(PostLoopScale, IntTy));
1465
0
    rememberInstruction(Result);
1466
0
  }
1467
272k
1468
272k
  // Re-apply any non-loop-dominating offset.
1469
272k
  if (PostLoopOffset) {
1470
1
    if (PointerType *PTy = dyn_cast<PointerType>(ExpandTy)) {
1471
1
      if (Result->getType()->isIntegerTy()) {
1472
1
        Value *Base = expandCodeFor(PostLoopOffset, ExpandTy);
1473
1
        Result = expandAddToGEP(SE.getUnknown(Result), PTy, IntTy, Base);
1474
1
      } else {
1475
0
        Result = expandAddToGEP(PostLoopOffset, PTy, IntTy, Result);
1476
0
      }
1477
1
    } else {
1478
0
      Result = InsertNoopCastOfTo(Result, IntTy);
1479
0
      Result = Builder.CreateAdd(Result,
1480
0
                                 expandCodeFor(PostLoopOffset, IntTy));
1481
0
      rememberInstruction(Result);
1482
0
    }
1483
1
  }
1484
272k
1485
272k
  return Result;
1486
272k
}
1487
1488
277k
Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
1489
277k
  if (!CanonicalMode) 
return expandAddRecExprLiterally(S)272k
;
1490
4.82k
1491
4.82k
  Type *Ty = SE.getEffectiveSCEVType(S->getType());
1492
4.82k
  const Loop *L = S->getLoop();
1493
4.82k
1494
4.82k
  // First check for an existing canonical IV in a suitable type.
1495
4.82k
  PHINode *CanonicalIV = nullptr;
1496
4.82k
  if (PHINode *PN = L->getCanonicalInductionVariable())
1497
4.24k
    if (SE.getTypeSizeInBits(PN->getType()) >= SE.getTypeSizeInBits(Ty))
1498
3.85k
      CanonicalIV = PN;
1499
4.82k
1500
4.82k
  // Rewrite an AddRec in terms of the canonical induction variable, if
1501
4.82k
  // its type is more narrow.
1502
4.82k
  if (CanonicalIV &&
1503
4.82k
      SE.getTypeSizeInBits(CanonicalIV->getType()) >
1504
3.85k
      SE.getTypeSizeInBits(Ty)) {
1505
50
    SmallVector<const SCEV *, 4> NewOps(S->getNumOperands());
1506
150
    for (unsigned i = 0, e = S->getNumOperands(); i != e; 
++i100
)
1507
100
      NewOps[i] = SE.getAnyExtendExpr(S->op_begin()[i], CanonicalIV->getType());
1508
50
    Value *V = expand(SE.getAddRecExpr(NewOps, S->getLoop(),
1509
50
                                       S->getNoWrapFlags(SCEV::FlagNW)));
1510
50
    BasicBlock::iterator NewInsertPt =
1511
50
        findInsertPointAfter(cast<Instruction>(V), Builder.GetInsertBlock());
1512
50
    V = expandCodeFor(SE.getTruncateExpr(SE.getUnknown(V), Ty), nullptr,
1513
50
                      &*NewInsertPt);
1514
50
    return V;
1515
50
  }
1516
4.77k
1517
4.77k
  // {X,+,F} --> X + {0,+,F}
1518
4.77k
  if (!S->getStart()->isZero()) {
1519
2.75k
    SmallVector<const SCEV *, 4> NewOps(S->op_begin(), S->op_end());
1520
2.75k
    NewOps[0] = SE.getConstant(Ty, 0);
1521
2.75k
    const SCEV *Rest = SE.getAddRecExpr(NewOps, L,
1522
2.75k
                                        S->getNoWrapFlags(SCEV::FlagNW));
1523
2.75k
1524
2.75k
    // Turn things like ptrtoint+arithmetic+inttoptr into GEP. See the
1525
2.75k
    // comments on expandAddToGEP for details.
1526
2.75k
    const SCEV *Base = S->getStart();
1527
2.75k
    // Dig into the expression to find the pointer base for a GEP.
1528
2.75k
    const SCEV *ExposedRest = Rest;
1529
2.75k
    ExposePointerBase(Base, ExposedRest, SE);
1530
2.75k
    // If we found a pointer, expand the AddRec with a GEP.
1531
2.75k
    if (PointerType *PTy = dyn_cast<PointerType>(Base->getType())) {
1532
1.83k
      // Make sure the Base isn't something exotic, such as a multiplied
1533
1.83k
      // or divided pointer value. In those cases, the result type isn't
1534
1.83k
      // actually a pointer type.
1535
1.83k
      if (!isa<SCEVMulExpr>(Base) && !isa<SCEVUDivExpr>(Base)) {
1536
1.83k
        Value *StartV = expand(Base);
1537
1.83k
        assert(StartV->getType() == PTy && "Pointer type mismatch for GEP!");
1538
1.83k
        return expandAddToGEP(ExposedRest, PTy, Ty, StartV);
1539
1.83k
      }
1540
916
    }
1541
916
1542
916
    // Just do a normal add. Pre-expand the operands to suppress folding.
1543
916
    //
1544
916
    // The LHS and RHS values are factored out of the expand call to make the
1545
916
    // output independent of the argument evaluation order.
1546
916
    const SCEV *AddExprLHS = SE.getUnknown(expand(S->getStart()));
1547
916
    const SCEV *AddExprRHS = SE.getUnknown(expand(Rest));
1548
916
    return expand(SE.getAddExpr(AddExprLHS, AddExprRHS));
1549
916
  }
1550
2.01k
1551
2.01k
  // If we don't yet have a canonical IV, create one.
1552
2.01k
  if (!CanonicalIV) {
1553
438
    // Create and insert the PHI node for the induction variable in the
1554
438
    // specified loop.
1555
438
    BasicBlock *Header = L->getHeader();
1556
438
    pred_iterator HPB = pred_begin(Header), HPE = pred_end(Header);
1557
438
    CanonicalIV = PHINode::Create(Ty, std::distance(HPB, HPE), "indvar",
1558
438
                                  &Header->front());
1559
438
    rememberInstruction(CanonicalIV);
1560
438
1561
438
    SmallSet<BasicBlock *, 4> PredSeen;
1562
438
    Constant *One = ConstantInt::get(Ty, 1);
1563
1.32k
    for (pred_iterator HPI = HPB; HPI != HPE; 
++HPI885
) {
1564
885
      BasicBlock *HP = *HPI;
1565
885
      if (!PredSeen.insert(HP).second) {
1566
0
        // There must be an incoming value for each predecessor, even the
1567
0
        // duplicates!
1568
0
        CanonicalIV->addIncoming(CanonicalIV->getIncomingValueForBlock(HP), HP);
1569
0
        continue;
1570
0
      }
1571
885
1572
885
      if (L->contains(HP)) {
1573
440
        // Insert a unit add instruction right before the terminator
1574
440
        // corresponding to the back-edge.
1575
440
        Instruction *Add = BinaryOperator::CreateAdd(CanonicalIV, One,
1576
440
                                                     "indvar.next",
1577
440
                                                     HP->getTerminator());
1578
440
        Add->setDebugLoc(HP->getTerminator()->getDebugLoc());
1579
440
        rememberInstruction(Add);
1580
440
        CanonicalIV->addIncoming(Add, HP);
1581
445
      } else {
1582
445
        CanonicalIV->addIncoming(Constant::getNullValue(Ty), HP);
1583
445
      }
1584
885
    }
1585
438
  }
1586
2.01k
1587
2.01k
  // {0,+,1} --> Insert a canonical induction variable into the loop!
1588
2.01k
  if (S->isAffine() && S->getOperand(1)->isOne()) {
1589
1.35k
    assert(Ty == SE.getEffectiveSCEVType(CanonicalIV->getType()) &&
1590
1.35k
           "IVs with types different from the canonical IV should "
1591
1.35k
           "already have been handled!");
1592
1.35k
    return CanonicalIV;
1593
1.35k
  }
1594
660
1595
660
  // {0,+,F} --> {0,+,1} * F
1596
660
1597
660
  // If this is a simple linear addrec, emit it now as a special case.
1598
660
  if (S->isAffine())    // {0,+,F} --> i*F
1599
660
    return
1600
660
      expand(SE.getTruncateOrNoop(
1601
660
        SE.getMulExpr(SE.getUnknown(CanonicalIV),
1602
660
                      SE.getNoopOrAnyExtend(S->getOperand(1),
1603
660
                                            CanonicalIV->getType())),
1604
660
        Ty));
1605
0
1606
0
  // If this is a chain of recurrences, turn it into a closed form, using the
1607
0
  // folders, then expandCodeFor the closed form.  This allows the folders to
1608
0
  // simplify the expression without having to build a bunch of special code
1609
0
  // into this folder.
1610
0
  const SCEV *IH = SE.getUnknown(CanonicalIV);   // Get I as a "symbolic" SCEV.
1611
0
1612
0
  // Promote S up to the canonical IV type, if the cast is foldable.
1613
0
  const SCEV *NewS = S;
1614
0
  const SCEV *Ext = SE.getNoopOrAnyExtend(S, CanonicalIV->getType());
1615
0
  if (isa<SCEVAddRecExpr>(Ext))
1616
0
    NewS = Ext;
1617
0
1618
0
  const SCEV *V = cast<SCEVAddRecExpr>(NewS)->evaluateAtIteration(IH, SE);
1619
0
  //cerr << "Evaluated: " << *this << "\n     to: " << *V << "\n";
1620
0
1621
0
  // Truncate the result down to the original type, if needed.
1622
0
  const SCEV *T = SE.getTruncateOrNoop(V, Ty);
1623
0
  return expand(T);
1624
0
}
1625
1626
998
Value *SCEVExpander::visitTruncateExpr(const SCEVTruncateExpr *S) {
1627
998
  Type *Ty = SE.getEffectiveSCEVType(S->getType());
1628
998
  Value *V = expandCodeFor(S->getOperand(),
1629
998
                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
1630
998
  Value *I = Builder.CreateTrunc(V, Ty);
1631
998
  rememberInstruction(I);
1632
998
  return I;
1633
998
}
1634
1635
4.20k
Value *SCEVExpander::visitZeroExtendExpr(const SCEVZeroExtendExpr *S) {
1636
4.20k
  Type *Ty = SE.getEffectiveSCEVType(S->getType());
1637
4.20k
  Value *V = expandCodeFor(S->getOperand(),
1638
4.20k
                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
1639
4.20k
  Value *I = Builder.CreateZExt(V, Ty);
1640
4.20k
  rememberInstruction(I);
1641
4.20k
  return I;
1642
4.20k
}
1643
1644
3.88k
Value *SCEVExpander::visitSignExtendExpr(const SCEVSignExtendExpr *S) {
1645
3.88k
  Type *Ty = SE.getEffectiveSCEVType(S->getType());
1646
3.88k
  Value *V = expandCodeFor(S->getOperand(),
1647
3.88k
                           SE.getEffectiveSCEVType(S->getOperand()->getType()));
1648
3.88k
  Value *I = Builder.CreateSExt(V, Ty);
1649
3.88k
  rememberInstruction(I);
1650
3.88k
  return I;
1651
3.88k
}
1652
1653
889
Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) {
1654
889
  Value *LHS = expand(S->getOperand(S->getNumOperands()-1));
1655
889
  Type *Ty = LHS->getType();
1656
1.80k
  for (int i = S->getNumOperands()-2; i >= 0; 
--i911
) {
1657
911
    // In the case of mixed integer and pointer types, do the
1658
911
    // rest of the comparisons as integer.
1659
911
    Type *OpTy = S->getOperand(i)->getType();
1660
911
    if (OpTy->isIntegerTy() != Ty->isIntegerTy()) {
1661
0
      Ty = SE.getEffectiveSCEVType(Ty);
1662
0
      LHS = InsertNoopCastOfTo(LHS, Ty);
1663
0
    }
1664
911
    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
1665
911
    Value *ICmp = Builder.CreateICmpSGT(LHS, RHS);
1666
911
    rememberInstruction(ICmp);
1667
911
    Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smax");
1668
911
    rememberInstruction(Sel);
1669
911
    LHS = Sel;
1670
911
  }
1671
889
  // In the case of mixed integer and pointer types, cast the
1672
889
  // final result back to the pointer type.
1673
889
  if (LHS->getType() != S->getType())
1674
0
    LHS = InsertNoopCastOfTo(LHS, S->getType());
1675
889
  return LHS;
1676
889
}
1677
1678
277
Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) {
1679
277
  Value *LHS = expand(S->getOperand(S->getNumOperands()-1));
1680
277
  Type *Ty = LHS->getType();
1681
554
  for (int i = S->getNumOperands()-2; i >= 0; 
--i277
) {
1682
277
    // In the case of mixed integer and pointer types, do the
1683
277
    // rest of the comparisons as integer.
1684
277
    Type *OpTy = S->getOperand(i)->getType();
1685
277
    if (OpTy->isIntegerTy() != Ty->isIntegerTy()) {
1686
0
      Ty = SE.getEffectiveSCEVType(Ty);
1687
0
      LHS = InsertNoopCastOfTo(LHS, Ty);
1688
0
    }
1689
277
    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
1690
277
    Value *ICmp = Builder.CreateICmpUGT(LHS, RHS);
1691
277
    rememberInstruction(ICmp);
1692
277
    Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umax");
1693
277
    rememberInstruction(Sel);
1694
277
    LHS = Sel;
1695
277
  }
1696
277
  // In the case of mixed integer and pointer types, cast the
1697
277
  // final result back to the pointer type.
1698
277
  if (LHS->getType() != S->getType())
1699
7
    LHS = InsertNoopCastOfTo(LHS, S->getType());
1700
277
  return LHS;
1701
277
}
1702
1703
312
Value *SCEVExpander::visitSMinExpr(const SCEVSMinExpr *S) {
1704
312
  Value *LHS = expand(S->getOperand(S->getNumOperands() - 1));
1705
312
  Type *Ty = LHS->getType();
1706
658
  for (int i = S->getNumOperands() - 2; i >= 0; 
--i346
) {
1707
346
    // In the case of mixed integer and pointer types, do the
1708
346
    // rest of the comparisons as integer.
1709
346
    Type *OpTy = S->getOperand(i)->getType();
1710
346
    if (OpTy->isIntegerTy() != Ty->isIntegerTy()) {
1711
0
      Ty = SE.getEffectiveSCEVType(Ty);
1712
0
      LHS = InsertNoopCastOfTo(LHS, Ty);
1713
0
    }
1714
346
    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
1715
346
    Value *ICmp = Builder.CreateICmpSLT(LHS, RHS);
1716
346
    rememberInstruction(ICmp);
1717
346
    Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smin");
1718
346
    rememberInstruction(Sel);
1719
346
    LHS = Sel;
1720
346
  }
1721
312
  // In the case of mixed integer and pointer types, cast the
1722
312
  // final result back to the pointer type.
1723
312
  if (LHS->getType() != S->getType())
1724
0
    LHS = InsertNoopCastOfTo(LHS, S->getType());
1725
312
  return LHS;
1726
312
}
1727
1728
322
Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) {
1729
322
  Value *LHS = expand(S->getOperand(S->getNumOperands() - 1));
1730
322
  Type *Ty = LHS->getType();
1731
654
  for (int i = S->getNumOperands() - 2; i >= 0; 
--i332
) {
1732
332
    // In the case of mixed integer and pointer types, do the
1733
332
    // rest of the comparisons as integer.
1734
332
    Type *OpTy = S->getOperand(i)->getType();
1735
332
    if (OpTy->isIntegerTy() != Ty->isIntegerTy()) {
1736
49
      Ty = SE.getEffectiveSCEVType(Ty);
1737
49
      LHS = InsertNoopCastOfTo(LHS, Ty);
1738
49
    }
1739
332
    Value *RHS = expandCodeFor(S->getOperand(i), Ty);
1740
332
    Value *ICmp = Builder.CreateICmpULT(LHS, RHS);
1741
332
    rememberInstruction(ICmp);
1742
332
    Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umin");
1743
332
    rememberInstruction(Sel);
1744
332
    LHS = Sel;
1745
332
  }
1746
322
  // In the case of mixed integer and pointer types, cast the
1747
322
  // final result back to the pointer type.
1748
322
  if (LHS->getType() != S->getType())
1749
1
    LHS = InsertNoopCastOfTo(LHS, S->getType());
1750
322
  return LHS;
1751
322
}
1752
1753
Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty,
1754
447k
                                   Instruction *IP) {
1755
447k
  setInsertPoint(IP);
1756
447k
  return expandCodeFor(SH, Ty);
1757
447k
}
1758
1759
2.73M
Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty) {
1760
2.73M
  // Expand the code for this SCEV.
1761
2.73M
  Value *V = expand(SH);
1762
2.73M
  if (Ty) {
1763
2.01M
    assert(SE.getTypeSizeInBits(Ty) == SE.getTypeSizeInBits(SH->getType()) &&
1764
2.01M
           "non-trivial casts should be done with the SCEVs directly!");
1765
2.01M
    V = InsertNoopCastOfTo(V, Ty);
1766
2.01M
  }
1767
2.73M
  return V;
1768
2.73M
}
1769
1770
ScalarEvolution::ValueOffsetPair
1771
SCEVExpander::FindValueInExprValueMap(const SCEV *S,
1772
2.70M
                                      const Instruction *InsertPt) {
1773
2.70M
  SetVector<ScalarEvolution::ValueOffsetPair> *Set = SE.getSCEVValues(S);
1774
2.70M
  // If the expansion is not in CanonicalMode, and the SCEV contains any
1775
2.70M
  // sub scAddRecExpr type SCEV, it is required to expand the SCEV literally.
1776
2.70M
  if (CanonicalMode || 
!SE.containsAddRecurrence(S)2.48M
) {
1777
2.42M
    // If S is scConstant, it may be worse to reuse an existing Value.
1778
2.42M
    if (S->getSCEVType() != scConstant && 
Set1.90M
) {
1779
292k
      // Choose a Value from the set which dominates the insertPt.
1780
292k
      // insertPt should be inside the Value's parent loop so as not to break
1781
292k
      // the LCSSA form.
1782
328k
      for (auto const &VOPair : *Set) {
1783
328k
        Value *V = VOPair.first;
1784
328k
        ConstantInt *Offset = VOPair.second;
1785
328k
        Instruction *EntInst = nullptr;
1786
328k
        if (V && isa<Instruction>(V) && 
(EntInst = cast<Instruction>(V))233k
&&
1787
328k
            
S->getType() == V->getType()233k
&&
1788
328k
            
EntInst->getFunction() == InsertPt->getFunction()222k
&&
1789
328k
            
SE.DT.dominates(EntInst, InsertPt)222k
&&
1790
328k
            
(194k
SE.LI.getLoopFor(EntInst->getParent()) == nullptr194k
||
1791
194k
             
SE.LI.getLoopFor(EntInst->getParent())->contains(InsertPt)65.2k
))
1792
193k
          return {V, Offset};
1793
328k
      }
1794
292k
    }
1795
2.42M
  }
1796
2.70M
  
return {nullptr, nullptr}2.51M
;
1797
2.70M
}
1798
1799
// The expansion of SCEV will either reuse a previous Value in ExprValueMap,
1800
// or expand the SCEV literally. Specifically, if the expansion is in LSRMode,
1801
// and the SCEV contains any sub scAddRecExpr type SCEV, it will be expanded
1802
// literally, to prevent LSR's transformed SCEV from being reverted. Otherwise,
1803
// the expansion will try to reuse Value from ExprValueMap, and only when it
1804
// fails, expand the SCEV literally.
1805
3.58M
Value *SCEVExpander::expand(const SCEV *S) {
1806
3.58M
  // Compute an insertion point for this SCEV object. Hoist the instructions
1807
3.58M
  // as far out in the loop nest as possible.
1808
3.58M
  Instruction *InsertPt = &*Builder.GetInsertPoint();
1809
3.58M
1810
3.58M
  // We can move insertion point only if there is no div or rem operations
1811
3.58M
  // otherwise we are risky to move it over the check for zero denominator.
1812
3.58M
  auto SafeToHoist = [](const SCEV *S) {
1813
7.21M
    return !SCEVExprContains(S, [](const SCEV *S) {
1814
7.21M
              if (const auto *D = dyn_cast<SCEVUDivExpr>(S)) {
1815
59.6k
                if (const auto *SC = dyn_cast<SCEVConstant>(D->getRHS()))
1816
59.2k
                  // Division by non-zero constants can be hoisted.
1817
59.2k
                  return SC->getValue()->isZero();
1818
398
                // All other divisions should not be moved as they may be
1819
398
                // divisions by zero and should be kept within the
1820
398
                // conditions of the surrounding loops that guard their
1821
398
                // execution (see PR35406).
1822
398
                return true;
1823
398
              }
1824
7.15M
              return false;
1825
7.15M
            });
1826
3.58M
  };
1827
3.58M
  if (SafeToHoist(S)) {
1828
3.58M
    for (Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock());;
1829
4.53M
         
L = L->getParentLoop()945k
) {
1830
4.53M
      if (SE.isLoopInvariant(S, L)) {
1831
1.78M
        if (!L) 
break843k
;
1832
945k
        if (BasicBlock *Preheader = L->getLoopPreheader())
1833
944k
          InsertPt = Preheader->getTerminator();
1834
188
        else
1835
188
          // LSR sets the insertion point for AddRec start/step values to the
1836
188
          // block start to simplify value reuse, even though it's an invalid
1837
188
          // position. SCEVExpander must correct for this in all cases.
1838
188
          InsertPt = &*L->getHeader()->getFirstInsertionPt();
1839
2.74M
      } else {
1840
2.74M
        // If the SCEV is computable at this level, insert it into the header
1841
2.74M
        // after the PHIs (and after any other instructions that we've inserted
1842
2.74M
        // there) so that it is guaranteed to dominate any user inside the loop.
1843
2.74M
        if (L && 
SE.hasComputableLoopEvolution(S, L)2.29M
&&
!PostIncLoops.count(L)497k
)
1844
411k
          InsertPt = &*L->getHeader()->getFirstInsertionPt();
1845
2.99M
        while (InsertPt->getIterator() != Builder.GetInsertPoint() &&
1846
2.99M
               
(816k
isInsertedInstruction(InsertPt)816k
||
1847
816k
                
isa<DbgInfoIntrinsic>(InsertPt)562k
))
1848
253k
          InsertPt = &*std::next(InsertPt->getIterator());
1849
2.74M
        break;
1850
2.74M
      }
1851
4.53M
    }
1852
3.58M
  }
1853
3.58M
1854
3.58M
  // IndVarSimplify sometimes sets the insertion point at the block start, even
1855
3.58M
  // when there are PHIs at that point.  We must correct for this.
1856
3.58M
  if (isa<PHINode>(*InsertPt))
1857
8
    InsertPt = &*InsertPt->getParent()->getFirstInsertionPt();
1858
3.58M
1859
3.58M
  // Check to see if we already expanded this here.
1860
3.58M
  auto I = InsertedExpressions.find(std::make_pair(S, InsertPt));
1861
3.58M
  if (I != InsertedExpressions.end())
1862
898k
    return I->second;
1863
2.68M
1864
2.68M
  SCEVInsertPointGuard Guard(Builder, this);
1865
2.68M
  Builder.SetInsertPoint(InsertPt);
1866
2.68M
1867
2.68M
  // Expand the expression into instructions.
1868
2.68M
  ScalarEvolution::ValueOffsetPair VO = FindValueInExprValueMap(S, InsertPt);
1869
2.68M
  Value *V = VO.first;
1870
2.68M
1871
2.68M
  if (!V)
1872
2.49M
    V = visit(S);
1873
190k
  else if (VO.second) {
1874
627
    if (PointerType *Vty = dyn_cast<PointerType>(V->getType())) {
1875
2
      Type *Ety = Vty->getPointerElementType();
1876
2
      int64_t Offset = VO.second->getSExtValue();
1877
2
      int64_t ESize = SE.getTypeSizeInBits(Ety);
1878
2
      if ((Offset * 8) % ESize == 0) {
1879
1
        ConstantInt *Idx =
1880
1
            ConstantInt::getSigned(VO.second->getType(), -(Offset * 8) / ESize);
1881
1
        V = Builder.CreateGEP(Ety, V, Idx, "scevgep");
1882
1
      } else {
1883
1
        ConstantInt *Idx =
1884
1
            ConstantInt::getSigned(VO.second->getType(), -Offset);
1885
1
        unsigned AS = Vty->getAddressSpace();
1886
1
        V = Builder.CreateBitCast(V, Type::getInt8PtrTy(SE.getContext(), AS));
1887
1
        V = Builder.CreateGEP(Type::getInt8Ty(SE.getContext()), V, Idx,
1888
1
                              "uglygep");
1889
1
        V = Builder.CreateBitCast(V, Vty);
1890
1
      }
1891
625
    } else {
1892
625
      V = Builder.CreateSub(V, VO.second);
1893
625
    }
1894
627
  }
1895
2.68M
  // Remember the expanded value for this SCEV at this location.
1896
2.68M
  //
1897
2.68M
  // This is independent of PostIncLoops. The mapped value simply materializes
1898
2.68M
  // the expression at this insertion point. If the mapped value happened to be
1899
2.68M
  // a postinc expansion, it could be reused by a non-postinc user, but only if
1900
2.68M
  // its insertion point was already at the head of the loop.
1901
2.68M
  InsertedExpressions[std::make_pair(S, InsertPt)] = V;
1902
2.68M
  return V;
1903
2.68M
}
1904
1905
1.19M
void SCEVExpander::rememberInstruction(Value *I) {
1906
1.19M
  if (!PostIncLoops.empty())
1907
95.5k
    InsertedPostIncValues.insert(I);
1908
1.10M
  else
1909
1.10M
    InsertedValues.insert(I);
1910
1.19M
}
1911
1912
/// getOrInsertCanonicalInductionVariable - This method returns the
1913
/// canonical induction variable of the specified type for the specified
1914
/// loop (inserting one if there is none).  A canonical induction variable
1915
/// starts at zero and steps by one on each iteration.
1916
PHINode *
1917
SCEVExpander::getOrInsertCanonicalInductionVariable(const Loop *L,
1918
0
                                                    Type *Ty) {
1919
0
  assert(Ty->isIntegerTy() && "Can only insert integer induction variables!");
1920
0
1921
0
  // Build a SCEV for {0,+,1}<L>.
1922
0
  // Conservatively use FlagAnyWrap for now.
1923
0
  const SCEV *H = SE.getAddRecExpr(SE.getConstant(Ty, 0),
1924
0
                                   SE.getConstant(Ty, 1), L, SCEV::FlagAnyWrap);
1925
0
1926
0
  // Emit code for it.
1927
0
  SCEVInsertPointGuard Guard(Builder, this);
1928
0
  PHINode *V =
1929
0
      cast<PHINode>(expandCodeFor(H, nullptr, &L->getHeader()->front()));
1930
0
1931
0
  return V;
1932
0
}
1933
1934
/// replaceCongruentIVs - Check for congruent phis in this loop header and
1935
/// replace them with their most canonical representative. Return the number of
1936
/// phis eliminated.
1937
///
1938
/// This does not depend on any SCEVExpander state but should be used in
1939
/// the same context that SCEVExpander is used.
1940
unsigned
1941
SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
1942
                                  SmallVectorImpl<WeakTrackingVH> &DeadInsts,
1943
428k
                                  const TargetTransformInfo *TTI) {
1944
428k
  // Find integer phis in order of increasing width.
1945
428k
  SmallVector<PHINode*, 8> Phis;
1946
428k
  for (PHINode &PN : L->getHeader()->phis())
1947
642k
    Phis.push_back(&PN);
1948
428k
1949
428k
  if (TTI)
1950
208k
    llvm::sort(Phis, [](Value *LHS, Value *RHS) {
1951
182k
      // Put pointers at the back and make sure pointer < pointer = false.
1952
182k
      if (!LHS->getType()->isIntegerTy() || 
!RHS->getType()->isIntegerTy()83.6k
)
1953
132k
        return RHS->getType()->isIntegerTy() && 
!LHS->getType()->isIntegerTy()62.4k
;
1954
50.5k
      return RHS->getType()->getPrimitiveSizeInBits() <
1955
50.5k
             LHS->getType()->getPrimitiveSizeInBits();
1956
50.5k
    });
1957
428k
1958
428k
  unsigned NumElim = 0;
1959
428k
  DenseMap<const SCEV *, PHINode *> ExprToIVMap;
1960
428k
  // Process phis from wide to narrow. Map wide phis to their truncation
1961
428k
  // so narrow phis can reuse them.
1962
642k
  for (PHINode *Phi : Phis) {
1963
642k
    auto SimplifyPHINode = [&](PHINode *PN) -> Value * {
1964
642k
      if (Value *V = SimplifyInstruction(PN, {DL, &SE.TLI, &SE.DT, &SE.AC}))
1965
14
        return V;
1966
642k
      if (!SE.isSCEVable(PN->getType()))
1967
27.1k
        return nullptr;
1968
615k
      auto *Const = dyn_cast<SCEVConstant>(SE.getSCEV(PN));
1969
615k
      if (!Const)
1970
615k
        return nullptr;
1971
4
      return Const->getValue();
1972
4
    };
1973
642k
1974
642k
    // Fold constant phis. They may be congruent to other constant phis and
1975
642k
    // would confuse the logic below that expects proper IVs.
1976
642k
    if (Value *V = SimplifyPHINode(Phi)) {
1977
18
      if (V->getType() != Phi->getType())
1978
1
        continue;
1979
17
      Phi->replaceAllUsesWith(V);
1980
17
      DeadInsts.emplace_back(Phi);
1981
17
      ++NumElim;
1982
17
      DEBUG_WITH_TYPE(DebugType, dbgs()
1983
17
                      << "INDVARS: Eliminated constant iv: " << *Phi << '\n');
1984
17
      continue;
1985
17
    }
1986
642k
1987
642k
    if (!SE.isSCEVable(Phi->getType()))
1988
27.1k
      continue;
1989
615k
1990
615k
    PHINode *&OrigPhiRef = ExprToIVMap[SE.getSCEV(Phi)];
1991
615k
    if (!OrigPhiRef) {
1992
611k
      OrigPhiRef = Phi;
1993
611k
      if (Phi->getType()->isIntegerTy() && 
TTI399k
&&
1994
611k
          
TTI->isTruncateFree(Phi->getType(), Phis.back()->getType())181k
) {
1995
20.6k
        // This phi can be freely truncated to the narrowest phi type. Map the
1996
20.6k
        // truncated expression to it so it will be reused for narrow types.
1997
20.6k
        const SCEV *TruncExpr =
1998
20.6k
          SE.getTruncateExpr(SE.getSCEV(Phi), Phis.back()->getType());
1999
20.6k
        ExprToIVMap[TruncExpr] = Phi;
2000
20.6k
      }
2001
611k
      continue;
2002
611k
    }
2003
3.71k
2004
3.71k
    // Replacing a pointer phi with an integer phi or vice-versa doesn't make
2005
3.71k
    // sense.
2006
3.71k
    if (OrigPhiRef->getType()->isPointerTy() != Phi->getType()->isPointerTy())
2007
2
      continue;
2008
3.71k
2009
3.71k
    if (BasicBlock *LatchBlock = L->getLoopLatch()) {
2010
3.71k
      Instruction *OrigInc = dyn_cast<Instruction>(
2011
3.71k
          OrigPhiRef->getIncomingValueForBlock(LatchBlock));
2012
3.71k
      Instruction *IsomorphicInc =
2013
3.71k
          dyn_cast<Instruction>(Phi->getIncomingValueForBlock(LatchBlock));
2014
3.71k
2015
3.71k
      if (OrigInc && 
IsomorphicInc3.70k
) {
2016
3.70k
        // If this phi has the same width but is more canonical, replace the
2017
3.70k
        // original with it. As part of the "more canonical" determination,
2018
3.70k
        // respect a prior decision to use an IV chain.
2019
3.70k
        if (OrigPhiRef->getType() == Phi->getType() &&
2020
3.70k
            
!(3.29k
ChainedPhis.count(Phi)3.29k
||
2021
3.29k
              isExpandedAddRecExprPHI(OrigPhiRef, OrigInc, L)) &&
2022
3.70k
            
(16
ChainedPhis.count(Phi)16
||
2023
16
             isExpandedAddRecExprPHI(Phi, IsomorphicInc, L))) {
2024
10
          std::swap(OrigPhiRef, Phi);
2025
10
          std::swap(OrigInc, IsomorphicInc);
2026
10
        }
2027
3.70k
        // Replacing the congruent phi is sufficient because acyclic
2028
3.70k
        // redundancy elimination, CSE/GVN, should handle the
2029
3.70k
        // rest. However, once SCEV proves that a phi is congruent,
2030
3.70k
        // it's often the head of an IV user cycle that is isomorphic
2031
3.70k
        // with the original phi. It's worth eagerly cleaning up the
2032
3.70k
        // common case of a single IV increment so that DeleteDeadPHIs
2033
3.70k
        // can remove cycles that had postinc uses.
2034
3.70k
        const SCEV *TruncExpr =
2035
3.70k
            SE.getTruncateOrNoop(SE.getSCEV(OrigInc), IsomorphicInc->getType());
2036
3.70k
        if (OrigInc != IsomorphicInc &&
2037
3.70k
            
TruncExpr == SE.getSCEV(IsomorphicInc)3.45k
&&
2038
3.70k
            
SE.LI.replacementPreservesLCSSAForm(IsomorphicInc, OrigInc)3.45k
&&
2039
3.70k
            
hoistIVInc(OrigInc, IsomorphicInc)3.45k
) {
2040
3.29k
          DEBUG_WITH_TYPE(DebugType,
2041
3.29k
                          dbgs() << "INDVARS: Eliminated congruent iv.inc: "
2042
3.29k
                                 << *IsomorphicInc << '\n');
2043
3.29k
          Value *NewInc = OrigInc;
2044
3.29k
          if (OrigInc->getType() != IsomorphicInc->getType()) {
2045
287
            Instruction *IP = nullptr;
2046
287
            if (PHINode *PN = dyn_cast<PHINode>(OrigInc))
2047
1
              IP = &*PN->getParent()->getFirstInsertionPt();
2048
286
            else
2049
286
              IP = OrigInc->getNextNode();
2050
287
2051
287
            IRBuilder<> Builder(IP);
2052
287
            Builder.SetCurrentDebugLocation(IsomorphicInc->getDebugLoc());
2053
287
            NewInc = Builder.CreateTruncOrBitCast(
2054
287
                OrigInc, IsomorphicInc->getType(), IVName);
2055
287
          }
2056
3.29k
          IsomorphicInc->replaceAllUsesWith(NewInc);
2057
3.29k
          DeadInsts.emplace_back(IsomorphicInc);
2058
3.29k
        }
2059
3.70k
      }
2060
3.71k
    }
2061
3.71k
    DEBUG_WITH_TYPE(DebugType, dbgs() << "INDVARS: Eliminated congruent iv: "
2062
3.71k
                                      << *Phi << '\n');
2063
3.71k
    ++NumElim;
2064
3.71k
    Value *NewIV = OrigPhiRef;
2065
3.71k
    if (OrigPhiRef->getType() != Phi->getType()) {
2066
413
      IRBuilder<> Builder(&*L->getHeader()->getFirstInsertionPt());
2067
413
      Builder.SetCurrentDebugLocation(Phi->getDebugLoc());
2068
413
      NewIV = Builder.CreateTruncOrBitCast(OrigPhiRef, Phi->getType(), IVName);
2069
413
    }
2070
3.71k
    Phi->replaceAllUsesWith(NewIV);
2071
3.71k
    DeadInsts.emplace_back(Phi);
2072
3.71k
  }
2073
428k
  return NumElim;
2074
428k
}
2075
2076
Value *SCEVExpander::getExactExistingExpansion(const SCEV *S,
2077
0
                                               const Instruction *At, Loop *L) {
2078
0
  Optional<ScalarEvolution::ValueOffsetPair> VO =
2079
0
      getRelatedExistingExpansion(S, At, L);
2080
0
  if (VO && VO.getValue().second == nullptr)
2081
0
    return VO.getValue().first;
2082
0
  return nullptr;
2083
0
}
2084
2085
Optional<ScalarEvolution::ValueOffsetPair>
2086
SCEVExpander::getRelatedExistingExpansion(const SCEV *S, const Instruction *At,
2087
20.0k
                                          Loop *L) {
2088
20.0k
  using namespace llvm::PatternMatch;
2089
20.0k
2090
20.0k
  SmallVector<BasicBlock *, 4> ExitingBlocks;
2091
20.0k
  L->getExitingBlocks(ExitingBlocks);
2092
20.0k
2093
20.0k
  // Look for suitable value in simple conditions at the loop exits.
2094
21.4k
  for (BasicBlock *BB : ExitingBlocks) {
2095
21.4k
    ICmpInst::Predicate Pred;
2096
21.4k
    Instruction *LHS, *RHS;
2097
21.4k
    BasicBlock *TrueBB, *FalseBB;
2098
21.4k
2099
21.4k
    if (!match(BB->getTerminator(),
2100
21.4k
               m_Br(m_ICmp(Pred, m_Instruction(LHS), m_Instruction(RHS)),
2101
21.4k
                    TrueBB, FalseBB)))
2102
9.91k
      continue;
2103
11.4k
2104
11.4k
    if (SE.getSCEV(LHS) == S && 
SE.DT.dominates(LHS, At)1
)
2105
1
      return ScalarEvolution::ValueOffsetPair(LHS, nullptr);
2106
11.4k
2107
11.4k
    if (SE.getSCEV(RHS) == S && 
SE.DT.dominates(RHS, At)1.46k
)
2108
1.46k
      return ScalarEvolution::ValueOffsetPair(RHS, nullptr);
2109
11.4k
  }
2110
20.0k
2111
20.0k
  // Use expand's logic which is used for reusing a previous Value in
2112
20.0k
  // ExprValueMap.
2113
20.0k
  ScalarEvolution::ValueOffsetPair VO = FindValueInExprValueMap(S, At);
2114
18.5k
  if (VO.first)
2115
2.36k
    return VO;
2116
16.2k
2117
16.2k
  // There is potential to make this significantly smarter, but this simple
2118
16.2k
  // heuristic already gets some interesting cases.
2119
16.2k
2120
16.2k
  // Can not find suitable value.
2121
16.2k
  return None;
2122
16.2k
}
2123
2124
bool SCEVExpander::isHighCostExpansionHelper(
2125
    const SCEV *S, Loop *L, const Instruction *At,
2126
140k
    SmallPtrSetImpl<const SCEV *> &Processed) {
2127
140k
2128
140k
  // If we can find an existing value for this scev available at the point "At"
2129
140k
  // then consider the expression cheap.
2130
140k
  if (At && 
getRelatedExistingExpansion(S, At, L)19.6k
)
2131
3.83k
    return false;
2132
136k
2133
136k
  // Zero/One operand expressions
2134
136k
  switch (S->getSCEVType()) {
2135
136k
  case scUnknown:
2136
76.8k
  case scConstant:
2137
76.8k
    return false;
2138
76.8k
  case scTruncate:
2139
826
    return isHighCostExpansionHelper(cast<SCEVTruncateExpr>(S)->getOperand(),
2140
826
                                     L, At, Processed);
2141
76.8k
  case scZeroExtend:
2142
869
    return isHighCostExpansionHelper(cast<SCEVZeroExtendExpr>(S)->getOperand(),
2143
869
                                     L, At, Processed);
2144
76.8k
  case scSignExtend:
2145
3.59k
    return isHighCostExpansionHelper(cast<SCEVSignExtendExpr>(S)->getOperand(),
2146
3.59k
                                     L, At, Processed);
2147
54.3k
  }
2148
54.3k
2149
54.3k
  if (!Processed.insert(S).second)
2150
125
    return false;
2151
54.1k
2152
54.1k
  if (auto *UDivExpr = dyn_cast<SCEVUDivExpr>(S)) {
2153
1.76k
    // If the divisor is a power of two and the SCEV type fits in a native
2154
1.76k
    // integer (and the LHS not expensive), consider the division cheap
2155
1.76k
    // irrespective of whether it occurs in the user code since it can be
2156
1.76k
    // lowered into a right shift.
2157
1.76k
    if (auto *SC = dyn_cast<SCEVConstant>(UDivExpr->getRHS()))
2158
1.75k
      if (SC->getAPInt().isPowerOf2()) {
2159
1.42k
        if (isHighCostExpansionHelper(UDivExpr->getLHS(), L, At, Processed))
2160
27
          return true;
2161
1.39k
        const DataLayout &DL =
2162
1.39k
            L->getHeader()->getParent()->getParent()->getDataLayout();
2163
1.39k
        unsigned Width = cast<IntegerType>(UDivExpr->getType())->getBitWidth();
2164
1.39k
        return DL.isIllegalInteger(Width);
2165
1.39k
      }
2166
349
2167
349
    // UDivExpr is very likely a UDiv that ScalarEvolution's HowFarToZero or
2168
349
    // HowManyLessThans produced to compute a precise expression, rather than a
2169
349
    // UDiv from the user's code. If we can't find a UDiv in the code with some
2170
349
    // simple searching, assume the former consider UDivExpr expensive to
2171
349
    // compute.
2172
349
    BasicBlock *ExitingBB = L->getExitingBlock();
2173
349
    if (!ExitingBB)
2174
1
      return true;
2175
348
2176
348
    // At the beginning of this function we already tried to find existing value
2177
348
    // for plain 'S'. Now try to lookup 'S + 1' since it is common pattern
2178
348
    // involving division. This is just a simple search heuristic.
2179
348
    if (!At)
2180
22
      At = &ExitingBB->back();
2181
348
    if (!getRelatedExistingExpansion(
2182
348
            SE.getAddExpr(S, SE.getConstant(S->getType(), 1)), At, L))
2183
347
      return true;
2184
52.4k
  }
2185
52.4k
2186
52.4k
  // HowManyLessThans uses a Max expression whenever the loop is not guarded by
2187
52.4k
  // the exit condition.
2188
52.4k
  if (isa<SCEVMinMaxExpr>(S))
2189
12.7k
    return true;
2190
39.6k
2191
39.6k
  // Recurse past nary expressions, which commonly occur in the
2192
39.6k
  // BackedgeTakenCount. They may already exist in program code, and if not,
2193
39.6k
  // they are not too expensive rematerialize.
2194
39.6k
  if (const SCEVNAryExpr *NAry = dyn_cast<SCEVNAryExpr>(S)) {
2195
39.6k
    for (auto *Op : NAry->operands())
2196
84.0k
      if (isHighCostExpansionHelper(Op, L, At, Processed))
2197
13.6k
        return true;
2198
39.6k
  }
2199
39.6k
2200
39.6k
  // If we haven't recognized an expensive SCEV pattern, assume it's an
2201
39.6k
  // expression produced by program code.
2202
39.6k
  
return false26.0k
;
2203
39.6k
}
2204
2205
Value *SCEVExpander::expandCodeForPredicate(const SCEVPredicate *Pred,
2206
17.6k
                                            Instruction *IP) {
2207
17.6k
  assert(IP);
2208
17.6k
  switch (Pred->getKind()) {
2209
17.6k
  case SCEVPredicate::P_Union:
2210
17.1k
    return expandUnionPredicate(cast<SCEVUnionPredicate>(Pred), IP);
2211
17.6k
  case SCEVPredicate::P_Equal:
2212
183
    return expandEqualPredicate(cast<SCEVEqualPredicate>(Pred), IP);
2213
17.6k
  case SCEVPredicate::P_Wrap: {
2214
327
    auto *AddRecPred = cast<SCEVWrapPredicate>(Pred);
2215
327
    return expandWrapPredicate(AddRecPred, IP);
2216
0
  }
2217
0
  }
2218
0
  llvm_unreachable("Unknown SCEV predicate type");
2219
0
}
2220
2221
Value *SCEVExpander::expandEqualPredicate(const SCEVEqualPredicate *Pred,
2222
183
                                          Instruction *IP) {
2223
183
  Value *Expr0 = expandCodeFor(Pred->getLHS(), Pred->getLHS()->getType(), IP);
2224
183
  Value *Expr1 = expandCodeFor(Pred->getRHS(), Pred->getRHS()->getType(), IP);
2225
183
2226
183
  Builder.SetInsertPoint(IP);
2227
183
  auto *I = Builder.CreateICmpNE(Expr0, Expr1, "ident.check");
2228
183
  return I;
2229
183
}
2230
2231
Value *SCEVExpander::generateOverflowCheck(const SCEVAddRecExpr *AR,
2232
327
                                           Instruction *Loc, bool Signed) {
2233
327
  assert(AR->isAffine() && "Cannot generate RT check for "
2234
327
                           "non-affine expression");
2235
327
2236
327
  SCEVUnionPredicate Pred;
2237
327
  const SCEV *ExitCount =
2238
327
      SE.getPredicatedBackedgeTakenCount(AR->getLoop(), Pred);
2239
327
2240
327
  assert(ExitCount != SE.getCouldNotCompute() && "Invalid loop count");
2241
327
2242
327
  const SCEV *Step = AR->getStepRecurrence(SE);
2243
327
  const SCEV *Start = AR->getStart();
2244
327
2245
327
  Type *ARTy = AR->getType();
2246
327
  unsigned SrcBits = SE.getTypeSizeInBits(ExitCount->getType());
2247
327
  unsigned DstBits = SE.getTypeSizeInBits(ARTy);
2248
327
2249
327
  // The expression {Start,+,Step} has nusw/nssw if
2250
327
  //   Step < 0, Start - |Step| * Backedge <= Start
2251
327
  //   Step >= 0, Start + |Step| * Backedge > Start
2252
327
  // and |Step| * Backedge doesn't unsigned overflow.
2253
327
2254
327
  IntegerType *CountTy = IntegerType::get(Loc->getContext(), SrcBits);
2255
327
  Builder.SetInsertPoint(Loc);
2256
327
  Value *TripCountVal = expandCodeFor(ExitCount, CountTy, Loc);
2257
327
2258
327
  IntegerType *Ty =
2259
327
      IntegerType::get(Loc->getContext(), SE.getTypeSizeInBits(ARTy));
2260
327
  Type *ARExpandTy = DL.isNonIntegralPointerType(ARTy) ? 
ARTy2
:
Ty325
;
2261
327
2262
327
  Value *StepValue = expandCodeFor(Step, Ty, Loc);
2263
327
  Value *NegStepValue = expandCodeFor(SE.getNegativeSCEV(Step), Ty, Loc);
2264
327
  Value *StartValue = expandCodeFor(Start, ARExpandTy, Loc);
2265
327
2266
327
  ConstantInt *Zero =
2267
327
      ConstantInt::get(Loc->getContext(), APInt::getNullValue(DstBits));
2268
327
2269
327
  Builder.SetInsertPoint(Loc);
2270
327
  // Compute |Step|
2271
327
  Value *StepCompare = Builder.CreateICmp(ICmpInst::ICMP_SLT, StepValue, Zero);
2272
327
  Value *AbsStep = Builder.CreateSelect(StepCompare, NegStepValue, StepValue);
2273
327
2274
327
  // Get the backedge taken count and truncate or extended to the AR type.
2275
327
  Value *TruncTripCount = Builder.CreateZExtOrTrunc(TripCountVal, Ty);
2276
327
  auto *MulF = Intrinsic::getDeclaration(Loc->getModule(),
2277
327
                                         Intrinsic::umul_with_overflow, Ty);
2278
327
2279
327
  // Compute |Step| * Backedge
2280
327
  CallInst *Mul = Builder.CreateCall(MulF, {AbsStep, TruncTripCount}, "mul");
2281
327
  Value *MulV = Builder.CreateExtractValue(Mul, 0, "mul.result");
2282
327
  Value *OfMul = Builder.CreateExtractValue(Mul, 1, "mul.overflow");
2283
327
2284
327
  // Compute:
2285
327
  //   Start + |Step| * Backedge < Start
2286
327
  //   Start - |Step| * Backedge > Start
2287
327
  Value *Add = nullptr, *Sub = nullptr;
2288
327
  if (PointerType *ARPtrTy = dyn_cast<PointerType>(ARExpandTy)) {
2289
2
    const SCEV *MulS = SE.getSCEV(MulV);
2290
2
    const SCEV *NegMulS = SE.getNegativeSCEV(MulS);
2291
2
    Add = Builder.CreateBitCast(expandAddToGEP(MulS, ARPtrTy, Ty, StartValue),
2292
2
                                ARPtrTy);
2293
2
    Sub = Builder.CreateBitCast(
2294
2
        expandAddToGEP(NegMulS, ARPtrTy, Ty, StartValue), ARPtrTy);
2295
325
  } else {
2296
325
    Add = Builder.CreateAdd(StartValue, MulV);
2297
325
    Sub = Builder.CreateSub(StartValue, MulV);
2298
325
  }
2299
327
2300
327
  Value *EndCompareGT = Builder.CreateICmp(
2301
327
      Signed ? 
ICmpInst::ICMP_SGT165
:
ICmpInst::ICMP_UGT162
, Sub, StartValue);
2302
327
2303
327
  Value *EndCompareLT = Builder.CreateICmp(
2304
327
      Signed ? 
ICmpInst::ICMP_SLT165
:
ICmpInst::ICMP_ULT162
, Add, StartValue);
2305
327
2306
327
  // Select the answer based on the sign of Step.
2307
327
  Value *EndCheck =
2308
327
      Builder.CreateSelect(StepCompare, EndCompareGT, EndCompareLT);
2309
327
2310
327
  // If the backedge taken count type is larger than the AR type,
2311
327
  // check that we don't drop any bits by truncating it. If we are
2312
327
  // dropping bits, then we have overflow (unless the step is zero).
2313
327
  if (SE.getTypeSizeInBits(CountTy) > SE.getTypeSizeInBits(Ty)) {
2314
238
    auto MaxVal = APInt::getMaxValue(DstBits).zext(SrcBits);
2315
238
    auto *BackedgeCheck =
2316
238
        Builder.CreateICmp(ICmpInst::ICMP_UGT, TripCountVal,
2317
238
                           ConstantInt::get(Loc->getContext(), MaxVal));
2318
238
    BackedgeCheck = Builder.CreateAnd(
2319
238
        BackedgeCheck, Builder.CreateICmp(ICmpInst::ICMP_NE, StepValue, Zero));
2320
238
2321
238
    EndCheck = Builder.CreateOr(EndCheck, BackedgeCheck);
2322
238
  }
2323
327
2324
327
  EndCheck = Builder.CreateOr(EndCheck, OfMul);
2325
327
  return EndCheck;
2326
327
}
2327
2328
Value *SCEVExpander::expandWrapPredicate(const SCEVWrapPredicate *Pred,
2329
327
                                         Instruction *IP) {
2330
327
  const auto *A = cast<SCEVAddRecExpr>(Pred->getExpr());
2331
327
  Value *NSSWCheck = nullptr, *NUSWCheck = nullptr;
2332
327
2333
327
  // Add a check for NUSW
2334
327
  if (Pred->getFlags() & SCEVWrapPredicate::IncrementNUSW)
2335
162
    NUSWCheck = generateOverflowCheck(A, IP, false);
2336
327
2337
327
  // Add a check for NSSW
2338
327
  if (Pred->getFlags() & SCEVWrapPredicate::IncrementNSSW)
2339
165
    NSSWCheck = generateOverflowCheck(A, IP, true);
2340
327
2341
327
  if (NUSWCheck && 
NSSWCheck162
)
2342
0
    return Builder.CreateOr(NUSWCheck, NSSWCheck);
2343
327
2344
327
  if (NUSWCheck)
2345
162
    return NUSWCheck;
2346
165
2347
165
  if (NSSWCheck)
2348
165
    return NSSWCheck;
2349
0
2350
0
  return ConstantInt::getFalse(IP->getContext());
2351
0
}
2352
2353
Value *SCEVExpander::expandUnionPredicate(const SCEVUnionPredicate *Union,
2354
17.1k
                                          Instruction *IP) {
2355
17.1k
  auto *BoolType = IntegerType::get(IP->getContext(), 1);
2356
17.1k
  Value *Check = ConstantInt::getNullValue(BoolType);
2357
17.1k
2358
17.1k
  // Loop over all checks in this set.
2359
17.1k
  for (auto Pred : Union->getPredicates()) {
2360
510
    auto *NextCheck = expandCodeForPredicate(Pred, IP);
2361
510
    Builder.SetInsertPoint(IP);
2362
510
    Check = Builder.CreateOr(Check, NextCheck);
2363
510
  }
2364
17.1k
2365
17.1k
  return Check;
2366
17.1k
}
2367
2368
namespace {
2369
// Search for a SCEV subexpression that is not safe to expand.  Any expression
2370
// that may expand to a !isSafeToSpeculativelyExecute value is unsafe, namely
2371
// UDiv expressions. We don't know if the UDiv is derived from an IR divide
2372
// instruction, but the important thing is that we prove the denominator is
2373
// nonzero before expansion.
2374
//
2375
// IVUsers already checks that IV-derived expressions are safe. So this check is
2376
// only needed when the expression includes some subexpression that is not IV
2377
// derived.
2378
//
2379
// Currently, we only allow division by a nonzero constant here. If this is
2380
// inadequate, we could easily allow division by SCEVUnknown by using
2381
// ValueTracking to check isKnownNonZero().
2382
//
2383
// We cannot generally expand recurrences unless the step dominates the loop
2384
// header. The expander handles the special case of affine recurrences by
2385
// scaling the recurrence outside the loop, but this technique isn't generally
2386
// applicable. Expanding a nested recurrence outside a loop requires computing
2387
// binomial coefficients. This could be done, but the recurrence has to be in a
2388
// perfectly reduced form, which can't be guaranteed.
2389
struct SCEVFindUnsafe {
2390
  ScalarEvolution &SE;
2391
  bool IsUnsafe;
2392
2393
545k
  SCEVFindUnsafe(ScalarEvolution &se): SE(se), IsUnsafe(false) {}
2394
2395
2.66M
  bool follow(const SCEV *S) {
2396
2.66M
    if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(S)) {
2397
25.8k
      const SCEVConstant *SC = dyn_cast<SCEVConstant>(D->getRHS());
2398
25.8k
      if (!SC || 
SC->getValue()->isZero()25.6k
) {
2399
263
        IsUnsafe = true;
2400
263
        return false;
2401
263
      }
2402
2.66M
    }
2403
2.66M
    if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S)) {
2404
472k
      const SCEV *Step = AR->getStepRecurrence(SE);
2405
472k
      if (!AR->isAffine() && 
!SE.dominates(Step, AR->getLoop()->getHeader())11
) {
2406
0
        IsUnsafe = true;
2407
0
        return false;
2408
0
      }
2409
2.66M
    }
2410
2.66M
    return true;
2411
2.66M
  }
2412
2.66M
  bool isDone() const { return IsUnsafe; }
2413
};
2414
}
2415
2416
namespace llvm {
2417
545k
bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE) {
2418
545k
  SCEVFindUnsafe Search(SE);
2419
545k
  visitAll(S, Search);
2420
545k
  return !Search.IsUnsafe;
2421
545k
}
2422
2423
bool isSafeToExpandAt(const SCEV *S, const Instruction *InsertionPoint,
2424
1.50k
                      ScalarEvolution &SE) {
2425
1.50k
  if (!isSafeToExpand(S, SE))
2426
14
    return false;
2427
1.49k
  // We have to prove that the expanded site of S dominates InsertionPoint.
2428
1.49k
  // This is easy when not in the same block, but hard when S is an instruction
2429
1.49k
  // to be expanded somewhere inside the same block as our insertion point.
2430
1.49k
  // What we really need here is something analogous to an OrderedBasicBlock,
2431
1.49k
  // but for the moment, we paper over the problem by handling two common and
2432
1.49k
  // cheap to check cases.
2433
1.49k
  if (SE.properlyDominates(S, InsertionPoint->getParent()))
2434
1.38k
    return true;
2435
106
  if (SE.dominates(S, InsertionPoint->getParent())) {
2436
104
    if (InsertionPoint->getParent()->getTerminator() == InsertionPoint)
2437
100
      return true;
2438
4
    if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S))
2439
4
      for (const Value *V : InsertionPoint->operand_values())
2440
12
        if (V == U->getValue())
2441
0
          return true;
2442
4
  }
2443
106
  
return false6
;
2444
106
}
2445
}