Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- PromoteMemoryToRegister.cpp - Convert allocas to registers ---------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file promotes memory references to be register references.  It promotes
10
// alloca instructions which only have loads and stores as uses.  An alloca is
11
// transformed by using iterated dominator frontiers to place PHI nodes, then
12
// traversing the function in depth-first order to rewrite loads and stores as
13
// appropriate.
14
//
15
//===----------------------------------------------------------------------===//
16
17
#include "llvm/ADT/ArrayRef.h"
18
#include "llvm/ADT/DenseMap.h"
19
#include "llvm/ADT/STLExtras.h"
20
#include "llvm/ADT/SmallPtrSet.h"
21
#include "llvm/ADT/SmallVector.h"
22
#include "llvm/ADT/Statistic.h"
23
#include "llvm/ADT/TinyPtrVector.h"
24
#include "llvm/ADT/Twine.h"
25
#include "llvm/Analysis/AssumptionCache.h"
26
#include "llvm/Analysis/InstructionSimplify.h"
27
#include "llvm/Analysis/IteratedDominanceFrontier.h"
28
#include "llvm/Transforms/Utils/Local.h"
29
#include "llvm/Analysis/ValueTracking.h"
30
#include "llvm/IR/BasicBlock.h"
31
#include "llvm/IR/CFG.h"
32
#include "llvm/IR/Constant.h"
33
#include "llvm/IR/Constants.h"
34
#include "llvm/IR/DIBuilder.h"
35
#include "llvm/IR/DerivedTypes.h"
36
#include "llvm/IR/Dominators.h"
37
#include "llvm/IR/Function.h"
38
#include "llvm/IR/InstrTypes.h"
39
#include "llvm/IR/Instruction.h"
40
#include "llvm/IR/Instructions.h"
41
#include "llvm/IR/IntrinsicInst.h"
42
#include "llvm/IR/Intrinsics.h"
43
#include "llvm/IR/LLVMContext.h"
44
#include "llvm/IR/Module.h"
45
#include "llvm/IR/Type.h"
46
#include "llvm/IR/User.h"
47
#include "llvm/Support/Casting.h"
48
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
49
#include <algorithm>
50
#include <cassert>
51
#include <iterator>
52
#include <utility>
53
#include <vector>
54
55
using namespace llvm;
56
57
#define DEBUG_TYPE "mem2reg"
58
59
STATISTIC(NumLocalPromoted, "Number of alloca's promoted within one block");
60
STATISTIC(NumSingleStore,   "Number of alloca's promoted with a single store");
61
STATISTIC(NumDeadAlloca,    "Number of dead alloca's removed");
62
STATISTIC(NumPHIInsert,     "Number of PHI nodes inserted");
63
64
148k
bool llvm::isAllocaPromotable(const AllocaInst *AI) {
65
148k
  // FIXME: If the memory unit is of pointer or integer type, we can permit
66
148k
  // assignments to subsections of the memory unit.
67
148k
  unsigned AS = AI->getType()->getAddressSpace();
68
148k
69
148k
  // Only allow direct and non-volatile loads and stores...
70
220k
  for (const User *U : AI->users()) {
71
220k
    if (const LoadInst *LI = dyn_cast<LoadInst>(U)) {
72
54.6k
      // Note that atomic loads can be transformed; atomic semantics do
73
54.6k
      // not have any meaning for a local alloca.
74
54.6k
      if (LI->isVolatile())
75
166
        return false;
76
165k
    } else if (const StoreInst *SI = dyn_cast<StoreInst>(U)) {
77
45.4k
      if (SI->getOperand(0) == AI)
78
353
        return false; // Don't allow a store OF the AI, only INTO the AI.
79
45.0k
      // Note that atomic stores can be transformed; atomic semantics do
80
45.0k
      // not have any meaning for a local alloca.
81
45.0k
      if (SI->isVolatile())
82
43
        return false;
83
119k
    } else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
84
1.21k
      if (!II->isLifetimeStartOrEnd())
85
4
        return false;
86
118k
    } else if (const BitCastInst *BCI = dyn_cast<BitCastInst>(U)) {
87
15.7k
      if (BCI->getType() != Type::getInt8PtrTy(U->getContext(), AS))
88
5.70k
        return false;
89
10.0k
      if (!onlyUsedByLifetimeMarkers(BCI))
90
9.70k
        return false;
91
103k
    } else if (const GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(U)) {
92
27.6k
      if (GEPI->getType() != Type::getInt8PtrTy(U->getContext(), AS))
93
15.7k
        return false;
94
11.8k
      if (!GEPI->hasAllZeroIndices())
95
10.3k
        return false;
96
1.52k
      if (!onlyUsedByLifetimeMarkers(GEPI))
97
1.51k
        return false;
98
75.4k
    } else {
99
75.4k
      return false;
100
75.4k
    }
101
220k
  }
102
148k
103
148k
  
return true29.9k
;
104
148k
}
105
106
namespace {
107
108
struct AllocaInfo {
109
  SmallVector<BasicBlock *, 32> DefiningBlocks;
110
  SmallVector<BasicBlock *, 32> UsingBlocks;
111
112
  StoreInst *OnlyStore;
113
  BasicBlock *OnlyBlock;
114
  bool OnlyUsedInOneBlock;
115
116
  TinyPtrVector<DbgVariableIntrinsic *> DbgDeclares;
117
118
950k
  void clear() {
119
950k
    DefiningBlocks.clear();
120
950k
    UsingBlocks.clear();
121
950k
    OnlyStore = nullptr;
122
950k
    OnlyBlock = nullptr;
123
950k
    OnlyUsedInOneBlock = true;
124
950k
    DbgDeclares.clear();
125
950k
  }
126
127
  /// Scan the uses of the specified alloca, filling in the AllocaInfo used
128
  /// by the rest of the pass to reason about the uses of this alloca.
129
950k
  void AnalyzeAlloca(AllocaInst *AI) {
130
950k
    clear();
131
950k
132
950k
    // As we scan the uses of the alloca instruction, keep track of stores,
133
950k
    // and decide whether all of the loads and stores to the alloca are within
134
950k
    // the same basic block.
135
4.13M
    for (auto UI = AI->user_begin(), E = AI->user_end(); UI != E;) {
136
3.17M
      Instruction *User = cast<Instruction>(*UI++);
137
3.17M
138
3.17M
      if (StoreInst *SI = dyn_cast<StoreInst>(User)) {
139
1.26M
        // Remember the basic blocks which define new values for the alloca
140
1.26M
        DefiningBlocks.push_back(SI->getParent());
141
1.26M
        OnlyStore = SI;
142
1.91M
      } else {
143
1.91M
        LoadInst *LI = cast<LoadInst>(User);
144
1.91M
        // Otherwise it must be a load instruction, keep track of variable
145
1.91M
        // reads.
146
1.91M
        UsingBlocks.push_back(LI->getParent());
147
1.91M
      }
148
3.17M
149
3.17M
      if (OnlyUsedInOneBlock) {
150
2.10M
        if (!OnlyBlock)
151
950k
          OnlyBlock = User->getParent();
152
1.15M
        else if (OnlyBlock != User->getParent())
153
395k
          OnlyUsedInOneBlock = false;
154
2.10M
      }
155
3.17M
    }
156
950k
157
950k
    DbgDeclares = FindDbgAddrUses(AI);
158
950k
  }
159
};
160
161
/// Data package used by RenamePass().
162
struct RenamePassData {
163
  using ValVector = std::vector<Value *>;
164
  using LocationVector = std::vector<DebugLoc>;
165
166
  RenamePassData(BasicBlock *B, BasicBlock *P, ValVector V, LocationVector L)
167
478k
      : BB(B), Pred(P), Values(std::move(V)), Locations(std::move(L)) {}
168
169
  BasicBlock *BB;
170
  BasicBlock *Pred;
171
  ValVector Values;
172
  LocationVector Locations;
173
};
174
175
/// This assigns and keeps a per-bb relative ordering of load/store
176
/// instructions in the block that directly load or store an alloca.
177
///
178
/// This functionality is important because it avoids scanning large basic
179
/// blocks multiple times when promoting many allocas in the same block.
180
class LargeBlockInfo {
181
  /// For each instruction that we track, keep the index of the
182
  /// instruction.
183
  ///
184
  /// The index starts out as the number of the instruction from the start of
185
  /// the block.
186
  DenseMap<const Instruction *, unsigned> InstNumbers;
187
188
public:
189
190
  /// This code only looks at accesses to allocas.
191
3.13M
  static bool isInterestingInstruction(const Instruction *I) {
192
3.13M
    return (isa<LoadInst>(I) && 
isa<AllocaInst>(I->getOperand(0))665k
) ||
193
3.13M
           
(2.62M
isa<StoreInst>(I)2.62M
&&
isa<AllocaInst>(I->getOperand(1))460k
);
194
3.13M
  }
195
196
  /// Get or calculate the index of the specified instruction.
197
542k
  unsigned getInstructionIndex(const Instruction *I) {
198
542k
    assert(isInterestingInstruction(I) &&
199
542k
           "Not a load/store to/from an alloca?");
200
542k
201
542k
    // If we already have this instruction number, return it.
202
542k
    DenseMap<const Instruction *, unsigned>::iterator It = InstNumbers.find(I);
203
542k
    if (It != InstNumbers.end())
204
420k
      return It->second;
205
122k
206
122k
    // Scan the whole block to get the instruction.  This accumulates
207
122k
    // information for every interesting instruction in the block, in order to
208
122k
    // avoid gratuitus rescans.
209
122k
    const BasicBlock *BB = I->getParent();
210
122k
    unsigned InstNo = 0;
211
122k
    for (const Instruction &BBI : *BB)
212
3.13M
      if (isInterestingInstruction(&BBI))
213
899k
        InstNumbers[&BBI] = InstNo++;
214
122k
    It = InstNumbers.find(I);
215
122k
216
122k
    assert(It != InstNumbers.end() && "Didn't insert instruction?");
217
122k
    return It->second;
218
122k
  }
219
220
2.26M
  void deleteValue(const Instruction *I) { InstNumbers.erase(I); }
221
222
53.9k
  void clear() { InstNumbers.clear(); }
223
};
224
225
struct PromoteMem2Reg {
226
  /// The alloca instructions being promoted.
227
  std::vector<AllocaInst *> Allocas;
228
229
  DominatorTree &DT;
230
  DIBuilder DIB;
231
232
  /// A cache of @llvm.assume intrinsics used by SimplifyInstruction.
233
  AssumptionCache *AC;
234
235
  const SimplifyQuery SQ;
236
237
  /// Reverse mapping of Allocas.
238
  DenseMap<AllocaInst *, unsigned> AllocaLookup;
239
240
  /// The PhiNodes we're adding.
241
  ///
242
  /// That map is used to simplify some Phi nodes as we iterate over it, so
243
  /// it should have deterministic iterators.  We could use a MapVector, but
244
  /// since we already maintain a map from BasicBlock* to a stable numbering
245
  /// (BBNumbers), the DenseMap is more efficient (also supports removal).
246
  DenseMap<std::pair<unsigned, unsigned>, PHINode *> NewPhiNodes;
247
248
  /// For each PHI node, keep track of which entry in Allocas it corresponds
249
  /// to.
250
  DenseMap<PHINode *, unsigned> PhiToAllocaMap;
251
252
  /// For each alloca, we keep track of the dbg.declare intrinsic that
253
  /// describes it, if any, so that we can convert it to a dbg.value
254
  /// intrinsic if the alloca gets promoted.
255
  SmallVector<TinyPtrVector<DbgVariableIntrinsic *>, 8> AllocaDbgDeclares;
256
257
  /// The set of basic blocks the renamer has already visited.
258
  SmallPtrSet<BasicBlock *, 16> Visited;
259
260
  /// Contains a stable numbering of basic blocks to avoid non-determinstic
261
  /// behavior.
262
  DenseMap<BasicBlock *, unsigned> BBNumbers;
263
264
  /// Lazily compute the number of predecessors a block has.
265
  DenseMap<const BasicBlock *, unsigned> BBNumPreds;
266
267
public:
268
  PromoteMem2Reg(ArrayRef<AllocaInst *> Allocas, DominatorTree &DT,
269
                 AssumptionCache *AC)
270
      : Allocas(Allocas.begin(), Allocas.end()), DT(DT),
271
        DIB(*DT.getRoot()->getParent()->getParent(), /*AllowUnresolved*/ false),
272
        AC(AC), SQ(DT.getRoot()->getParent()->getParent()->getDataLayout(),
273
281k
                   nullptr, &DT, AC) {}
274
275
  void run();
276
277
private:
278
820k
  void RemoveFromAllocasList(unsigned &AllocaIdx) {
279
820k
    Allocas[AllocaIdx] = Allocas.back();
280
820k
    Allocas.pop_back();
281
820k
    --AllocaIdx;
282
820k
  }
283
284
332k
  unsigned getNumPreds(const BasicBlock *BB) {
285
332k
    unsigned &NP = BBNumPreds[BB];
286
332k
    if (NP == 0)
287
137k
      NP = pred_size(BB) + 1;
288
332k
    return NP - 1;
289
332k
  }
290
291
  void ComputeLiveInBlocks(AllocaInst *AI, AllocaInfo &Info,
292
                           const SmallPtrSetImpl<BasicBlock *> &DefBlocks,
293
                           SmallPtrSetImpl<BasicBlock *> &LiveInBlocks);
294
  void RenamePass(BasicBlock *BB, BasicBlock *Pred,
295
                  RenamePassData::ValVector &IncVals,
296
                  RenamePassData::LocationVector &IncLocs,
297
                  std::vector<RenamePassData> &Worklist);
298
  bool QueuePhiNode(BasicBlock *BB, unsigned AllocaIdx, unsigned &Version);
299
};
300
301
} // end anonymous namespace
302
303
/// Given a LoadInst LI this adds assume(LI != null) after it.
304
4
static void addAssumeNonNull(AssumptionCache *AC, LoadInst *LI) {
305
4
  Function *AssumeIntrinsic =
306
4
      Intrinsic::getDeclaration(LI->getModule(), Intrinsic::assume);
307
4
  ICmpInst *LoadNotNull = new ICmpInst(ICmpInst::ICMP_NE, LI,
308
4
                                       Constant::getNullValue(LI->getType()));
309
4
  LoadNotNull->insertAfter(LI);
310
4
  CallInst *CI = CallInst::Create(AssumeIntrinsic, {LoadNotNull});
311
4
  CI->insertAfter(LoadNotNull);
312
4
  AC->registerAssumption(CI);
313
4
}
314
315
957k
static void removeLifetimeIntrinsicUsers(AllocaInst *AI) {
316
957k
  // Knowing that this alloca is promotable, we know that it's safe to kill all
317
957k
  // instructions except for load and store.
318
957k
319
4.89M
  for (auto UI = AI->user_begin(), UE = AI->user_end(); UI != UE;) {
320
3.93M
    Instruction *I = cast<Instruction>(*UI);
321
3.93M
    ++UI;
322
3.93M
    if (isa<LoadInst>(I) || 
isa<StoreInst>(I)2.01M
)
323
3.17M
      continue;
324
753k
325
753k
    if (!I->getType()->isVoidTy()) {
326
717k
      // The only users of this bitcast/GEP instruction are lifetime intrinsics.
327
717k
      // Follow the use/def chain to erase them now instead of leaving it for
328
717k
      // dead code elimination later.
329
1.43M
      for (auto UUI = I->user_begin(), UUE = I->user_end(); UUI != UUE;) {
330
717k
        Instruction *Inst = cast<Instruction>(*UUI);
331
717k
        ++UUI;
332
717k
        Inst->eraseFromParent();
333
717k
      }
334
717k
    }
335
753k
    I->eraseFromParent();
336
753k
  }
337
957k
}
338
339
/// Rewrite as many loads as possible given a single store.
340
///
341
/// When there is only a single store, we can use the domtree to trivially
342
/// replace all of the dominated loads with the stored value. Do so, and return
343
/// true if this has successfully promoted the alloca entirely. If this returns
344
/// false there were some loads which were not dominated by the single store
345
/// and thus must be phi-ed with undef. We fall back to the standard alloca
346
/// promotion algorithm in that case.
347
static bool rewriteSingleStoreAlloca(AllocaInst *AI, AllocaInfo &Info,
348
                                     LargeBlockInfo &LBI, const DataLayout &DL,
349
809k
                                     DominatorTree &DT, AssumptionCache *AC) {
350
809k
  StoreInst *OnlyStore = Info.OnlyStore;
351
809k
  bool StoringGlobalVal = !isa<Instruction>(OnlyStore->getOperand(0));
352
809k
  BasicBlock *StoreBB = OnlyStore->getParent();
353
809k
  int StoreIndex = -1;
354
809k
355
809k
  // Clear out UsingBlocks.  We will reconstruct it here if needed.
356
809k
  Info.UsingBlocks.clear();
357
809k
358
3.03M
  for (auto UI = AI->user_begin(), E = AI->user_end(); UI != E;) {
359
2.22M
    Instruction *UserInst = cast<Instruction>(*UI++);
360
2.22M
    if (UserInst == OnlyStore)
361
809k
      continue;
362
1.41M
    LoadInst *LI = cast<LoadInst>(UserInst);
363
1.41M
364
1.41M
    // Okay, if we have a load from the alloca, we want to replace it with the
365
1.41M
    // only value stored to the alloca.  We can do this if the value is
366
1.41M
    // dominated by the store.  If not, we use the rest of the mem2reg machinery
367
1.41M
    // to insert the phi nodes as needed.
368
1.41M
    if (!StoringGlobalVal) { // Non-instructions are always dominated.
369
539k
      if (LI->getParent() == StoreBB) {
370
270k
        // If we have a use that is in the same block as the store, compare the
371
270k
        // indices of the two instructions to see which one came first.  If the
372
270k
        // load came before the store, we can't handle it.
373
270k
        if (StoreIndex == -1)
374
225k
          StoreIndex = LBI.getInstructionIndex(OnlyStore);
375
270k
376
270k
        if (unsigned(StoreIndex) > LBI.getInstructionIndex(LI)) {
377
153
          // Can't handle this load, bail out.
378
153
          Info.UsingBlocks.push_back(StoreBB);
379
153
          continue;
380
153
        }
381
268k
      } else if (!DT.dominates(StoreBB, LI->getParent())) {
382
2.39k
        // If the load and store are in different blocks, use BB dominance to
383
2.39k
        // check their relationships.  If the store doesn't dom the use, bail
384
2.39k
        // out.
385
2.39k
        Info.UsingBlocks.push_back(LI->getParent());
386
2.39k
        continue;
387
2.39k
      }
388
1.41M
    }
389
1.41M
390
1.41M
    // Otherwise, we *can* safely rewrite this load.
391
1.41M
    Value *ReplVal = OnlyStore->getOperand(0);
392
1.41M
    // If the replacement value is the load, this must occur in unreachable
393
1.41M
    // code.
394
1.41M
    if (ReplVal == LI)
395
0
      ReplVal = UndefValue::get(LI->getType());
396
1.41M
397
1.41M
    // If the load was marked as nonnull we don't want to lose
398
1.41M
    // that information when we erase this Load. So we preserve
399
1.41M
    // it with an assume.
400
1.41M
    if (AC && 
LI->getMetadata(LLVMContext::MD_nonnull)1.41M
&&
401
1.41M
        
!isKnownNonZero(ReplVal, DL, 0, AC, LI, &DT)3
)
402
2
      addAssumeNonNull(AC, LI);
403
1.41M
404
1.41M
    LI->replaceAllUsesWith(ReplVal);
405
1.41M
    LI->eraseFromParent();
406
1.41M
    LBI.deleteValue(LI);
407
1.41M
  }
408
809k
409
809k
  // Finally, after the scan, check to see if the store is all that is left.
410
809k
  if (!Info.UsingBlocks.empty())
411
980
    return false; // If not, we'll have to fall back for the remainder.
412
808k
413
808k
  // Record debuginfo for the store and remove the declaration's
414
808k
  // debuginfo.
415
808k
  for (DbgVariableIntrinsic *DII : Info.DbgDeclares) {
416
73
    DIBuilder DIB(*AI->getModule(), /*AllowUnresolved*/ false);
417
73
    ConvertDebugDeclareToDebugValue(DII, Info.OnlyStore, DIB);
418
73
    DII->eraseFromParent();
419
73
  }
420
808k
  // Remove the (now dead) store and alloca.
421
808k
  Info.OnlyStore->eraseFromParent();
422
808k
  LBI.deleteValue(Info.OnlyStore);
423
808k
424
808k
  AI->eraseFromParent();
425
808k
  return true;
426
808k
}
427
428
/// Many allocas are only used within a single basic block.  If this is the
429
/// case, avoid traversing the CFG and inserting a lot of potentially useless
430
/// PHI nodes by just performing a single linear pass over the basic block
431
/// using the Alloca.
432
///
433
/// If we cannot promote this alloca (because it is read before it is written),
434
/// return false.  This is necessary in cases where, due to control flow, the
435
/// alloca is undefined only on some control flow paths.  e.g. code like
436
/// this is correct in LLVM IR:
437
///  // A is an alloca with no stores so far
438
///  for (...) {
439
///    int t = *A;
440
///    if (!first_iteration)
441
///      use(t);
442
///    *A = 42;
443
///  }
444
static bool promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
445
                                     LargeBlockInfo &LBI,
446
                                     const DataLayout &DL,
447
                                     DominatorTree &DT,
448
5.94k
                                     AssumptionCache *AC) {
449
5.94k
  // The trickiest case to handle is when we have large blocks. Because of this,
450
5.94k
  // this code is optimized assuming that large blocks happen.  This does not
451
5.94k
  // significantly pessimize the small block case.  This uses LargeBlockInfo to
452
5.94k
  // make it efficient to get the index of various operations in the block.
453
5.94k
454
5.94k
  // Walk the use-def list of the alloca, getting the locations of all stores.
455
5.94k
  using StoresByIndexTy = SmallVector<std::pair<unsigned, StoreInst *>, 64>;
456
5.94k
  StoresByIndexTy StoresByIndex;
457
5.94k
458
5.94k
  for (User *U : AI->users())
459
47.7k
    if (StoreInst *SI = dyn_cast<StoreInst>(U))
460
18.8k
      StoresByIndex.push_back(std::make_pair(LBI.getInstructionIndex(SI), SI));
461
5.94k
462
5.94k
  // Sort the stores by their index, making it efficient to do a lookup with a
463
5.94k
  // binary search.
464
5.94k
  llvm::sort(StoresByIndex, less_first());
465
5.94k
466
5.94k
  // Walk all of the loads from this alloca, replacing them with the nearest
467
5.94k
  // store above them, if any.
468
52.3k
  for (auto UI = AI->user_begin(), E = AI->user_end(); UI != E;) {
469
46.8k
    LoadInst *LI = dyn_cast<LoadInst>(*UI++);
470
46.8k
    if (!LI)
471
18.5k
      continue;
472
28.2k
473
28.2k
    unsigned LoadIdx = LBI.getInstructionIndex(LI);
474
28.2k
475
28.2k
    // Find the nearest store that has a lower index than this load.
476
28.2k
    StoresByIndexTy::iterator I = llvm::lower_bound(
477
28.2k
        StoresByIndex,
478
28.2k
        std::make_pair(LoadIdx, static_cast<StoreInst *>(nullptr)),
479
28.2k
        less_first());
480
28.2k
    if (I == StoresByIndex.begin()) {
481
1.63k
      if (StoresByIndex.empty())
482
1.19k
        // If there are no stores, the load takes the undef value.
483
1.19k
        LI->replaceAllUsesWith(UndefValue::get(LI->getType()));
484
439
      else
485
439
        // There is no store before this load, bail out (load may be affected
486
439
        // by the following stores - see main comment).
487
439
        return false;
488
26.6k
    } else {
489
26.6k
      // Otherwise, there was a store before this load, the load takes its value.
490
26.6k
      // Note, if the load was marked as nonnull we don't want to lose that
491
26.6k
      // information when we erase it. So we preserve it with an assume.
492
26.6k
      Value *ReplVal = std::prev(I)->second->getOperand(0);
493
26.6k
      if (AC && 
LI->getMetadata(LLVMContext::MD_nonnull)26.1k
&&
494
26.6k
          
!isKnownNonZero(ReplVal, DL, 0, AC, LI, &DT)1
)
495
1
        addAssumeNonNull(AC, LI);
496
26.6k
497
26.6k
      // If the replacement value is the load, this must occur in unreachable
498
26.6k
      // code.
499
26.6k
      if (ReplVal == LI)
500
1
        ReplVal = UndefValue::get(LI->getType());
501
26.6k
502
26.6k
      LI->replaceAllUsesWith(ReplVal);
503
26.6k
    }
504
28.2k
505
28.2k
    LI->eraseFromParent();
506
27.8k
    LBI.deleteValue(LI);
507
27.8k
  }
508
5.94k
509
5.94k
  // Remove the (now dead) stores and alloca.
510
23.1k
  
while (5.51k
!AI->use_empty()) {
511
17.5k
    StoreInst *SI = cast<StoreInst>(AI->user_back());
512
17.5k
    // Record debuginfo for the store before removing it.
513
17.5k
    for (DbgVariableIntrinsic *DII : Info.DbgDeclares) {
514
20
      DIBuilder DIB(*AI->getModule(), /*AllowUnresolved*/ false);
515
20
      ConvertDebugDeclareToDebugValue(DII, SI, DIB);
516
20
    }
517
17.5k
    SI->eraseFromParent();
518
17.5k
    LBI.deleteValue(SI);
519
17.5k
  }
520
5.51k
521
5.51k
  AI->eraseFromParent();
522
5.51k
523
5.51k
  // The alloca's debuginfo can be removed as well.
524
5.51k
  for (DbgVariableIntrinsic *DII : Info.DbgDeclares)
525
12
    DII->eraseFromParent();
526
5.51k
527
5.51k
  ++NumLocalPromoted;
528
5.51k
  return true;
529
5.94k
}
530
531
281k
void PromoteMem2Reg::run() {
532
281k
  Function &F = *DT.getRoot()->getParent();
533
281k
534
281k
  AllocaDbgDeclares.resize(Allocas.size());
535
281k
536
281k
  AllocaInfo Info;
537
281k
  LargeBlockInfo LBI;
538
281k
  ForwardIDFCalculator IDF(DT);
539
281k
540
1.23M
  for (unsigned AllocaNum = 0; AllocaNum != Allocas.size(); 
++AllocaNum957k
) {
541
957k
    AllocaInst *AI = Allocas[AllocaNum];
542
957k
543
957k
    assert(isAllocaPromotable(AI) && "Cannot promote non-promotable alloca!");
544
957k
    assert(AI->getParent()->getParent() == &F &&
545
957k
           "All allocas should be in the same function, which is same as DF!");
546
957k
547
957k
    removeLifetimeIntrinsicUsers(AI);
548
957k
549
957k
    if (AI->use_empty()) {
550
6.37k
      // If there are no uses of the alloca, just delete it now.
551
6.37k
      AI->eraseFromParent();
552
6.37k
553
6.37k
      // Remove the alloca from the Allocas list, since it has been processed
554
6.37k
      RemoveFromAllocasList(AllocaNum);
555
6.37k
      ++NumDeadAlloca;
556
6.37k
      continue;
557
6.37k
    }
558
950k
559
950k
    // Calculate the set of read and write-locations for each alloca.  This is
560
950k
    // analogous to finding the 'uses' and 'definitions' of each variable.
561
950k
    Info.AnalyzeAlloca(AI);
562
950k
563
950k
    // If there is only a single store to this value, replace any loads of
564
950k
    // it that are directly dominated by the definition with the value stored.
565
950k
    if (Info.DefiningBlocks.size() == 1) {
566
809k
      if (rewriteSingleStoreAlloca(AI, Info, LBI, SQ.DL, DT, AC)) {
567
808k
        // The alloca has been processed, move on.
568
808k
        RemoveFromAllocasList(AllocaNum);
569
808k
        ++NumSingleStore;
570
808k
        continue;
571
808k
      }
572
142k
    }
573
142k
574
142k
    // If the alloca is only read and written in one basic block, just perform a
575
142k
    // linear sweep over the block to eliminate it.
576
142k
    if (Info.OnlyUsedInOneBlock &&
577
142k
        
promoteSingleBlockAlloca(AI, Info, LBI, SQ.DL, DT, AC)5.94k
) {
578
5.51k
      // The alloca has been processed, move on.
579
5.51k
      RemoveFromAllocasList(AllocaNum);
580
5.51k
      continue;
581
5.51k
    }
582
137k
583
137k
    // If we haven't computed a numbering for the BB's in the function, do so
584
137k
    // now.
585
137k
    if (BBNumbers.empty()) {
586
53.9k
      unsigned ID = 0;
587
53.9k
      for (auto &BB : F)
588
901k
        BBNumbers[&BB] = ID++;
589
53.9k
    }
590
137k
591
137k
    // Remember the dbg.declare intrinsic describing this alloca, if any.
592
137k
    if (!Info.DbgDeclares.empty())
593
14
      AllocaDbgDeclares[AllocaNum] = Info.DbgDeclares;
594
137k
595
137k
    // Keep the reverse mapping of the 'Allocas' array for the rename pass.
596
137k
    AllocaLookup[Allocas[AllocaNum]] = AllocaNum;
597
137k
598
137k
    // At this point, we're committed to promoting the alloca using IDF's, and
599
137k
    // the standard SSA construction algorithm.  Determine which blocks need PHI
600
137k
    // nodes and see if we can optimize out some work by avoiding insertion of
601
137k
    // dead phi nodes.
602
137k
603
137k
    // Unique the set of defining blocks for efficient lookup.
604
137k
    SmallPtrSet<BasicBlock *, 32> DefBlocks(Info.DefiningBlocks.begin(),
605
137k
                                            Info.DefiningBlocks.end());
606
137k
607
137k
    // Determine which blocks the value is live in.  These are blocks which lead
608
137k
    // to uses.
609
137k
    SmallPtrSet<BasicBlock *, 32> LiveInBlocks;
610
137k
    ComputeLiveInBlocks(AI, Info, DefBlocks, LiveInBlocks);
611
137k
612
137k
    // At this point, we're committed to promoting the alloca using IDF's, and
613
137k
    // the standard SSA construction algorithm.  Determine which blocks need phi
614
137k
    // nodes and see if we can optimize out some work by avoiding insertion of
615
137k
    // dead phi nodes.
616
137k
    IDF.setLiveInBlocks(LiveInBlocks);
617
137k
    IDF.setDefiningBlocks(DefBlocks);
618
137k
    SmallVector<BasicBlock *, 32> PHIBlocks;
619
137k
    IDF.calculate(PHIBlocks);
620
242k
    llvm::sort(PHIBlocks, [this](BasicBlock *A, BasicBlock *B) {
621
242k
      return BBNumbers.find(A)->second < BBNumbers.find(B)->second;
622
242k
    });
623
137k
624
137k
    unsigned CurrentVersion = 0;
625
137k
    for (BasicBlock *BB : PHIBlocks)
626
195k
      QueuePhiNode(BB, AllocaNum, CurrentVersion);
627
137k
  }
628
281k
629
281k
  if (Allocas.empty())
630
227k
    return; // All of the allocas must have been trivial!
631
53.9k
632
53.9k
  LBI.clear();
633
53.9k
634
53.9k
  // Set the incoming values for the basic block to be null values for all of
635
53.9k
  // the alloca's.  We do this in case there is a load of a value that has not
636
53.9k
  // been stored yet.  In this case, it will get this null value.
637
53.9k
  RenamePassData::ValVector Values(Allocas.size());
638
191k
  for (unsigned i = 0, e = Allocas.size(); i != e; 
++i137k
)
639
137k
    Values[i] = UndefValue::get(Allocas[i]->getAllocatedType());
640
53.9k
641
53.9k
  // When handling debug info, treat all incoming values as if they have unknown
642
53.9k
  // locations until proven otherwise.
643
53.9k
  RenamePassData::LocationVector Locations(Allocas.size());
644
53.9k
645
53.9k
  // Walks all basic blocks in the function performing the SSA rename algorithm
646
53.9k
  // and inserting the phi nodes we marked as necessary
647
53.9k
  std::vector<RenamePassData> RenamePassWorkList;
648
53.9k
  RenamePassWorkList.emplace_back(&F.front(), nullptr, std::move(Values),
649
53.9k
                                  std::move(Locations));
650
478k
  do {
651
478k
    RenamePassData RPD = std::move(RenamePassWorkList.back());
652
478k
    RenamePassWorkList.pop_back();
653
478k
    // RenamePass may add new worklist entries.
654
478k
    RenamePass(RPD.BB, RPD.Pred, RPD.Values, RPD.Locations, RenamePassWorkList);
655
478k
  } while (!RenamePassWorkList.empty());
656
53.9k
657
53.9k
  // The renamer uses the Visited set to avoid infinite loops.  Clear it now.
658
53.9k
  Visited.clear();
659
53.9k
660
53.9k
  // Remove the allocas themselves from the function.
661
137k
  for (Instruction *A : Allocas) {
662
137k
    // If there are any uses of the alloca instructions left, they must be in
663
137k
    // unreachable basic blocks that were not processed by walking the dominator
664
137k
    // tree. Just delete the users now.
665
137k
    if (!A->use_empty())
666
1.06k
      A->replaceAllUsesWith(UndefValue::get(A->getType()));
667
137k
    A->eraseFromParent();
668
137k
  }
669
53.9k
670
53.9k
  // Remove alloca's dbg.declare instrinsics from the function.
671
53.9k
  for (auto &Declares : AllocaDbgDeclares)
672
443k
    for (auto *DII : Declares)
673
14
      DII->eraseFromParent();
674
53.9k
675
53.9k
  // Loop over all of the PHI nodes and see if there are any that we can get
676
53.9k
  // rid of because they merge all of the same incoming values.  This can
677
53.9k
  // happen due to undef values coming into the PHI nodes.  This process is
678
53.9k
  // iterative, because eliminating one PHI node can cause others to be removed.
679
53.9k
  bool EliminatedAPHI = true;
680
109k
  while (EliminatedAPHI) {
681
55.7k
    EliminatedAPHI = false;
682
55.7k
683
55.7k
    // Iterating over NewPhiNodes is deterministic, so it is safe to try to
684
55.7k
    // simplify and RAUW them as we go.  If it was not, we could add uses to
685
55.7k
    // the values we replace with in a non-deterministic order, thus creating
686
55.7k
    // non-deterministic def->use chains.
687
55.7k
    for (DenseMap<std::pair<unsigned, unsigned>, PHINode *>::iterator
688
55.7k
             I = NewPhiNodes.begin(),
689
55.7k
             E = NewPhiNodes.end();
690
269k
         I != E;) {
691
213k
      PHINode *PN = I->second;
692
213k
693
213k
      // If this PHI node merges one value and/or undefs, get the value.
694
213k
      if (Value *V = SimplifyInstruction(PN, SQ)) {
695
2.15k
        PN->replaceAllUsesWith(V);
696
2.15k
        PN->eraseFromParent();
697
2.15k
        NewPhiNodes.erase(I++);
698
2.15k
        EliminatedAPHI = true;
699
2.15k
        continue;
700
2.15k
      }
701
211k
      ++I;
702
211k
    }
703
55.7k
  }
704
53.9k
705
53.9k
  // At this point, the renamer has added entries to PHI nodes for all reachable
706
53.9k
  // code.  Unfortunately, there may be unreachable blocks which the renamer
707
53.9k
  // hasn't traversed.  If this is the case, the PHI nodes may not
708
53.9k
  // have incoming values for all predecessors.  Loop over all PHI nodes we have
709
53.9k
  // created, inserting undef values if they are missing any incoming values.
710
53.9k
  for (DenseMap<std::pair<unsigned, unsigned>, PHINode *>::iterator
711
53.9k
           I = NewPhiNodes.begin(),
712
53.9k
           E = NewPhiNodes.end();
713
247k
       I != E; 
++I193k
) {
714
193k
    // We want to do this once per basic block.  As such, only process a block
715
193k
    // when we find the PHI that is the first entry in the block.
716
193k
    PHINode *SomePHI = I->second;
717
193k
    BasicBlock *BB = SomePHI->getParent();
718
193k
    if (&BB->front() != SomePHI)
719
56.0k
      continue;
720
137k
721
137k
    // Only do work here if there the PHI nodes are missing incoming values.  We
722
137k
    // know that all PHI nodes that were inserted in a block will have the same
723
137k
    // number of incoming values, so we can just check any of them.
724
137k
    if (SomePHI->getNumIncomingValues() == getNumPreds(BB))
725
137k
      continue;
726
27
727
27
    // Get the preds for BB.
728
27
    SmallVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
729
27
730
27
    // Ok, now we know that all of the PHI nodes are missing entries for some
731
27
    // basic blocks.  Start by sorting the incoming predecessors for efficient
732
27
    // access.
733
974
    auto CompareBBNumbers = [this](BasicBlock *A, BasicBlock *B) {
734
974
      return BBNumbers.find(A)->second < BBNumbers.find(B)->second;
735
974
    };
736
27
    llvm::sort(Preds, CompareBBNumbers);
737
27
738
27
    // Now we loop through all BB's which have entries in SomePHI and remove
739
27
    // them from the Preds list.
740
157
    for (unsigned i = 0, e = SomePHI->getNumIncomingValues(); i != e; 
++i130
) {
741
130
      // Do a log(n) search of the Preds list for the entry we want.
742
130
      SmallVectorImpl<BasicBlock *>::iterator EntIt = llvm::lower_bound(
743
130
          Preds, SomePHI->getIncomingBlock(i), CompareBBNumbers);
744
130
      assert(EntIt != Preds.end() && *EntIt == SomePHI->getIncomingBlock(i) &&
745
130
             "PHI node has entry for a block which is not a predecessor!");
746
130
747
130
      // Remove the entry
748
130
      Preds.erase(EntIt);
749
130
    }
750
27
751
27
    // At this point, the blocks left in the preds list must have dummy
752
27
    // entries inserted into every PHI nodes for the block.  Update all the phi
753
27
    // nodes in this block that we are inserting (there could be phis before
754
27
    // mem2reg runs).
755
27
    unsigned NumBadPreds = SomePHI->getNumIncomingValues();
756
27
    BasicBlock::iterator BBI = BB->begin();
757
66
    while ((SomePHI = dyn_cast<PHINode>(BBI++)) &&
758
66
           
SomePHI->getNumIncomingValues() == NumBadPreds64
) {
759
39
      Value *UndefVal = UndefValue::get(SomePHI->getType());
760
39
      for (BasicBlock *Pred : Preds)
761
101
        SomePHI->addIncoming(UndefVal, Pred);
762
39
    }
763
27
  }
764
53.9k
765
53.9k
  NewPhiNodes.clear();
766
53.9k
}
767
768
/// Determine which blocks the value is live in.
769
///
770
/// These are blocks which lead to uses.  Knowing this allows us to avoid
771
/// inserting PHI nodes into blocks which don't lead to uses (thus, the
772
/// inserted phi nodes would be dead).
773
void PromoteMem2Reg::ComputeLiveInBlocks(
774
    AllocaInst *AI, AllocaInfo &Info,
775
    const SmallPtrSetImpl<BasicBlock *> &DefBlocks,
776
137k
    SmallPtrSetImpl<BasicBlock *> &LiveInBlocks) {
777
137k
  // To determine liveness, we must iterate through the predecessors of blocks
778
137k
  // where the def is live.  Blocks are added to the worklist if we need to
779
137k
  // check their predecessors.  Start with all the using blocks.
780
137k
  SmallVector<BasicBlock *, 64> LiveInBlockWorklist(Info.UsingBlocks.begin(),
781
137k
                                                    Info.UsingBlocks.end());
782
137k
783
137k
  // If any of the using blocks is also a definition block, check to see if the
784
137k
  // definition occurs before or after the use.  If it happens before the use,
785
137k
  // the value isn't really live-in.
786
616k
  for (unsigned i = 0, e = LiveInBlockWorklist.size(); i != e; 
++i479k
) {
787
479k
    BasicBlock *BB = LiveInBlockWorklist[i];
788
479k
    if (!DefBlocks.count(BB))
789
292k
      continue;
790
186k
791
186k
    // Okay, this is a block that both uses and defines the value.  If the first
792
186k
    // reference to the alloca is a def (store), then we know it isn't live-in.
793
1.67M
    
for (BasicBlock::iterator I = BB->begin();; 186k
++I1.49M
) {
794
1.67M
      if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
795
203k
        if (SI->getOperand(1) != AI)
796
159k
          continue;
797
43.5k
798
43.5k
        // We found a store to the alloca before a load.  The alloca is not
799
43.5k
        // actually live-in here.
800
43.5k
        LiveInBlockWorklist[i] = LiveInBlockWorklist.back();
801
43.5k
        LiveInBlockWorklist.pop_back();
802
43.5k
        --i;
803
43.5k
        --e;
804
43.5k
        break;
805
43.5k
      }
806
1.47M
807
1.47M
      if (LoadInst *LI = dyn_cast<LoadInst>(I))
808
625k
        // Okay, we found a load before a store to the alloca.  It is actually
809
625k
        // live into this block.
810
625k
        if (LI->getOperand(0) == AI)
811
143k
          break;
812
1.47M
    }
813
186k
  }
814
137k
815
137k
  // Now that we have a set of blocks where the phi is live-in, recursively add
816
137k
  // their predecessors until we find the full region the value is live.
817
2.27M
  while (!LiveInBlockWorklist.empty()) {
818
2.13M
    BasicBlock *BB = LiveInBlockWorklist.pop_back_val();
819
2.13M
820
2.13M
    // The block really is live in here, insert it into the set.  If already in
821
2.13M
    // the set, then it has already been processed.
822
2.13M
    if (!LiveInBlocks.insert(BB).second)
823
781k
      continue;
824
1.35M
825
1.35M
    // Since the value is live into BB, it is either defined in a predecessor or
826
1.35M
    // live into it to.  Add the preds to the worklist unless they are a
827
1.35M
    // defining block.
828
2.07M
    
for (BasicBlock *P : predecessors(BB))1.35M
{
829
2.07M
      // The value is not live into a predecessor if it defines the value.
830
2.07M
      if (DefBlocks.count(P))
831
377k
        continue;
832
1.70M
833
1.70M
      // Otherwise it is, add to the worklist.
834
1.70M
      LiveInBlockWorklist.push_back(P);
835
1.70M
    }
836
1.35M
  }
837
137k
}
838
839
/// Queue a phi-node to be added to a basic-block for a specific Alloca.
840
///
841
/// Returns true if there wasn't already a phi-node for that variable
842
bool PromoteMem2Reg::QueuePhiNode(BasicBlock *BB, unsigned AllocaNo,
843
195k
                                  unsigned &Version) {
844
195k
  // Look up the basic-block in question.
845
195k
  PHINode *&PN = NewPhiNodes[std::make_pair(BBNumbers[BB], AllocaNo)];
846
195k
847
195k
  // If the BB already has a phi node added for the i'th alloca then we're done!
848
195k
  if (PN)
849
0
    return false;
850
195k
851
195k
  // Create a PhiNode using the dereferenced type... and add the phi-node to the
852
195k
  // BasicBlock.
853
195k
  PN = PHINode::Create(Allocas[AllocaNo]->getAllocatedType(), getNumPreds(BB),
854
195k
                       Allocas[AllocaNo]->getName() + "." + Twine(Version++),
855
195k
                       &BB->front());
856
195k
  ++NumPHIInsert;
857
195k
  PhiToAllocaMap[PN] = AllocaNo;
858
195k
  return true;
859
195k
}
860
861
/// Update the debug location of a phi. \p ApplyMergedLoc indicates whether to
862
/// create a merged location incorporating \p DL, or to set \p DL directly.
863
static void updateForIncomingValueLocation(PHINode *PN, DebugLoc DL,
864
485k
                                           bool ApplyMergedLoc) {
865
485k
  if (ApplyMergedLoc)
866
289k
    PN->applyMergedLocation(PN->getDebugLoc(), DL);
867
195k
  else
868
195k
    PN->setDebugLoc(DL);
869
485k
}
870
871
/// Recursively traverse the CFG of the function, renaming loads and
872
/// stores to the allocas which we are promoting.
873
///
874
/// IncomingVals indicates what value each Alloca contains on exit from the
875
/// predecessor block Pred.
876
void PromoteMem2Reg::RenamePass(BasicBlock *BB, BasicBlock *Pred,
877
                                RenamePassData::ValVector &IncomingVals,
878
                                RenamePassData::LocationVector &IncomingLocs,
879
478k
                                std::vector<RenamePassData> &Worklist) {
880
1.30M
NextIteration:
881
1.30M
  // If we are inserting any phi nodes into this BB, they will already be in the
882
1.30M
  // block.
883
1.30M
  if (PHINode *APN = dyn_cast<PHINode>(BB->begin())) {
884
425k
    // If we have PHI nodes to update, compute the number of edges from Pred to
885
425k
    // BB.
886
425k
    if (PhiToAllocaMap.count(APN)) {
887
332k
      // We want to be able to distinguish between PHI nodes being inserted by
888
332k
      // this invocation of mem2reg from those phi nodes that already existed in
889
332k
      // the IR before mem2reg was run.  We determine that APN is being inserted
890
332k
      // because it is missing incoming edges.  All other PHI nodes being
891
332k
      // inserted by this pass of mem2reg will have the same number of incoming
892
332k
      // operands so far.  Remember this count.
893
332k
      unsigned NewPHINumOperands = APN->getNumOperands();
894
332k
895
332k
      unsigned NumEdges = std::count(succ_begin(Pred), succ_end(Pred), BB);
896
332k
      assert(NumEdges && "Must be at least one edge from Pred to BB!");
897
332k
898
332k
      // Add entries for all the phis.
899
332k
      BasicBlock::iterator PNI = BB->begin();
900
485k
      do {
901
485k
        unsigned AllocaNo = PhiToAllocaMap[APN];
902
485k
903
485k
        // Update the location of the phi node.
904
485k
        updateForIncomingValueLocation(APN, IncomingLocs[AllocaNo],
905
485k
                                       APN->getNumIncomingValues() > 0);
906
485k
907
485k
        // Add N incoming values to the PHI node.
908
971k
        for (unsigned i = 0; i != NumEdges; 
++i486k
)
909
486k
          APN->addIncoming(IncomingVals[AllocaNo], Pred);
910
485k
911
485k
        // The currently active variable for this block is now the PHI.
912
485k
        IncomingVals[AllocaNo] = APN;
913
485k
        for (DbgVariableIntrinsic *DII : AllocaDbgDeclares[AllocaNo])
914
28
          ConvertDebugDeclareToDebugValue(DII, APN, DIB);
915
485k
916
485k
        // Get the next phi node.
917
485k
        ++PNI;
918
485k
        APN = dyn_cast<PHINode>(PNI);
919
485k
        if (!APN)
920
318k
          break;
921
167k
922
167k
        // Verify that it is missing entries.  If not, it is not being inserted
923
167k
        // by this mem2reg invocation so we want to ignore it.
924
167k
      } while (APN->getNumOperands() == NewPHINumOperands);
925
332k
    }
926
425k
  }
927
1.30M
928
1.30M
  // Don't revisit blocks.
929
1.30M
  
if (1.13M
!Visited.insert(BB).second1.13M
)
930
404k
    return;
931
733k
932
5.99M
  
for (BasicBlock::iterator II = BB->begin(); 733k
!II->isTerminator();) {
933
5.26M
    Instruction *I = &*II++; // get the instruction, increment iterator
934
5.26M
935
5.26M
    if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
936
1.04M
      AllocaInst *Src = dyn_cast<AllocaInst>(LI->getPointerOperand());
937
1.04M
      if (!Src)
938
548k
        continue;
939
501k
940
501k
      DenseMap<AllocaInst *, unsigned>::iterator AI = AllocaLookup.find(Src);
941
501k
      if (AI == AllocaLookup.end())
942
23.3k
        continue;
943
477k
944
477k
      Value *V = IncomingVals[AI->second];
945
477k
946
477k
      // If the load was marked as nonnull we don't want to lose
947
477k
      // that information when we erase this Load. So we preserve
948
477k
      // it with an assume.
949
477k
      if (AC && 
LI->getMetadata(LLVMContext::MD_nonnull)477k
&&
950
477k
          
!isKnownNonZero(V, SQ.DL, 0, AC, LI, &DT)1
)
951
1
        addAssumeNonNull(AC, LI);
952
477k
953
477k
      // Anything using the load now uses the current value.
954
477k
      LI->replaceAllUsesWith(V);
955
477k
      BB->getInstList().erase(LI);
956
4.21M
    } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
957
671k
      // Delete this instruction and mark the name as the current holder of the
958
671k
      // value
959
671k
      AllocaInst *Dest = dyn_cast<AllocaInst>(SI->getPointerOperand());
960
671k
      if (!Dest)
961
220k
        continue;
962
450k
963
450k
      DenseMap<AllocaInst *, unsigned>::iterator ai = AllocaLookup.find(Dest);
964
450k
      if (ai == AllocaLookup.end())
965
14.8k
        continue;
966
435k
967
435k
      // what value were we writing?
968
435k
      unsigned AllocaNo = ai->second;
969
435k
      IncomingVals[AllocaNo] = SI->getOperand(0);
970
435k
971
435k
      // Record debuginfo for the store before removing it.
972
435k
      IncomingLocs[AllocaNo] = SI->getDebugLoc();
973
435k
      for (DbgVariableIntrinsic *DII : AllocaDbgDeclares[ai->second])
974
31
        ConvertDebugDeclareToDebugValue(DII, SI, DIB);
975
435k
      BB->getInstList().erase(SI);
976
435k
    }
977
5.26M
  }
978
733k
979
733k
  // 'Recurse' to our successors.
980
733k
  succ_iterator I = succ_begin(BB), E = succ_end(BB);
981
733k
  if (I == E)
982
73.7k
    return;
983
659k
984
659k
  // Keep track of the successors so we don't visit the same successor twice
985
659k
  SmallPtrSet<BasicBlock *, 8> VisitedSuccs;
986
659k
987
659k
  // Handle the first successor without using the worklist.
988
659k
  VisitedSuccs.insert(*I);
989
659k
  Pred = BB;
990
659k
  BB = *I;
991
659k
  ++I;
992
659k
993
1.08M
  for (; I != E; 
++I428k
)
994
428k
    if (VisitedSuccs.insert(*I).second)
995
424k
      Worklist.emplace_back(*I, Pred, IncomingVals, IncomingLocs);
996
659k
997
659k
  goto NextIteration;
998
659k
}
999
1000
void llvm::PromoteMemToReg(ArrayRef<AllocaInst *> Allocas, DominatorTree &DT,
1001
281k
                           AssumptionCache *AC) {
1002
281k
  // If there is nothing to do, bail out...
1003
281k
  if (Allocas.empty())
1004
0
    return;
1005
281k
1006
281k
  PromoteMem2Reg(Allocas, DT, AC).run();
1007
281k
}