Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/Scalar/NewGVN.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- NewGVN.cpp - Global Value Numbering Pass ---------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// This file implements the new LLVM's Global Value Numbering pass.
11
/// GVN partitions values computed by a function into congruence classes.
12
/// Values ending up in the same congruence class are guaranteed to be the same
13
/// for every execution of the program. In that respect, congruency is a
14
/// compile-time approximation of equivalence of values at runtime.
15
/// The algorithm implemented here uses a sparse formulation and it's based
16
/// on the ideas described in the paper:
17
/// "A Sparse Algorithm for Predicated Global Value Numbering" from
18
/// Karthik Gargi.
19
///
20
/// A brief overview of the algorithm: The algorithm is essentially the same as
21
/// the standard RPO value numbering algorithm (a good reference is the paper
22
/// "SCC based value numbering" by L. Taylor Simpson) with one major difference:
23
/// The RPO algorithm proceeds, on every iteration, to process every reachable
24
/// block and every instruction in that block.  This is because the standard RPO
25
/// algorithm does not track what things have the same value number, it only
26
/// tracks what the value number of a given operation is (the mapping is
27
/// operation -> value number).  Thus, when a value number of an operation
28
/// changes, it must reprocess everything to ensure all uses of a value number
29
/// get updated properly.  In constrast, the sparse algorithm we use *also*
30
/// tracks what operations have a given value number (IE it also tracks the
31
/// reverse mapping from value number -> operations with that value number), so
32
/// that it only needs to reprocess the instructions that are affected when
33
/// something's value number changes.  The vast majority of complexity and code
34
/// in this file is devoted to tracking what value numbers could change for what
35
/// instructions when various things happen.  The rest of the algorithm is
36
/// devoted to performing symbolic evaluation, forward propagation, and
37
/// simplification of operations based on the value numbers deduced so far
38
///
39
/// In order to make the GVN mostly-complete, we use a technique derived from
40
/// "Detection of Redundant Expressions: A Complete and Polynomial-time
41
/// Algorithm in SSA" by R.R. Pai.  The source of incompleteness in most SSA
42
/// based GVN algorithms is related to their inability to detect equivalence
43
/// between phi of ops (IE phi(a+b, c+d)) and op of phis (phi(a,c) + phi(b, d)).
44
/// We resolve this issue by generating the equivalent "phi of ops" form for
45
/// each op of phis we see, in a way that only takes polynomial time to resolve.
46
///
47
/// We also do not perform elimination by using any published algorithm.  All
48
/// published algorithms are O(Instructions). Instead, we use a technique that
49
/// is O(number of operations with the same value number), enabling us to skip
50
/// trying to eliminate things that have unique value numbers.
51
//
52
//===----------------------------------------------------------------------===//
53
54
#include "llvm/Transforms/Scalar/NewGVN.h"
55
#include "llvm/ADT/ArrayRef.h"
56
#include "llvm/ADT/BitVector.h"
57
#include "llvm/ADT/DenseMap.h"
58
#include "llvm/ADT/DenseMapInfo.h"
59
#include "llvm/ADT/DenseSet.h"
60
#include "llvm/ADT/DepthFirstIterator.h"
61
#include "llvm/ADT/GraphTraits.h"
62
#include "llvm/ADT/Hashing.h"
63
#include "llvm/ADT/PointerIntPair.h"
64
#include "llvm/ADT/PostOrderIterator.h"
65
#include "llvm/ADT/SmallPtrSet.h"
66
#include "llvm/ADT/SmallVector.h"
67
#include "llvm/ADT/SparseBitVector.h"
68
#include "llvm/ADT/Statistic.h"
69
#include "llvm/ADT/iterator_range.h"
70
#include "llvm/Analysis/AliasAnalysis.h"
71
#include "llvm/Analysis/AssumptionCache.h"
72
#include "llvm/Analysis/CFGPrinter.h"
73
#include "llvm/Analysis/ConstantFolding.h"
74
#include "llvm/Analysis/GlobalsModRef.h"
75
#include "llvm/Analysis/InstructionSimplify.h"
76
#include "llvm/Analysis/MemoryBuiltins.h"
77
#include "llvm/Analysis/MemorySSA.h"
78
#include "llvm/Analysis/TargetLibraryInfo.h"
79
#include "llvm/Transforms/Utils/Local.h"
80
#include "llvm/IR/Argument.h"
81
#include "llvm/IR/BasicBlock.h"
82
#include "llvm/IR/Constant.h"
83
#include "llvm/IR/Constants.h"
84
#include "llvm/IR/Dominators.h"
85
#include "llvm/IR/Function.h"
86
#include "llvm/IR/InstrTypes.h"
87
#include "llvm/IR/Instruction.h"
88
#include "llvm/IR/Instructions.h"
89
#include "llvm/IR/IntrinsicInst.h"
90
#include "llvm/IR/Intrinsics.h"
91
#include "llvm/IR/LLVMContext.h"
92
#include "llvm/IR/Type.h"
93
#include "llvm/IR/Use.h"
94
#include "llvm/IR/User.h"
95
#include "llvm/IR/Value.h"
96
#include "llvm/Pass.h"
97
#include "llvm/Support/Allocator.h"
98
#include "llvm/Support/ArrayRecycler.h"
99
#include "llvm/Support/Casting.h"
100
#include "llvm/Support/CommandLine.h"
101
#include "llvm/Support/Debug.h"
102
#include "llvm/Support/DebugCounter.h"
103
#include "llvm/Support/ErrorHandling.h"
104
#include "llvm/Support/PointerLikeTypeTraits.h"
105
#include "llvm/Support/raw_ostream.h"
106
#include "llvm/Transforms/Scalar.h"
107
#include "llvm/Transforms/Scalar/GVNExpression.h"
108
#include "llvm/Transforms/Utils/PredicateInfo.h"
109
#include "llvm/Transforms/Utils/VNCoercion.h"
110
#include <algorithm>
111
#include <cassert>
112
#include <cstdint>
113
#include <iterator>
114
#include <map>
115
#include <memory>
116
#include <set>
117
#include <string>
118
#include <tuple>
119
#include <utility>
120
#include <vector>
121
122
using namespace llvm;
123
using namespace llvm::GVNExpression;
124
using namespace llvm::VNCoercion;
125
126
#define DEBUG_TYPE "newgvn"
127
128
STATISTIC(NumGVNInstrDeleted, "Number of instructions deleted");
129
STATISTIC(NumGVNBlocksDeleted, "Number of blocks deleted");
130
STATISTIC(NumGVNOpsSimplified, "Number of Expressions simplified");
131
STATISTIC(NumGVNPhisAllSame, "Number of PHIs whos arguments are all the same");
132
STATISTIC(NumGVNMaxIterations,
133
          "Maximum Number of iterations it took to converge GVN");
134
STATISTIC(NumGVNLeaderChanges, "Number of leader changes");
135
STATISTIC(NumGVNSortedLeaderChanges, "Number of sorted leader changes");
136
STATISTIC(NumGVNAvoidedSortedLeaderChanges,
137
          "Number of avoided sorted leader changes");
138
STATISTIC(NumGVNDeadStores, "Number of redundant/dead stores eliminated");
139
STATISTIC(NumGVNPHIOfOpsCreated, "Number of PHI of ops created");
140
STATISTIC(NumGVNPHIOfOpsEliminations,
141
          "Number of things eliminated using PHI of ops");
142
DEBUG_COUNTER(VNCounter, "newgvn-vn",
143
              "Controls which instructions are value numbered");
144
DEBUG_COUNTER(PHIOfOpsCounter, "newgvn-phi",
145
              "Controls which instructions we create phi of ops for");
146
// Currently store defining access refinement is too slow due to basicaa being
147
// egregiously slow.  This flag lets us keep it working while we work on this
148
// issue.
149
static cl::opt<bool> EnableStoreRefinement("enable-store-refinement",
150
                                           cl::init(false), cl::Hidden);
151
152
/// Currently, the generation "phi of ops" can result in correctness issues.
153
static cl::opt<bool> EnablePhiOfOps("enable-phi-of-ops", cl::init(true),
154
                                    cl::Hidden);
155
156
//===----------------------------------------------------------------------===//
157
//                                GVN Pass
158
//===----------------------------------------------------------------------===//
159
160
// Anchor methods.
161
namespace llvm {
162
namespace GVNExpression {
163
164
0
Expression::~Expression() = default;
165
0
BasicExpression::~BasicExpression() = default;
166
0
CallExpression::~CallExpression() = default;
167
0
LoadExpression::~LoadExpression() = default;
168
0
StoreExpression::~StoreExpression() = default;
169
0
AggregateValueExpression::~AggregateValueExpression() = default;
170
0
PHIExpression::~PHIExpression() = default;
171
172
} // end namespace GVNExpression
173
} // end namespace llvm
174
175
namespace {
176
177
// Tarjan's SCC finding algorithm with Nuutila's improvements
178
// SCCIterator is actually fairly complex for the simple thing we want.
179
// It also wants to hand us SCC's that are unrelated to the phi node we ask
180
// about, and have us process them there or risk redoing work.
181
// Graph traits over a filter iterator also doesn't work that well here.
182
// This SCC finder is specialized to walk use-def chains, and only follows
183
// instructions,
184
// not generic values (arguments, etc).
185
struct TarjanSCC {
186
333
  TarjanSCC() : Components(1) {}
187
188
171
  void Start(const Instruction *Start) {
189
171
    if (Root.lookup(Start) == 0)
190
119
      FindSCC(Start);
191
171
  }
192
193
171
  const SmallPtrSetImpl<const Value *> &getComponentFor(const Value *V) const {
194
171
    unsigned ComponentID = ValueToComponent.lookup(V);
195
171
196
171
    assert(ComponentID > 0 &&
197
171
           "Asking for a component for a value we never processed");
198
171
    return Components[ComponentID];
199
171
  }
200
201
private:
202
443
  void FindSCC(const Instruction *I) {
203
443
    Root[I] = ++DFSNum;
204
443
    // Store the DFS Number we had before it possibly gets incremented.
205
443
    unsigned int OurDFS = DFSNum;
206
816
    for (auto &Op : I->operands()) {
207
816
      if (auto *InstOp = dyn_cast<Instruction>(Op)) {
208
450
        if (Root.lookup(Op) == 0)
209
324
          FindSCC(InstOp);
210
450
        if (!InComponent.count(Op))
211
143
          Root[I] = std::min(Root.lookup(I), Root.lookup(Op));
212
450
      }
213
816
    }
214
443
    // See if we really were the root of a component, by seeing if we still have
215
443
    // our DFSNumber.  If we do, we are the root of the component, and we have
216
443
    // completed a component. If we do not, we are not the root of a component,
217
443
    // and belong on the component stack.
218
443
    if (Root.lookup(I) == OurDFS) {
219
362
      unsigned ComponentID = Components.size();
220
362
      Components.resize(Components.size() + 1);
221
362
      auto &Component = Components.back();
222
362
      Component.insert(I);
223
362
      LLVM_DEBUG(dbgs() << "Component root is " << *I << "\n");
224
362
      InComponent.insert(I);
225
362
      ValueToComponent[I] = ComponentID;
226
362
      // Pop a component off the stack and label it.
227
443
      while (!Stack.empty() && 
Root.lookup(Stack.back()) >= OurDFS95
) {
228
81
        auto *Member = Stack.back();
229
81
        LLVM_DEBUG(dbgs() << "Component member is " << *Member << "\n");
230
81
        Component.insert(Member);
231
81
        InComponent.insert(Member);
232
81
        ValueToComponent[Member] = ComponentID;
233
81
        Stack.pop_back();
234
81
      }
235
362
    } else {
236
81
      // Part of a component, push to stack
237
81
      Stack.push_back(I);
238
81
    }
239
443
  }
240
241
  unsigned int DFSNum = 1;
242
  SmallPtrSet<const Value *, 8> InComponent;
243
  DenseMap<const Value *, unsigned int> Root;
244
  SmallVector<const Value *, 8> Stack;
245
246
  // Store the components as vector of ptr sets, because we need the topo order
247
  // of SCC's, but not individual member order
248
  SmallVector<SmallPtrSet<const Value *, 8>, 8> Components;
249
250
  DenseMap<const Value *, unsigned> ValueToComponent;
251
};
252
253
// Congruence classes represent the set of expressions/instructions
254
// that are all the same *during some scope in the function*.
255
// That is, because of the way we perform equality propagation, and
256
// because of memory value numbering, it is not correct to assume
257
// you can willy-nilly replace any member with any other at any
258
// point in the function.
259
//
260
// For any Value in the Member set, it is valid to replace any dominated member
261
// with that Value.
262
//
263
// Every congruence class has a leader, and the leader is used to symbolize
264
// instructions in a canonical way (IE every operand of an instruction that is a
265
// member of the same congruence class will always be replaced with leader
266
// during symbolization).  To simplify symbolization, we keep the leader as a
267
// constant if class can be proved to be a constant value.  Otherwise, the
268
// leader is the member of the value set with the smallest DFS number.  Each
269
// congruence class also has a defining expression, though the expression may be
270
// null.  If it exists, it can be used for forward propagation and reassociation
271
// of values.
272
273
// For memory, we also track a representative MemoryAccess, and a set of memory
274
// members for MemoryPhis (which have no real instructions). Note that for
275
// memory, it seems tempting to try to split the memory members into a
276
// MemoryCongruenceClass or something.  Unfortunately, this does not work
277
// easily.  The value numbering of a given memory expression depends on the
278
// leader of the memory congruence class, and the leader of memory congruence
279
// class depends on the value numbering of a given memory expression.  This
280
// leads to wasted propagation, and in some cases, missed optimization.  For
281
// example: If we had value numbered two stores together before, but now do not,
282
// we move them to a new value congruence class.  This in turn will move at one
283
// of the memorydefs to a new memory congruence class.  Which in turn, affects
284
// the value numbering of the stores we just value numbered (because the memory
285
// congruence class is part of the value number).  So while theoretically
286
// possible to split them up, it turns out to be *incredibly* complicated to get
287
// it to work right, because of the interdependency.  While structurally
288
// slightly messier, it is algorithmically much simpler and faster to do what we
289
// do here, and track them both at once in the same class.
290
// Note: The default iterators for this class iterate over values
291
class CongruenceClass {
292
public:
293
  using MemberType = Value;
294
  using MemberSet = SmallPtrSet<MemberType *, 4>;
295
  using MemoryMemberType = MemoryPhi;
296
  using MemoryMemberSet = SmallPtrSet<const MemoryMemberType *, 2>;
297
298
0
  explicit CongruenceClass(unsigned ID) : ID(ID) {}
299
  CongruenceClass(unsigned ID, Value *Leader, const Expression *E)
300
3.38k
      : ID(ID), RepLeader(Leader), DefiningExpr(E) {}
301
302
0
  unsigned getID() const { return ID; }
303
304
  // True if this class has no members left.  This is mainly used for assertion
305
  // purposes, and for skipping empty classes.
306
3.38k
  bool isDead() const {
307
3.38k
    // If it's both dead from a value perspective, and dead from a memory
308
3.38k
    // perspective, it's really dead.
309
3.38k
    return empty() && 
memory_empty()1.01k
;
310
3.38k
  }
311
312
  // Leader functions
313
12.9k
  Value *getLeader() const { return RepLeader; }
314
2.19k
  void setLeader(Value *Leader) { RepLeader = Leader; }
315
2.85k
  const std::pair<Value *, unsigned int> &getNextLeader() const {
316
2.85k
    return NextLeader;
317
2.85k
  }
318
224
  void resetNextLeader() { NextLeader = {nullptr, ~0}; }
319
1.04k
  void addPossibleNextLeader(std::pair<Value *, unsigned int> LeaderPair) {
320
1.04k
    if (LeaderPair.second < NextLeader.second)
321
773
      NextLeader = LeaderPair;
322
1.04k
  }
323
324
10.7k
  Value *getStoredValue() const { return RepStoredValue; }
325
267
  void setStoredValue(Value *Leader) { RepStoredValue = Leader; }
326
2.50k
  const MemoryAccess *getMemoryLeader() const { return RepMemoryAccess; }
327
1.25k
  void setMemoryLeader(const MemoryAccess *Leader) { RepMemoryAccess = Leader; }
328
329
  // Forward propagation info
330
604
  const Expression *getDefiningExpr() const { return DefiningExpr; }
331
332
  // Value member set
333
8.70k
  bool empty() const { return Members.empty(); }
334
1.52k
  unsigned size() const { return Members.size(); }
335
3.87k
  MemberSet::const_iterator begin() const { return Members.begin(); }
336
3.83k
  MemberSet::const_iterator end() const { return Members.end(); }
337
5.77k
  void insert(MemberType *M) { Members.insert(M); }
338
2.84k
  void erase(MemberType *M) { Members.erase(M); }
339
3.13k
  void swap(MemberSet &Other) { Members.swap(Other); }
340
341
  // Memory member set
342
1.05k
  bool memory_empty() const { return MemoryMembers.empty(); }
343
8
  unsigned memory_size() const { return MemoryMembers.size(); }
344
459
  MemoryMemberSet::const_iterator memory_begin() const {
345
459
    return MemoryMembers.begin();
346
459
  }
347
453
  MemoryMemberSet::const_iterator memory_end() const {
348
453
    return MemoryMembers.end();
349
453
  }
350
453
  iterator_range<MemoryMemberSet::const_iterator> memory() const {
351
453
    return make_range(memory_begin(), memory_end());
352
453
  }
353
354
335
  void memory_insert(const MemoryMemberType *M) { MemoryMembers.insert(M); }
355
197
  void memory_erase(const MemoryMemberType *M) { MemoryMembers.erase(M); }
356
357
  // Store count
358
2.64k
  unsigned getStoreCount() const { return StoreCount; }
359
540
  void incStoreCount() { ++StoreCount; }
360
280
  void decStoreCount() {
361
280
    assert(StoreCount != 0 && "Store count went negative");
362
280
    --StoreCount;
363
280
  }
364
365
  // True if this class has no memory members.
366
37
  bool definesNoMemory() const { return StoreCount == 0 && 
memory_empty()33
; }
367
368
  // Return true if two congruence classes are equivalent to each other. This
369
  // means that every field but the ID number and the dead field are equivalent.
370
0
  bool isEquivalentTo(const CongruenceClass *Other) const {
371
0
    if (!Other)
372
0
      return false;
373
0
    if (this == Other)
374
0
      return true;
375
0
376
0
    if (std::tie(StoreCount, RepLeader, RepStoredValue, RepMemoryAccess) !=
377
0
        std::tie(Other->StoreCount, Other->RepLeader, Other->RepStoredValue,
378
0
                 Other->RepMemoryAccess))
379
0
      return false;
380
0
    if (DefiningExpr != Other->DefiningExpr)
381
0
      if (!DefiningExpr || !Other->DefiningExpr ||
382
0
          *DefiningExpr != *Other->DefiningExpr)
383
0
        return false;
384
0
385
0
    if (Members.size() != Other->Members.size())
386
0
      return false;
387
0
388
0
    return all_of(Members,
389
0
                  [&](const Value *V) { return Other->Members.count(V); });
390
0
  }
391
392
private:
393
  unsigned ID;
394
395
  // Representative leader.
396
  Value *RepLeader = nullptr;
397
398
  // The most dominating leader after our current leader, because the member set
399
  // is not sorted and is expensive to keep sorted all the time.
400
  std::pair<Value *, unsigned int> NextLeader = {nullptr, ~0U};
401
402
  // If this is represented by a store, the value of the store.
403
  Value *RepStoredValue = nullptr;
404
405
  // If this class contains MemoryDefs or MemoryPhis, this is the leading memory
406
  // access.
407
  const MemoryAccess *RepMemoryAccess = nullptr;
408
409
  // Defining Expression.
410
  const Expression *DefiningExpr = nullptr;
411
412
  // Actual members of this class.
413
  MemberSet Members;
414
415
  // This is the set of MemoryPhis that exist in the class. MemoryDefs and
416
  // MemoryUses have real instructions representing them, so we only need to
417
  // track MemoryPhis here.
418
  MemoryMemberSet MemoryMembers;
419
420
  // Number of stores in this congruence class.
421
  // This is used so we can detect store equivalence changes properly.
422
  int StoreCount = 0;
423
};
424
425
} // end anonymous namespace
426
427
namespace llvm {
428
429
struct ExactEqualsExpression {
430
  const Expression &E;
431
432
338
  explicit ExactEqualsExpression(const Expression &E) : E(E) {}
433
434
338
  hash_code getComputedHash() const { return E.getComputedHash(); }
435
436
371
  bool operator==(const Expression &Other) const {
437
371
    return E.exactlyEquals(Other);
438
371
  }
439
};
440
441
template <> struct DenseMapInfo<const Expression *> {
442
17.4k
  static const Expression *getEmptyKey() {
443
17.4k
    auto Val = static_cast<uintptr_t>(-1);
444
17.4k
    Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
445
17.4k
    return reinterpret_cast<const Expression *>(Val);
446
17.4k
  }
447
448
16.8k
  static const Expression *getTombstoneKey() {
449
16.8k
    auto Val = static_cast<uintptr_t>(~1U);
450
16.8k
    Val <<= PointerLikeTypeTraits<const Expression *>::NumLowBitsAvailable;
451
16.8k
    return reinterpret_cast<const Expression *>(Val);
452
16.8k
  }
453
454
3.79k
  static unsigned getHashValue(const Expression *E) {
455
3.79k
    return E->getComputedHash();
456
3.79k
  }
457
458
338
  static unsigned getHashValue(const ExactEqualsExpression &E) {
459
338
    return E.getComputedHash();
460
338
  }
461
462
462
  static bool isEqual(const ExactEqualsExpression &LHS, const Expression *RHS) {
463
462
    if (RHS == getTombstoneKey() || 
RHS == getEmptyKey()407
)
464
91
      return false;
465
371
    return LHS == *RHS;
466
371
  }
467
468
38.0k
  static bool isEqual(const Expression *LHS, const Expression *RHS) {
469
38.0k
    if (LHS == RHS)
470
32.3k
      return true;
471
5.69k
    if (LHS == getTombstoneKey() || 
RHS == getTombstoneKey()5.48k
||
472
5.69k
        
LHS == getEmptyKey()4.82k
||
RHS == getEmptyKey()4.82k
)
473
4.41k
      return false;
474
1.27k
    // Compare hashes before equality.  This is *not* what the hashtable does,
475
1.27k
    // since it is computing it modulo the number of buckets, whereas we are
476
1.27k
    // using the full hash keyspace.  Since the hashes are precomputed, this
477
1.27k
    // check is *much* faster than equality.
478
1.27k
    if (LHS->getComputedHash() != RHS->getComputedHash())
479
432
      return false;
480
846
    return *LHS == *RHS;
481
846
  }
482
};
483
484
} // end namespace llvm
485
486
namespace {
487
488
class NewGVN {
489
  Function &F;
490
  DominatorTree *DT;
491
  const TargetLibraryInfo *TLI;
492
  AliasAnalysis *AA;
493
  MemorySSA *MSSA;
494
  MemorySSAWalker *MSSAWalker;
495
  const DataLayout &DL;
496
  std::unique_ptr<PredicateInfo> PredInfo;
497
498
  // These are the only two things the create* functions should have
499
  // side-effects on due to allocating memory.
500
  mutable BumpPtrAllocator ExpressionAllocator;
501
  mutable ArrayRecycler<Value *> ArgRecycler;
502
  mutable TarjanSCC SCCFinder;
503
  const SimplifyQuery SQ;
504
505
  // Number of function arguments, used by ranking
506
  unsigned int NumFuncArgs;
507
508
  // RPOOrdering of basic blocks
509
  DenseMap<const DomTreeNode *, unsigned> RPOOrdering;
510
511
  // Congruence class info.
512
513
  // This class is called INITIAL in the paper. It is the class everything
514
  // startsout in, and represents any value. Being an optimistic analysis,
515
  // anything in the TOP class has the value TOP, which is indeterminate and
516
  // equivalent to everything.
517
  CongruenceClass *TOPClass;
518
  std::vector<CongruenceClass *> CongruenceClasses;
519
  unsigned NextCongruenceNum;
520
521
  // Value Mappings.
522
  DenseMap<Value *, CongruenceClass *> ValueToClass;
523
  DenseMap<Value *, const Expression *> ValueToExpression;
524
525
  // Value PHI handling, used to make equivalence between phi(op, op) and
526
  // op(phi, phi).
527
  // These mappings just store various data that would normally be part of the
528
  // IR.
529
  SmallPtrSet<const Instruction *, 8> PHINodeUses;
530
531
  DenseMap<const Value *, bool> OpSafeForPHIOfOps;
532
533
  // Map a temporary instruction we created to a parent block.
534
  DenseMap<const Value *, BasicBlock *> TempToBlock;
535
536
  // Map between the already in-program instructions and the temporary phis we
537
  // created that they are known equivalent to.
538
  DenseMap<const Value *, PHINode *> RealToTemp;
539
540
  // In order to know when we should re-process instructions that have
541
  // phi-of-ops, we track the set of expressions that they needed as
542
  // leaders. When we discover new leaders for those expressions, we process the
543
  // associated phi-of-op instructions again in case they have changed.  The
544
  // other way they may change is if they had leaders, and those leaders
545
  // disappear.  However, at the point they have leaders, there are uses of the
546
  // relevant operands in the created phi node, and so they will get reprocessed
547
  // through the normal user marking we perform.
548
  mutable DenseMap<const Value *, SmallPtrSet<Value *, 2>> AdditionalUsers;
549
  DenseMap<const Expression *, SmallPtrSet<Instruction *, 2>>
550
      ExpressionToPhiOfOps;
551
552
  // Map from temporary operation to MemoryAccess.
553
  DenseMap<const Instruction *, MemoryUseOrDef *> TempToMemory;
554
555
  // Set of all temporary instructions we created.
556
  // Note: This will include instructions that were just created during value
557
  // numbering.  The way to test if something is using them is to check
558
  // RealToTemp.
559
  DenseSet<Instruction *> AllTempInstructions;
560
561
  // This is the set of instructions to revisit on a reachability change.  At
562
  // the end of the main iteration loop it will contain at least all the phi of
563
  // ops instructions that will be changed to phis, as well as regular phis.
564
  // During the iteration loop, it may contain other things, such as phi of ops
565
  // instructions that used edge reachability to reach a result, and so need to
566
  // be revisited when the edge changes, independent of whether the phi they
567
  // depended on changes.
568
  DenseMap<BasicBlock *, SparseBitVector<>> RevisitOnReachabilityChange;
569
570
  // Mapping from predicate info we used to the instructions we used it with.
571
  // In order to correctly ensure propagation, we must keep track of what
572
  // comparisons we used, so that when the values of the comparisons change, we
573
  // propagate the information to the places we used the comparison.
574
  mutable DenseMap<const Value *, SmallPtrSet<Instruction *, 2>>
575
      PredicateToUsers;
576
577
  // the same reasoning as PredicateToUsers.  When we skip MemoryAccesses for
578
  // stores, we no longer can rely solely on the def-use chains of MemorySSA.
579
  mutable DenseMap<const MemoryAccess *, SmallPtrSet<MemoryAccess *, 2>>
580
      MemoryToUsers;
581
582
  // A table storing which memorydefs/phis represent a memory state provably
583
  // equivalent to another memory state.
584
  // We could use the congruence class machinery, but the MemoryAccess's are
585
  // abstract memory states, so they can only ever be equivalent to each other,
586
  // and not to constants, etc.
587
  DenseMap<const MemoryAccess *, CongruenceClass *> MemoryAccessToClass;
588
589
  // We could, if we wanted, build MemoryPhiExpressions and
590
  // MemoryVariableExpressions, etc, and value number them the same way we value
591
  // number phi expressions.  For the moment, this seems like overkill.  They
592
  // can only exist in one of three states: they can be TOP (equal to
593
  // everything), Equivalent to something else, or unique.  Because we do not
594
  // create expressions for them, we need to simulate leader change not just
595
  // when they change class, but when they change state.  Note: We can do the
596
  // same thing for phis, and avoid having phi expressions if we wanted, We
597
  // should eventually unify in one direction or the other, so this is a little
598
  // bit of an experiment in which turns out easier to maintain.
599
  enum MemoryPhiState { MPS_Invalid, MPS_TOP, MPS_Equivalent, MPS_Unique };
600
  DenseMap<const MemoryPhi *, MemoryPhiState> MemoryPhiState;
601
602
  enum InstCycleState { ICS_Unknown, ICS_CycleFree, ICS_Cycle };
603
  mutable DenseMap<const Instruction *, InstCycleState> InstCycleState;
604
605
  // Expression to class mapping.
606
  using ExpressionClassMap = DenseMap<const Expression *, CongruenceClass *>;
607
  ExpressionClassMap ExpressionToClass;
608
609
  // We have a single expression that represents currently DeadExpressions.
610
  // For dead expressions we can prove will stay dead, we mark them with
611
  // DFS number zero.  However, it's possible in the case of phi nodes
612
  // for us to assume/prove all arguments are dead during fixpointing.
613
  // We use DeadExpression for that case.
614
  DeadExpression *SingletonDeadExpression = nullptr;
615
616
  // Which values have changed as a result of leader changes.
617
  SmallPtrSet<Value *, 8> LeaderChanges;
618
619
  // Reachability info.
620
  using BlockEdge = BasicBlockEdge;
621
  DenseSet<BlockEdge> ReachableEdges;
622
  SmallPtrSet<const BasicBlock *, 8> ReachableBlocks;
623
624
  // This is a bitvector because, on larger functions, we may have
625
  // thousands of touched instructions at once (entire blocks,
626
  // instructions with hundreds of uses, etc).  Even with optimization
627
  // for when we mark whole blocks as touched, when this was a
628
  // SmallPtrSet or DenseSet, for some functions, we spent >20% of all
629
  // the time in GVN just managing this list.  The bitvector, on the
630
  // other hand, efficiently supports test/set/clear of both
631
  // individual and ranges, as well as "find next element" This
632
  // enables us to use it as a worklist with essentially 0 cost.
633
  BitVector TouchedInstructions;
634
635
  DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
636
637
#ifndef NDEBUG
638
  // Debugging for how many times each block and instruction got processed.
639
  DenseMap<const Value *, unsigned> ProcessedCount;
640
#endif
641
642
  // DFS info.
643
  // This contains a mapping from Instructions to DFS numbers.
644
  // The numbering starts at 1. An instruction with DFS number zero
645
  // means that the instruction is dead.
646
  DenseMap<const Value *, unsigned> InstrDFS;
647
648
  // This contains the mapping DFS numbers to instructions.
649
  SmallVector<Value *, 32> DFSToInstr;
650
651
  // Deletion info.
652
  SmallPtrSet<Instruction *, 8> InstructionsToErase;
653
654
public:
655
  NewGVN(Function &F, DominatorTree *DT, AssumptionCache *AC,
656
         TargetLibraryInfo *TLI, AliasAnalysis *AA, MemorySSA *MSSA,
657
         const DataLayout &DL)
658
      : F(F), DT(DT), TLI(TLI), AA(AA), MSSA(MSSA), DL(DL),
659
        PredInfo(make_unique<PredicateInfo>(F, *DT, *AC)),
660
333
        SQ(DL, TLI, DT, AC, /*CtxI=*/nullptr, /*UseInstrInfo=*/false) {}
661
662
  bool runGVN();
663
664
private:
665
  // Expression handling.
666
  const Expression *createExpression(Instruction *) const;
667
  const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *,
668
                                           Instruction *) const;
669
670
  // Our canonical form for phi arguments is a pair of incoming value, incoming
671
  // basic block.
672
  using ValPair = std::pair<Value *, BasicBlock *>;
673
674
  PHIExpression *createPHIExpression(ArrayRef<ValPair>, const Instruction *,
675
                                     BasicBlock *, bool &HasBackEdge,
676
                                     bool &OriginalOpsConstant) const;
677
  const DeadExpression *createDeadExpression() const;
678
  const VariableExpression *createVariableExpression(Value *) const;
679
  const ConstantExpression *createConstantExpression(Constant *) const;
680
  const Expression *createVariableOrConstant(Value *V) const;
681
  const UnknownExpression *createUnknownExpression(Instruction *) const;
682
  const StoreExpression *createStoreExpression(StoreInst *,
683
                                               const MemoryAccess *) const;
684
  LoadExpression *createLoadExpression(Type *, Value *, LoadInst *,
685
                                       const MemoryAccess *) const;
686
  const CallExpression *createCallExpression(CallInst *,
687
                                             const MemoryAccess *) const;
688
  const AggregateValueExpression *
689
  createAggregateValueExpression(Instruction *) const;
690
  bool setBasicExpressionInfo(Instruction *, BasicExpression *) const;
691
692
  // Congruence class handling.
693
3.38k
  CongruenceClass *createCongruenceClass(Value *Leader, const Expression *E) {
694
3.38k
    auto *result = new CongruenceClass(NextCongruenceNum++, Leader, E);
695
3.38k
    CongruenceClasses.emplace_back(result);
696
3.38k
    return result;
697
3.38k
  }
698
699
447
  CongruenceClass *createMemoryClass(MemoryAccess *MA) {
700
447
    auto *CC = createCongruenceClass(nullptr, nullptr);
701
447
    CC->setMemoryLeader(MA);
702
447
    return CC;
703
447
  }
704
705
146
  CongruenceClass *ensureLeaderOfMemoryClass(MemoryAccess *MA) {
706
146
    auto *CC = getMemoryClass(MA);
707
146
    if (CC->getMemoryLeader() != MA)
708
114
      CC = createMemoryClass(MA);
709
146
    return CC;
710
146
  }
711
712
477
  CongruenceClass *createSingletonCongruenceClass(Value *Member) {
713
477
    CongruenceClass *CClass = createCongruenceClass(Member, nullptr);
714
477
    CClass->insert(Member);
715
477
    ValueToClass[Member] = CClass;
716
477
    return CClass;
717
477
  }
718
719
  void initializeCongruenceClasses(Function &F);
720
  const Expression *makePossiblePHIOfOps(Instruction *,
721
                                         SmallPtrSetImpl<Value *> &);
722
  Value *findLeaderForInst(Instruction *ValueOp,
723
                           SmallPtrSetImpl<Value *> &Visited,
724
                           MemoryAccess *MemAccess, Instruction *OrigInst,
725
                           BasicBlock *PredBB);
726
  bool OpIsSafeForPHIOfOpsHelper(Value *V, const BasicBlock *PHIBlock,
727
                                 SmallPtrSetImpl<const Value *> &Visited,
728
                                 SmallVectorImpl<Instruction *> &Worklist);
729
  bool OpIsSafeForPHIOfOps(Value *Op, const BasicBlock *PHIBlock,
730
                           SmallPtrSetImpl<const Value *> &);
731
  void addPhiOfOps(PHINode *Op, BasicBlock *BB, Instruction *ExistingValue);
732
  void removePhiOfOps(Instruction *I, PHINode *PHITemp);
733
734
  // Value number an Instruction or MemoryPhi.
735
  void valueNumberMemoryPhi(MemoryPhi *);
736
  void valueNumberInstruction(Instruction *);
737
738
  // Symbolic evaluation.
739
  const Expression *checkSimplificationResults(Expression *, Instruction *,
740
                                               Value *) const;
741
  const Expression *performSymbolicEvaluation(Value *,
742
                                              SmallPtrSetImpl<Value *> &) const;
743
  const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,
744
                                                Instruction *,
745
                                                MemoryAccess *) const;
746
  const Expression *performSymbolicLoadEvaluation(Instruction *) const;
747
  const Expression *performSymbolicStoreEvaluation(Instruction *) const;
748
  const Expression *performSymbolicCallEvaluation(Instruction *) const;
749
  void sortPHIOps(MutableArrayRef<ValPair> Ops) const;
750
  const Expression *performSymbolicPHIEvaluation(ArrayRef<ValPair>,
751
                                                 Instruction *I,
752
                                                 BasicBlock *PHIBlock) const;
753
  const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
754
  const Expression *performSymbolicCmpEvaluation(Instruction *) const;
755
  const Expression *performSymbolicPredicateInfoEvaluation(Instruction *) const;
756
757
  // Congruence finding.
758
  bool someEquivalentDominates(const Instruction *, const Instruction *) const;
759
  Value *lookupOperandLeader(Value *) const;
760
  CongruenceClass *getClassForExpression(const Expression *E) const;
761
  void performCongruenceFinding(Instruction *, const Expression *);
762
  void moveValueToNewCongruenceClass(Instruction *, const Expression *,
763
                                     CongruenceClass *, CongruenceClass *);
764
  void moveMemoryToNewCongruenceClass(Instruction *, MemoryAccess *,
765
                                      CongruenceClass *, CongruenceClass *);
766
  Value *getNextValueLeader(CongruenceClass *) const;
767
  const MemoryAccess *getNextMemoryLeader(CongruenceClass *) const;
768
  bool setMemoryClass(const MemoryAccess *From, CongruenceClass *To);
769
  CongruenceClass *getMemoryClass(const MemoryAccess *MA) const;
770
  const MemoryAccess *lookupMemoryLeader(const MemoryAccess *) const;
771
  bool isMemoryAccessTOP(const MemoryAccess *) const;
772
773
  // Ranking
774
  unsigned int getRank(const Value *) const;
775
  bool shouldSwapOperands(const Value *, const Value *) const;
776
777
  // Reachability handling.
778
  void updateReachableEdge(BasicBlock *, BasicBlock *);
779
  void processOutgoingEdges(Instruction *, BasicBlock *);
780
  Value *findConditionEquivalence(Value *) const;
781
782
  // Elimination.
783
  struct ValueDFS;
784
  void convertClassToDFSOrdered(const CongruenceClass &,
785
                                SmallVectorImpl<ValueDFS> &,
786
                                DenseMap<const Value *, unsigned int> &,
787
                                SmallPtrSetImpl<Instruction *> &) const;
788
  void convertClassToLoadsAndStores(const CongruenceClass &,
789
                                    SmallVectorImpl<ValueDFS> &) const;
790
791
  bool eliminateInstructions(Function &);
792
  void replaceInstruction(Instruction *, Value *);
793
  void markInstructionForDeletion(Instruction *);
794
  void deleteInstructionsInBlock(BasicBlock *);
795
  Value *findPHIOfOpsLeader(const Expression *, const Instruction *,
796
                            const BasicBlock *) const;
797
798
  // New instruction creation.
799
0
  void handleNewInstruction(Instruction *) {}
800
801
  // Various instruction touch utilities
802
  template <typename Map, typename KeyType, typename Func>
803
  void for_each_found(Map &, const KeyType &, Func);
804
  template <typename Map, typename KeyType>
805
  void touchAndErase(Map &, const KeyType &);
806
  void markUsersTouched(Value *);
807
  void markMemoryUsersTouched(const MemoryAccess *);
808
  void markMemoryDefTouched(const MemoryAccess *);
809
  void markPredicateUsersTouched(Instruction *);
810
  void markValueLeaderChangeTouched(CongruenceClass *CC);
811
  void markMemoryLeaderChangeTouched(CongruenceClass *CC);
812
  void markPhiOfOpsChanged(const Expression *E);
813
  void addPredicateUsers(const PredicateBase *, Instruction *) const;
814
  void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const;
815
  void addAdditionalUsers(Value *To, Value *User) const;
816
817
  // Main loop of value numbering
818
  void iterateTouchedInstructions();
819
820
  // Utilities.
821
  void cleanupTables();
822
  std::pair<unsigned, unsigned> assignDFSNumbers(BasicBlock *, unsigned);
823
  void updateProcessedCount(const Value *V);
824
  void verifyMemoryCongruency() const;
825
  void verifyIterationSettled(Function &F);
826
  void verifyStoreExpressions() const;
827
  bool singleReachablePHIPath(SmallPtrSet<const MemoryAccess *, 8> &,
828
                              const MemoryAccess *, const MemoryAccess *) const;
829
  BasicBlock *getBlockForValue(Value *V) const;
830
  void deleteExpression(const Expression *E) const;
831
  MemoryUseOrDef *getMemoryAccess(const Instruction *) const;
832
  MemoryAccess *getDefiningAccess(const MemoryAccess *) const;
833
  MemoryPhi *getMemoryAccess(const BasicBlock *) const;
834
  template <class T, class Range> T *getMinDFSOfRange(const Range &) const;
835
836
8.63k
  unsigned InstrToDFSNum(const Value *V) const {
837
8.63k
    assert(isa<Instruction>(V) && "This should not be used for MemoryAccesses");
838
8.63k
    return InstrDFS.lookup(V);
839
8.63k
  }
840
841
162
  unsigned InstrToDFSNum(const MemoryAccess *MA) const {
842
162
    return MemoryToDFSNum(MA);
843
162
  }
844
845
5.14k
  Value *InstrFromDFSNum(unsigned DFSNum) { return DFSToInstr[DFSNum]; }
846
847
  // Given a MemoryAccess, return the relevant instruction DFS number.  Note:
848
  // This deliberately takes a value so it can be used with Use's, which will
849
  // auto-convert to Value's but not to MemoryAccess's.
850
1.16k
  unsigned MemoryToDFSNum(const Value *MA) const {
851
1.16k
    assert(isa<MemoryAccess>(MA) &&
852
1.16k
           "This should not be used with instructions");
853
1.16k
    return isa<MemoryUseOrDef>(MA)
854
1.16k
               ? 
InstrToDFSNum(cast<MemoryUseOrDef>(MA)->getMemoryInst())701
855
1.16k
               : 
InstrDFS.lookup(MA)461
;
856
1.16k
  }
857
858
  bool isCycleFree(const Instruction *) const;
859
  bool isBackedge(BasicBlock *From, BasicBlock *To) const;
860
861
  // Debug counter info.  When verifying, we have to reset the value numbering
862
  // debug counter to the same state it started in to get the same results.
863
  int64_t StartingVNCounter;
864
};
865
866
} // end anonymous namespace
867
868
template <typename T>
869
471
static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
870
471
  if (!isa<LoadExpression>(RHS) && 
!isa<StoreExpression>(RHS)250
)
871
0
    return false;
872
471
  return LHS.MemoryExpression::equals(RHS);
873
471
}
NewGVN.cpp:bool equalsLoadStoreHelper<llvm::GVNExpression::LoadExpression>(llvm::GVNExpression::LoadExpression const&, llvm::GVNExpression::Expression const&)
Line
Count
Source
869
245
static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
870
245
  if (!isa<LoadExpression>(RHS) && 
!isa<StoreExpression>(RHS)87
)
871
0
    return false;
872
245
  return LHS.MemoryExpression::equals(RHS);
873
245
}
NewGVN.cpp:bool equalsLoadStoreHelper<llvm::GVNExpression::StoreExpression>(llvm::GVNExpression::StoreExpression const&, llvm::GVNExpression::Expression const&)
Line
Count
Source
869
226
static bool equalsLoadStoreHelper(const T &LHS, const Expression &RHS) {
870
226
  if (!isa<LoadExpression>(RHS) && 
!isa<StoreExpression>(RHS)163
)
871
0
    return false;
872
226
  return LHS.MemoryExpression::equals(RHS);
873
226
}
874
875
245
bool LoadExpression::equals(const Expression &Other) const {
876
245
  return equalsLoadStoreHelper(*this, Other);
877
245
}
878
879
226
bool StoreExpression::equals(const Expression &Other) const {
880
226
  if (!equalsLoadStoreHelper(*this, Other))
881
30
    return false;
882
196
  // Make sure that store vs store includes the value operand.
883
196
  if (const auto *S = dyn_cast<StoreExpression>(&Other))
884
133
    if (getStoredValue() != S->getStoredValue())
885
40
      return false;
886
156
  return true;
887
156
}
888
889
// Determine if the edge From->To is a backedge
890
858
bool NewGVN::isBackedge(BasicBlock *From, BasicBlock *To) const {
891
858
  return From == To ||
892
858
         RPOOrdering.lookup(DT->getNode(From)) >=
893
815
             RPOOrdering.lookup(DT->getNode(To));
894
858
}
895
896
#ifndef NDEBUG
897
static std::string getBlockName(const BasicBlock *B) {
898
  return DOTGraphTraits<const Function *>::getSimpleNodeLabel(B, nullptr);
899
}
900
#endif
901
902
// Get a MemoryAccess for an instruction, fake or real.
903
7.45k
MemoryUseOrDef *NewGVN::getMemoryAccess(const Instruction *I) const {
904
7.45k
  auto *Result = MSSA->getMemoryAccess(I);
905
7.45k
  return Result ? 
Result2.81k
:
TempToMemory.lookup(I)4.63k
;
906
7.45k
}
907
908
// Get a MemoryPhi for a basic block. These are all real.
909
1.60k
MemoryPhi *NewGVN::getMemoryAccess(const BasicBlock *BB) const {
910
1.60k
  return MSSA->getMemoryAccess(BB);
911
1.60k
}
912
913
// Get the basic block from an instruction/memory value.
914
7.26k
BasicBlock *NewGVN::getBlockForValue(Value *V) const {
915
7.26k
  if (auto *I = dyn_cast<Instruction>(V)) {
916
7.02k
    auto *Parent = I->getParent();
917
7.02k
    if (Parent)
918
6.93k
      return Parent;
919
94
    Parent = TempToBlock.lookup(V);
920
94
    assert(Parent && "Every fake instruction should have a block");
921
94
    return Parent;
922
94
  }
923
239
924
239
  auto *MP = dyn_cast<MemoryPhi>(V);
925
239
  assert(MP && "Should have been an instruction or a MemoryPhi");
926
239
  return MP->getBlock();
927
239
}
928
929
// Delete a definitely dead expression, so it can be reused by the expression
930
// allocator.  Some of these are not in creation functions, so we have to accept
931
// const versions.
932
861
void NewGVN::deleteExpression(const Expression *E) const {
933
861
  assert(isa<BasicExpression>(E));
934
861
  auto *BE = cast<BasicExpression>(E);
935
861
  const_cast<BasicExpression *>(BE)->deallocateOperands(ArgRecycler);
936
861
  ExpressionAllocator.Deallocate(E);
937
861
}
938
939
// If V is a predicateinfo copy, get the thing it is a copy of.
940
1.02k
static Value *getCopyOf(const Value *V) {
941
1.02k
  if (auto *II = dyn_cast<IntrinsicInst>(V))
942
71
    if (II->getIntrinsicID() == Intrinsic::ssa_copy)
943
71
      return II->getOperand(0);
944
951
  return nullptr;
945
951
}
946
947
// Return true if V is really PN, even accounting for predicateinfo copies.
948
956
static bool isCopyOfPHI(const Value *V, const PHINode *PN) {
949
956
  return V == PN || 
getCopyOf(V) == PN945
;
950
956
}
951
952
77
static bool isCopyOfAPHI(const Value *V) {
953
77
  auto *CO = getCopyOf(V);
954
77
  return CO && 
isa<PHINode>(CO)8
;
955
77
}
956
957
// Sort PHI Operands into a canonical order.  What we use here is an RPO
958
// order. The BlockInstRange numbers are generated in an RPO walk of the basic
959
// blocks.
960
523
void NewGVN::sortPHIOps(MutableArrayRef<ValPair> Ops) const {
961
554
  llvm::sort(Ops, [&](const ValPair &P1, const ValPair &P2) {
962
554
    return BlockInstRange.lookup(P1.second).first <
963
554
           BlockInstRange.lookup(P2.second).first;
964
554
  });
965
523
}
966
967
// Return true if V is a value that will always be available (IE can
968
// be placed anywhere) in the function.  We don't do globals here
969
// because they are often worse to put in place.
970
2.39k
static bool alwaysAvailable(Value *V) {
971
2.39k
  return isa<Constant>(V) || 
isa<Argument>(V)2.06k
;
972
2.39k
}
973
974
// Create a PHIExpression from an array of {incoming edge, value} pairs.  I is
975
// the original instruction we are creating a PHIExpression for (but may not be
976
// a phi node). We require, as an invariant, that all the PHIOperands in the
977
// same block are sorted the same way. sortPHIOps will sort them into a
978
// canonical order.
979
PHIExpression *NewGVN::createPHIExpression(ArrayRef<ValPair> PHIOperands,
980
                                           const Instruction *I,
981
                                           BasicBlock *PHIBlock,
982
                                           bool &HasBackedge,
983
523
                                           bool &OriginalOpsConstant) const {
984
523
  unsigned NumOps = PHIOperands.size();
985
523
  auto *E = new (ExpressionAllocator) PHIExpression(NumOps, PHIBlock);
986
523
987
523
  E->allocateOperands(ArgRecycler, ExpressionAllocator);
988
523
  E->setType(PHIOperands.begin()->first->getType());
989
523
  E->setOpcode(Instruction::PHI);
990
523
991
523
  // Filter out unreachable phi operands.
992
1.05k
  auto Filtered = make_filter_range(PHIOperands, [&](const ValPair &P) {
993
1.05k
    auto *BB = P.second;
994
1.05k
    if (auto *PHIOp = dyn_cast<PHINode>(I))
995
956
      if (isCopyOfPHI(P.first, PHIOp))
996
13
        return false;
997
1.03k
    if (!ReachableEdges.count({BB, PHIBlock}))
998
180
      return false;
999
859
    // Things in TOPClass are equivalent to everything.
1000
859
    if (ValueToClass.lookup(P.first) == TOPClass)
1001
0
      return false;
1002
859
    OriginalOpsConstant = OriginalOpsConstant && 
isa<Constant>(P.first)694
;
1003
859
    HasBackedge = HasBackedge || 
isBackedge(BB, PHIBlock)858
;
1004
859
    return lookupOperandLeader(P.first) != I;
1005
859
  });
1006
523
  std::transform(Filtered.begin(), Filtered.end(), op_inserter(E),
1007
850
                 [&](const ValPair &P) -> Value * {
1008
850
                   return lookupOperandLeader(P.first);
1009
850
                 });
1010
523
  return E;
1011
523
}
1012
1013
// Set basic expression info (Arguments, type, opcode) for Expression
1014
// E from Instruction I in block B.
1015
1.68k
bool NewGVN::setBasicExpressionInfo(Instruction *I, BasicExpression *E) const {
1016
1.68k
  bool AllConstant = true;
1017
1.68k
  if (auto *GEP = dyn_cast<GetElementPtrInst>(I))
1018
238
    E->setType(GEP->getSourceElementType());
1019
1.44k
  else
1020
1.44k
    E->setType(I->getType());
1021
1.68k
  E->setOpcode(I->getOpcode());
1022
1.68k
  E->allocateOperands(ArgRecycler, ExpressionAllocator);
1023
1.68k
1024
1.68k
  // Transform the operand array into an operand leader array, and keep track of
1025
1.68k
  // whether all members are constant.
1026
3.44k
  std::transform(I->op_begin(), I->op_end(), op_inserter(E), [&](Value *O) {
1027
3.44k
    auto Operand = lookupOperandLeader(O);
1028
3.44k
    AllConstant = AllConstant && 
isa<Constant>(Operand)2.15k
;
1029
3.44k
    return Operand;
1030
3.44k
  });
1031
1.68k
1032
1.68k
  return AllConstant;
1033
1.68k
}
1034
1035
const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,
1036
                                                 Value *Arg1, Value *Arg2,
1037
12
                                                 Instruction *I) const {
1038
12
  auto *E = new (ExpressionAllocator) BasicExpression(2);
1039
12
1040
12
  E->setType(T);
1041
12
  E->setOpcode(Opcode);
1042
12
  E->allocateOperands(ArgRecycler, ExpressionAllocator);
1043
12
  if (Instruction::isCommutative(Opcode)) {
1044
10
    // Ensure that commutative instructions that only differ by a permutation
1045
10
    // of their operands get the same value number by sorting the operand value
1046
10
    // numbers.  Since all commutative instructions have two operands it is more
1047
10
    // efficient to sort by hand rather than using, say, std::sort.
1048
10
    if (shouldSwapOperands(Arg1, Arg2))
1049
4
      std::swap(Arg1, Arg2);
1050
10
  }
1051
12
  E->op_push_back(lookupOperandLeader(Arg1));
1052
12
  E->op_push_back(lookupOperandLeader(Arg2));
1053
12
1054
12
  Value *V = SimplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), SQ);
1055
12
  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1056
3
    return SimplifiedE;
1057
9
  return E;
1058
9
}
1059
1060
// Take a Value returned by simplification of Expression E/Instruction
1061
// I, and see if it resulted in a simpler expression. If so, return
1062
// that expression.
1063
const Expression *NewGVN::checkSimplificationResults(Expression *E,
1064
                                                     Instruction *I,
1065
1.61k
                                                     Value *V) const {
1066
1.61k
  if (!V)
1067
1.20k
    return nullptr;
1068
412
  if (auto *C = dyn_cast<Constant>(V)) {
1069
343
    if (I)
1070
343
      LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1071
343
                        << " constant " << *C << "\n");
1072
343
    NumGVNOpsSimplified++;
1073
343
    assert(isa<BasicExpression>(E) &&
1074
343
           "We should always have had a basic expression here");
1075
343
    deleteExpression(E);
1076
343
    return createConstantExpression(C);
1077
343
  } else 
if (69
isa<Argument>(V)69
||
isa<GlobalVariable>(V)58
) {
1078
11
    if (I)
1079
11
      LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1080
11
                        << " variable " << *V << "\n");
1081
11
    deleteExpression(E);
1082
11
    return createVariableExpression(V);
1083
11
  }
1084
58
1085
58
  CongruenceClass *CC = ValueToClass.lookup(V);
1086
58
  if (CC) {
1087
58
    if (CC->getLeader() && CC->getLeader() != I) {
1088
58
      // If we simplified to something else, we need to communicate
1089
58
      // that we're users of the value we simplified to.
1090
58
      if (I != V) {
1091
57
        // Don't add temporary instructions to the user lists.
1092
57
        if (!AllTempInstructions.count(I))
1093
45
          addAdditionalUsers(V, I);
1094
57
      }
1095
58
      return createVariableOrConstant(CC->getLeader());
1096
58
    }
1097
0
    if (CC->getDefiningExpr()) {
1098
0
      // If we simplified to something else, we need to communicate
1099
0
      // that we're users of the value we simplified to.
1100
0
      if (I != V) {
1101
0
        // Don't add temporary instructions to the user lists.
1102
0
        if (!AllTempInstructions.count(I))
1103
0
          addAdditionalUsers(V, I);
1104
0
      }
1105
0
1106
0
      if (I)
1107
0
        LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
1108
0
                          << " expression " << *CC->getDefiningExpr() << "\n");
1109
0
      NumGVNOpsSimplified++;
1110
0
      deleteExpression(E);
1111
0
      return CC->getDefiningExpr();
1112
0
    }
1113
0
  }
1114
0
1115
0
  return nullptr;
1116
0
}
1117
1118
// Create a value expression from the instruction I, replacing operands with
1119
// their leaders.
1120
1121
1.62k
const Expression *NewGVN::createExpression(Instruction *I) const {
1122
1.62k
  auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());
1123
1.62k
1124
1.62k
  bool AllConstant = setBasicExpressionInfo(I, E);
1125
1.62k
1126
1.62k
  if (I->isCommutative()) {
1127
484
    // Ensure that commutative instructions that only differ by a permutation
1128
484
    // of their operands get the same value number by sorting the operand value
1129
484
    // numbers.  Since all commutative instructions have two operands it is more
1130
484
    // efficient to sort by hand rather than using, say, std::sort.
1131
484
    assert(I->getNumOperands() == 2 && "Unsupported commutative instruction!");
1132
484
    if (shouldSwapOperands(E->getOperand(0), E->getOperand(1)))
1133
265
      E->swapOperands(0, 1);
1134
484
  }
1135
1.62k
  // Perform simplification.
1136
1.62k
  if (auto *CI = dyn_cast<CmpInst>(I)) {
1137
541
    // Sort the operand value numbers so x<y and y>x get the same value
1138
541
    // number.
1139
541
    CmpInst::Predicate Predicate = CI->getPredicate();
1140
541
    if (shouldSwapOperands(E->getOperand(0), E->getOperand(1))) {
1141
406
      E->swapOperands(0, 1);
1142
406
      Predicate = CmpInst::getSwappedPredicate(Predicate);
1143
406
    }
1144
541
    E->setOpcode((CI->getOpcode() << 8) | Predicate);
1145
541
    // TODO: 25% of our time is spent in SimplifyCmpInst with pointer operands
1146
541
    assert(I->getOperand(0)->getType() == I->getOperand(1)->getType() &&
1147
541
           "Wrong types on cmp instruction");
1148
541
    assert((E->getOperand(0)->getType() == I->getOperand(0)->getType() &&
1149
541
            E->getOperand(1)->getType() == I->getOperand(1)->getType()));
1150
541
    Value *V =
1151
541
        SimplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1), SQ);
1152
541
    if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1153
76
      return SimplifiedE;
1154
1.08k
  } else if (isa<SelectInst>(I)) {
1155
29
    if (isa<Constant>(E->getOperand(0)) ||
1156
29
        
E->getOperand(1) == E->getOperand(2)23
) {
1157
13
      assert(E->getOperand(1)->getType() == I->getOperand(1)->getType() &&
1158
13
             E->getOperand(2)->getType() == I->getOperand(2)->getType());
1159
13
      Value *V = SimplifySelectInst(E->getOperand(0), E->getOperand(1),
1160
13
                                    E->getOperand(2), SQ);
1161
13
      if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1162
13
        return SimplifiedE;
1163
1.05k
    }
1164
1.05k
  } else if (I->isBinaryOp()) {
1165
630
    Value *V =
1166
630
        SimplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1), SQ);
1167
630
    if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1168
224
      return SimplifiedE;
1169
425
  } else if (auto *CI = dyn_cast<CastInst>(I)) {
1170
179
    Value *V =
1171
179
        SimplifyCastInst(CI->getOpcode(), E->getOperand(0), CI->getType(), SQ);
1172
179
    if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1173
33
      return SimplifiedE;
1174
246
  } else if (isa<GetElementPtrInst>(I)) {
1175
238
    Value *V = SimplifyGEPInst(
1176
238
        E->getType(), ArrayRef<Value *>(E->op_begin(), E->op_end()), SQ);
1177
238
    if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1178
63
      return SimplifiedE;
1179
8
  } else if (AllConstant) {
1180
2
    // We don't bother trying to simplify unless all of the operands
1181
2
    // were constant.
1182
2
    // TODO: There are a lot of Simplify*'s we could call here, if we
1183
2
    // wanted to.  The original motivating case for this code was a
1184
2
    // zext i1 false to i8, which we don't have an interface to
1185
2
    // simplify (IE there is no SimplifyZExt).
1186
2
1187
2
    SmallVector<Constant *, 8> C;
1188
2
    for (Value *Arg : E->operands())
1189
4
      C.emplace_back(cast<Constant>(Arg));
1190
2
1191
2
    if (Value *V = ConstantFoldInstOperands(I, C, DL, TLI))
1192
0
      if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
1193
0
        return SimplifiedE;
1194
1.21k
  }
1195
1.21k
  return E;
1196
1.21k
}
1197
1198
const AggregateValueExpression *
1199
3
NewGVN::createAggregateValueExpression(Instruction *I) const {
1200
3
  if (auto *II = dyn_cast<InsertValueInst>(I)) {
1201
0
    auto *E = new (ExpressionAllocator)
1202
0
        AggregateValueExpression(I->getNumOperands(), II->getNumIndices());
1203
0
    setBasicExpressionInfo(I, E);
1204
0
    E->allocateIntOperands(ExpressionAllocator);
1205
0
    std::copy(II->idx_begin(), II->idx_end(), int_op_inserter(E));
1206
0
    return E;
1207
3
  } else if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1208
3
    auto *E = new (ExpressionAllocator)
1209
3
        AggregateValueExpression(I->getNumOperands(), EI->getNumIndices());
1210
3
    setBasicExpressionInfo(EI, E);
1211
3
    E->allocateIntOperands(ExpressionAllocator);
1212
3
    std::copy(EI->idx_begin(), EI->idx_end(), int_op_inserter(E));
1213
3
    return E;
1214
3
  }
1215
0
  llvm_unreachable("Unhandled type of aggregate value operation");
1216
0
}
1217
1218
0
const DeadExpression *NewGVN::createDeadExpression() const {
1219
0
  // DeadExpression has no arguments and all DeadExpression's are the same,
1220
0
  // so we only need one of them.
1221
0
  return SingletonDeadExpression;
1222
0
}
1223
1224
285
const VariableExpression *NewGVN::createVariableExpression(Value *V) const {
1225
285
  auto *E = new (ExpressionAllocator) VariableExpression(V);
1226
285
  E->setOpcode(V->getValueID());
1227
285
  return E;
1228
285
}
1229
1230
399
const Expression *NewGVN::createVariableOrConstant(Value *V) const {
1231
399
  if (auto *C = dyn_cast<Constant>(V))
1232
125
    return createConstantExpression(C);
1233
274
  return createVariableExpression(V);
1234
274
}
1235
1236
572
const ConstantExpression *NewGVN::createConstantExpression(Constant *C) const {
1237
572
  auto *E = new (ExpressionAllocator) ConstantExpression(C);
1238
572
  E->setOpcode(C->getValueID());
1239
572
  return E;
1240
572
}
1241
1242
278
const UnknownExpression *NewGVN::createUnknownExpression(Instruction *I) const {
1243
278
  auto *E = new (ExpressionAllocator) UnknownExpression(I);
1244
278
  E->setOpcode(I->getOpcode());
1245
278
  return E;
1246
278
}
1247
1248
const CallExpression *
1249
54
NewGVN::createCallExpression(CallInst *CI, const MemoryAccess *MA) const {
1250
54
  // FIXME: Add operand bundles for calls.
1251
54
  auto *E =
1252
54
      new (ExpressionAllocator) CallExpression(CI->getNumOperands(), CI, MA);
1253
54
  setBasicExpressionInfo(CI, E);
1254
54
  return E;
1255
54
}
1256
1257
// Return true if some equivalent of instruction Inst dominates instruction U.
1258
bool NewGVN::someEquivalentDominates(const Instruction *Inst,
1259
8
                                     const Instruction *U) const {
1260
8
  auto *CC = ValueToClass.lookup(Inst);
1261
8
   // This must be an instruction because we are only called from phi nodes
1262
8
  // in the case that the value it needs to check against is an instruction.
1263
8
1264
8
  // The most likely candidates for dominance are the leader and the next leader.
1265
8
  // The leader or nextleader will dominate in all cases where there is an
1266
8
  // equivalent that is higher up in the dom tree.
1267
8
  // We can't *only* check them, however, because the
1268
8
  // dominator tree could have an infinite number of non-dominating siblings
1269
8
  // with instructions that are in the right congruence class.
1270
8
  //       A
1271
8
  // B C D E F G
1272
8
  // |
1273
8
  // H
1274
8
  // Instruction U could be in H,  with equivalents in every other sibling.
1275
8
  // Depending on the rpo order picked, the leader could be the equivalent in
1276
8
  // any of these siblings.
1277
8
  if (!CC)
1278
0
    return false;
1279
8
  if (alwaysAvailable(CC->getLeader()))
1280
0
    return true;
1281
8
  if (DT->dominates(cast<Instruction>(CC->getLeader()), U))
1282
4
    return true;
1283
4
  if (CC->getNextLeader().first &&
1284
4
      
DT->dominates(cast<Instruction>(CC->getNextLeader().first), U)2
)
1285
1
    return true;
1286
5
  
return llvm::any_of(*CC, [&](const Value *Member) 3
{
1287
5
    return Member != CC->getLeader() &&
1288
5
           
DT->dominates(cast<Instruction>(Member), U)2
;
1289
5
  });
1290
3
}
1291
1292
// See if we have a congruence class and leader for this operand, and if so,
1293
// return it. Otherwise, return the operand itself.
1294
8.58k
Value *NewGVN::lookupOperandLeader(Value *V) const {
1295
8.58k
  CongruenceClass *CC = ValueToClass.lookup(V);
1296
8.58k
  if (CC) {
1297
5.51k
    // Everything in TOP is represented by undef, as it can be any value.
1298
5.51k
    // We do have to make sure we get the type right though, so we can't set the
1299
5.51k
    // RepLeader to undef.
1300
5.51k
    if (CC == TOPClass)
1301
0
      return UndefValue::get(V->getType());
1302
5.51k
    return CC->getStoredValue() ? 
CC->getStoredValue()189
:
CC->getLeader()5.32k
;
1303
5.51k
  }
1304
3.07k
1305
3.07k
  return V;
1306
3.07k
}
1307
1308
1.22k
const MemoryAccess *NewGVN::lookupMemoryLeader(const MemoryAccess *MA) const {
1309
1.22k
  auto *CC = getMemoryClass(MA);
1310
1.22k
  assert(CC->getMemoryLeader() &&
1311
1.22k
         "Every MemoryAccess should be mapped to a congruence class with a "
1312
1.22k
         "representative memory access");
1313
1.22k
  return CC->getMemoryLeader();
1314
1.22k
}
1315
1316
// Return true if the MemoryAccess is really equivalent to everything. This is
1317
// equivalent to the lattice value "TOP" in most lattices.  This is the initial
1318
// state of all MemoryAccesses.
1319
495
bool NewGVN::isMemoryAccessTOP(const MemoryAccess *MA) const {
1320
495
  return getMemoryClass(MA) == TOPClass;
1321
495
}
1322
1323
LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,
1324
                                             LoadInst *LI,
1325
502
                                             const MemoryAccess *MA) const {
1326
502
  auto *E =
1327
502
      new (ExpressionAllocator) LoadExpression(1, LI, lookupMemoryLeader(MA));
1328
502
  E->allocateOperands(ArgRecycler, ExpressionAllocator);
1329
502
  E->setType(LoadType);
1330
502
1331
502
  // Give store and loads same opcode so they value number together.
1332
502
  E->setOpcode(0);
1333
502
  E->op_push_back(PointerOp);
1334
502
  if (LI)
1335
502
    E->setAlignment(LI->getAlignment());
1336
502
1337
502
  // TODO: Value number heap versions. We may be able to discover
1338
502
  // things alias analysis can't on it's own (IE that a store and a
1339
502
  // load have the same value, and thus, it isn't clobbering the load).
1340
502
  return E;
1341
502
}
1342
1343
const StoreExpression *
1344
604
NewGVN::createStoreExpression(StoreInst *SI, const MemoryAccess *MA) const {
1345
604
  auto *StoredValueLeader = lookupOperandLeader(SI->getValueOperand());
1346
604
  auto *E = new (ExpressionAllocator)
1347
604
      StoreExpression(SI->getNumOperands(), SI, StoredValueLeader, MA);
1348
604
  E->allocateOperands(ArgRecycler, ExpressionAllocator);
1349
604
  E->setType(SI->getValueOperand()->getType());
1350
604
1351
604
  // Give store and loads same opcode so they value number together.
1352
604
  E->setOpcode(0);
1353
604
  E->op_push_back(lookupOperandLeader(SI->getPointerOperand()));
1354
604
1355
604
  // TODO: Value number heap versions. We may be able to discover
1356
604
  // things alias analysis can't on it's own (IE that a store and a
1357
604
  // load have the same value, and thus, it isn't clobbering the load).
1358
604
  return E;
1359
604
}
1360
1361
321
const Expression *NewGVN::performSymbolicStoreEvaluation(Instruction *I) const {
1362
321
  // Unlike loads, we never try to eliminate stores, so we do not check if they
1363
321
  // are simple and avoid value numbering them.
1364
321
  auto *SI = cast<StoreInst>(I);
1365
321
  auto *StoreAccess = getMemoryAccess(SI);
1366
321
  // Get the expression, if any, for the RHS of the MemoryDef.
1367
321
  const MemoryAccess *StoreRHS = StoreAccess->getDefiningAccess();
1368
321
  if (EnableStoreRefinement)
1369
3
    StoreRHS = MSSAWalker->getClobberingMemoryAccess(StoreAccess);
1370
321
  // If we bypassed the use-def chains, make sure we add a use.
1371
321
  StoreRHS = lookupMemoryLeader(StoreRHS);
1372
321
  if (StoreRHS != StoreAccess->getDefiningAccess())
1373
52
    addMemoryUsers(StoreRHS, StoreAccess);
1374
321
  // If we are defined by ourselves, use the live on entry def.
1375
321
  if (StoreRHS == StoreAccess)
1376
1
    StoreRHS = MSSA->getLiveOnEntryDef();
1377
321
1378
321
  if (SI->isSimple()) {
1379
313
    // See if we are defined by a previous store expression, it already has a
1380
313
    // value, and it's the same value as our current store. FIXME: Right now, we
1381
313
    // only do this for simple stores, we should expand to cover memcpys, etc.
1382
313
    const auto *LastStore = createStoreExpression(SI, StoreRHS);
1383
313
    const auto *LastCC = ExpressionToClass.lookup(LastStore);
1384
313
    // We really want to check whether the expression we matched was a store. No
1385
313
    // easy way to do that. However, we can check that the class we found has a
1386
313
    // store, which, assuming the value numbering state is not corrupt, is
1387
313
    // sufficient, because we must also be equivalent to that store's expression
1388
313
    // for it to be in the same class as the load.
1389
313
    if (LastCC && 
LastCC->getStoredValue() == LastStore->getStoredValue()60
)
1390
22
      return LastStore;
1391
291
    // Also check if our value operand is defined by a load of the same memory
1392
291
    // location, and the memory state is the same as it was then (otherwise, it
1393
291
    // could have been overwritten later. See test32 in
1394
291
    // transforms/DeadStoreElimination/simple.ll).
1395
291
    if (auto *LI = dyn_cast<LoadInst>(LastStore->getStoredValue()))
1396
37
      if ((lookupOperandLeader(LI->getPointerOperand()) ==
1397
37
           LastStore->getOperand(0)) &&
1398
37
          (lookupMemoryLeader(getMemoryAccess(LI)->getDefiningAccess()) ==
1399
9
           StoreRHS))
1400
8
        return LastStore;
1401
283
    deleteExpression(LastStore);
1402
283
  }
1403
321
1404
321
  // If the store is not equivalent to anything, value number it as a store that
1405
321
  // produces a unique memory state (instead of using it's MemoryUse, we use
1406
321
  // it's MemoryDef).
1407
321
  
return createStoreExpression(SI, StoreAccess)291
;
1408
321
}
1409
1410
// See if we can extract the value of a loaded pointer from a load, a store, or
1411
// a memory instruction.
1412
const Expression *
1413
NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
1414
                                    LoadInst *LI, Instruction *DepInst,
1415
159
                                    MemoryAccess *DefiningAccess) const {
1416
159
  assert((!LI || LI->isSimple()) && "Not a simple load");
1417
159
  if (auto *DepSI = dyn_cast<StoreInst>(DepInst)) {
1418
100
    // Can't forward from non-atomic to atomic without violating memory model.
1419
100
    // Also don't need to coerce if they are the same type, we will just
1420
100
    // propagate.
1421
100
    if (LI->isAtomic() > DepSI->isAtomic() ||
1422
100
        LoadType == DepSI->getValueOperand()->getType())
1423
78
      return nullptr;
1424
22
    int Offset = analyzeLoadFromClobberingStore(LoadType, LoadPtr, DepSI, DL);
1425
22
    if (Offset >= 0) {
1426
3
      if (auto *C = dyn_cast<Constant>(
1427
1
              lookupOperandLeader(DepSI->getValueOperand()))) {
1428
1
        LLVM_DEBUG(dbgs() << "Coercing load from store " << *DepSI
1429
1
                          << " to constant " << *C << "\n");
1430
1
        return createConstantExpression(
1431
1
            getConstantStoreValueForLoad(C, Offset, LoadType, DL));
1432
1
      }
1433
59
    }
1434
59
  } else if (auto *DepLI = dyn_cast<LoadInst>(DepInst)) {
1435
0
    // Can't forward from non-atomic to atomic without violating memory model.
1436
0
    if (LI->isAtomic() > DepLI->isAtomic())
1437
0
      return nullptr;
1438
0
    int Offset = analyzeLoadFromClobberingLoad(LoadType, LoadPtr, DepLI, DL);
1439
0
    if (Offset >= 0) {
1440
0
      // We can coerce a constant load into a load.
1441
0
      if (auto *C = dyn_cast<Constant>(lookupOperandLeader(DepLI)))
1442
0
        if (auto *PossibleConstant =
1443
0
                getConstantLoadValueForLoad(C, Offset, LoadType, DL)) {
1444
0
          LLVM_DEBUG(dbgs() << "Coercing load from load " << *LI
1445
0
                            << " to constant " << *PossibleConstant << "\n");
1446
0
          return createConstantExpression(PossibleConstant);
1447
0
        }
1448
59
    }
1449
59
  } else if (auto *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {
1450
10
    int Offset = analyzeLoadFromClobberingMemInst(LoadType, LoadPtr, DepMI, DL);
1451
10
    if (Offset >= 0) {
1452
10
      if (auto *PossibleConstant =
1453
10
              getConstantMemInstValueForLoad(DepMI, Offset, LoadType, DL)) {
1454
10
        LLVM_DEBUG(dbgs() << "Coercing load from meminst " << *DepMI
1455
10
                          << " to constant " << *PossibleConstant << "\n");
1456
10
        return createConstantExpression(PossibleConstant);
1457
10
      }
1458
70
    }
1459
10
  }
1460
70
1461
70
  // All of the below are only true if the loaded pointer is produced
1462
70
  // by the dependent instruction.
1463
70
  if (LoadPtr != lookupOperandLeader(DepInst) &&
1464
70
      
!AA->isMustAlias(LoadPtr, DepInst)66
)
1465
64
    return nullptr;
1466
6
  // If this load really doesn't depend on anything, then we must be loading an
1467
6
  // undef value.  This can happen when loading for a fresh allocation with no
1468
6
  // intervening stores, for example.  Note that this is only true in the case
1469
6
  // that the result of the allocation is pointer equal to the load ptr.
1470
6
  if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI)) {
1471
2
    return createConstantExpression(UndefValue::get(LoadType));
1472
2
  }
1473
4
  // If this load occurs either right after a lifetime begin,
1474
4
  // then the loaded value is undefined.
1475
4
  else if (auto *II = dyn_cast<IntrinsicInst>(DepInst)) {
1476
0
    if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1477
0
      return createConstantExpression(UndefValue::get(LoadType));
1478
4
  }
1479
4
  // If this load follows a calloc (which zero initializes memory),
1480
4
  // then the loaded value is zero
1481
4
  else if (isCallocLikeFn(DepInst, TLI)) {
1482
1
    return createConstantExpression(Constant::getNullValue(LoadType));
1483
1
  }
1484
3
1485
3
  return nullptr;
1486
3
}
1487
1488
530
const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
1489
530
  auto *LI = cast<LoadInst>(I);
1490
530
1491
530
  // We can eliminate in favor of non-simple loads, but we won't be able to
1492
530
  // eliminate the loads themselves.
1493
530
  if (!LI->isSimple())
1494
4
    return nullptr;
1495
526
1496
526
  Value *LoadAddressLeader = lookupOperandLeader(LI->getPointerOperand());
1497
526
  // Load of undef is undef.
1498
526
  if (isa<UndefValue>(LoadAddressLeader))
1499
10
    return createConstantExpression(UndefValue::get(LI->getType()));
1500
516
  MemoryAccess *OriginalAccess = getMemoryAccess(I);
1501
516
  MemoryAccess *DefiningAccess =
1502
516
      MSSAWalker->getClobberingMemoryAccess(OriginalAccess);
1503
516
1504
516
  if (!MSSA->isLiveOnEntryDef(DefiningAccess)) {
1505
312
    if (auto *MD = dyn_cast<MemoryDef>(DefiningAccess)) {
1506
159
      Instruction *DefiningInst = MD->getMemoryInst();
1507
159
      // If the defining instruction is not reachable, replace with undef.
1508
159
      if (!ReachableBlocks.count(DefiningInst->getParent()))
1509
0
        return createConstantExpression(UndefValue::get(LI->getType()));
1510
159
      // This will handle stores and memory insts.  We only do if it the
1511
159
      // defining access has a different type, or it is a pointer produced by
1512
159
      // certain memory operations that cause the memory to have a fixed value
1513
159
      // (IE things like calloc).
1514
159
      if (const auto *CoercionResult =
1515
14
              performSymbolicLoadCoercion(LI->getType(), LoadAddressLeader, LI,
1516
14
                                          DefiningInst, DefiningAccess))
1517
14
        return CoercionResult;
1518
502
    }
1519
312
  }
1520
502
1521
502
  const auto *LE = createLoadExpression(LI->getType(), LoadAddressLeader, LI,
1522
502
                                        DefiningAccess);
1523
502
  // If our MemoryLeader is not our defining access, add a use to the
1524
502
  // MemoryLeader, so that we get reprocessed when it changes.
1525
502
  if (LE->getMemoryLeader() != DefiningAccess)
1526
83
    addMemoryUsers(LE->getMemoryLeader(), OriginalAccess);
1527
502
  return LE;
1528
502
}
1529
1530
const Expression *
1531
140
NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const {
1532
140
  auto *PI = PredInfo->getPredicateInfoFor(I);
1533
140
  if (!PI)
1534
0
    return nullptr;
1535
140
1536
140
  LLVM_DEBUG(dbgs() << "Found predicate info from instruction !\n");
1537
140
1538
140
  auto *PWC = dyn_cast<PredicateWithCondition>(PI);
1539
140
  if (!PWC)
1540
0
    return nullptr;
1541
140
1542
140
  auto *CopyOf = I->getOperand(0);
1543
140
  auto *Cond = PWC->Condition;
1544
140
1545
140
  // If this a copy of the condition, it must be either true or false depending
1546
140
  // on the predicate info type and edge.
1547
140
  if (CopyOf == Cond) {
1548
21
    // We should not need to add predicate users because the predicate info is
1549
21
    // already a use of this operand.
1550
21
    if (isa<PredicateAssume>(PI))
1551
14
      return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1552
7
    if (auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1553
5
      if (PBranch->TrueEdge)
1554
4
        return createConstantExpression(ConstantInt::getTrue(Cond->getType()));
1555
1
      return createConstantExpression(ConstantInt::getFalse(Cond->getType()));
1556
1
    }
1557
2
    if (auto *PSwitch = dyn_cast<PredicateSwitch>(PI))
1558
2
      return createConstantExpression(cast<Constant>(PSwitch->CaseValue));
1559
119
  }
1560
119
1561
119
  // Not a copy of the condition, so see what the predicates tell us about this
1562
119
  // value.  First, though, we check to make sure the value is actually a copy
1563
119
  // of one of the condition operands. It's possible, in certain cases, for it
1564
119
  // to be a copy of a predicateinfo copy. In particular, if two branch
1565
119
  // operations use the same condition, and one branch dominates the other, we
1566
119
  // will end up with a copy of a copy.  This is currently a small deficiency in
1567
119
  // predicateinfo.  What will end up happening here is that we will value
1568
119
  // number both copies the same anyway.
1569
119
1570
119
  // Everything below relies on the condition being a comparison.
1571
119
  auto *Cmp = dyn_cast<CmpInst>(Cond);
1572
119
  if (!Cmp)
1573
0
    return nullptr;
1574
119
1575
119
  if (CopyOf != Cmp->getOperand(0) && 
CopyOf != Cmp->getOperand(1)23
) {
1576
2
    LLVM_DEBUG(dbgs() << "Copy is not of any condition operands!\n");
1577
2
    return nullptr;
1578
2
  }
1579
117
  Value *FirstOp = lookupOperandLeader(Cmp->getOperand(0));
1580
117
  Value *SecondOp = lookupOperandLeader(Cmp->getOperand(1));
1581
117
  bool SwappedOps = false;
1582
117
  // Sort the ops.
1583
117
  if (shouldSwapOperands(FirstOp, SecondOp)) {
1584
74
    std::swap(FirstOp, SecondOp);
1585
74
    SwappedOps = true;
1586
74
  }
1587
117
  CmpInst::Predicate Predicate =
1588
117
      SwappedOps ? 
Cmp->getSwappedPredicate()74
:
Cmp->getPredicate()43
;
1589
117
1590
117
  if (isa<PredicateAssume>(PI)) {
1591
11
    // If we assume the operands are equal, then they are equal.
1592
11
    if (Predicate == CmpInst::ICMP_EQ) {
1593
7
      addPredicateUsers(PI, I);
1594
7
      addAdditionalUsers(SwappedOps ? 
Cmp->getOperand(1)6
:
Cmp->getOperand(0)1
,
1595
7
                         I);
1596
7
      return createVariableOrConstant(FirstOp);
1597
7
    }
1598
110
  }
1599
110
  if (const auto *PBranch = dyn_cast<PredicateBranch>(PI)) {
1600
106
    // If we are *not* a copy of the comparison, we may equal to the other
1601
106
    // operand when the predicate implies something about equality of
1602
106
    // operations.  In particular, if the comparison is true/false when the
1603
106
    // operands are equal, and we are on the right edge, we know this operation
1604
106
    // is equal to something.
1605
106
    if ((PBranch->TrueEdge && 
Predicate == CmpInst::ICMP_EQ55
) ||
1606
106
        
(86
!PBranch->TrueEdge86
&&
Predicate == CmpInst::ICMP_NE51
)) {
1607
21
      addPredicateUsers(PI, I);
1608
21
      addAdditionalUsers(SwappedOps ? 
Cmp->getOperand(1)12
:
Cmp->getOperand(0)9
,
1609
21
                         I);
1610
21
      return createVariableOrConstant(FirstOp);
1611
21
    }
1612
85
    // Handle the special case of floating point.
1613
85
    if (((PBranch->TrueEdge && 
Predicate == CmpInst::FCMP_OEQ35
) ||
1614
85
         
(82
!PBranch->TrueEdge82
&&
Predicate == CmpInst::FCMP_UNE50
)) &&
1615
85
        
isa<ConstantFP>(FirstOp)6
&&
!cast<ConstantFP>(FirstOp)->isZero()4
) {
1616
2
      addPredicateUsers(PI, I);
1617
2
      addAdditionalUsers(SwappedOps ? Cmp->getOperand(1) : 
Cmp->getOperand(0)0
,
1618
2
                         I);
1619
2
      return createConstantExpression(cast<Constant>(FirstOp));
1620
2
    }
1621
87
  }
1622
87
  return nullptr;
1623
87
}
1624
1625
// Evaluate read only and pure calls, and create an expression result.
1626
403
const Expression *NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
1627
403
  auto *CI = cast<CallInst>(I);
1628
403
  if (auto *II = dyn_cast<IntrinsicInst>(I)) {
1629
214
    // Intrinsics with the returned attribute are copies of arguments.
1630
214
    if (auto *ReturnedValue = II->getReturnedArgOperand()) {
1631
140
      if (II->getIntrinsicID() == Intrinsic::ssa_copy)
1632
140
        if (const auto *Result = performSymbolicPredicateInfoEvaluation(I))
1633
51
          return Result;
1634
89
      return createVariableOrConstant(ReturnedValue);
1635
89
    }
1636
214
  }
1637
263
  if (AA->doesNotAccessMemory(CI)) {
1638
23
    return createCallExpression(CI, TOPClass->getMemoryLeader());
1639
240
  } else if (AA->onlyReadsMemory(CI)) {
1640
31
    MemoryAccess *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(CI);
1641
31
    return createCallExpression(CI, DefiningAccess);
1642
31
  }
1643
209
  return nullptr;
1644
209
}
1645
1646
// Retrieve the memory class for a given MemoryAccess.
1647
1.96k
CongruenceClass *NewGVN::getMemoryClass(const MemoryAccess *MA) const {
1648
1.96k
  auto *Result = MemoryAccessToClass.lookup(MA);
1649
1.96k
  assert(Result && "Should have found memory class");
1650
1.96k
  return Result;
1651
1.96k
}
1652
1653
// Update the MemoryAccess equivalence table to say that From is equal to To,
1654
// and return true if this is different from what already existed in the table.
1655
bool NewGVN::setMemoryClass(const MemoryAccess *From,
1656
705
                            CongruenceClass *NewClass) {
1657
705
  assert(NewClass &&
1658
705
         "Every MemoryAccess should be getting mapped to a non-null class");
1659
705
  LLVM_DEBUG(dbgs() << "Setting " << *From);
1660
705
  LLVM_DEBUG(dbgs() << " equivalent to congruence class ");
1661
705
  LLVM_DEBUG(dbgs() << NewClass->getID()
1662
705
                    << " with current MemoryAccess leader ");
1663
705
  LLVM_DEBUG(dbgs() << *NewClass->getMemoryLeader() << "\n");
1664
705
1665
705
  auto LookupResult = MemoryAccessToClass.find(From);
1666
705
  bool Changed = false;
1667
705
  // If it's already in the table, see if the value changed.
1668
705
  if (LookupResult != MemoryAccessToClass.end()) {
1669
705
    auto *OldClass = LookupResult->second;
1670
705
    if (OldClass != NewClass) {
1671
663
      // If this is a phi, we have to handle memory member updates.
1672
663
      if (auto *MP = dyn_cast<MemoryPhi>(From)) {
1673
197
        OldClass->memory_erase(MP);
1674
197
        NewClass->memory_insert(MP);
1675
197
        // This may have killed the class if it had no non-memory members
1676
197
        if (OldClass->getMemoryLeader() == From) {
1677
3
          if (OldClass->definesNoMemory()) {
1678
3
            OldClass->setMemoryLeader(nullptr);
1679
3
          } else {
1680
0
            OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
1681
0
            LLVM_DEBUG(dbgs() << "Memory class leader change for class "
1682
0
                              << OldClass->getID() << " to "
1683
0
                              << *OldClass->getMemoryLeader()
1684
0
                              << " due to removal of a memory member " << *From
1685
0
                              << "\n");
1686
0
            markMemoryLeaderChangeTouched(OldClass);
1687
0
          }
1688
3
        }
1689
197
      }
1690
663
      // It wasn't equivalent before, and now it is.
1691
663
      LookupResult->second = NewClass;
1692
663
      Changed = true;
1693
663
    }
1694
705
  }
1695
705
1696
705
  return Changed;
1697
705
}
1698
1699
// Determine if a instruction is cycle-free.  That means the values in the
1700
// instruction don't depend on any expressions that can change value as a result
1701
// of the instruction.  For example, a non-cycle free instruction would be v =
1702
// phi(0, v+1).
1703
233
bool NewGVN::isCycleFree(const Instruction *I) const {
1704
233
  // In order to compute cycle-freeness, we do SCC finding on the instruction,
1705
233
  // and see what kind of SCC it ends up in.  If it is a singleton, it is
1706
233
  // cycle-free.  If it is not in a singleton, it is only cycle free if the
1707
233
  // other members are all phi nodes (as they do not compute anything, they are
1708
233
  // copies).
1709
233
  auto ICS = InstCycleState.lookup(I);
1710
233
  if (ICS == ICS_Unknown) {
1711
171
    SCCFinder.Start(I);
1712
171
    auto &SCC = SCCFinder.getComponentFor(I);
1713
171
    // It's cycle free if it's size 1 or the SCC is *only* phi nodes.
1714
171
    if (SCC.size() == 1)
1715
94
      InstCycleState.insert({I, ICS_CycleFree});
1716
77
    else {
1717
115
      bool AllPhis = llvm::all_of(SCC, [](const Value *V) {
1718
115
        return isa<PHINode>(V) || 
isCopyOfAPHI(V)77
;
1719
115
      });
1720
77
      ICS = AllPhis ? 
ICS_CycleFree2
:
ICS_Cycle75
;
1721
77
      for (auto *Member : SCC)
1722
186
        if (auto *MemberPhi = dyn_cast<PHINode>(Member))
1723
92
          InstCycleState.insert({MemberPhi, ICS});
1724
77
    }
1725
171
  }
1726
233
  if (ICS == ICS_Cycle)
1727
79
    return false;
1728
154
  return true;
1729
154
}
1730
1731
// Evaluate PHI nodes symbolically and create an expression result.
1732
const Expression *
1733
NewGVN::performSymbolicPHIEvaluation(ArrayRef<ValPair> PHIOps,
1734
                                     Instruction *I,
1735
523
                                     BasicBlock *PHIBlock) const {
1736
523
  // True if one of the incoming phi edges is a backedge.
1737
523
  bool HasBackedge = false;
1738
523
  // All constant tracks the state of whether all the *original* phi operands
1739
523
  // This is really shorthand for "this phi cannot cycle due to forward
1740
523
  // change in value of the phi is guaranteed not to later change the value of
1741
523
  // the phi. IE it can't be v = phi(undef, v+1)
1742
523
  bool OriginalOpsConstant = true;
1743
523
  auto *E = cast<PHIExpression>(createPHIExpression(
1744
523
      PHIOps, I, PHIBlock, HasBackedge, OriginalOpsConstant));
1745
523
  // We match the semantics of SimplifyPhiNode from InstructionSimplify here.
1746
523
  // See if all arguments are the same.
1747
523
  // We track if any were undef because they need special handling.
1748
523
  bool HasUndef = false;
1749
1.12k
  auto Filtered = make_filter_range(E->operands(), [&](Value *Arg) {
1750
1.12k
    if (isa<UndefValue>(Arg)) {
1751
68
      HasUndef = true;
1752
68
      return false;
1753
68
    }
1754
1.06k
    return true;
1755
1.06k
  });
1756
523
  // If we are left with no operands, it's dead.
1757
523
  if (empty(Filtered)) {
1758
26
    // If it has undef at this point, it means there are no-non-undef arguments,
1759
26
    // and thus, the value of the phi node must be undef.
1760
26
    if (HasUndef) {
1761
26
      LLVM_DEBUG(
1762
26
          dbgs() << "PHI Node " << *I
1763
26
                 << " has no non-undef arguments, valuing it as undef\n");
1764
26
      return createConstantExpression(UndefValue::get(I->getType()));
1765
26
    }
1766
0
1767
0
    LLVM_DEBUG(dbgs() << "No arguments of PHI node " << *I << " are live\n");
1768
0
    deleteExpression(E);
1769
0
    return createDeadExpression();
1770
0
  }
1771
497
  Value *AllSameValue = *(Filtered.begin());
1772
497
  ++Filtered.begin();
1773
497
  // Can't use std::equal here, sadly, because filter.begin moves.
1774
779
  if (
llvm::all_of(Filtered, [&](Value *Arg) 497
{ return Arg == AllSameValue; })) {
1775
243
    // In LLVM's non-standard representation of phi nodes, it's possible to have
1776
243
    // phi nodes with cycles (IE dependent on other phis that are .... dependent
1777
243
    // on the original phi node), especially in weird CFG's where some arguments
1778
243
    // are unreachable, or uninitialized along certain paths.  This can cause
1779
243
    // infinite loops during evaluation. We work around this by not trying to
1780
243
    // really evaluate them independently, but instead using a variable
1781
243
    // expression to say if one is equivalent to the other.
1782
243
    // We also special case undef, so that if we have an undef, we can't use the
1783
243
    // common value unless it dominates the phi block.
1784
243
    if (HasUndef) {
1785
30
      // If we have undef and at least one other value, this is really a
1786
30
      // multivalued phi, and we need to know if it's cycle free in order to
1787
30
      // evaluate whether we can ignore the undef.  The other parts of this are
1788
30
      // just shortcuts.  If there is no backedge, or all operands are
1789
30
      // constants, it also must be cycle free.
1790
30
      if (HasBackedge && 
!OriginalOpsConstant21
&&
1791
30
          
!isa<UndefValue>(AllSameValue)18
&&
!isCycleFree(I)18
)
1792
7
        return E;
1793
23
1794
23
      // Only have to check for instructions
1795
23
      if (auto *AllSameInst = dyn_cast<Instruction>(AllSameValue))
1796
8
        if (!someEquivalentDominates(AllSameInst, I))
1797
3
          return E;
1798
233
    }
1799
233
    // Can't simplify to something that comes later in the iteration.
1800
233
    // Otherwise, when and if it changes congruence class, we will never catch
1801
233
    // up. We will always be a class behind it.
1802
233
    if (isa<Instruction>(AllSameValue) &&
1803
233
        
InstrToDFSNum(AllSameValue) > InstrToDFSNum(I)94
)
1804
9
      return E;
1805
224
    NumGVNPhisAllSame++;
1806
224
    LLVM_DEBUG(dbgs() << "Simplified PHI node " << *I << " to " << *AllSameValue
1807
224
                      << "\n");
1808
224
    deleteExpression(E);
1809
224
    return createVariableOrConstant(AllSameValue);
1810
224
  }
1811
254
  return E;
1812
254
}
1813
1814
const Expression *
1815
15
NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) const {
1816
15
  if (auto *EI = dyn_cast<ExtractValueInst>(I)) {
1817
15
    auto *WO = dyn_cast<WithOverflowInst>(EI->getAggregateOperand());
1818
15
    if (WO && 
EI->getNumIndices() == 112
&&
*EI->idx_begin() == 012
)
1819
12
      // EI is an extract from one of our with.overflow intrinsics. Synthesize
1820
12
      // a semantically equivalent expression instead of an extract value
1821
12
      // expression.
1822
12
      return createBinaryExpression(WO->getBinaryOp(), EI->getType(),
1823
12
                                    WO->getLHS(), WO->getRHS(), I);
1824
3
  }
1825
3
1826
3
  return createAggregateValueExpression(I);
1827
3
}
1828
1829
400
const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
1830
400
  assert(isa<CmpInst>(I) && "Expected a cmp instruction.");
1831
400
1832
400
  auto *CI = cast<CmpInst>(I);
1833
400
  // See if our operands are equal to those of a previous predicate, and if so,
1834
400
  // if it implies true or false.
1835
400
  auto Op0 = lookupOperandLeader(CI->getOperand(0));
1836
400
  auto Op1 = lookupOperandLeader(CI->getOperand(1));
1837
400
  auto OurPredicate = CI->getPredicate();
1838
400
  if (shouldSwapOperands(Op0, Op1)) {
1839
279
    std::swap(Op0, Op1);
1840
279
    OurPredicate = CI->getSwappedPredicate();
1841
279
  }
1842
400
1843
400
  // Avoid processing the same info twice.
1844
400
  const PredicateBase *LastPredInfo = nullptr;
1845
400
  // See if we know something about the comparison itself, like it is the target
1846
400
  // of an assume.
1847
400
  auto *CmpPI = PredInfo->getPredicateInfoFor(I);
1848
400
  if (dyn_cast_or_null<PredicateAssume>(CmpPI))
1849
0
    return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1850
400
1851
400
  if (Op0 == Op1) {
1852
21
    // This condition does not depend on predicates, no need to add users
1853
21
    if (CI->isTrueWhenEqual())
1854
15
      return createConstantExpression(ConstantInt::getTrue(CI->getType()));
1855
6
    else if (CI->isFalseWhenEqual())
1856
6
      return createConstantExpression(ConstantInt::getFalse(CI->getType()));
1857
379
  }
1858
379
1859
379
  // NOTE: Because we are comparing both operands here and below, and using
1860
379
  // previous comparisons, we rely on fact that predicateinfo knows to mark
1861
379
  // comparisons that use renamed operands as users of the earlier comparisons.
1862
379
  // It is *not* enough to just mark predicateinfo renamed operands as users of
1863
379
  // the earlier comparisons, because the *other* operand may have changed in a
1864
379
  // previous iteration.
1865
379
  // Example:
1866
379
  // icmp slt %a, %b
1867
379
  // %b.0 = ssa.copy(%b)
1868
379
  // false branch:
1869
379
  // icmp slt %c, %b.0
1870
379
1871
379
  // %c and %a may start out equal, and thus, the code below will say the second
1872
379
  // %icmp is false.  c may become equal to something else, and in that case the
1873
379
  // %second icmp *must* be reexamined, but would not if only the renamed
1874
379
  // %operands are considered users of the icmp.
1875
379
1876
379
  // *Currently* we only check one level of comparisons back, and only mark one
1877
379
  // level back as touched when changes happen.  If you modify this code to look
1878
379
  // back farther through comparisons, you *must* mark the appropriate
1879
379
  // comparisons as users in PredicateInfo.cpp, or you will cause bugs.  See if
1880
379
  // we know something just from the operands themselves
1881
379
1882
379
  // See if our operands have predicate info, so that we may be able to derive
1883
379
  // something from a previous comparison.
1884
748
  
for (const auto &Op : CI->operands())379
{
1885
748
    auto *PI = PredInfo->getPredicateInfoFor(Op);
1886
748
    if (const auto *PBranch = dyn_cast_or_null<PredicateBranch>(PI)) {
1887
48
      if (PI == LastPredInfo)
1888
0
        continue;
1889
48
      LastPredInfo = PI;
1890
48
      // In phi of ops cases, we may have predicate info that we are evaluating
1891
48
      // in a different context.
1892
48
      if (!DT->dominates(PBranch->To, getBlockForValue(I)))
1893
5
        continue;
1894
43
      // TODO: Along the false edge, we may know more things too, like
1895
43
      // icmp of
1896
43
      // same operands is false.
1897
43
      // TODO: We only handle actual comparison conditions below, not
1898
43
      // and/or.
1899
43
      auto *BranchCond = dyn_cast<CmpInst>(PBranch->Condition);
1900
43
      if (!BranchCond)
1901
0
        continue;
1902
43
      auto *BranchOp0 = lookupOperandLeader(BranchCond->getOperand(0));
1903
43
      auto *BranchOp1 = lookupOperandLeader(BranchCond->getOperand(1));
1904
43
      auto BranchPredicate = BranchCond->getPredicate();
1905
43
      if (shouldSwapOperands(BranchOp0, BranchOp1)) {
1906
27
        std::swap(BranchOp0, BranchOp1);
1907
27
        BranchPredicate = BranchCond->getSwappedPredicate();
1908
27
      }
1909
43
      if (BranchOp0 == Op0 && 
BranchOp1 == Op122
) {
1910
11
        if (PBranch->TrueEdge) {
1911
6
          // If we know the previous predicate is true and we are in the true
1912
6
          // edge then we may be implied true or false.
1913
6
          if (CmpInst::isImpliedTrueByMatchingCmp(BranchPredicate,
1914
6
                                                  OurPredicate)) {
1915
2
            addPredicateUsers(PI, I);
1916
2
            return createConstantExpression(
1917
2
                ConstantInt::getTrue(CI->getType()));
1918
2
          }
1919
4
1920
4
          if (CmpInst::isImpliedFalseByMatchingCmp(BranchPredicate,
1921
4
                                                   OurPredicate)) {
1922
3
            addPredicateUsers(PI, I);
1923
3
            return createConstantExpression(
1924
3
                ConstantInt::getFalse(CI->getType()));
1925
3
          }
1926
5
        } else {
1927
5
          // Just handle the ne and eq cases, where if we have the same
1928
5
          // operands, we may know something.
1929
5
          if (BranchPredicate == OurPredicate) {
1930
5
            addPredicateUsers(PI, I);
1931
5
            // Same predicate, same ops,we know it was false, so this is false.
1932
5
            return createConstantExpression(
1933
5
                ConstantInt::getFalse(CI->getType()));
1934
5
          } else 
if (0
BranchPredicate ==
1935
0
                     CmpInst::getInversePredicate(OurPredicate)) {
1936
0
            addPredicateUsers(PI, I);
1937
0
            // Inverse predicate, we know the other was false, so this is true.
1938
0
            return createConstantExpression(
1939
0
                ConstantInt::getTrue(CI->getType()));
1940
0
          }
1941
5
        }
1942
11
      }
1943
43
    }
1944
748
  }
1945
379
  // Create expression will take care of simplifyCmpInst
1946
379
  
return createExpression(I)369
;
1947
379
}
1948
1949
// Substitute and symbolize the value before value numbering.
1950
const Expression *
1951
NewGVN::performSymbolicEvaluation(Value *V,
1952
3.28k
                                  SmallPtrSetImpl<Value *> &Visited) const {
1953
3.28k
  const Expression *E = nullptr;
1954
3.28k
  if (auto *C = dyn_cast<Constant>(V))
1955
0
    E = createConstantExpression(C);
1956
3.28k
  else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
1957
0
    E = createVariableExpression(V);
1958
3.28k
  } else {
1959
3.28k
    // TODO: memory intrinsics.
1960
3.28k
    // TODO: Some day, we should do the forward propagation and reassociation
1961
3.28k
    // parts of the algorithm.
1962
3.28k
    auto *I = cast<Instruction>(V);
1963
3.28k
    switch (I->getOpcode()) {
1964
3.28k
    case Instruction::ExtractValue:
1965
15
    case Instruction::InsertValue:
1966
15
      E = performSymbolicAggrValueEvaluation(I);
1967
15
      break;
1968
476
    case Instruction::PHI: {
1969
476
      SmallVector<ValPair, 3> Ops;
1970
476
      auto *PN = cast<PHINode>(I);
1971
1.43k
      for (unsigned i = 0; i < PN->getNumOperands(); 
++i956
)
1972
956
        Ops.push_back({PN->getIncomingValue(i), PN->getIncomingBlock(i)});
1973
476
      // Sort to ensure the invariant createPHIExpression requires is met.
1974
476
      sortPHIOps(Ops);
1975
476
      E = performSymbolicPHIEvaluation(Ops, I, getBlockForValue(I));
1976
476
    } break;
1977
403
    case Instruction::Call:
1978
403
      E = performSymbolicCallEvaluation(I);
1979
403
      break;
1980
321
    case Instruction::Store:
1981
321
      E = performSymbolicStoreEvaluation(I);
1982
321
      break;
1983
530
    case Instruction::Load:
1984
530
      E = performSymbolicLoadEvaluation(I);
1985
530
      break;
1986
82
    case Instruction::BitCast:
1987
82
    case Instruction::AddrSpaceCast:
1988
82
      E = createExpression(I);
1989
82
      break;
1990
400
    case Instruction::ICmp:
1991
400
    case Instruction::FCmp:
1992
400
      E = performSymbolicCmpEvaluation(I);
1993
400
      break;
1994
991
    case Instruction::FNeg:
1995
991
    case Instruction::Add:
1996
991
    case Instruction::FAdd:
1997
991
    case Instruction::Sub:
1998
991
    case Instruction::FSub:
1999
991
    case Instruction::Mul:
2000
991
    case Instruction::FMul:
2001
991
    case Instruction::UDiv:
2002
991
    case Instruction::SDiv:
2003
991
    case Instruction::FDiv:
2004
991
    case Instruction::URem:
2005
991
    case Instruction::SRem:
2006
991
    case Instruction::FRem:
2007
991
    case Instruction::Shl:
2008
991
    case Instruction::LShr:
2009
991
    case Instruction::AShr:
2010
991
    case Instruction::And:
2011
991
    case Instruction::Or:
2012
991
    case Instruction::Xor:
2013
991
    case Instruction::Trunc:
2014
991
    case Instruction::ZExt:
2015
991
    case Instruction::SExt:
2016
991
    case Instruction::FPToUI:
2017
991
    case Instruction::FPToSI:
2018
991
    case Instruction::UIToFP:
2019
991
    case Instruction::SIToFP:
2020
991
    case Instruction::FPTrunc:
2021
991
    case Instruction::FPExt:
2022
991
    case Instruction::PtrToInt:
2023
991
    case Instruction::IntToPtr:
2024
991
    case Instruction::Select:
2025
991
    case Instruction::ExtractElement:
2026
991
    case Instruction::InsertElement:
2027
991
    case Instruction::ShuffleVector:
2028
991
    case Instruction::GetElementPtr:
2029
991
      E = createExpression(I);
2030
991
      break;
2031
991
    default:
2032
63
      return nullptr;
2033
3.21k
    }
2034
3.21k
  }
2035
3.21k
  return E;
2036
3.21k
}
2037
2038
// Look up a container in a map, and then call a function for each thing in the
2039
// found container.
2040
template <typename Map, typename KeyType, typename Func>
2041
void NewGVN::for_each_found(Map &M, const KeyType &Key, Func F) {
2042
  const auto Result = M.find_as(Key);
2043
  if (Result != M.end())
2044
    for (typename Map::mapped_type::value_type Mapped : Result->second)
2045
      F(Mapped);
2046
}
2047
2048
// Look up a container of values/instructions in a map, and touch all the
2049
// instructions in the container.  Then erase value from the map.
2050
template <typename Map, typename KeyType>
2051
6.65k
void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
2052
6.65k
  const auto Result = M.find_as(Key);
2053
6.65k
  if (Result != M.end()) {
2054
58
    for (const typename Map::mapped_type::value_type Mapped : Result->second)
2055
68
      TouchedInstructions.set(InstrToDFSNum(Mapped));
2056
58
    M.erase(Result);
2057
58
  }
2058
6.65k
}
NewGVN.cpp:void (anonymous namespace)::NewGVN::touchAndErase<llvm::DenseMap<llvm::MemoryAccess const*, llvm::SmallPtrSet<llvm::MemoryAccess*, 2u>, llvm::DenseMapInfo<llvm::MemoryAccess const*>, llvm::detail::DenseMapPair<llvm::MemoryAccess const*, llvm::SmallPtrSet<llvm::MemoryAccess*, 2u> > >, llvm::MemoryAccess const*>(llvm::DenseMap<llvm::MemoryAccess const*, llvm::SmallPtrSet<llvm::MemoryAccess*, 2u>, llvm::DenseMapInfo<llvm::MemoryAccess const*>, llvm::detail::DenseMapPair<llvm::MemoryAccess const*, llvm::SmallPtrSet<llvm::MemoryAccess*, 2u> > >&, llvm::MemoryAccess const* const&)
Line
Count
Source
2051
674
void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
2052
674
  const auto Result = M.find_as(Key);
2053
674
  if (Result != M.end()) {
2054
7
    for (const typename Map::mapped_type::value_type Mapped : Result->second)
2055
16
      TouchedInstructions.set(InstrToDFSNum(Mapped));
2056
7
    M.erase(Result);
2057
7
  }
2058
674
}
NewGVN.cpp:void (anonymous namespace)::NewGVN::touchAndErase<llvm::DenseMap<llvm::GVNExpression::Expression const*, llvm::SmallPtrSet<llvm::Instruction*, 2u>, llvm::DenseMapInfo<llvm::GVNExpression::Expression const*>, llvm::detail::DenseMapPair<llvm::GVNExpression::Expression const*, llvm::SmallPtrSet<llvm::Instruction*, 2u> > >, llvm::GVNExpression::Expression const*>(llvm::DenseMap<llvm::GVNExpression::Expression const*, llvm::SmallPtrSet<llvm::Instruction*, 2u>, llvm::DenseMapInfo<llvm::GVNExpression::Expression const*>, llvm::detail::DenseMapPair<llvm::GVNExpression::Expression const*, llvm::SmallPtrSet<llvm::Instruction*, 2u> > >&, llvm::GVNExpression::Expression const* const&)
Line
Count
Source
2051
2.83k
void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
2052
2.83k
  const auto Result = M.find_as(Key);
2053
2.83k
  if (Result != M.end()) {
2054
26
    for (const typename Map::mapped_type::value_type Mapped : Result->second)
2055
26
      TouchedInstructions.set(InstrToDFSNum(Mapped));
2056
26
    M.erase(Result);
2057
26
  }
2058
2.83k
}
NewGVN.cpp:void (anonymous namespace)::NewGVN::touchAndErase<llvm::DenseMap<llvm::Value const*, llvm::SmallPtrSet<llvm::Value*, 2u>, llvm::DenseMapInfo<llvm::Value const*>, llvm::detail::DenseMapPair<llvm::Value const*, llvm::SmallPtrSet<llvm::Value*, 2u> > >, llvm::Value*>(llvm::DenseMap<llvm::Value const*, llvm::SmallPtrSet<llvm::Value*, 2u>, llvm::DenseMapInfo<llvm::Value const*>, llvm::detail::DenseMapPair<llvm::Value const*, llvm::SmallPtrSet<llvm::Value*, 2u> > >&, llvm::Value* const&)
Line
Count
Source
2051
2.84k
void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
2052
2.84k
  const auto Result = M.find_as(Key);
2053
2.84k
  if (Result != M.end()) {
2054
23
    for (const typename Map::mapped_type::value_type Mapped : Result->second)
2055
24
      TouchedInstructions.set(InstrToDFSNum(Mapped));
2056
23
    M.erase(Result);
2057
23
  }
2058
2.84k
}
NewGVN.cpp:void (anonymous namespace)::NewGVN::touchAndErase<llvm::DenseMap<llvm::Value const*, llvm::SmallPtrSet<llvm::Instruction*, 2u>, llvm::DenseMapInfo<llvm::Value const*>, llvm::detail::DenseMapPair<llvm::Value const*, llvm::SmallPtrSet<llvm::Instruction*, 2u> > >, llvm::Instruction*>(llvm::DenseMap<llvm::Value const*, llvm::SmallPtrSet<llvm::Instruction*, 2u>, llvm::DenseMapInfo<llvm::Value const*>, llvm::detail::DenseMapPair<llvm::Value const*, llvm::SmallPtrSet<llvm::Instruction*, 2u> > >&, llvm::Instruction* const&)
Line
Count
Source
2051
301
void NewGVN::touchAndErase(Map &M, const KeyType &Key) {
2052
301
  const auto Result = M.find_as(Key);
2053
301
  if (Result != M.end()) {
2054
2
    for (const typename Map::mapped_type::value_type Mapped : Result->second)
2055
2
      TouchedInstructions.set(InstrToDFSNum(Mapped));
2056
2
    M.erase(Result);
2057
2
  }
2058
301
}
2059
2060
234
void NewGVN::addAdditionalUsers(Value *To, Value *User) const {
2061
234
  assert(User && To != User);
2062
234
  if (isa<Instruction>(To))
2063
145
    AdditionalUsers[To].insert(User);
2064
234
}
2065
2066
2.84k
void NewGVN::markUsersTouched(Value *V) {
2067
2.84k
  // Now mark the users as touched.
2068
3.07k
  for (auto *User : V->users()) {
2069
3.07k
    assert(isa<Instruction>(User) && "Use of value not within an instruction?");
2070
3.07k
    TouchedInstructions.set(InstrToDFSNum(User));
2071
3.07k
  }
2072
2.84k
  touchAndErase(AdditionalUsers, V);
2073
2.84k
}
2074
2075
135
void NewGVN::addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const {
2076
135
  LLVM_DEBUG(dbgs() << "Adding memory user " << *U << " to " << *To << "\n");
2077
135
  MemoryToUsers[To].insert(U);
2078
135
}
2079
2080
11
void NewGVN::markMemoryDefTouched(const MemoryAccess *MA) {
2081
11
  TouchedInstructions.set(MemoryToDFSNum(MA));
2082
11
}
2083
2084
1.19k
void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) {
2085
1.19k
  if (isa<MemoryUse>(MA))
2086
523
    return;
2087
674
  for (auto U : MA->users())
2088
989
    TouchedInstructions.set(MemoryToDFSNum(U));
2089
674
  touchAndErase(MemoryToUsers, MA);
2090
674
}
2091
2092
// Add I to the set of users of a given predicate.
2093
40
void NewGVN::addPredicateUsers(const PredicateBase *PB, Instruction *I) const {
2094
40
  // Don't add temporary instructions to the user lists.
2095
40
  if (AllTempInstructions.count(I))
2096
5
    return;
2097
35
2098
35
  if (auto *PBranch = dyn_cast<PredicateBranch>(PB))
2099
28
    PredicateToUsers[PBranch->Condition].insert(I);
2100
7
  else if (auto *PAssume = dyn_cast<PredicateAssume>(PB))
2101
7
    PredicateToUsers[PAssume->Condition].insert(I);
2102
35
}
2103
2104
// Touch all the predicates that depend on this instruction.
2105
301
void NewGVN::markPredicateUsersTouched(Instruction *I) {
2106
301
  touchAndErase(PredicateToUsers, I);
2107
301
}
2108
2109
// Mark users affected by a memory leader change.
2110
451
void NewGVN::markMemoryLeaderChangeTouched(CongruenceClass *CC) {
2111
451
  for (auto M : CC->memory())
2112
11
    markMemoryDefTouched(M);
2113
451
}
2114
2115
// Touch the instructions that need to be updated after a congruence class has a
2116
// leader change, and mark changed values.
2117
60
void NewGVN::markValueLeaderChangeTouched(CongruenceClass *CC) {
2118
89
  for (auto M : *CC) {
2119
89
    if (auto *I = dyn_cast<Instruction>(M))
2120
89
      TouchedInstructions.set(InstrToDFSNum(I));
2121
89
    LeaderChanges.insert(M);
2122
89
  }
2123
60
}
2124
2125
// Give a range of things that have instruction DFS numbers, this will return
2126
// the member of the range with the smallest dfs number.
2127
template <class T, class Range>
2128
7
T *NewGVN::getMinDFSOfRange(const Range &R) const {
2129
7
  std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
2130
14
  for (const auto X : R) {
2131
14
    auto DFSNum = InstrToDFSNum(X);
2132
14
    if (DFSNum < MinDFS.second)
2133
8
      MinDFS = {X, DFSNum};
2134
14
  }
2135
7
  return MinDFS.first;
2136
7
}
NewGVN.cpp:llvm::Value* (anonymous namespace)::NewGVN::getMinDFSOfRange<llvm::Value, llvm::iterator_range<llvm::filter_iterator_impl<llvm::SmallPtrSetIterator<llvm::Value*>, (anonymous namespace)::NewGVN::getNextMemoryLeader((anonymous namespace)::CongruenceClass*) const::$_8, std::__1::forward_iterator_tag> > >(llvm::iterator_range<llvm::filter_iterator_impl<llvm::SmallPtrSetIterator<llvm::Value*>, (anonymous namespace)::NewGVN::getNextMemoryLeader((anonymous namespace)::CongruenceClass*) const::$_8, std::__1::forward_iterator_tag> > const&) const
Line
Count
Source
2128
1
T *NewGVN::getMinDFSOfRange(const Range &R) const {
2129
1
  std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
2130
1
  for (const auto X : R) {
2131
1
    auto DFSNum = InstrToDFSNum(X);
2132
1
    if (DFSNum < MinDFS.second)
2133
1
      MinDFS = {X, DFSNum};
2134
1
  }
2135
1
  return MinDFS.first;
2136
1
}
NewGVN.cpp:llvm::MemoryPhi const* (anonymous namespace)::NewGVN::getMinDFSOfRange<llvm::MemoryPhi const, llvm::iterator_range<llvm::SmallPtrSetIterator<llvm::MemoryPhi const*> > >(llvm::iterator_range<llvm::SmallPtrSetIterator<llvm::MemoryPhi const*> > const&) const
Line
Count
Source
2128
2
T *NewGVN::getMinDFSOfRange(const Range &R) const {
2129
2
  std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
2130
4
  for (const auto X : R) {
2131
4
    auto DFSNum = InstrToDFSNum(X);
2132
4
    if (DFSNum < MinDFS.second)
2133
2
      MinDFS = {X, DFSNum};
2134
4
  }
2135
2
  return MinDFS.first;
2136
2
}
NewGVN.cpp:llvm::Value* (anonymous namespace)::NewGVN::getMinDFSOfRange<llvm::Value, (anonymous namespace)::CongruenceClass>((anonymous namespace)::CongruenceClass const&) const
Line
Count
Source
2128
4
T *NewGVN::getMinDFSOfRange(const Range &R) const {
2129
4
  std::pair<T *, unsigned> MinDFS = {nullptr, ~0U};
2130
9
  for (const auto X : R) {
2131
9
    auto DFSNum = InstrToDFSNum(X);
2132
9
    if (DFSNum < MinDFS.second)
2133
5
      MinDFS = {X, DFSNum};
2134
9
  }
2135
4
  return MinDFS.first;
2136
4
}
2137
2138
// This function returns the MemoryAccess that should be the next leader of
2139
// congruence class CC, under the assumption that the current leader is going to
2140
// disappear.
2141
12
const MemoryAccess *NewGVN::getNextMemoryLeader(CongruenceClass *CC) const {
2142
12
  // TODO: If this ends up to slow, we can maintain a next memory leader like we
2143
12
  // do for regular leaders.
2144
12
  // Make sure there will be a leader to find.
2145
12
  assert(!CC->definesNoMemory() && "Can't get next leader if there is none");
2146
12
  if (CC->getStoreCount() > 0) {
2147
4
    if (auto *NL = dyn_cast_or_null<StoreInst>(CC->getNextLeader().first))
2148
3
      return getMemoryAccess(NL);
2149
1
    // Find the store with the minimum DFS number.
2150
1
    auto *V = getMinDFSOfRange<Value>(make_filter_range(
2151
2
        *CC, [&](const Value *V) { return isa<StoreInst>(V); }));
2152
1
    return getMemoryAccess(cast<StoreInst>(V));
2153
1
  }
2154
8
  assert(CC->getStoreCount() == 0);
2155
8
2156
8
  // Given our assertion, hitting this part must mean
2157
8
  // !OldClass->memory_empty()
2158
8
  if (CC->memory_size() == 1)
2159
6
    return *CC->memory_begin();
2160
2
  return getMinDFSOfRange<const MemoryPhi>(CC->memory());
2161
2
}
2162
2163
// This function returns the next value leader of a congruence class, under the
2164
// assumption that the current leader is going away.  This should end up being
2165
// the next most dominating member.
2166
51
Value *NewGVN::getNextValueLeader(CongruenceClass *CC) const {
2167
51
  // We don't need to sort members if there is only 1, and we don't care about
2168
51
  // sorting the TOP class because everything either gets out of it or is
2169
51
  // unreachable.
2170
51
2171
51
  if (CC->size() == 1 || 
CC == TOPClass11
) {
2172
40
    return *(CC->begin());
2173
40
  } else 
if (11
CC->getNextLeader().first11
) {
2174
7
    ++NumGVNAvoidedSortedLeaderChanges;
2175
7
    return CC->getNextLeader().first;
2176
7
  } else {
2177
4
    ++NumGVNSortedLeaderChanges;
2178
4
    // NOTE: If this ends up to slow, we can maintain a dual structure for
2179
4
    // member testing/insertion, or keep things mostly sorted, and sort only
2180
4
    // here, or use SparseBitVector or ....
2181
4
    return getMinDFSOfRange<Value>(*CC);
2182
4
  }
2183
51
}
2184
2185
// Move a MemoryAccess, currently in OldClass, to NewClass, including updates to
2186
// the memory members, etc for the move.
2187
//
2188
// The invariants of this function are:
2189
//
2190
// - I must be moving to NewClass from OldClass
2191
// - The StoreCount of OldClass and NewClass is expected to have been updated
2192
//   for I already if it is a store.
2193
// - The OldClass memory leader has not been updated yet if I was the leader.
2194
void NewGVN::moveMemoryToNewCongruenceClass(Instruction *I,
2195
                                            MemoryAccess *InstMA,
2196
                                            CongruenceClass *OldClass,
2197
458
                                            CongruenceClass *NewClass) {
2198
458
  // If the leader is I, and we had a representative MemoryAccess, it should
2199
458
  // be the MemoryAccess of OldClass.
2200
458
  assert((!InstMA || !OldClass->getMemoryLeader() ||
2201
458
          OldClass->getLeader() != I ||
2202
458
          MemoryAccessToClass.lookup(OldClass->getMemoryLeader()) ==
2203
458
              MemoryAccessToClass.lookup(InstMA)) &&
2204
458
         "Representative MemoryAccess mismatch");
2205
458
  // First, see what happens to the new class
2206
458
  if (!NewClass->getMemoryLeader()) {
2207
439
    // Should be a new class, or a store becoming a leader of a new class.
2208
439
    assert(NewClass->size() == 1 ||
2209
439
           (isa<StoreInst>(I) && NewClass->getStoreCount() == 1));
2210
439
    NewClass->setMemoryLeader(InstMA);
2211
439
    // Mark it touched if we didn't just create a singleton
2212
439
    LLVM_DEBUG(dbgs() << "Memory class leader change for class "
2213
439
                      << NewClass->getID()
2214
439
                      << " due to new memory instruction becoming leader\n");
2215
439
    markMemoryLeaderChangeTouched(NewClass);
2216
439
  }
2217
458
  setMemoryClass(InstMA, NewClass);
2218
458
  // Now, fixup the old class if necessary
2219
458
  if (OldClass->getMemoryLeader() == InstMA) {
2220
34
    if (!OldClass->definesNoMemory()) {
2221
12
      OldClass->setMemoryLeader(getNextMemoryLeader(OldClass));
2222
12
      LLVM_DEBUG(dbgs() << "Memory class leader change for class "
2223
12
                        << OldClass->getID() << " to "
2224
12
                        << *OldClass->getMemoryLeader()
2225
12
                        << " due to removal of old leader " << *InstMA << "\n");
2226
12
      markMemoryLeaderChangeTouched(OldClass);
2227
12
    } else
2228
22
      OldClass->setMemoryLeader(nullptr);
2229
34
  }
2230
458
}
2231
2232
// Move a value, currently in OldClass, to be part of NewClass
2233
// Update OldClass and NewClass for the move (including changing leaders, etc).
2234
void NewGVN::moveValueToNewCongruenceClass(Instruction *I, const Expression *E,
2235
                                           CongruenceClass *OldClass,
2236
2.83k
                                           CongruenceClass *NewClass) {
2237
2.83k
  if (I == OldClass->getNextLeader().first)
2238
173
    OldClass->resetNextLeader();
2239
2.83k
2240
2.83k
  OldClass->erase(I);
2241
2.83k
  NewClass->insert(I);
2242
2.83k
2243
2.83k
  if (NewClass->getLeader() != I)
2244
1.04k
    NewClass->addPossibleNextLeader({I, InstrToDFSNum(I)});
2245
2.83k
  // Handle our special casing of stores.
2246
2.83k
  if (auto *SI = dyn_cast<StoreInst>(I)) {
2247
280
    OldClass->decStoreCount();
2248
280
    // Okay, so when do we want to make a store a leader of a class?
2249
280
    // If we have a store defined by an earlier load, we want the earlier load
2250
280
    // to lead the class.
2251
280
    // If we have a store defined by something else, we want the store to lead
2252
280
    // the class so everything else gets the "something else" as a value.
2253
280
    // If we have a store as the single member of the class, we want the store
2254
280
    // as the leader
2255
280
    if (NewClass->getStoreCount() == 0 && 
!NewClass->getStoredValue()262
) {
2256
9
      // If it's a store expression we are using, it means we are not equivalent
2257
9
      // to something earlier.
2258
9
      if (auto *SE = dyn_cast<StoreExpression>(E)) {
2259
9
        NewClass->setStoredValue(SE->getStoredValue());
2260
9
        markValueLeaderChangeTouched(NewClass);
2261
9
        // Shift the new class leader to be the store
2262
9
        LLVM_DEBUG(dbgs() << "Changing leader of congruence class "
2263
9
                          << NewClass->getID() << " from "
2264
9
                          << *NewClass->getLeader() << " to  " << *SI
2265
9
                          << " because store joined class\n");
2266
9
        // If we changed the leader, we have to mark it changed because we don't
2267
9
        // know what it will do to symbolic evaluation.
2268
9
        NewClass->setLeader(SI);
2269
9
      }
2270
9
      // We rely on the code below handling the MemoryAccess change.
2271
9
    }
2272
280
    NewClass->incStoreCount();
2273
280
  }
2274
2.83k
  // True if there is no memory instructions left in a class that had memory
2275
2.83k
  // instructions before.
2276
2.83k
2277
2.83k
  // If it's not a memory use, set the MemoryAccess equivalence
2278
2.83k
  auto *InstMA = dyn_cast_or_null<MemoryDef>(getMemoryAccess(I));
2279
2.83k
  if (InstMA)
2280
458
    moveMemoryToNewCongruenceClass(I, InstMA, OldClass, NewClass);
2281
2.83k
  ValueToClass[I] = NewClass;
2282
2.83k
  // See if we destroyed the class or need to swap leaders.
2283
2.83k
  if (OldClass->empty() && 
OldClass != TOPClass554
) {
2284
302
    if (OldClass->getDefiningExpr()) {
2285
302
      LLVM_DEBUG(dbgs() << "Erasing expression " << *OldClass->getDefiningExpr()
2286
302
                        << " from table\n");
2287
302
      // We erase it as an exact expression to make sure we don't just erase an
2288
302
      // equivalent one.
2289
302
      auto Iter = ExpressionToClass.find_as(
2290
302
          ExactEqualsExpression(*OldClass->getDefiningExpr()));
2291
302
      if (Iter != ExpressionToClass.end())
2292
295
        ExpressionToClass.erase(Iter);
2293
#ifdef EXPENSIVE_CHECKS
2294
      assert(
2295
          (*OldClass->getDefiningExpr() != *E || ExpressionToClass.lookup(E)) &&
2296
          "We erased the expression we just inserted, which should not happen");
2297
#endif
2298
    }
2299
2.52k
  } else if (OldClass->getLeader() == I) {
2300
51
    // When the leader changes, the value numbering of
2301
51
    // everything may change due to symbolization changes, so we need to
2302
51
    // reprocess.
2303
51
    LLVM_DEBUG(dbgs() << "Value class leader change for class "
2304
51
                      << OldClass->getID() << "\n");
2305
51
    ++NumGVNLeaderChanges;
2306
51
    // Destroy the stored value if there are no more stores to represent it.
2307
51
    // Note that this is basically clean up for the expression removal that
2308
51
    // happens below.  If we remove stores from a class, we may leave it as a
2309
51
    // class of equivalent memory phis.
2310
51
    if (OldClass->getStoreCount() == 0) {
2311
46
      if (OldClass->getStoredValue())
2312
5
        OldClass->setStoredValue(nullptr);
2313
46
    }
2314
51
    OldClass->setLeader(getNextValueLeader(OldClass));
2315
51
    OldClass->resetNextLeader();
2316
51
    markValueLeaderChangeTouched(OldClass);
2317
51
  }
2318
2.83k
}
2319
2320
// For a given expression, mark the phi of ops instructions that could have
2321
// changed as a result.
2322
2.83k
void NewGVN::markPhiOfOpsChanged(const Expression *E) {
2323
2.83k
  touchAndErase(ExpressionToPhiOfOps, E);
2324
2.83k
}
2325
2326
// Perform congruence finding on a given value numbering expression.
2327
3.09k
void NewGVN::performCongruenceFinding(Instruction *I, const Expression *E) {
2328
3.09k
  // This is guaranteed to return something, since it will at least find
2329
3.09k
  // TOP.
2330
3.09k
2331
3.09k
  CongruenceClass *IClass = ValueToClass.lookup(I);
2332
3.09k
  assert(IClass && "Should have found a IClass");
2333
3.09k
  // Dead classes should have been eliminated from the mapping.
2334
3.09k
  assert(!IClass->isDead() && "Found a dead class");
2335
3.09k
2336
3.09k
  CongruenceClass *EClass = nullptr;
2337
3.09k
  if (const auto *VE = dyn_cast<VariableExpression>(E)) {
2338
261
    EClass = ValueToClass.lookup(VE->getVariableValue());
2339
2.82k
  } else if (isa<DeadExpression>(E)) {
2340
0
    EClass = TOPClass;
2341
0
  }
2342
3.09k
  if (!EClass) {
2343
2.82k
    auto lookupResult = ExpressionToClass.insert({E, nullptr});
2344
2.82k
2345
2.82k
    // If it's not in the value table, create a new congruence class.
2346
2.82k
    if (lookupResult.second) {
2347
2.13k
      CongruenceClass *NewClass = createCongruenceClass(nullptr, E);
2348
2.13k
      auto place = lookupResult.first;
2349
2.13k
      place->second = NewClass;
2350
2.13k
2351
2.13k
      // Constants and variables should always be made the leader.
2352
2.13k
      if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2353
341
        NewClass->setLeader(CE->getConstantValue());
2354
1.78k
      } else if (const auto *SE = dyn_cast<StoreExpression>(E)) {
2355
253
        StoreInst *SI = SE->getStoreInst();
2356
253
        NewClass->setLeader(SI);
2357
253
        NewClass->setStoredValue(SE->getStoredValue());
2358
253
        // The RepMemoryAccess field will be filled in properly by the
2359
253
        // moveValueToNewCongruenceClass call.
2360
1.53k
      } else {
2361
1.53k
        NewClass->setLeader(I);
2362
1.53k
      }
2363
2.13k
      assert(!isa<VariableExpression>(E) &&
2364
2.13k
             "VariableExpression should have been handled already");
2365
2.13k
2366
2.13k
      EClass = NewClass;
2367
2.13k
      LLVM_DEBUG(dbgs() << "Created new congruence class for " << *I
2368
2.13k
                        << " using expression " << *E << " at "
2369
2.13k
                        << NewClass->getID() << " and leader "
2370
2.13k
                        << *(NewClass->getLeader()));
2371
2.13k
      if (NewClass->getStoredValue())
2372
2.13k
        LLVM_DEBUG(dbgs() << " and stored value "
2373
2.13k
                          << *(NewClass->getStoredValue()));
2374
2.13k
      LLVM_DEBUG(dbgs() << "\n");
2375
2.13k
    } else {
2376
699
      EClass = lookupResult.first->second;
2377
699
      if (isa<ConstantExpression>(E))
2378
699
        assert((isa<Constant>(EClass->getLeader()) ||
2379
699
                (EClass->getStoredValue() &&
2380
699
                 isa<Constant>(EClass->getStoredValue()))) &&
2381
699
               "Any class with a constant expression should have a "
2382
699
               "constant leader");
2383
699
2384
699
      assert(EClass && "Somehow don't have an eclass");
2385
699
2386
699
      assert(!EClass->isDead() && "We accidentally looked up a dead class");
2387
699
    }
2388
2.82k
  }
2389
3.09k
  bool ClassChanged = IClass != EClass;
2390
3.09k
  bool LeaderChanged = LeaderChanges.erase(I);
2391
3.09k
  if (ClassChanged || 
LeaderChanged260
) {
2392
2.84k
    LLVM_DEBUG(dbgs() << "New class " << EClass->getID() << " for expression "
2393
2.84k
                      << *E << "\n");
2394
2.84k
    if (ClassChanged) {
2395
2.83k
      moveValueToNewCongruenceClass(I, E, IClass, EClass);
2396
2.83k
      markPhiOfOpsChanged(E);
2397
2.83k
    }
2398
2.84k
2399
2.84k
    markUsersTouched(I);
2400
2.84k
    if (MemoryAccess *MA = getMemoryAccess(I))
2401
986
      markMemoryUsersTouched(MA);
2402
2.84k
    if (auto *CI = dyn_cast<CmpInst>(I))
2403
301
      markPredicateUsersTouched(CI);
2404
2.84k
  }
2405
3.09k
  // If we changed the class of the store, we want to ensure nothing finds the
2406
3.09k
  // old store expression.  In particular, loads do not compare against stored
2407
3.09k
  // value, so they will find old store expressions (and associated class
2408
3.09k
  // mappings) if we leave them in the table.
2409
3.09k
  if (ClassChanged && 
isa<StoreInst>(I)2.83k
) {
2410
280
    auto *OldE = ValueToExpression.lookup(I);
2411
280
    // It could just be that the old class died. We don't want to erase it if we
2412
280
    // just moved classes.
2413
280
    if (OldE && 
isa<StoreExpression>(OldE)36
&&
*E != *OldE36
) {
2414
36
      // Erase this as an exact expression to ensure we don't erase expressions
2415
36
      // equivalent to it.
2416
36
      auto Iter = ExpressionToClass.find_as(ExactEqualsExpression(*OldE));
2417
36
      if (Iter != ExpressionToClass.end())
2418
7
        ExpressionToClass.erase(Iter);
2419
36
    }
2420
280
  }
2421
3.09k
  ValueToExpression[I] = E;
2422
3.09k
}
2423
2424
// Process the fact that Edge (from, to) is reachable, including marking
2425
// any newly reachable blocks and instructions for processing.
2426
1.18k
void NewGVN::updateReachableEdge(BasicBlock *From, BasicBlock *To) {
2427
1.18k
  // Check if the Edge was reachable before.
2428
1.18k
  if (ReachableEdges.insert({From, To}).second) {
2429
1.10k
    // If this block wasn't reachable before, all instructions are touched.
2430
1.10k
    if (ReachableBlocks.insert(To).second) {
2431
815
      LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)
2432
815
                        << " marked reachable\n");
2433
815
      const auto &InstRange = BlockInstRange.lookup(To);
2434
815
      TouchedInstructions.set(InstRange.first, InstRange.second);
2435
815
    } else {
2436
293
      LLVM_DEBUG(dbgs() << "Block " << getBlockName(To)
2437
293
                        << " was reachable, but new edge {"
2438
293
                        << getBlockName(From) << "," << getBlockName(To)
2439
293
                        << "} to it found\n");
2440
293
2441
293
      // We've made an edge reachable to an existing block, which may
2442
293
      // impact predicates. Otherwise, only mark the phi nodes as touched, as
2443
293
      // they are the only thing that depend on new edges. Anything using their
2444
293
      // values will get propagated to if necessary.
2445
293
      if (MemoryAccess *MemPhi = getMemoryAccess(To))
2446
142
        TouchedInstructions.set(InstrToDFSNum(MemPhi));
2447
293
2448
293
      // FIXME: We should just add a union op on a Bitvector and
2449
293
      // SparseBitVector.  We can do it word by word faster than we are doing it
2450
293
      // here.
2451
293
      for (auto InstNum : RevisitOnReachabilityChange[To])
2452
197
        TouchedInstructions.set(InstNum);
2453
293
    }
2454
1.10k
  }
2455
1.18k
}
2456
2457
// Given a predicate condition (from a switch, cmp, or whatever) and a block,
2458
// see if we know some constant value for it already.
2459
417
Value *NewGVN::findConditionEquivalence(Value *Cond) const {
2460
417
  auto Result = lookupOperandLeader(Cond);
2461
417
  return isa<Constant>(Result) ? 
Result199
:
nullptr218
;
2462
417
}
2463
2464
// Process the outgoing edges of a block for reachability.
2465
1.20k
void NewGVN::processOutgoingEdges(Instruction *TI, BasicBlock *B) {
2466
1.20k
  // Evaluate reachability of terminator instruction.
2467
1.20k
  BranchInst *BR;
2468
1.20k
  if ((BR = dyn_cast<BranchInst>(TI)) && 
BR->isConditional()802
) {
2469
399
    Value *Cond = BR->getCondition();
2470
399
    Value *CondEvaluated = findConditionEquivalence(Cond);
2471
399
    if (!CondEvaluated) {
2472
212
      if (auto *I = dyn_cast<Instruction>(Cond)) {
2473
183
        const Expression *E = createExpression(I);
2474
183
        if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
2475
0
          CondEvaluated = CE->getConstantValue();
2476
0
        }
2477
183
      } else 
if (29
isa<ConstantInt>(Cond)29
) {
2478
0
        CondEvaluated = Cond;
2479
0
      }
2480
212
    }
2481
399
    ConstantInt *CI;
2482
399
    BasicBlock *TrueSucc = BR->getSuccessor(0);
2483
399
    BasicBlock *FalseSucc = BR->getSuccessor(1);
2484
399
    if (CondEvaluated && 
(CI = dyn_cast<ConstantInt>(CondEvaluated))187
) {
2485
88
      if (CI->isOne()) {
2486
36
        LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TI
2487
36
                          << " evaluated to true\n");
2488
36
        updateReachableEdge(B, TrueSucc);
2489
52
      } else if (CI->isZero()) {
2490
52
        LLVM_DEBUG(dbgs() << "Condition for Terminator " << *TI
2491
52
                          << " evaluated to false\n");
2492
52
        updateReachableEdge(B, FalseSucc);
2493
52
      }
2494
311
    } else {
2495
311
      updateReachableEdge(B, TrueSucc);
2496
311
      updateReachableEdge(B, FalseSucc);
2497
311
    }
2498
802
  } else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
2499
18
    // For switches, propagate the case values into the case
2500
18
    // destinations.
2501
18
2502
18
    Value *SwitchCond = SI->getCondition();
2503
18
    Value *CondEvaluated = findConditionEquivalence(SwitchCond);
2504
18
    // See if we were able to turn this switch statement into a constant.
2505
18
    if (CondEvaluated && 
isa<ConstantInt>(CondEvaluated)12
) {
2506
5
      auto *CondVal = cast<ConstantInt>(CondEvaluated);
2507
5
      // We should be able to get case value for this.
2508
5
      auto Case = *SI->findCaseValue(CondVal);
2509
5
      if (Case.getCaseSuccessor() == SI->getDefaultDest()) {
2510
4
        // We proved the value is outside of the range of the case.
2511
4
        // We can't do anything other than mark the default dest as reachable,
2512
4
        // and go home.
2513
4
        updateReachableEdge(B, SI->getDefaultDest());
2514
4
        return;
2515
4
      }
2516
1
      // Now get where it goes and mark it reachable.
2517
1
      BasicBlock *TargetBlock = Case.getCaseSuccessor();
2518
1
      updateReachableEdge(B, TargetBlock);
2519
13
    } else {
2520
62
      for (unsigned i = 0, e = SI->getNumSuccessors(); i != e; 
++i49
) {
2521
49
        BasicBlock *TargetBlock = SI->getSuccessor(i);
2522
49
        updateReachableEdge(B, TargetBlock);
2523
49
      }
2524
13
    }
2525
784
  } else {
2526
784
    // Otherwise this is either unconditional, or a type we have no
2527
784
    // idea about. Just mark successors as reachable.
2528
1.20k
    for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; 
++i421
) {
2529
421
      BasicBlock *TargetBlock = TI->getSuccessor(i);
2530
421
      updateReachableEdge(B, TargetBlock);
2531
421
    }
2532
784
2533
784
    // This also may be a memory defining terminator, in which case, set it
2534
784
    // equivalent only to itself.
2535
784
    //
2536
784
    auto *MA = getMemoryAccess(TI);
2537
784
    if (MA && 
!isa<MemoryUse>(MA)9
) {
2538
9
      auto *CC = ensureLeaderOfMemoryClass(MA);
2539
9
      if (setMemoryClass(MA, CC))
2540
8
        markMemoryUsersTouched(MA);
2541
9
    }
2542
784
  }
2543
1.20k
}
2544
2545
// Remove the PHI of Ops PHI for I
2546
8
void NewGVN::removePhiOfOps(Instruction *I, PHINode *PHITemp) {
2547
8
  InstrDFS.erase(PHITemp);
2548
8
  // It's still a temp instruction. We keep it in the array so it gets erased.
2549
8
  // However, it's no longer used by I, or in the block
2550
8
  TempToBlock.erase(PHITemp);
2551
8
  RealToTemp.erase(I);
2552
8
  // We don't remove the users from the phi node uses. This wastes a little
2553
8
  // time, but such is life.  We could use two sets to track which were there
2554
8
  // are the start of NewGVN, and which were added, but right nowt he cost of
2555
8
  // tracking is more than the cost of checking for more phi of ops.
2556
8
}
2557
2558
// Add PHI Op in BB as a PHI of operations version of ExistingValue.
2559
void NewGVN::addPhiOfOps(PHINode *Op, BasicBlock *BB,
2560
36
                         Instruction *ExistingValue) {
2561
36
  InstrDFS[Op] = InstrToDFSNum(ExistingValue);
2562
36
  AllTempInstructions.insert(Op);
2563
36
  TempToBlock[Op] = BB;
2564
36
  RealToTemp[ExistingValue] = Op;
2565
36
  // Add all users to phi node use, as they are now uses of the phi of ops phis
2566
36
  // and may themselves be phi of ops.
2567
36
  for (auto *U : ExistingValue->users())
2568
38
    if (auto *UI = dyn_cast<Instruction>(U))
2569
38
      PHINodeUses.insert(UI);
2570
36
}
2571
2572
551
static bool okayForPHIOfOps(const Instruction *I) {
2573
551
  if (!EnablePhiOfOps)
2574
0
    return false;
2575
551
  return isa<BinaryOperator>(I) || 
isa<SelectInst>(I)334
||
isa<CmpInst>(I)320
||
2576
551
         
isa<LoadInst>(I)232
;
2577
551
}
2578
2579
bool NewGVN::OpIsSafeForPHIOfOpsHelper(
2580
    Value *V, const BasicBlock *PHIBlock,
2581
    SmallPtrSetImpl<const Value *> &Visited,
2582
190
    SmallVectorImpl<Instruction *> &Worklist) {
2583
190
2584
190
  if (!isa<Instruction>(V))
2585
111
    return true;
2586
79
  auto OISIt = OpSafeForPHIOfOps.find(V);
2587
79
  if (OISIt != OpSafeForPHIOfOps.end())
2588
39
    return OISIt->second;
2589
40
2590
40
  // Keep walking until we either dominate the phi block, or hit a phi, or run
2591
40
  // out of things to check.
2592
40
  if (DT->properlyDominates(getBlockForValue(V), PHIBlock)) {
2593
18
    OpSafeForPHIOfOps.insert({V, true});
2594
18
    return true;
2595
18
  }
2596
22
  // PHI in the same block.
2597
22
  if (isa<PHINode>(V) && 
getBlockForValue(V) == PHIBlock6
) {
2598
5
    OpSafeForPHIOfOps.insert({V, false});
2599
5
    return false;
2600
5
  }
2601
17
2602
17
  auto *OrigI = cast<Instruction>(V);
2603
32
  for (auto *Op : OrigI->operand_values()) {
2604
32
    if (!isa<Instruction>(Op))
2605
16
      continue;
2606
16
    // Stop now if we find an unsafe operand.
2607
16
    auto OISIt = OpSafeForPHIOfOps.find(OrigI);
2608
16
    if (OISIt != OpSafeForPHIOfOps.end()) {
2609
0
      if (!OISIt->second) {
2610
0
        OpSafeForPHIOfOps.insert({V, false});
2611
0
        return false;
2612
0
      }
2613
0
      continue;
2614
0
    }
2615
16
    if (!Visited.insert(Op).second)
2616
0
      continue;
2617
16
    Worklist.push_back(cast<Instruction>(Op));
2618
16
  }
2619
17
  return true;
2620
17
}
2621
2622
// Return true if this operand will be safe to use for phi of ops.
2623
//
2624
// The reason some operands are unsafe is that we are not trying to recursively
2625
// translate everything back through phi nodes.  We actually expect some lookups
2626
// of expressions to fail.  In particular, a lookup where the expression cannot
2627
// exist in the predecessor.  This is true even if the expression, as shown, can
2628
// be determined to be constant.
2629
bool NewGVN::OpIsSafeForPHIOfOps(Value *V, const BasicBlock *PHIBlock,
2630
178
                                 SmallPtrSetImpl<const Value *> &Visited) {
2631
178
  SmallVector<Instruction *, 4> Worklist;
2632
178
  if (!OpIsSafeForPHIOfOpsHelper(V, PHIBlock, Visited, Worklist))
2633
0
    return false;
2634
183
  
while (178
!Worklist.empty()) {
2635
12
    auto *I = Worklist.pop_back_val();
2636
12
    if (!OpIsSafeForPHIOfOpsHelper(I, PHIBlock, Visited, Worklist))
2637
7
      return false;
2638
12
  }
2639
178
  OpSafeForPHIOfOps.insert({V, true});
2640
171
  return true;
2641
178
}
2642
2643
// Try to find a leader for instruction TransInst, which is a phi translated
2644
// version of something in our original program.  Visited is used to ensure we
2645
// don't infinite loop during translations of cycles.  OrigInst is the
2646
// instruction in the original program, and PredBB is the predecessor we
2647
// translated it through.
2648
Value *NewGVN::findLeaderForInst(Instruction *TransInst,
2649
                                 SmallPtrSetImpl<Value *> &Visited,
2650
                                 MemoryAccess *MemAccess, Instruction *OrigInst,
2651
193
                                 BasicBlock *PredBB) {
2652
193
  unsigned IDFSNum = InstrToDFSNum(OrigInst);
2653
193
  // Make sure it's marked as a temporary instruction.
2654
193
  AllTempInstructions.insert(TransInst);
2655
193
  // and make sure anything that tries to add it's DFS number is
2656
193
  // redirected to the instruction we are making a phi of ops
2657
193
  // for.
2658
193
  TempToBlock.insert({TransInst, PredBB});
2659
193
  InstrDFS.insert({TransInst, IDFSNum});
2660
193
2661
193
  const Expression *E = performSymbolicEvaluation(TransInst, Visited);
2662
193
  InstrDFS.erase(TransInst);
2663
193
  AllTempInstructions.erase(TransInst);
2664
193
  TempToBlock.erase(TransInst);
2665
193
  if (MemAccess)
2666
22
    TempToMemory.erase(TransInst);
2667
193
  if (!E)
2668
0
    return nullptr;
2669
193
  auto *FoundVal = findPHIOfOpsLeader(E, OrigInst, PredBB);
2670
193
  if (!FoundVal) {
2671
81
    ExpressionToPhiOfOps[E].insert(OrigInst);
2672
81
    LLVM_DEBUG(dbgs() << "Cannot find phi of ops operand for " << *TransInst
2673
81
                      << " in block " << getBlockName(PredBB) << "\n");
2674
81
    return nullptr;
2675
81
  }
2676
112
  if (auto *SI = dyn_cast<StoreInst>(FoundVal))
2677
1
    FoundVal = SI->getValueOperand();
2678
112
  return FoundVal;
2679
112
}
2680
2681
// When we see an instruction that is an op of phis, generate the equivalent phi
2682
// of ops form.
2683
const Expression *
2684
NewGVN::makePossiblePHIOfOps(Instruction *I,
2685
223
                             SmallPtrSetImpl<Value *> &Visited) {
2686
223
  if (!okayForPHIOfOps(I))
2687
8
    return nullptr;
2688
215
2689
215
  if (!Visited.insert(I).second)
2690
0
    return nullptr;
2691
215
  // For now, we require the instruction be cycle free because we don't
2692
215
  // *always* create a phi of ops for instructions that could be done as phi
2693
215
  // of ops, we only do it if we think it is useful.  If we did do it all the
2694
215
  // time, we could remove the cycle free check.
2695
215
  if (!isCycleFree(I))
2696
72
    return nullptr;
2697
143
2698
143
  SmallPtrSet<const Value *, 8> ProcessedPHIs;
2699
143
  // TODO: We don't do phi translation on memory accesses because it's
2700
143
  // complicated. For a load, we'd need to be able to simulate a new memoryuse,
2701
143
  // which we don't have a good way of doing ATM.
2702
143
  auto *MemAccess = getMemoryAccess(I);
2703
143
  // If the memory operation is defined by a memory operation this block that
2704
143
  // isn't a MemoryPhi, transforming the pointer backwards through a scalar phi
2705
143
  // can't help, as it would still be killed by that memory operation.
2706
143
  if (MemAccess && 
!isa<MemoryPhi>(MemAccess->getDefiningAccess())19
&&
2707
143
      
MemAccess->getDefiningAccess()->getBlock() == I->getParent()8
)
2708
0
    return nullptr;
2709
143
2710
143
  // Convert op of phis to phi of ops
2711
143
  SmallPtrSet<const Value *, 10> VisitedOps;
2712
143
  SmallVector<Value *, 4> Ops(I->operand_values());
2713
143
  BasicBlock *SamePHIBlock = nullptr;
2714
143
  PHINode *OpPHI = nullptr;
2715
143
  if (!DebugCounter::shouldExecute(PHIOfOpsCounter))
2716
0
    return nullptr;
2717
272
  
for (auto *Op : Ops)143
{
2718
272
    if (!isa<PHINode>(Op)) {
2719
139
      auto *ValuePHI = RealToTemp.lookup(Op);
2720
139
      if (!ValuePHI)
2721
124
        continue;
2722
15
      LLVM_DEBUG(dbgs() << "Found possible dependent phi of ops\n");
2723
15
      Op = ValuePHI;
2724
15
    }
2725
272
    OpPHI = cast<PHINode>(Op);
2726
148
    if (!SamePHIBlock) {
2727
140
      SamePHIBlock = getBlockForValue(OpPHI);
2728
140
    } else 
if (8
SamePHIBlock != getBlockForValue(OpPHI)8
) {
2729
3
      LLVM_DEBUG(
2730
3
          dbgs()
2731
3
          << "PHIs for operands are not all in the same block, aborting\n");
2732
3
      return nullptr;
2733
3
    }
2734
145
    // No point in doing this for one-operand phis.
2735
145
    if (OpPHI->getNumOperands() == 1) {
2736
2
      OpPHI = nullptr;
2737
2
      continue;
2738
2
    }
2739
145
  }
2740
143
2741
143
  
if (140
!OpPHI140
)
2742
5
    return nullptr;
2743
135
2744
135
  SmallVector<ValPair, 4> PHIOps;
2745
135
  SmallPtrSet<Value *, 4> Deps;
2746
135
  auto *PHIBlock = getBlockForValue(OpPHI);
2747
135
  RevisitOnReachabilityChange[PHIBlock].reset(InstrToDFSNum(I));
2748
258
  for (unsigned PredNum = 0; PredNum < OpPHI->getNumOperands(); 
++PredNum123
) {
2749
211
    auto *PredBB = OpPHI->getIncomingBlock(PredNum);
2750
211
    Value *FoundVal = nullptr;
2751
211
    SmallPtrSet<Value *, 4> CurrentDeps;
2752
211
    // We could just skip unreachable edges entirely but it's tricky to do
2753
211
    // with rewriting existing phi nodes.
2754
211
    if (ReachableEdges.count({PredBB, PHIBlock})) {
2755
200
      // Clone the instruction, create an expression from it that is
2756
200
      // translated back into the predecessor, and see if we have a leader.
2757
200
      Instruction *ValueOp = I->clone();
2758
200
      if (MemAccess)
2759
22
        TempToMemory.insert({ValueOp, MemAccess});
2760
200
      bool SafeForPHIOfOps = true;
2761
200
      VisitedOps.clear();
2762
387
      for (auto &Op : ValueOp->operands()) {
2763
387
        auto *OrigOp = &*Op;
2764
387
        // When these operand changes, it could change whether there is a
2765
387
        // leader for us or not, so we have to add additional users.
2766
387
        if (isa<PHINode>(Op)) {
2767
186
          Op = Op->DoPHITranslation(PHIBlock, PredBB);
2768
186
          if (Op != OrigOp && Op != I)
2769
186
            CurrentDeps.insert(Op);
2770
201
        } else if (auto *ValuePHI = RealToTemp.lookup(Op)) {
2771
22
          if (getBlockForValue(ValuePHI) == PHIBlock)
2772
22
            Op = ValuePHI->getIncomingValueForBlock(PredBB);
2773
22
        }
2774
387
        // If we phi-translated the op, it must be safe.
2775
387
        SafeForPHIOfOps =
2776
387
            SafeForPHIOfOps &&
2777
387
            
(380
Op != OrigOp380
||
OpIsSafeForPHIOfOps(Op, PHIBlock, VisitedOps)178
);
2778
387
      }
2779
200
      // FIXME: For those things that are not safe we could generate
2780
200
      // expressions all the way down, and see if this comes out to a
2781
200
      // constant.  For anything where that is true, and unsafe, we should
2782
200
      // have made a phi-of-ops (or value numbered it equivalent to something)
2783
200
      // for the pieces already.
2784
200
      FoundVal = !SafeForPHIOfOps ? 
nullptr7
2785
200
                                  : findLeaderForInst(ValueOp, Visited,
2786
193
                                                      MemAccess, I, PredBB);
2787
200
      ValueOp->deleteValue();
2788
200
      if (!FoundVal) {
2789
88
        // We failed to find a leader for the current ValueOp, but this might
2790
88
        // change in case of the translated operands change.
2791
88
        if (SafeForPHIOfOps)
2792
81
          for (auto Dep : CurrentDeps)
2793
77
            addAdditionalUsers(Dep, I);
2794
88
2795
88
        return nullptr;
2796
88
      }
2797
112
      Deps.insert(CurrentDeps.begin(), CurrentDeps.end());
2798
112
    } else {
2799
11
      LLVM_DEBUG(dbgs() << "Skipping phi of ops operand for incoming block "
2800
11
                        << getBlockName(PredBB)
2801
11
                        << " because the block is unreachable\n");
2802
11
      FoundVal = UndefValue::get(I->getType());
2803
11
      RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I));
2804
11
    }
2805
211
2806
211
    PHIOps.push_back({FoundVal, PredBB});
2807
123
    LLVM_DEBUG(dbgs() << "Found phi of ops operand " << *FoundVal << " in "
2808
123
                      << getBlockName(PredBB) << "\n");
2809
123
  }
2810
135
  
for (auto Dep : Deps)47
2811
82
    addAdditionalUsers(Dep, I);
2812
47
  sortPHIOps(PHIOps);
2813
47
  auto *E = performSymbolicPHIEvaluation(PHIOps, I, PHIBlock);
2814
47
  if (isa<ConstantExpression>(E) || 
isa<VariableExpression>(E)40
) {
2815
10
    LLVM_DEBUG(
2816
10
        dbgs()
2817
10
        << "Not creating real PHI of ops because it simplified to existing "
2818
10
           "value or constant\n");
2819
10
    return E;
2820
10
  }
2821
37
  auto *ValuePHI = RealToTemp.lookup(I);
2822
37
  bool NewPHI = false;
2823
37
  if (!ValuePHI) {
2824
36
    ValuePHI =
2825
36
        PHINode::Create(I->getType(), OpPHI->getNumOperands(), "phiofops");
2826
36
    addPhiOfOps(ValuePHI, PHIBlock, I);
2827
36
    NewPHI = true;
2828
36
    NumGVNPHIOfOpsCreated++;
2829
36
  }
2830
37
  if (NewPHI) {
2831
36
    for (auto PHIOp : PHIOps)
2832
74
      ValuePHI->addIncoming(PHIOp.first, PHIOp.second);
2833
36
  } else {
2834
1
    TempToBlock[ValuePHI] = PHIBlock;
2835
1
    unsigned int i = 0;
2836
2
    for (auto PHIOp : PHIOps) {
2837
2
      ValuePHI->setIncomingValue(i, PHIOp.first);
2838
2
      ValuePHI->setIncomingBlock(i, PHIOp.second);
2839
2
      ++i;
2840
2
    }
2841
1
  }
2842
37
  RevisitOnReachabilityChange[PHIBlock].set(InstrToDFSNum(I));
2843
37
  LLVM_DEBUG(dbgs() << "Created phi of ops " << *ValuePHI << " for " << *I
2844
37
                    << "\n");
2845
37
2846
37
  return E;
2847
37
}
2848
2849
// The algorithm initially places the values of the routine in the TOP
2850
// congruence class. The leader of TOP is the undetermined value `undef`.
2851
// When the algorithm has finished, values still in TOP are unreachable.
2852
333
void NewGVN::initializeCongruenceClasses(Function &F) {
2853
333
  NextCongruenceNum = 0;
2854
333
2855
333
  // Note that even though we use the live on entry def as a representative
2856
333
  // MemoryAccess, it is *not* the same as the actual live on entry def. We
2857
333
  // have no real equivalemnt to undef for MemoryAccesses, and so we really
2858
333
  // should be checking whether the MemoryAccess is top if we want to know if it
2859
333
  // is equivalent to everything.  Otherwise, what this really signifies is that
2860
333
  // the access "it reaches all the way back to the beginning of the function"
2861
333
2862
333
  // Initialize all other instructions to be in TOP class.
2863
333
  TOPClass = createCongruenceClass(nullptr, nullptr);
2864
333
  TOPClass->setMemoryLeader(MSSA->getLiveOnEntryDef());
2865
333
  //  The live on entry def gets put into it's own class
2866
333
  MemoryAccessToClass[MSSA->getLiveOnEntryDef()] =
2867
333
      createMemoryClass(MSSA->getLiveOnEntryDef());
2868
333
2869
1.31k
  for (auto DTN : nodes(DT)) {
2870
1.31k
    BasicBlock *BB = DTN->getBlock();
2871
1.31k
    // All MemoryAccesses are equivalent to live on entry to start. They must
2872
1.31k
    // be initialized to something so that initial changes are noticed. For
2873
1.31k
    // the maximal answer, we initialize them all to be the same as
2874
1.31k
    // liveOnEntry.
2875
1.31k
    auto *MemoryBlockDefs = MSSA->getBlockDefs(BB);
2876
1.31k
    if (MemoryBlockDefs)
2877
588
      
for (const auto &Def : *MemoryBlockDefs)410
{
2878
588
        MemoryAccessToClass[&Def] = TOPClass;
2879
588
        auto *MD = dyn_cast<MemoryDef>(&Def);
2880
588
        // Insert the memory phis into the member list.
2881
588
        if (!MD) {
2882
138
          const MemoryPhi *MP = cast<MemoryPhi>(&Def);
2883
138
          TOPClass->memory_insert(MP);
2884
138
          MemoryPhiState.insert({MP, MPS_TOP});
2885
138
        }
2886
588
2887
588
        if (MD && 
isa<StoreInst>(MD->getMemoryInst())450
)
2888
260
          TOPClass->incStoreCount();
2889
588
      }
2890
1.31k
2891
1.31k
    // FIXME: This is trying to discover which instructions are uses of phi
2892
1.31k
    // nodes.  We should move this into one of the myriad of places that walk
2893
1.31k
    // all the operands already.
2894
3.77k
    for (auto &I : *BB) {
2895
3.77k
      if (isa<PHINode>(&I))
2896
230
        for (auto *U : I.users())
2897
337
          if (auto *UInst = dyn_cast<Instruction>(U))
2898
337
            if (InstrToDFSNum(UInst) != 0 && 
okayForPHIOfOps(UInst)328
)
2899
138
              PHINodeUses.insert(UInst);
2900
3.77k
      // Don't insert void terminators into the class. We don't value number
2901
3.77k
      // them, and they just end up sitting in TOP.
2902
3.77k
      if (I.isTerminator() && 
I.getType()->isVoidTy()1.31k
)
2903
1.30k
        continue;
2904
2.46k
      TOPClass->insert(&I);
2905
2.46k
      ValueToClass[&I] = TOPClass;
2906
2.46k
    }
2907
1.31k
  }
2908
333
2909
333
  // Initialize arguments to be in their own unique congruence classes
2910
333
  for (auto &FA : F.args())
2911
477
    createSingletonCongruenceClass(&FA);
2912
333
}
2913
2914
333
void NewGVN::cleanupTables() {
2915
3.72k
  for (unsigned i = 0, e = CongruenceClasses.size(); i != e; 
++i3.38k
) {
2916
3.38k
    LLVM_DEBUG(dbgs() << "Congruence class " << CongruenceClasses[i]->getID()
2917
3.38k
                      << " has " << CongruenceClasses[i]->size()
2918
3.38k
                      << " members\n");
2919
3.38k
    // Make sure we delete the congruence class (probably worth switching to
2920
3.38k
    // a unique_ptr at some point.
2921
3.38k
    delete CongruenceClasses[i];
2922
3.38k
    CongruenceClasses[i] = nullptr;
2923
3.38k
  }
2924
333
2925
333
  // Destroy the value expressions
2926
333
  SmallVector<Instruction *, 8> TempInst(AllTempInstructions.begin(),
2927
333
                                         AllTempInstructions.end());
2928
333
  AllTempInstructions.clear();
2929
333
2930
333
  // We have to drop all references for everything first, so there are no uses
2931
333
  // left as we delete them.
2932
333
  for (auto *I : TempInst) {
2933
8
    I->dropAllReferences();
2934
8
  }
2935
333
2936
341
  while (!TempInst.empty()) {
2937
8
    auto *I = TempInst.back();
2938
8
    TempInst.pop_back();
2939
8
    I->deleteValue();
2940
8
  }
2941
333
2942
333
  ValueToClass.clear();
2943
333
  ArgRecycler.clear(ExpressionAllocator);
2944
333
  ExpressionAllocator.Reset();
2945
333
  CongruenceClasses.clear();
2946
333
  ExpressionToClass.clear();
2947
333
  ValueToExpression.clear();
2948
333
  RealToTemp.clear();
2949
333
  AdditionalUsers.clear();
2950
333
  ExpressionToPhiOfOps.clear();
2951
333
  TempToBlock.clear();
2952
333
  TempToMemory.clear();
2953
333
  PHINodeUses.clear();
2954
333
  OpSafeForPHIOfOps.clear();
2955
333
  ReachableBlocks.clear();
2956
333
  ReachableEdges.clear();
2957
#ifndef NDEBUG
2958
  ProcessedCount.clear();
2959
#endif
2960
  InstrDFS.clear();
2961
333
  InstructionsToErase.clear();
2962
333
  DFSToInstr.clear();
2963
333
  BlockInstRange.clear();
2964
333
  TouchedInstructions.clear();
2965
333
  MemoryAccessToClass.clear();
2966
333
  PredicateToUsers.clear();
2967
333
  MemoryToUsers.clear();
2968
333
  RevisitOnReachabilityChange.clear();
2969
333
}
2970
2971
// Assign local DFS number mapping to instructions, and leave space for Value
2972
// PHI's.
2973
std::pair<unsigned, unsigned> NewGVN::assignDFSNumbers(BasicBlock *B,
2974
1.31k
                                                       unsigned Start) {
2975
1.31k
  unsigned End = Start;
2976
1.31k
  if (MemoryAccess *MemPhi = getMemoryAccess(B)) {
2977
138
    InstrDFS[MemPhi] = End++;
2978
138
    DFSToInstr.emplace_back(MemPhi);
2979
138
  }
2980
1.31k
2981
1.31k
  // Then the real block goes next.
2982
3.77k
  for (auto &I : *B) {
2983
3.77k
    // There's no need to call isInstructionTriviallyDead more than once on
2984
3.77k
    // an instruction. Therefore, once we know that an instruction is dead
2985
3.77k
    // we change its DFS number so that it doesn't get value numbered.
2986
3.77k
    if (isInstructionTriviallyDead(&I, TLI)) {
2987
105
      InstrDFS[&I] = 0;
2988
105
      LLVM_DEBUG(dbgs() << "Skipping trivially dead instruction " << I << "\n");
2989
105
      markInstructionForDeletion(&I);
2990
105
      continue;
2991
105
    }
2992
3.67k
    if (isa<PHINode>(&I))
2993
225
      RevisitOnReachabilityChange[B].set(End);
2994
3.67k
    InstrDFS[&I] = End++;
2995
3.67k
    DFSToInstr.emplace_back(&I);
2996
3.67k
  }
2997
1.31k
2998
1.31k
  // All of the range functions taken half-open ranges (open on the end side).
2999
1.31k
  // So we do not subtract one from count, because at this point it is one
3000
1.31k
  // greater than the last instruction.
3001
1.31k
  return std::make_pair(Start, End);
3002
1.31k
}
3003
3004
5.68k
void NewGVN::updateProcessedCount(const Value *V) {
3005
#ifndef NDEBUG
3006
  if (ProcessedCount.count(V) == 0) {
3007
    ProcessedCount.insert({V, 1});
3008
  } else {
3009
    ++ProcessedCount[V];
3010
    assert(ProcessedCount[V] < 100 &&
3011
           "Seem to have processed the same Value a lot");
3012
  }
3013
#endif
3014
}
3015
3016
// Evaluate MemoryPhi nodes symbolically, just like PHI nodes
3017
238
void NewGVN::valueNumberMemoryPhi(MemoryPhi *MP) {
3018
238
  // If all the arguments are the same, the MemoryPhi has the same value as the
3019
238
  // argument.  Filter out unreachable blocks and self phis from our operands.
3020
238
  // TODO: We could do cycle-checking on the memory phis to allow valueizing for
3021
238
  // self-phi checking.
3022
238
  const BasicBlock *PHIBlock = MP->getBlock();
3023
503
  auto Filtered = make_filter_range(MP->operands(), [&](const Use &U) {
3024
503
    return cast<MemoryAccess>(U) != MP &&
3025
503
           
!isMemoryAccessTOP(cast<MemoryAccess>(U))495
&&
3026
503
           
ReachableEdges.count({MP->getIncomingBlock(U), PHIBlock})414
;
3027
503
  });
3028
238
  // If all that is left is nothing, our memoryphi is undef. We keep it as
3029
238
  // InitialClass.  Note: The only case this should happen is if we have at
3030
238
  // least one self-argument.
3031
238
  if (Filtered.begin() == Filtered.end()) {
3032
0
    if (setMemoryClass(MP, TOPClass))
3033
0
      markMemoryUsersTouched(MP);
3034
0
    return;
3035
0
  }
3036
238
3037
238
  // Transform the remaining operands into operand leaders.
3038
238
  // FIXME: mapped_iterator should have a range version.
3039
393
  
auto LookupFunc = [&](const Use &U) 238
{
3040
393
    return lookupMemoryLeader(cast<MemoryAccess>(U));
3041
393
  };
3042
238
  auto MappedBegin = map_iterator(Filtered.begin(), LookupFunc);
3043
238
  auto MappedEnd = map_iterator(Filtered.end(), LookupFunc);
3044
238
3045
238
  // and now check if all the elements are equal.
3046
238
  // Sadly, we can't use std::equals since these are random access iterators.
3047
238
  const auto *AllSameValue = *MappedBegin;
3048
238
  ++MappedBegin;
3049
238
  bool AllEqual = std::all_of(
3050
238
      MappedBegin, MappedEnd,
3051
238
      [&AllSameValue](const MemoryAccess *V) 
{ return V == AllSameValue; }155
);
3052
238
3053
238
  if (AllEqual)
3054
238
    LLVM_DEBUG(dbgs() << "Memory Phi value numbered to " << *AllSameValue
3055
238
                      << "\n");
3056
238
  else
3057
238
    LLVM_DEBUG(dbgs() << "Memory Phi value numbered to itself\n");
3058
238
  // If it's equal to something, it's in that class. Otherwise, it has to be in
3059
238
  // a class where it is the leader (other things may be equivalent to it, but
3060
238
  // it needs to start off in its own class, which means it must have been the
3061
238
  // leader, and it can't have stopped being the leader because it was never
3062
238
  // removed).
3063
238
  CongruenceClass *CC =
3064
238
      AllEqual ? 
getMemoryClass(AllSameValue)101
:
ensureLeaderOfMemoryClass(MP)137
;
3065
238
  auto OldState = MemoryPhiState.lookup(MP);
3066
238
  assert(OldState != MPS_Invalid && "Invalid memory phi state");
3067
238
  auto NewState = AllEqual ? 
MPS_Equivalent101
:
MPS_Unique137
;
3068
238
  MemoryPhiState[MP] = NewState;
3069
238
  if (setMemoryClass(MP, CC) || 
OldState != NewState41
)
3070
203
    markMemoryUsersTouched(MP);
3071
238
}
3072
3073
// Value number a single instruction, symbolically evaluating, performing
3074
// congruence finding, and updating mappings.
3075
4.28k
void NewGVN::valueNumberInstruction(Instruction *I) {
3076
4.28k
  LLVM_DEBUG(dbgs() << "Processing instruction " << *I << "\n");
3077
4.28k
  if (!I->isTerminator()) {
3078
3.08k
    const Expression *Symbolized = nullptr;
3079
3.08k
    SmallPtrSet<Value *, 2> Visited;
3080
3.08k
    if (DebugCounter::shouldExecute(VNCounter)) {
3081
3.08k
      Symbolized = performSymbolicEvaluation(I, Visited);
3082
3.08k
      // Make a phi of ops if necessary
3083
3.08k
      if (Symbolized && 
!isa<ConstantExpression>(Symbolized)2.81k
&&
3084
3.08k
          
!isa<VariableExpression>(Symbolized)2.33k
&&
PHINodeUses.count(I)2.07k
) {
3085
223
        auto *PHIE = makePossiblePHIOfOps(I, Visited);
3086
223
        // If we created a phi of ops, use it.
3087
223
        // If we couldn't create one, make sure we don't leave one lying around
3088
223
        if (PHIE) {
3089
47
          Symbolized = PHIE;
3090
176
        } else if (auto *Op = RealToTemp.lookup(I)) {
3091
8
          removePhiOfOps(I, Op);
3092
8
        }
3093
223
      }
3094
3.08k
    } else {
3095
0
      // Mark the instruction as unused so we don't value number it again.
3096
0
      InstrDFS[I] = 0;
3097
0
    }
3098
3.08k
    // If we couldn't come up with a symbolic expression, use the unknown
3099
3.08k
    // expression
3100
3.08k
    if (Symbolized == nullptr)
3101
276
      Symbolized = createUnknownExpression(I);
3102
3.08k
    performCongruenceFinding(I, Symbolized);
3103
3.08k
  } else {
3104
1.20k
    // Handle terminators that return values. All of them produce values we
3105
1.20k
    // don't currently understand.  We don't place non-value producing
3106
1.20k
    // terminators in a class.
3107
1.20k
    if (!I->getType()->isVoidTy()) {
3108
2
      auto *Symbolized = createUnknownExpression(I);
3109
2
      performCongruenceFinding(I, Symbolized);
3110
2
    }
3111
1.20k
    processOutgoingEdges(I, I->getParent());
3112
1.20k
  }
3113
4.28k
}
3114
3115
// Check if there is a path, using single or equal argument phi nodes, from
3116
// First to Second.
3117
bool NewGVN::singleReachablePHIPath(
3118
    SmallPtrSet<const MemoryAccess *, 8> &Visited, const MemoryAccess *First,
3119
0
    const MemoryAccess *Second) const {
3120
0
  if (First == Second)
3121
0
    return true;
3122
0
  if (MSSA->isLiveOnEntryDef(First))
3123
0
    return false;
3124
0
3125
0
  // This is not perfect, but as we're just verifying here, we can live with
3126
0
  // the loss of precision. The real solution would be that of doing strongly
3127
0
  // connected component finding in this routine, and it's probably not worth
3128
0
  // the complexity for the time being. So, we just keep a set of visited
3129
0
  // MemoryAccess and return true when we hit a cycle.
3130
0
  if (Visited.count(First))
3131
0
    return true;
3132
0
  Visited.insert(First);
3133
0
3134
0
  const auto *EndDef = First;
3135
0
  for (auto *ChainDef : optimized_def_chain(First)) {
3136
0
    if (ChainDef == Second)
3137
0
      return true;
3138
0
    if (MSSA->isLiveOnEntryDef(ChainDef))
3139
0
      return false;
3140
0
    EndDef = ChainDef;
3141
0
  }
3142
0
  auto *MP = cast<MemoryPhi>(EndDef);
3143
0
  auto ReachableOperandPred = [&](const Use &U) {
3144
0
    return ReachableEdges.count({MP->getIncomingBlock(U), MP->getBlock()});
3145
0
  };
3146
0
  auto FilteredPhiArgs =
3147
0
      make_filter_range(MP->operands(), ReachableOperandPred);
3148
0
  SmallVector<const Value *, 32> OperandList;
3149
0
  llvm::copy(FilteredPhiArgs, std::back_inserter(OperandList));
3150
0
  bool Okay = is_splat(OperandList);
3151
0
  if (Okay)
3152
0
    return singleReachablePHIPath(Visited, cast<MemoryAccess>(OperandList[0]),
3153
0
                                  Second);
3154
0
  return false;
3155
0
}
3156
3157
// Verify the that the memory equivalence table makes sense relative to the
3158
// congruence classes.  Note that this checking is not perfect, and is currently
3159
// subject to very rare false negatives. It is only useful for
3160
// testing/debugging.
3161
333
void NewGVN::verifyMemoryCongruency() const {
3162
#ifndef NDEBUG
3163
  // Verify that the memory table equivalence and memory member set match
3164
  for (const auto *CC : CongruenceClasses) {
3165
    if (CC == TOPClass || CC->isDead())
3166
      continue;
3167
    if (CC->getStoreCount() != 0) {
3168
      assert((CC->getStoredValue() || !isa<StoreInst>(CC->getLeader())) &&
3169
             "Any class with a store as a leader should have a "
3170
             "representative stored value");
3171
      assert(CC->getMemoryLeader() &&
3172
             "Any congruence class with a store should have a "
3173
             "representative access");
3174
    }
3175
3176
    if (CC->getMemoryLeader())
3177
      assert(MemoryAccessToClass.lookup(CC->getMemoryLeader()) == CC &&
3178
             "Representative MemoryAccess does not appear to be reverse "
3179
             "mapped properly");
3180
    for (auto M : CC->memory())
3181
      assert(MemoryAccessToClass.lookup(M) == CC &&
3182
             "Memory member does not appear to be reverse mapped properly");
3183
  }
3184
3185
  // Anything equivalent in the MemoryAccess table should be in the same
3186
  // congruence class.
3187
3188
  // Filter out the unreachable and trivially dead entries, because they may
3189
  // never have been updated if the instructions were not processed.
3190
  auto ReachableAccessPred =
3191
      [&](const std::pair<const MemoryAccess *, CongruenceClass *> Pair) {
3192
        bool Result = ReachableBlocks.count(Pair.first->getBlock());
3193
        if (!Result || MSSA->isLiveOnEntryDef(Pair.first) ||
3194
            MemoryToDFSNum(Pair.first) == 0)
3195
          return false;
3196
        if (auto *MemDef = dyn_cast<MemoryDef>(Pair.first))
3197
          return !isInstructionTriviallyDead(MemDef->getMemoryInst());
3198
3199
        // We could have phi nodes which operands are all trivially dead,
3200
        // so we don't process them.
3201
        if (auto *MemPHI = dyn_cast<MemoryPhi>(Pair.first)) {
3202
          for (auto &U : MemPHI->incoming_values()) {
3203
            if (auto *I = dyn_cast<Instruction>(&*U)) {
3204
              if (!isInstructionTriviallyDead(I))
3205
                return true;
3206
            }
3207
          }
3208
          return false;
3209
        }
3210
3211
        return true;
3212
      };
3213
3214
  auto Filtered = make_filter_range(MemoryAccessToClass, ReachableAccessPred);
3215
  for (auto KV : Filtered) {
3216
    if (auto *FirstMUD = dyn_cast<MemoryUseOrDef>(KV.first)) {
3217
      auto *SecondMUD = dyn_cast<MemoryUseOrDef>(KV.second->getMemoryLeader());
3218
      if (FirstMUD && SecondMUD) {
3219
        SmallPtrSet<const MemoryAccess *, 8> VisitedMAS;
3220
        assert((singleReachablePHIPath(VisitedMAS, FirstMUD, SecondMUD) ||
3221
                ValueToClass.lookup(FirstMUD->getMemoryInst()) ==
3222
                    ValueToClass.lookup(SecondMUD->getMemoryInst())) &&
3223
               "The instructions for these memory operations should have "
3224
               "been in the same congruence class or reachable through"
3225
               "a single argument phi");
3226
      }
3227
    } else if (auto *FirstMP = dyn_cast<MemoryPhi>(KV.first)) {
3228
      // We can only sanely verify that MemoryDefs in the operand list all have
3229
      // the same class.
3230
      auto ReachableOperandPred = [&](const Use &U) {
3231
        return ReachableEdges.count(
3232
                   {FirstMP->getIncomingBlock(U), FirstMP->getBlock()}) &&
3233
               isa<MemoryDef>(U);
3234
3235
      };
3236
      // All arguments should in the same class, ignoring unreachable arguments
3237
      auto FilteredPhiArgs =
3238
          make_filter_range(FirstMP->operands(), ReachableOperandPred);
3239
      SmallVector<const CongruenceClass *, 16> PhiOpClasses;
3240
      std::transform(FilteredPhiArgs.begin(), FilteredPhiArgs.end(),
3241
                     std::back_inserter(PhiOpClasses), [&](const Use &U) {
3242
                       const MemoryDef *MD = cast<MemoryDef>(U);
3243
                       return ValueToClass.lookup(MD->getMemoryInst());
3244
                     });
3245
      assert(is_splat(PhiOpClasses) &&
3246
             "All MemoryPhi arguments should be in the same class");
3247
    }
3248
  }
3249
#endif
3250
}
3251
3252
// Verify that the sparse propagation we did actually found the maximal fixpoint
3253
// We do this by storing the value to class mapping, touching all instructions,
3254
// and redoing the iteration to see if anything changed.
3255
333
void NewGVN::verifyIterationSettled(Function &F) {
3256
#ifndef NDEBUG
3257
  LLVM_DEBUG(dbgs() << "Beginning iteration verification\n");
3258
  if (DebugCounter::isCounterSet(VNCounter))
3259
    DebugCounter::setCounterValue(VNCounter, StartingVNCounter);
3260
3261
  // Note that we have to store the actual classes, as we may change existing
3262
  // classes during iteration.  This is because our memory iteration propagation
3263
  // is not perfect, and so may waste a little work.  But it should generate
3264
  // exactly the same congruence classes we have now, with different IDs.
3265
  std::map<const Value *, CongruenceClass> BeforeIteration;
3266
3267
  for (auto &KV : ValueToClass) {
3268
    if (auto *I = dyn_cast<Instruction>(KV.first))
3269
      // Skip unused/dead instructions.
3270
      if (InstrToDFSNum(I) == 0)
3271
        continue;
3272
    BeforeIteration.insert({KV.first, *KV.second});
3273
  }
3274
3275
  TouchedInstructions.set();
3276
  TouchedInstructions.reset(0);
3277
  iterateTouchedInstructions();
3278
  DenseSet<std::pair<const CongruenceClass *, const CongruenceClass *>>
3279
      EqualClasses;
3280
  for (const auto &KV : ValueToClass) {
3281
    if (auto *I = dyn_cast<Instruction>(KV.first))
3282
      // Skip unused/dead instructions.
3283
      if (InstrToDFSNum(I) == 0)
3284
        continue;
3285
    // We could sink these uses, but i think this adds a bit of clarity here as
3286
    // to what we are comparing.
3287
    auto *BeforeCC = &BeforeIteration.find(KV.first)->second;
3288
    auto *AfterCC = KV.second;
3289
    // Note that the classes can't change at this point, so we memoize the set
3290
    // that are equal.
3291
    if (!EqualClasses.count({BeforeCC, AfterCC})) {
3292
      assert(BeforeCC->isEquivalentTo(AfterCC) &&
3293
             "Value number changed after main loop completed!");
3294
      EqualClasses.insert({BeforeCC, AfterCC});
3295
    }
3296
  }
3297
#endif
3298
}
3299
3300
// Verify that for each store expression in the expression to class mapping,
3301
// only the latest appears, and multiple ones do not appear.
3302
// Because loads do not use the stored value when doing equality with stores,
3303
// if we don't erase the old store expressions from the table, a load can find
3304
// a no-longer valid StoreExpression.
3305
333
void NewGVN::verifyStoreExpressions() const {
3306
#ifndef NDEBUG
3307
  // This is the only use of this, and it's not worth defining a complicated
3308
  // densemapinfo hash/equality function for it.
3309
  std::set<
3310
      std::pair<const Value *,
3311
                std::tuple<const Value *, const CongruenceClass *, Value *>>>
3312
      StoreExpressionSet;
3313
  for (const auto &KV : ExpressionToClass) {
3314
    if (auto *SE = dyn_cast<StoreExpression>(KV.first)) {
3315
      // Make sure a version that will conflict with loads is not already there
3316
      auto Res = StoreExpressionSet.insert(
3317
          {SE->getOperand(0), std::make_tuple(SE->getMemoryLeader(), KV.second,
3318
                                              SE->getStoredValue())});
3319
      bool Okay = Res.second;
3320
      // It's okay to have the same expression already in there if it is
3321
      // identical in nature.
3322
      // This can happen when the leader of the stored value changes over time.
3323
      if (!Okay)
3324
        Okay = (std::get<1>(Res.first->second) == KV.second) &&
3325
               (lookupOperandLeader(std::get<2>(Res.first->second)) ==
3326
                lookupOperandLeader(SE->getStoredValue()));
3327
      assert(Okay && "Stored expression conflict exists in expression table");
3328
      auto *ValueExpr = ValueToExpression.lookup(SE->getStoreInst());
3329
      assert(ValueExpr && ValueExpr->equals(*SE) &&
3330
             "StoreExpression in ExpressionToClass is not latest "
3331
             "StoreExpression for value");
3332
    }
3333
  }
3334
#endif
3335
}
3336
3337
// This is the main value numbering loop, it iterates over the initial touched
3338
// instruction set, propagating value numbers, marking things touched, etc,
3339
// until the set of touched instructions is completely empty.
3340
333
void NewGVN::iterateTouchedInstructions() {
3341
333
  unsigned int Iterations = 0;
3342
333
  // Figure out where touchedinstructions starts
3343
333
  int FirstInstr = TouchedInstructions.find_first();
3344
333
  // Nothing set, nothing to iterate, just return.
3345
333
  if (FirstInstr == -1)
3346
0
    return;
3347
333
  const BasicBlock *LastBlock = getBlockForValue(InstrFromDFSNum(FirstInstr));
3348
833
  while (TouchedInstructions.any()) {
3349
500
    ++Iterations;
3350
500
    // Walk through all the instructions in all the blocks in RPO.
3351
500
    // TODO: As we hit a new block, we should push and pop equalities into a
3352
500
    // table lookupOperandLeader can use, to catch things PredicateInfo
3353
500
    // might miss, like edge-only equivalences.
3354
4.61k
    for (unsigned InstrNum : TouchedInstructions.set_bits()) {
3355
4.61k
3356
4.61k
      // This instruction was found to be dead. We don't bother looking
3357
4.61k
      // at it again.
3358
4.61k
      if (InstrNum == 0) {
3359
51
        TouchedInstructions.reset(InstrNum);
3360
51
        continue;
3361
51
      }
3362
4.55k
3363
4.55k
      Value *V = InstrFromDFSNum(InstrNum);
3364
4.55k
      const BasicBlock *CurrBlock = getBlockForValue(V);
3365
4.55k
3366
4.55k
      // If we hit a new block, do reachability processing.
3367
4.55k
      if (CurrBlock != LastBlock) {
3368
1.18k
        LastBlock = CurrBlock;
3369
1.18k
        bool BlockReachable = ReachableBlocks.count(CurrBlock);
3370
1.18k
        const auto &CurrInstRange = BlockInstRange.lookup(CurrBlock);
3371
1.18k
3372
1.18k
        // If it's not reachable, erase any touched instructions and move on.
3373
1.18k
        if (!BlockReachable) {
3374
32
          TouchedInstructions.reset(CurrInstRange.first, CurrInstRange.second);
3375
32
          LLVM_DEBUG(dbgs() << "Skipping instructions in block "
3376
32
                            << getBlockName(CurrBlock)
3377
32
                            << " because it is unreachable\n");
3378
32
          continue;
3379
32
        }
3380
1.15k
        updateProcessedCount(CurrBlock);
3381
1.15k
      }
3382
4.55k
      // Reset after processing (because we may mark ourselves as touched when
3383
4.55k
      // we propagate equalities).
3384
4.55k
      TouchedInstructions.reset(InstrNum);
3385
4.52k
3386
4.52k
      if (auto *MP = dyn_cast<MemoryPhi>(V)) {
3387
238
        LLVM_DEBUG(dbgs() << "Processing MemoryPhi " << *MP << "\n");
3388
238
        valueNumberMemoryPhi(MP);
3389
4.28k
      } else if (auto *I = dyn_cast<Instruction>(V)) {
3390
4.28k
        valueNumberInstruction(I);
3391
4.28k
      } else {
3392
0
        llvm_unreachable("Should have been a MemoryPhi or Instruction");
3393
0
      }
3394
4.52k
      updateProcessedCount(V);
3395
4.52k
    }
3396
500
  }
3397
333
  NumGVNMaxIterations = std::max(NumGVNMaxIterations.getValue(), Iterations);
3398
333
}
3399
3400
// This is the main transformation entry point.
3401
333
bool NewGVN::runGVN() {
3402
333
  if (DebugCounter::isCounterSet(VNCounter))
3403
0
    StartingVNCounter = DebugCounter::getCounterValue(VNCounter);
3404
333
  bool Changed = false;
3405
333
  NumFuncArgs = F.arg_size();
3406
333
  MSSAWalker = MSSA->getWalker();
3407
333
  SingletonDeadExpression = new (ExpressionAllocator) DeadExpression();
3408
333
3409
333
  // Count number of instructions for sizing of hash tables, and come
3410
333
  // up with a global dfs numbering for instructions.
3411
333
  unsigned ICount = 1;
3412
333
  // Add an empty instruction to account for the fact that we start at 1
3413
333
  DFSToInstr.emplace_back(nullptr);
3414
333
  // Note: We want ideal RPO traversal of the blocks, which is not quite the
3415
333
  // same as dominator tree order, particularly with regard whether backedges
3416
333
  // get visited first or second, given a block with multiple successors.
3417
333
  // If we visit in the wrong order, we will end up performing N times as many
3418
333
  // iterations.
3419
333
  // The dominator tree does guarantee that, for a given dom tree node, it's
3420
333
  // parent must occur before it in the RPO ordering. Thus, we only need to sort
3421
333
  // the siblings.
3422
333
  ReversePostOrderTraversal<Function *> RPOT(&F);
3423
333
  unsigned Counter = 0;
3424
1.31k
  for (auto &B : RPOT) {
3425
1.31k
    auto *Node = DT->getNode(B);
3426
1.31k
    assert(Node && "RPO and Dominator tree should have same reachability");
3427
1.31k
    RPOOrdering[Node] = ++Counter;
3428
1.31k
  }
3429
333
  // Sort dominator tree children arrays into RPO.
3430
1.31k
  for (auto &B : RPOT) {
3431
1.31k
    auto *Node = DT->getNode(B);
3432
1.31k
    if (Node->getChildren().size() > 1)
3433
313
      llvm::sort(Node->begin(), Node->end(),
3434
575
                 [&](const DomTreeNode *A, const DomTreeNode *B) {
3435
575
                   return RPOOrdering[A] < RPOOrdering[B];
3436
575
                 });
3437
1.31k
  }
3438
333
3439
333
  // Now a standard depth first ordering of the domtree is equivalent to RPO.
3440
1.31k
  for (auto DTN : depth_first(DT->getRootNode())) {
3441
1.31k
    BasicBlock *B = DTN->getBlock();
3442
1.31k
    const auto &BlockRange = assignDFSNumbers(B, ICount);
3443
1.31k
    BlockInstRange.insert({B, BlockRange});
3444
1.31k
    ICount += BlockRange.second - BlockRange.first;
3445
1.31k
  }
3446
333
  initializeCongruenceClasses(F);
3447
333
3448
333
  TouchedInstructions.resize(ICount);
3449
333
  // Ensure we don't end up resizing the expressionToClass map, as
3450
333
  // that can be quite expensive. At most, we have one expression per
3451
333
  // instruction.
3452
333
  ExpressionToClass.reserve(ICount);
3453
333
3454
333
  // Initialize the touched instructions to include the entry block.
3455
333
  const auto &InstRange = BlockInstRange.lookup(&F.getEntryBlock());
3456
333
  TouchedInstructions.set(InstRange.first, InstRange.second);
3457
333
  LLVM_DEBUG(dbgs() << "Block " << getBlockName(&F.getEntryBlock())
3458
333
                    << " marked reachable\n");
3459
333
  ReachableBlocks.insert(&F.getEntryBlock());
3460
333
3461
333
  iterateTouchedInstructions();
3462
333
  verifyMemoryCongruency();
3463
333
  verifyIterationSettled(F);
3464
333
  verifyStoreExpressions();
3465
333
3466
333
  Changed |= eliminateInstructions(F);
3467
333
3468
333
  // Delete all instructions marked for deletion.
3469
813
  for (Instruction *ToErase : InstructionsToErase) {
3470
813
    if (!ToErase->use_empty())
3471
15
      ToErase->replaceAllUsesWith(UndefValue::get(ToErase->getType()));
3472
813
3473
813
    assert(ToErase->getParent() &&
3474
813
           "BB containing ToErase deleted unexpectedly!");
3475
813
    ToErase->eraseFromParent();
3476
813
  }
3477
333
  Changed |= !InstructionsToErase.empty();
3478
333
3479
333
  // Delete all unreachable blocks.
3480
1.32k
  auto UnreachableBlockPred = [&](const BasicBlock &BB) {
3481
1.32k
    return !ReachableBlocks.count(&BB);
3482
1.32k
  };
3483
333
3484
333
  for (auto &BB : make_filter_range(F, UnreachableBlockPred)) {
3485
181
    LLVM_DEBUG(dbgs() << "We believe block " << getBlockName(&BB)
3486
181
                      << " is unreachable\n");
3487
181
    deleteInstructionsInBlock(&BB);
3488
181
    Changed = true;
3489
181
  }
3490
333
3491
333
  cleanupTables();
3492
333
  return Changed;
3493
333
}
3494
3495
struct NewGVN::ValueDFS {
3496
  int DFSIn = 0;
3497
  int DFSOut = 0;
3498
  int LocalNum = 0;
3499
3500
  // Only one of Def and U will be set.
3501
  // The bool in the Def tells us whether the Def is the stored value of a
3502
  // store.
3503
  PointerIntPair<Value *, 1, bool> Def;
3504
  Use *U = nullptr;
3505
3506
1.64k
  bool operator<(const ValueDFS &Other) const {
3507
1.64k
    // It's not enough that any given field be less than - we have sets
3508
1.64k
    // of fields that need to be evaluated together to give a proper ordering.
3509
1.64k
    // For example, if you have;
3510
1.64k
    // DFS (1, 3)
3511
1.64k
    // Val 0
3512
1.64k
    // DFS (1, 2)
3513
1.64k
    // Val 50
3514
1.64k
    // We want the second to be less than the first, but if we just go field
3515
1.64k
    // by field, we will get to Val 0 < Val 50 and say the first is less than
3516
1.64k
    // the second. We only want it to be less than if the DFS orders are equal.
3517
1.64k
    //
3518
1.64k
    // Each LLVM instruction only produces one value, and thus the lowest-level
3519
1.64k
    // differentiator that really matters for the stack (and what we use as as a
3520
1.64k
    // replacement) is the local dfs number.
3521
1.64k
    // Everything else in the structure is instruction level, and only affects
3522
1.64k
    // the order in which we will replace operands of a given instruction.
3523
1.64k
    //
3524
1.64k
    // For a given instruction (IE things with equal dfsin, dfsout, localnum),
3525
1.64k
    // the order of replacement of uses does not matter.
3526
1.64k
    // IE given,
3527
1.64k
    //  a = 5
3528
1.64k
    //  b = a + a
3529
1.64k
    // When you hit b, you will have two valuedfs with the same dfsin, out, and
3530
1.64k
    // localnum.
3531
1.64k
    // The .val will be the same as well.
3532
1.64k
    // The .u's will be different.
3533
1.64k
    // You will replace both, and it does not matter what order you replace them
3534
1.64k
    // in (IE whether you replace operand 2, then operand 1, or operand 1, then
3535
1.64k
    // operand 2).
3536
1.64k
    // Similarly for the case of same dfsin, dfsout, localnum, but different
3537
1.64k
    // .val's
3538
1.64k
    //  a = 5
3539
1.64k
    //  b  = 6
3540
1.64k
    //  c = a + b
3541
1.64k
    // in c, we will a valuedfs for a, and one for b,with everything the same
3542
1.64k
    // but .val  and .u.
3543
1.64k
    // It does not matter what order we replace these operands in.
3544
1.64k
    // You will always end up with the same IR, and this is guaranteed.
3545
1.64k
    return std::tie(DFSIn, DFSOut, LocalNum, Def, U) <
3546
1.64k
           std::tie(Other.DFSIn, Other.DFSOut, Other.LocalNum, Other.Def,
3547
1.64k
                    Other.U);
3548
1.64k
  }
3549
};
3550
3551
// This function converts the set of members for a congruence class from values,
3552
// to sets of defs and uses with associated DFS info.  The total number of
3553
// reachable uses for each value is stored in UseCount, and instructions that
3554
// seem
3555
// dead (have no non-dead uses) are stored in ProbablyDead.
3556
void NewGVN::convertClassToDFSOrdered(
3557
    const CongruenceClass &Dense, SmallVectorImpl<ValueDFS> &DFSOrderedSet,
3558
    DenseMap<const Value *, unsigned int> &UseCounts,
3559
303
    SmallPtrSetImpl<Instruction *> &ProbablyDead) const {
3560
642
  for (auto D : Dense) {
3561
642
    // First add the value.
3562
642
    BasicBlock *BB = getBlockForValue(D);
3563
642
    // Constants are handled prior to ever calling this function, so
3564
642
    // we should only be left with instructions as members.
3565
642
    assert(BB && "Should have figured out a basic block for value");
3566
642
    ValueDFS VDDef;
3567
642
    DomTreeNode *DomNode = DT->getNode(BB);
3568
642
    VDDef.DFSIn = DomNode->getDFSNumIn();
3569
642
    VDDef.DFSOut = DomNode->getDFSNumOut();
3570
642
    // If it's a store, use the leader of the value operand, if it's always
3571
642
    // available, or the value operand.  TODO: We could do dominance checks to
3572
642
    // find a dominating leader, but not worth it ATM.
3573
642
    if (auto *SI = dyn_cast<StoreInst>(D)) {
3574
31
      auto Leader = lookupOperandLeader(SI->getValueOperand());
3575
31
      if (alwaysAvailable(Leader)) {
3576
1
        VDDef.Def.setPointer(Leader);
3577
30
      } else {
3578
30
        VDDef.Def.setPointer(SI->getValueOperand());
3579
30
        VDDef.Def.setInt(true);
3580
30
      }
3581
611
    } else {
3582
611
      VDDef.Def.setPointer(D);
3583
611
    }
3584
642
    assert(isa<Instruction>(D) &&
3585
642
           "The dense set member should always be an instruction");
3586
642
    Instruction *Def = cast<Instruction>(D);
3587
642
    VDDef.LocalNum = InstrToDFSNum(D);
3588
642
    DFSOrderedSet.push_back(VDDef);
3589
642
    // If there is a phi node equivalent, add it
3590
642
    if (auto *PN = RealToTemp.lookup(Def)) {
3591
28
      auto *PHIE =
3592
28
          dyn_cast_or_null<PHIExpression>(ValueToExpression.lookup(Def));
3593
28
      if (PHIE) {
3594
28
        VDDef.Def.setInt(false);
3595
28
        VDDef.Def.setPointer(PN);
3596
28
        VDDef.LocalNum = 0;
3597
28
        DFSOrderedSet.push_back(VDDef);
3598
28
      }
3599
28
    }
3600
642
3601
642
    unsigned int UseCount = 0;
3602
642
    // Now add the uses.
3603
732
    for (auto &U : Def->uses()) {
3604
732
      if (auto *I = dyn_cast<Instruction>(U.getUser())) {
3605
732
        // Don't try to replace into dead uses
3606
732
        if (InstructionsToErase.count(I))
3607
94
          continue;
3608
638
        ValueDFS VDUse;
3609
638
        // Put the phi node uses in the incoming block.
3610
638
        BasicBlock *IBlock;
3611
638
        if (auto *P = dyn_cast<PHINode>(I)) {
3612
93
          IBlock = P->getIncomingBlock(U);
3613
93
          // Make phi node users appear last in the incoming block
3614
93
          // they are from.
3615
93
          VDUse.LocalNum = InstrDFS.size() + 1;
3616
545
        } else {
3617
545
          IBlock = getBlockForValue(I);
3618
545
          VDUse.LocalNum = InstrToDFSNum(I);
3619
545
        }
3620
638
3621
638
        // Skip uses in unreachable blocks, as we're going
3622
638
        // to delete them.
3623
638
        if (ReachableBlocks.count(IBlock) == 0)
3624
1
          continue;
3625
637
3626
637
        DomTreeNode *DomNode = DT->getNode(IBlock);
3627
637
        VDUse.DFSIn = DomNode->getDFSNumIn();
3628
637
        VDUse.DFSOut = DomNode->getDFSNumOut();
3629
637
        VDUse.U = &U;
3630
637
        ++UseCount;
3631
637
        DFSOrderedSet.emplace_back(VDUse);
3632
637
      }
3633
732
    }
3634
642
3635
642
    // If there are no uses, it's probably dead (but it may have side-effects,
3636
642
    // so not definitely dead. Otherwise, store the number of uses so we can
3637
642
    // track if it becomes dead later).
3638
642
    if (UseCount == 0)
3639
109
      ProbablyDead.insert(Def);
3640
533
    else
3641
533
      UseCounts[Def] = UseCount;
3642
642
  }
3643
303
}
3644
3645
// This function converts the set of members for a congruence class from values,
3646
// to the set of defs for loads and stores, with associated DFS info.
3647
void NewGVN::convertClassToLoadsAndStores(
3648
    const CongruenceClass &Dense,
3649
231
    SmallVectorImpl<ValueDFS> &LoadsAndStores) const {
3650
250
  for (auto D : Dense) {
3651
250
    if (!isa<LoadInst>(D) && 
!isa<StoreInst>(D)244
)
3652
0
      continue;
3653
250
3654
250
    BasicBlock *BB = getBlockForValue(D);
3655
250
    ValueDFS VD;
3656
250
    DomTreeNode *DomNode = DT->getNode(BB);
3657
250
    VD.DFSIn = DomNode->getDFSNumIn();
3658
250
    VD.DFSOut = DomNode->getDFSNumOut();
3659
250
    VD.Def.setPointer(D);
3660
250
3661
250
    // If it's an instruction, use the real local dfs number.
3662
250
    if (auto *I = dyn_cast<Instruction>(D))
3663
250
      VD.LocalNum = InstrToDFSNum(I);
3664
250
    else
3665
250
      
llvm_unreachable0
("Should have been an instruction");
3666
250
3667
250
    LoadsAndStores.emplace_back(VD);
3668
250
  }
3669
231
}
3670
3671
365
static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
3672
365
  patchReplacementInstruction(I, Repl);
3673
365
  I->replaceAllUsesWith(Repl);
3674
365
}
3675
3676
181
void NewGVN::deleteInstructionsInBlock(BasicBlock *BB) {
3677
181
  LLVM_DEBUG(dbgs() << "  BasicBlock Dead:" << *BB);
3678
181
  ++NumGVNBlocksDeleted;
3679
181
3680
181
  // Delete the instructions backwards, as it has a reduced likelihood of having
3681
181
  // to update as many def-use and use-def chains. Start after the terminator.
3682
181
  auto StartPoint = BB->rbegin();
3683
181
  ++StartPoint;
3684
181
  // Note that we explicitly recalculate BB->rend() on each iteration,
3685
181
  // as it may change when we remove the first instruction.
3686
239
  for (BasicBlock::reverse_iterator I(StartPoint); I != BB->rend();) {
3687
58
    Instruction &Inst = *I++;
3688
58
    if (!Inst.use_empty())
3689
10
      Inst.replaceAllUsesWith(UndefValue::get(Inst.getType()));
3690
58
    if (isa<LandingPadInst>(Inst))
3691
0
      continue;
3692
58
3693
58
    Inst.eraseFromParent();
3694
58
    ++NumGVNInstrDeleted;
3695
58
  }
3696
181
  // Now insert something that simplifycfg will turn into an unreachable.
3697
181
  Type *Int8Ty = Type::getInt8Ty(BB->getContext());
3698
181
  new StoreInst(UndefValue::get(Int8Ty),
3699
181
                Constant::getNullValue(Int8Ty->getPointerTo()),
3700
181
                BB->getTerminator());
3701
181
}
3702
3703
876
void NewGVN::markInstructionForDeletion(Instruction *I) {
3704
876
  LLVM_DEBUG(dbgs() << "Marking " << *I << " for deletion\n");
3705
876
  InstructionsToErase.insert(I);
3706
876
}
3707
3708
365
void NewGVN::replaceInstruction(Instruction *I, Value *V) {
3709
365
  LLVM_DEBUG(dbgs() << "Replacing " << *I << " with " << *V << "\n");
3710
365
  patchAndReplaceAllUsesWith(I, V);
3711
365
  // We save the actual erasing to avoid invalidating memory
3712
365
  // dependencies until we are done with everything.
3713
365
  markInstructionForDeletion(I);
3714
365
}
3715
3716
namespace {
3717
3718
// This is a stack that contains both the value and dfs info of where
3719
// that value is valid.
3720
class ValueDFSStack {
3721
public:
3722
1.32k
  Value *back() const { return ValueStack.back(); }
3723
0
  std::pair<int, int> dfs_back() const { return DFSStack.back(); }
3724
3725
585
  void push_back(Value *V, int DFSIn, int DFSOut) {
3726
585
    ValueStack.emplace_back(V);
3727
585
    DFSStack.emplace_back(DFSIn, DFSOut);
3728
585
  }
3729
3730
5.44k
  bool empty() const { return DFSStack.empty(); }
3731
3732
1.32k
  bool isInScope(int DFSIn, int DFSOut) const {
3733
1.32k
    if (empty())
3734
303
      return false;
3735
1.02k
    return DFSIn >= DFSStack.back().first && DFSOut <= DFSStack.back().second;
3736
1.02k
  }
3737
3738
585
  void popUntilDFSScope(int DFSIn, int DFSOut) {
3739
585
3740
585
    // These two should always be in sync at this point.
3741
585
    assert(ValueStack.size() == DFSStack.size() &&
3742
585
           "Mismatch between ValueStack and DFSStack");
3743
585
    while (
3744
636
        !DFSStack.empty() &&
3745
636
        
!(51
DFSIn >= DFSStack.back().first51
&&
DFSOut <= DFSStack.back().second51
)) {
3746
51
      DFSStack.pop_back();
3747
51
      ValueStack.pop_back();
3748
51
    }
3749
585
  }
3750
3751
private:
3752
  SmallVector<Value *, 8> ValueStack;
3753
  SmallVector<std::pair<int, int>, 8> DFSStack;
3754
};
3755
3756
} // end anonymous namespace
3757
3758
// Given an expression, get the congruence class for it.
3759
95
CongruenceClass *NewGVN::getClassForExpression(const Expression *E) const {
3760
95
  if (auto *VE = dyn_cast<VariableExpression>(E))
3761
1
    return ValueToClass.lookup(VE->getVariableValue());
3762
94
  else if (isa<DeadExpression>(E))
3763
0
    return TOPClass;
3764
94
  return ExpressionToClass.lookup(E);
3765
94
}
3766
3767
// Given a value and a basic block we are trying to see if it is available in,
3768
// see if the value has a leader available in that block.
3769
Value *NewGVN::findPHIOfOpsLeader(const Expression *E,
3770
                                  const Instruction *OrigInst,
3771
193
                                  const BasicBlock *BB) const {
3772
193
  // It would already be constant if we could make it constant
3773
193
  if (auto *CE = dyn_cast<ConstantExpression>(E))
3774
87
    return CE->getConstantValue();
3775
106
  if (auto *VE = dyn_cast<VariableExpression>(E)) {
3776
12
    auto *V = VE->getVariableValue();
3777
12
    if (alwaysAvailable(V) || DT->dominates(getBlockForValue(V), BB))
3778
11
      return VE->getVariableValue();
3779
95
  }
3780
95
3781
95
  auto *CC = getClassForExpression(E);
3782
95
  if (!CC)
3783
60
    return nullptr;
3784
35
  if (alwaysAvailable(CC->getLeader()))
3785
0
    return CC->getLeader();
3786
35
3787
39
  
for (auto Member : *CC)35
{
3788
39
    auto *MemberInst = dyn_cast<Instruction>(Member);
3789
39
    if (MemberInst == OrigInst)
3790
18
      continue;
3791
21
    // Anything that isn't an instruction is always available.
3792
21
    if (!MemberInst)
3793
0
      return Member;
3794
21
    if (DT->dominates(getBlockForValue(MemberInst), BB))
3795
14
      return Member;
3796
21
  }
3797
35
  
return nullptr21
;
3798
35
}
3799
3800
333
bool NewGVN::eliminateInstructions(Function &F) {
3801
333
  // This is a non-standard eliminator. The normal way to eliminate is
3802
333
  // to walk the dominator tree in order, keeping track of available
3803
333
  // values, and eliminating them.  However, this is mildly
3804
333
  // pointless. It requires doing lookups on every instruction,
3805
333
  // regardless of whether we will ever eliminate it.  For
3806
333
  // instructions part of most singleton congruence classes, we know we
3807
333
  // will never eliminate them.
3808
333
3809
333
  // Instead, this eliminator looks at the congruence classes directly, sorts
3810
333
  // them into a DFS ordering of the dominator tree, and then we just
3811
333
  // perform elimination straight on the sets by walking the congruence
3812
333
  // class member uses in order, and eliminate the ones dominated by the
3813
333
  // last member.   This is worst case O(E log E) where E = number of
3814
333
  // instructions in a single congruence class.  In theory, this is all
3815
333
  // instructions.   In practice, it is much faster, as most instructions are
3816
333
  // either in singleton congruence classes or can't possibly be eliminated
3817
333
  // anyway (if there are no overlapping DFS ranges in class).
3818
333
  // When we find something not dominated, it becomes the new leader
3819
333
  // for elimination purposes.
3820
333
  // TODO: If we wanted to be faster, We could remove any members with no
3821
333
  // overlapping ranges while sorting, as we will never eliminate anything
3822
333
  // with those members, as they don't dominate anything else in our set.
3823
333
3824
333
  bool AnythingReplaced = false;
3825
333
3826
333
  // Since we are going to walk the domtree anyway, and we can't guarantee the
3827
333
  // DFS numbers are updated, we compute some ourselves.
3828
333
  DT->updateDFSNumbers();
3829
333
3830
333
  // Go through all of our phi nodes, and kill the arguments associated with
3831
333
  // unreachable edges.
3832
333
  auto ReplaceUnreachablePHIArgs = [&](PHINode *PHI, BasicBlock *BB) {
3833
30
    for (auto &Operand : PHI->incoming_values())
3834
69
      if (!ReachableEdges.count({PHI->getIncomingBlock(Operand), BB})) {
3835
36
        LLVM_DEBUG(dbgs() << "Replacing incoming value of " << PHI
3836
36
                          << " for block "
3837
36
                          << getBlockName(PHI->getIncomingBlock(Operand))
3838
36
                          << " with undef due to it being unreachable\n");
3839
36
        Operand.set(UndefValue::get(PHI->getType()));
3840
36
      }
3841
30
  };
3842
333
  // Replace unreachable phi arguments.
3843
333
  // At this point, RevisitOnReachabilityChange only contains:
3844
333
  //
3845
333
  // 1. PHIs
3846
333
  // 2. Temporaries that will convert to PHIs
3847
333
  // 3. Operations that are affected by an unreachable edge but do not fit into
3848
333
  // 1 or 2 (rare).
3849
333
  // So it is a slight overshoot of what we want. We could make it exact by
3850
333
  // using two SparseBitVectors per block.
3851
333
  DenseMap<const BasicBlock *, unsigned> ReachablePredCount;
3852
333
  for (auto &KV : ReachableEdges)
3853
1.10k
    ReachablePredCount[KV.getEnd()]++;
3854
333
  for (auto &BBPair : RevisitOnReachabilityChange) {
3855
301
    for (auto InstNum : BBPair.second) {
3856
256
      auto *Inst = InstrFromDFSNum(InstNum);
3857
256
      auto *PHI = dyn_cast<PHINode>(Inst);
3858
256
      PHI = PHI ? 
PHI225
:
dyn_cast_or_null<PHINode>(RealToTemp.lookup(Inst))31
;
3859
256
      if (!PHI)
3860
3
        continue;
3861
253
      auto *BB = BBPair.first;
3862
253
      if (ReachablePredCount.lookup(BB) != PHI->getNumIncomingValues())
3863
30
        ReplaceUnreachablePHIArgs(PHI, BB);
3864
253
    }
3865
301
  }
3866
333
3867
333
  // Map to store the use counts
3868
333
  DenseMap<const Value *, unsigned int> UseCounts;
3869
3.38k
  for (auto *CC : reverse(CongruenceClasses)) {
3870
3.38k
    LLVM_DEBUG(dbgs() << "Eliminating in congruence class " << CC->getID()
3871
3.38k
                      << "\n");
3872
3.38k
    // Track the equivalent store info so we can decide whether to try
3873
3.38k
    // dead store elimination.
3874
3.38k
    SmallVector<ValueDFS, 8> PossibleDeadStores;
3875
3.38k
    SmallPtrSet<Instruction *, 8> ProbablyDead;
3876
3.38k
    if (CC->isDead() || 
CC->empty()2.48k
)
3877
1.01k
      continue;
3878
2.36k
    // Everything still in the TOP class is unreachable or dead.
3879
2.36k
    if (CC == TOPClass) {
3880
146
      for (auto M : *CC) {
3881
146
        auto *VTE = ValueToExpression.lookup(M);
3882
146
        if (VTE && 
isa<DeadExpression>(VTE)0
)
3883
0
          markInstructionForDeletion(cast<Instruction>(M));
3884
146
        assert((!ReachableBlocks.count(cast<Instruction>(M)->getParent()) ||
3885
146
                InstructionsToErase.count(cast<Instruction>(M))) &&
3886
146
               "Everything in TOP should be unreachable or dead at this "
3887
146
               "point");
3888
146
      }
3889
64
      continue;
3890
64
    }
3891
2.30k
3892
2.30k
    assert(CC->getLeader() && "We should have had a leader");
3893
2.30k
    // If this is a leader that is always available, and it's a
3894
2.30k
    // constant or has no equivalences, just replace everything with
3895
2.30k
    // it. We then update the congruence class with whatever members
3896
2.30k
    // are left.
3897
2.30k
    Value *Leader =
3898
2.30k
        CC->getStoredValue() ? 
CC->getStoredValue()231
:
CC->getLeader()2.07k
;
3899
2.30k
    if (alwaysAvailable(Leader)) {
3900
827
      CongruenceClass::MemberSet MembersLeft;
3901
983
      for (auto M : *CC) {
3902
983
        Value *Member = M;
3903
983
        // Void things have no uses we can replace.
3904
983
        if (Member == Leader || 
!isa<Instruction>(Member)506
||
3905
983
            
Member->getType()->isVoidTy()506
) {
3906
618
          MembersLeft.insert(Member);
3907
618
          continue;
3908
618
        }
3909
365
        LLVM_DEBUG(dbgs() << "Found replacement " << *(Leader) << " for "
3910
365
                          << *Member << "\n");
3911
365
        auto *I = cast<Instruction>(Member);
3912
365
        assert(Leader != I && "About to accidentally remove our leader");
3913
365
        replaceInstruction(I, Leader);
3914
365
        AnythingReplaced = true;
3915
365
      }
3916
827
      CC->swap(MembersLeft);
3917
1.47k
    } else {
3918
1.47k
      // If this is a singleton, we can skip it.
3919
1.47k
      if (CC->size() != 1 || 
RealToTemp.count(Leader)1.20k
) {
3920
303
        // This is a stack because equality replacement/etc may place
3921
303
        // constants in the middle of the member list, and we want to use
3922
303
        // those constant values in preference to the current leader, over
3923
303
        // the scope of those constants.
3924
303
        ValueDFSStack EliminationStack;
3925
303
3926
303
        // Convert the members to DFS ordered sets and then merge them.
3927
303
        SmallVector<ValueDFS, 8> DFSOrderedSet;
3928
303
        convertClassToDFSOrdered(*CC, DFSOrderedSet, UseCounts, ProbablyDead);
3929
303
3930
303
        // Sort the whole thing.
3931
303
        llvm::sort(DFSOrderedSet);
3932
1.30k
        for (auto &VD : DFSOrderedSet) {
3933
1.30k
          int MemberDFSIn = VD.DFSIn;
3934
1.30k
          int MemberDFSOut = VD.DFSOut;
3935
1.30k
          Value *Def = VD.Def.getPointer();
3936
1.30k
          bool FromStore = VD.Def.getInt();
3937
1.30k
          Use *U = VD.U;
3938
1.30k
          // We ignore void things because we can't get a value from them.
3939
1.30k
          if (Def && 
Def->getType()->isVoidTy()670
)
3940
0
            continue;
3941
1.30k
          auto *DefInst = dyn_cast_or_null<Instruction>(Def);
3942
1.30k
          if (DefInst && 
AllTempInstructions.count(DefInst)669
) {
3943
28
            auto *PN = cast<PHINode>(DefInst);
3944
28
3945
28
            // If this is a value phi and that's the expression we used, insert
3946
28
            // it into the program
3947
28
            // remove from temp instruction list.
3948
28
            AllTempInstructions.erase(PN);
3949
28
            auto *DefBlock = getBlockForValue(Def);
3950
28
            LLVM_DEBUG(dbgs() << "Inserting fully real phi of ops" << *Def
3951
28
                              << " into block "
3952
28
                              << getBlockName(getBlockForValue(Def)) << "\n");
3953
28
            PN->insertBefore(&DefBlock->front());
3954
28
            Def = PN;
3955
28
            NumGVNPHIOfOpsEliminations++;
3956
28
          }
3957
1.30k
3958
1.30k
          if (EliminationStack.empty()) {
3959
303
            LLVM_DEBUG(dbgs() << "Elimination Stack is empty\n");
3960
1.00k
          } else {
3961
1.00k
            LLVM_DEBUG(dbgs() << "Elimination Stack Top DFS numbers are ("
3962
1.00k
                              << EliminationStack.dfs_back().first << ","
3963
1.00k
                              << EliminationStack.dfs_back().second << ")\n");
3964
1.00k
          }
3965
1.30k
3966
1.30k
          LLVM_DEBUG(dbgs() << "Current DFS numbers are (" << MemberDFSIn << ","
3967
1.30k
                            << MemberDFSOut << ")\n");
3968
1.30k
          // First, we see if we are out of scope or empty.  If so,
3969
1.30k
          // and there equivalences, we try to replace the top of
3970
1.30k
          // stack with equivalences (if it's on the stack, it must
3971
1.30k
          // not have been eliminated yet).
3972
1.30k
          // Then we synchronize to our current scope, by
3973
1.30k
          // popping until we are back within a DFS scope that
3974
1.30k
          // dominates the current member.
3975
1.30k
          // Then, what happens depends on a few factors
3976
1.30k
          // If the stack is now empty, we need to push
3977
1.30k
          // If we have a constant or a local equivalence we want to
3978
1.30k
          // start using, we also push.
3979
1.30k
          // Otherwise, we walk along, processing members who are
3980
1.30k
          // dominated by this scope, and eliminate them.
3981
1.30k
          bool ShouldPush = Def && 
EliminationStack.empty()670
;
3982
1.30k
          bool OutOfScope =
3983
1.30k
              !EliminationStack.isInScope(MemberDFSIn, MemberDFSOut);
3984
1.30k
3985
1.30k
          if (OutOfScope || 
ShouldPush955
) {
3986
352
            // Sync to our current scope.
3987
352
            EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
3988
352
            bool ShouldPush = Def && EliminationStack.empty();
3989
352
            if (ShouldPush) {
3990
352
              EliminationStack.push_back(Def, MemberDFSIn, MemberDFSOut);
3991
352
            }
3992
352
          }
3993
1.30k
3994
1.30k
          // Skip the Def's, we only want to eliminate on their uses.  But mark
3995
1.30k
          // dominated defs as dead.
3996
1.30k
          if (Def) {
3997
670
            // For anything in this case, what and how we value number
3998
670
            // guarantees that any side-effets that would have occurred (ie
3999
670
            // throwing, etc) can be proven to either still occur (because it's
4000
670
            // dominated by something that has the same side-effects), or never
4001
670
            // occur.  Otherwise, we would not have been able to prove it value
4002
670
            // equivalent to something else. For these things, we can just mark
4003
670
            // it all dead.  Note that this is different from the "ProbablyDead"
4004
670
            // set, which may not be dominated by anything, and thus, are only
4005
670
            // easy to prove dead if they are also side-effect free. Note that
4006
670
            // because stores are put in terms of the stored value, we skip
4007
670
            // stored values here. If the stored value is really dead, it will
4008
670
            // still be marked for deletion when we process it in its own class.
4009
670
            if (!EliminationStack.empty() && Def != EliminationStack.back() &&
4010
670
                
isa<Instruction>(Def)314
&&
!FromStore314
)
4011
310
              markInstructionForDeletion(cast<Instruction>(Def));
4012
670
            continue;
4013
670
          }
4014
637
          // At this point, we know it is a Use we are trying to possibly
4015
637
          // replace.
4016
637
4017
637
          assert(isa<Instruction>(U->get()) &&
4018
637
                 "Current def should have been an instruction");
4019
637
          assert(isa<Instruction>(U->getUser()) &&
4020
637
                 "Current user should have been an instruction");
4021
637
4022
637
          // If the thing we are replacing into is already marked to be dead,
4023
637
          // this use is dead.  Note that this is true regardless of whether
4024
637
          // we have anything dominating the use or not.  We do this here
4025
637
          // because we are already walking all the uses anyway.
4026
637
          Instruction *InstUse = cast<Instruction>(U->getUser());
4027
637
          if (InstructionsToErase.count(InstUse)) {
4028
10
            auto &UseCount = UseCounts[U->get()];
4029
10
            if (--UseCount == 0) {
4030
3
              ProbablyDead.insert(cast<Instruction>(U->get()));
4031
3
            }
4032
10
          }
4033
637
4034
637
          // If we get to this point, and the stack is empty we must have a use
4035
637
          // with nothing we can use to eliminate this use, so just skip it.
4036
637
          if (EliminationStack.empty())
4037
0
            continue;
4038
637
4039
637
          Value *DominatingLeader = EliminationStack.back();
4040
637
4041
637
          auto *II = dyn_cast<IntrinsicInst>(DominatingLeader);
4042
637
          bool isSSACopy = II && 
II->getIntrinsicID() == Intrinsic::ssa_copy5
;
4043
637
          if (isSSACopy)
4044
2
            DominatingLeader = II->getOperand(0);
4045
637
4046
637
          // Don't replace our existing users with ourselves.
4047
637
          if (U->get() == DominatingLeader)
4048
366
            continue;
4049
271
          LLVM_DEBUG(dbgs()
4050
271
                     << "Found replacement " << *DominatingLeader << " for "
4051
271
                     << *U->get() << " in " << *(U->getUser()) << "\n");
4052
271
4053
271
          // If we replaced something in an instruction, handle the patching of
4054
271
          // metadata.  Skip this if we are replacing predicateinfo with its
4055
271
          // original operand, as we already know we can just drop it.
4056
271
          auto *ReplacedInst = cast<Instruction>(U->get());
4057
271
          auto *PI = PredInfo->getPredicateInfoFor(ReplacedInst);
4058
271
          if (!PI || 
DominatingLeader != PI->OriginalOp46
)
4059
229
            patchReplacementInstruction(ReplacedInst, DominatingLeader);
4060
271
          U->set(DominatingLeader);
4061
271
          // This is now a use of the dominating leader, which means if the
4062
271
          // dominating leader was dead, it's now live!
4063
271
          auto &LeaderUseCount = UseCounts[DominatingLeader];
4064
271
          // It's about to be alive again.
4065
271
          if (LeaderUseCount == 0 && 
isa<Instruction>(DominatingLeader)44
)
4066
43
            ProbablyDead.erase(cast<Instruction>(DominatingLeader));
4067
271
          // For copy instructions, we use their operand as a leader,
4068
271
          // which means we remove a user of the copy and it may become dead.
4069
271
          if (isSSACopy) {
4070
2
            unsigned &IIUseCount = UseCounts[II];
4071
2
            if (--IIUseCount == 0)
4072
2
              ProbablyDead.insert(II);
4073
2
          }
4074
271
          ++LeaderUseCount;
4075
271
          AnythingReplaced = true;
4076
271
        }
4077
303
      }
4078
1.47k
    }
4079
2.30k
4080
2.30k
    // At this point, anything still in the ProbablyDead set is actually dead if
4081
2.30k
    // would be trivially dead.
4082
2.30k
    for (auto *I : ProbablyDead)
4083
112
      if (wouldInstructionBeTriviallyDead(I))
4084
79
        markInstructionForDeletion(I);
4085
2.30k
4086
2.30k
    // Cleanup the congruence class.
4087
2.30k
    CongruenceClass::MemberSet MembersLeft;
4088
2.30k
    for (auto *Member : *CC)
4089
2.43k
      if (!isa<Instruction>(Member) ||
4090
2.43k
          
!InstructionsToErase.count(cast<Instruction>(Member))1.95k
)
4091
2.10k
        MembersLeft.insert(Member);
4092
2.30k
    CC->swap(MembersLeft);
4093
2.30k
4094
2.30k
    // If we have possible dead stores to look at, try to eliminate them.
4095
2.30k
    if (CC->getStoreCount() > 0) {
4096
231
      convertClassToLoadsAndStores(*CC, PossibleDeadStores);
4097
231
      llvm::sort(PossibleDeadStores);
4098
231
      ValueDFSStack EliminationStack;
4099
250
      for (auto &VD : PossibleDeadStores) {
4100
250
        int MemberDFSIn = VD.DFSIn;
4101
250
        int MemberDFSOut = VD.DFSOut;
4102
250
        Instruction *Member = cast<Instruction>(VD.Def.getPointer());
4103
250
        if (EliminationStack.empty() ||
4104
250
            
!EliminationStack.isInScope(MemberDFSIn, MemberDFSOut)19
) {
4105
233
          // Sync to our current scope.
4106
233
          EliminationStack.popUntilDFSScope(MemberDFSIn, MemberDFSOut);
4107
233
          if (EliminationStack.empty()) {
4108
233
            EliminationStack.push_back(Member, MemberDFSIn, MemberDFSOut);
4109
233
            continue;
4110
233
          }
4111
17
        }
4112
17
        // We already did load elimination, so nothing to do here.
4113
17
        if (isa<LoadInst>(Member))
4114
0
          continue;
4115
17
        assert(!EliminationStack.empty());
4116
17
        Instruction *Leader = cast<Instruction>(EliminationStack.back());
4117
17
        (void)Leader;
4118
17
        assert(DT->dominates(Leader->getParent(), Member->getParent()));
4119
17
        // Member is dominater by Leader, and thus dead
4120
17
        LLVM_DEBUG(dbgs() << "Marking dead store " << *Member
4121
17
                          << " that is dominated by " << *Leader << "\n");
4122
17
        markInstructionForDeletion(Member);
4123
17
        CC->erase(Member);
4124
17
        ++NumGVNDeadStores;
4125
17
      }
4126
231
    }
4127
2.30k
  }
4128
333
  return AnythingReplaced;
4129
333
}
4130
4131
// This function provides global ranking of operations so that we can place them
4132
// in a canonical order.  Note that rank alone is not necessarily enough for a
4133
// complete ordering, as constants all have the same rank.  However, generally,
4134
// we will simplify an operation with all constants so that it doesn't matter
4135
// what order they appear in.
4136
3.19k
unsigned int NewGVN::getRank(const Value *V) const {
4137
3.19k
  // Prefer constants to undef to anything else
4138
3.19k
  // Undef is a constant, have to check it first.
4139
3.19k
  // Prefer smaller constants to constantexprs
4140
3.19k
  if (isa<ConstantExpr>(V))
4141
44
    return 2;
4142
3.14k
  if (isa<UndefValue>(V))
4143
24
    return 1;
4144
3.12k
  if (isa<Constant>(V))
4145
1.29k
    return 0;
4146
1.82k
  else if (auto *A = dyn_cast<Argument>(V))
4147
532
    return 3 + A->getArgNo();
4148
1.29k
4149
1.29k
  // Need to shift the instruction DFS by number of arguments + 3 to account for
4150
1.29k
  // the constant and argument ranking above.
4151
1.29k
  unsigned Result = InstrToDFSNum(V);
4152
1.29k
  if (Result > 0)
4153
1.29k
    return 4 + NumFuncArgs + Result;
4154
0
  // Unreachable or something else, just return a really large number.
4155
0
  return ~0;
4156
0
}
4157
4158
// This is a function that says whether two commutative operations should
4159
// have their order swapped when canonicalizing.
4160
1.59k
bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
4161
1.59k
  // Because we only care about a total ordering, and don't rewrite expressions
4162
1.59k
  // in this order, we order by rank, which will give a strict weak ordering to
4163
1.59k
  // everything but constants, and then we order by pointer address.
4164
1.59k
  return std::make_pair(getRank(A), A) > std::make_pair(getRank(B), B);
4165
1.59k
}
4166
4167
namespace {
4168
4169
class NewGVNLegacyPass : public FunctionPass {
4170
public:
4171
  // Pass identification, replacement for typeid.
4172
  static char ID;
4173
4174
156
  NewGVNLegacyPass() : FunctionPass(ID) {
4175
156
    initializeNewGVNLegacyPassPass(*PassRegistry::getPassRegistry());
4176
156
  }
4177
4178
  bool runOnFunction(Function &F) override;
4179
4180
private:
4181
156
  void getAnalysisUsage(AnalysisUsage &AU) const override {
4182
156
    AU.addRequired<AssumptionCacheTracker>();
4183
156
    AU.addRequired<DominatorTreeWrapperPass>();
4184
156
    AU.addRequired<TargetLibraryInfoWrapperPass>();
4185
156
    AU.addRequired<MemorySSAWrapperPass>();
4186
156
    AU.addRequired<AAResultsWrapperPass>();
4187
156
    AU.addPreserved<DominatorTreeWrapperPass>();
4188
156
    AU.addPreserved<GlobalsAAWrapperPass>();
4189
156
  }
4190
};
4191
4192
} // end anonymous namespace
4193
4194
330
bool NewGVNLegacyPass::runOnFunction(Function &F) {
4195
330
  if (skipFunction(F))
4196
0
    return false;
4197
330
  return NewGVN(F, &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
4198
330
                &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
4199
330
                &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(),
4200
330
                &getAnalysis<AAResultsWrapperPass>().getAAResults(),
4201
330
                &getAnalysis<MemorySSAWrapperPass>().getMSSA(),
4202
330
                F.getParent()->getDataLayout())
4203
330
      .runGVN();
4204
330
}
4205
4206
char NewGVNLegacyPass::ID = 0;
4207
4208
36.0k
INITIALIZE_PASS_BEGIN(NewGVNLegacyPass, "newgvn", "Global Value Numbering",
4209
36.0k
                      false, false)
4210
36.0k
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
4211
36.0k
INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
4212
36.0k
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
4213
36.0k
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
4214
36.0k
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
4215
36.0k
INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
4216
36.0k
INITIALIZE_PASS_END(NewGVNLegacyPass, "newgvn", "Global Value Numbering", false,
4217
                    false)
4218
4219
// createGVNPass - The public interface to this file.
4220
0
FunctionPass *llvm::createNewGVNPass() { return new NewGVNLegacyPass(); }
4221
4222
3
PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) {
4223
3
  // Apparently the order in which we get these results matter for
4224
3
  // the old GVN (see Chandler's comment in GVN.cpp). I'll keep
4225
3
  // the same order here, just in case.
4226
3
  auto &AC = AM.getResult<AssumptionAnalysis>(F);
4227
3
  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
4228
3
  auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
4229
3
  auto &AA = AM.getResult<AAManager>(F);
4230
3
  auto &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
4231
3
  bool Changed =
4232
3
      NewGVN(F, &DT, &AC, &TLI, &AA, &MSSA, F.getParent()->getDataLayout())
4233
3
          .runGVN();
4234
3
  if (!Changed)
4235
0
    return PreservedAnalyses::all();
4236
3
  PreservedAnalyses PA;
4237
3
  PA.preserve<DominatorTreeAnalysis>();
4238
3
  PA.preserve<GlobalsAA>();
4239
3
  return PA;
4240
3
}