Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/Utils/Local.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- Local.cpp - Functions to perform local transformations -------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This family of functions perform various local transformations to the
10
// program.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "llvm/Transforms/Utils/Local.h"
15
#include "llvm/ADT/APInt.h"
16
#include "llvm/ADT/DenseMap.h"
17
#include "llvm/ADT/DenseMapInfo.h"
18
#include "llvm/ADT/DenseSet.h"
19
#include "llvm/ADT/Hashing.h"
20
#include "llvm/ADT/None.h"
21
#include "llvm/ADT/Optional.h"
22
#include "llvm/ADT/STLExtras.h"
23
#include "llvm/ADT/SetVector.h"
24
#include "llvm/ADT/SmallPtrSet.h"
25
#include "llvm/ADT/SmallVector.h"
26
#include "llvm/ADT/Statistic.h"
27
#include "llvm/ADT/TinyPtrVector.h"
28
#include "llvm/Analysis/ConstantFolding.h"
29
#include "llvm/Analysis/DomTreeUpdater.h"
30
#include "llvm/Analysis/EHPersonalities.h"
31
#include "llvm/Analysis/InstructionSimplify.h"
32
#include "llvm/Analysis/LazyValueInfo.h"
33
#include "llvm/Analysis/MemoryBuiltins.h"
34
#include "llvm/Analysis/MemorySSAUpdater.h"
35
#include "llvm/Analysis/TargetLibraryInfo.h"
36
#include "llvm/Analysis/ValueTracking.h"
37
#include "llvm/Analysis/VectorUtils.h"
38
#include "llvm/BinaryFormat/Dwarf.h"
39
#include "llvm/IR/Argument.h"
40
#include "llvm/IR/Attributes.h"
41
#include "llvm/IR/BasicBlock.h"
42
#include "llvm/IR/CFG.h"
43
#include "llvm/IR/CallSite.h"
44
#include "llvm/IR/Constant.h"
45
#include "llvm/IR/ConstantRange.h"
46
#include "llvm/IR/Constants.h"
47
#include "llvm/IR/DIBuilder.h"
48
#include "llvm/IR/DataLayout.h"
49
#include "llvm/IR/DebugInfoMetadata.h"
50
#include "llvm/IR/DebugLoc.h"
51
#include "llvm/IR/DerivedTypes.h"
52
#include "llvm/IR/Dominators.h"
53
#include "llvm/IR/Function.h"
54
#include "llvm/IR/GetElementPtrTypeIterator.h"
55
#include "llvm/IR/GlobalObject.h"
56
#include "llvm/IR/IRBuilder.h"
57
#include "llvm/IR/InstrTypes.h"
58
#include "llvm/IR/Instruction.h"
59
#include "llvm/IR/Instructions.h"
60
#include "llvm/IR/IntrinsicInst.h"
61
#include "llvm/IR/Intrinsics.h"
62
#include "llvm/IR/LLVMContext.h"
63
#include "llvm/IR/MDBuilder.h"
64
#include "llvm/IR/Metadata.h"
65
#include "llvm/IR/Module.h"
66
#include "llvm/IR/Operator.h"
67
#include "llvm/IR/PatternMatch.h"
68
#include "llvm/IR/Type.h"
69
#include "llvm/IR/Use.h"
70
#include "llvm/IR/User.h"
71
#include "llvm/IR/Value.h"
72
#include "llvm/IR/ValueHandle.h"
73
#include "llvm/Support/Casting.h"
74
#include "llvm/Support/Debug.h"
75
#include "llvm/Support/ErrorHandling.h"
76
#include "llvm/Support/KnownBits.h"
77
#include "llvm/Support/raw_ostream.h"
78
#include "llvm/Transforms/Utils/ValueMapper.h"
79
#include <algorithm>
80
#include <cassert>
81
#include <climits>
82
#include <cstdint>
83
#include <iterator>
84
#include <map>
85
#include <utility>
86
87
using namespace llvm;
88
using namespace llvm::PatternMatch;
89
90
#define DEBUG_TYPE "local"
91
92
STATISTIC(NumRemoved, "Number of unreachable basic blocks removed");
93
94
// Max recursion depth for collectBitParts used when detecting bswap and
95
// bitreverse idioms
96
static const unsigned BitPartRecursionMaxDepth = 64;
97
98
//===----------------------------------------------------------------------===//
99
//  Local constant propagation.
100
//
101
102
/// ConstantFoldTerminator - If a terminator instruction is predicated on a
103
/// constant value, convert it into an unconditional branch to the constant
104
/// destination.  This is a nontrivial operation because the successors of this
105
/// basic block must have their PHI nodes updated.
106
/// Also calls RecursivelyDeleteTriviallyDeadInstructions() on any branch/switch
107
/// conditions and indirectbr addresses this might make dead if
108
/// DeleteDeadConditions is true.
109
bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
110
                                  const TargetLibraryInfo *TLI,
111
77.3M
                                  DomTreeUpdater *DTU) {
112
77.3M
  Instruction *T = BB->getTerminator();
113
77.3M
  IRBuilder<> Builder(T);
114
77.3M
115
77.3M
  // Branch - See if we are conditional jumping on constant
116
77.3M
  if (auto *BI = dyn_cast<BranchInst>(T)) {
117
63.3M
    if (BI->isUnconditional()) 
return false27.2M
; // Can't optimize uncond branch
118
36.1M
    BasicBlock *Dest1 = BI->getSuccessor(0);
119
36.1M
    BasicBlock *Dest2 = BI->getSuccessor(1);
120
36.1M
121
36.1M
    if (auto *Cond = dyn_cast<ConstantInt>(BI->getCondition())) {
122
88.5k
      // Are we branching on constant?
123
88.5k
      // YES.  Change to unconditional branch...
124
88.5k
      BasicBlock *Destination = Cond->getZExtValue() ? 
Dest145.4k
:
Dest243.1k
;
125
88.5k
      BasicBlock *OldDest     = Cond->getZExtValue() ? 
Dest245.4k
:
Dest143.1k
;
126
88.5k
127
88.5k
      // Let the basic block know that we are letting go of it.  Based on this,
128
88.5k
      // it will adjust it's PHI nodes.
129
88.5k
      OldDest->removePredecessor(BB);
130
88.5k
131
88.5k
      // Replace the conditional branch with an unconditional one.
132
88.5k
      Builder.CreateBr(Destination);
133
88.5k
      BI->eraseFromParent();
134
88.5k
      if (DTU)
135
36.8k
        DTU->applyUpdatesPermissive({{DominatorTree::Delete, BB, OldDest}});
136
88.5k
      return true;
137
88.5k
    }
138
36.0M
139
36.0M
    if (Dest2 == Dest1) {       // Conditional branch to same location?
140
5.16k
      // This branch matches something like this:
141
5.16k
      //     br bool %cond, label %Dest, label %Dest
142
5.16k
      // and changes it into:  br label %Dest
143
5.16k
144
5.16k
      // Let the basic block know that we are letting go of one copy of it.
145
5.16k
      assert(BI->getParent() && "Terminator not inserted in block!");
146
5.16k
      Dest1->removePredecessor(BI->getParent());
147
5.16k
148
5.16k
      // Replace the conditional branch with an unconditional one.
149
5.16k
      Builder.CreateBr(Dest1);
150
5.16k
      Value *Cond = BI->getCondition();
151
5.16k
      BI->eraseFromParent();
152
5.16k
      if (DeleteDeadConditions)
153
5.15k
        RecursivelyDeleteTriviallyDeadInstructions(Cond, TLI);
154
5.16k
      return true;
155
5.16k
    }
156
36.0M
    return false;
157
36.0M
  }
158
13.9M
159
13.9M
  if (auto *SI = dyn_cast<SwitchInst>(T)) {
160
627k
    // If we are switching on a constant, we can convert the switch to an
161
627k
    // unconditional branch.
162
627k
    auto *CI = dyn_cast<ConstantInt>(SI->getCondition());
163
627k
    BasicBlock *DefaultDest = SI->getDefaultDest();
164
627k
    BasicBlock *TheOnlyDest = DefaultDest;
165
627k
166
627k
    // If the default is unreachable, ignore it when searching for TheOnlyDest.
167
627k
    if (isa<UnreachableInst>(DefaultDest->getFirstNonPHIOrDbg()) &&
168
627k
        
SI->getNumCases() > 010.7k
) {
169
10.7k
      TheOnlyDest = SI->case_begin()->getCaseSuccessor();
170
10.7k
    }
171
627k
172
627k
    // Figure out which case it goes to.
173
2.68M
    for (auto i = SI->case_begin(), e = SI->case_end(); i != e;) {
174
2.05M
      // Found case matching a constant operand?
175
2.05M
      if (i->getCaseValue() == CI) {
176
648
        TheOnlyDest = i->getCaseSuccessor();
177
648
        break;
178
648
      }
179
2.05M
180
2.05M
      // Check to see if this branch is going to the same place as the default
181
2.05M
      // dest.  If so, eliminate it as an explicit compare.
182
2.05M
      if (i->getCaseSuccessor() == DefaultDest) {
183
1.25k
        MDNode *MD = SI->getMetadata(LLVMContext::MD_prof);
184
1.25k
        unsigned NCases = SI->getNumCases();
185
1.25k
        // Fold the case metadata into the default if there will be any branches
186
1.25k
        // left, unless the metadata doesn't match the switch.
187
1.25k
        if (NCases > 1 && 
MD1.18k
&&
MD->getNumOperands() == 2 + NCases3
) {
188
3
          // Collect branch weights into a vector.
189
3
          SmallVector<uint32_t, 8> Weights;
190
14
          for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e;
191
11
               ++MD_i) {
192
11
            auto *CI = mdconst::extract<ConstantInt>(MD->getOperand(MD_i));
193
11
            Weights.push_back(CI->getValue().getZExtValue());
194
11
          }
195
3
          // Merge weight of this case to the default weight.
196
3
          unsigned idx = i->getCaseIndex();
197
3
          Weights[0] += Weights[idx+1];
198
3
          // Remove weight for this case.
199
3
          std::swap(Weights[idx+1], Weights.back());
200
3
          Weights.pop_back();
201
3
          SI->setMetadata(LLVMContext::MD_prof,
202
3
                          MDBuilder(BB->getContext()).
203
3
                          createBranchWeights(Weights));
204
3
        }
205
1.25k
        // Remove this entry.
206
1.25k
        BasicBlock *ParentBB = SI->getParent();
207
1.25k
        DefaultDest->removePredecessor(ParentBB);
208
1.25k
        i = SI->removeCase(i);
209
1.25k
        e = SI->case_end();
210
1.25k
        if (DTU)
211
4
          DTU->applyUpdatesPermissive(
212
4
              {{DominatorTree::Delete, ParentBB, DefaultDest}});
213
1.25k
        continue;
214
1.25k
      }
215
2.05M
216
2.05M
      // Otherwise, check to see if the switch only branches to one destination.
217
2.05M
      // We do this by reseting "TheOnlyDest" to null when we find two non-equal
218
2.05M
      // destinations.
219
2.05M
      if (i->getCaseSuccessor() != TheOnlyDest)
220
2.04M
        TheOnlyDest = nullptr;
221
2.05M
222
2.05M
      // Increment this iterator as we haven't removed the case.
223
2.05M
      ++i;
224
2.05M
    }
225
627k
226
627k
    if (CI && 
!TheOnlyDest940
) {
227
225
      // Branching on a constant, but not any of the cases, go to the default
228
225
      // successor.
229
225
      TheOnlyDest = SI->getDefaultDest();
230
225
    }
231
627k
232
627k
    // If we found a single destination that we can fold the switch into, do so
233
627k
    // now.
234
627k
    if (TheOnlyDest) {
235
3.67k
      // Insert the new branch.
236
3.67k
      Builder.CreateBr(TheOnlyDest);
237
3.67k
      BasicBlock *BB = SI->getParent();
238
3.67k
      std::vector <DominatorTree::UpdateType> Updates;
239
3.67k
      if (DTU)
240
550
        Updates.reserve(SI->getNumSuccessors() - 1);
241
3.67k
242
3.67k
      // Remove entries from PHI nodes which we no longer branch to...
243
11.5k
      for (BasicBlock *Succ : successors(SI)) {
244
11.5k
        // Found case matching a constant operand?
245
11.5k
        if (Succ == TheOnlyDest) {
246
3.67k
          TheOnlyDest = nullptr; // Don't modify the first branch to TheOnlyDest
247
7.86k
        } else {
248
7.86k
          Succ->removePredecessor(BB);
249
7.86k
          if (DTU)
250
1.30k
            Updates.push_back({DominatorTree::Delete, BB, Succ});
251
7.86k
        }
252
11.5k
      }
253
3.67k
254
3.67k
      // Delete the old switch.
255
3.67k
      Value *Cond = SI->getCondition();
256
3.67k
      SI->eraseFromParent();
257
3.67k
      if (DeleteDeadConditions)
258
3.60k
        RecursivelyDeleteTriviallyDeadInstructions(Cond, TLI);
259
3.67k
      if (DTU)
260
550
        DTU->applyUpdatesPermissive(Updates);
261
3.67k
      return true;
262
3.67k
    }
263
623k
264
623k
    if (SI->getNumCases() == 1) {
265
6.05k
      // Otherwise, we can fold this switch into a conditional branch
266
6.05k
      // instruction if it has only one non-default destination.
267
6.05k
      auto FirstCase = *SI->case_begin();
268
6.05k
      Value *Cond = Builder.CreateICmpEQ(SI->getCondition(),
269
6.05k
          FirstCase.getCaseValue(), "cond");
270
6.05k
271
6.05k
      // Insert the new branch.
272
6.05k
      BranchInst *NewBr = Builder.CreateCondBr(Cond,
273
6.05k
                                               FirstCase.getCaseSuccessor(),
274
6.05k
                                               SI->getDefaultDest());
275
6.05k
      MDNode *MD = SI->getMetadata(LLVMContext::MD_prof);
276
6.05k
      if (MD && 
MD->getNumOperands() == 36
) {
277
6
        ConstantInt *SICase =
278
6
            mdconst::dyn_extract<ConstantInt>(MD->getOperand(2));
279
6
        ConstantInt *SIDef =
280
6
            mdconst::dyn_extract<ConstantInt>(MD->getOperand(1));
281
6
        assert(SICase && SIDef);
282
6
        // The TrueWeight should be the weight for the single case of SI.
283
6
        NewBr->setMetadata(LLVMContext::MD_prof,
284
6
                        MDBuilder(BB->getContext()).
285
6
                        createBranchWeights(SICase->getValue().getZExtValue(),
286
6
                                            SIDef->getValue().getZExtValue()));
287
6
      }
288
6.05k
289
6.05k
      // Update make.implicit metadata to the newly-created conditional branch.
290
6.05k
      MDNode *MakeImplicitMD = SI->getMetadata(LLVMContext::MD_make_implicit);
291
6.05k
      if (MakeImplicitMD)
292
1
        NewBr->setMetadata(LLVMContext::MD_make_implicit, MakeImplicitMD);
293
6.05k
294
6.05k
      // Delete the old switch.
295
6.05k
      SI->eraseFromParent();
296
6.05k
      return true;
297
6.05k
    }
298
617k
    return false;
299
617k
  }
300
13.3M
301
13.3M
  if (auto *IBI = dyn_cast<IndirectBrInst>(T)) {
302
252
    // indirectbr blockaddress(@F, @BB) -> br label @BB
303
252
    if (auto *BA =
304
9
          dyn_cast<BlockAddress>(IBI->getAddress()->stripPointerCasts())) {
305
9
      BasicBlock *TheOnlyDest = BA->getBasicBlock();
306
9
      std::vector <DominatorTree::UpdateType> Updates;
307
9
      if (DTU)
308
5
        Updates.reserve(IBI->getNumDestinations() - 1);
309
9
310
9
      // Insert the new branch.
311
9
      Builder.CreateBr(TheOnlyDest);
312
9
313
30
      for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; 
++i21
) {
314
21
        if (IBI->getDestination(i) == TheOnlyDest) {
315
8
          TheOnlyDest = nullptr;
316
13
        } else {
317
13
          BasicBlock *ParentBB = IBI->getParent();
318
13
          BasicBlock *DestBB = IBI->getDestination(i);
319
13
          DestBB->removePredecessor(ParentBB);
320
13
          if (DTU)
321
4
            Updates.push_back({DominatorTree::Delete, ParentBB, DestBB});
322
13
        }
323
21
      }
324
9
      Value *Address = IBI->getAddress();
325
9
      IBI->eraseFromParent();
326
9
      if (DeleteDeadConditions)
327
9
        // Delete pointer cast instructions.
328
9
        RecursivelyDeleteTriviallyDeadInstructions(Address, TLI);
329
9
330
9
      // Also zap the blockaddress constant if there are no users remaining,
331
9
      // otherwise the destination is still marked as having its address taken.
332
9
      if (BA->use_empty())
333
6
        BA->destroyConstant();
334
9
335
9
      // If we didn't find our destination in the IBI successor list, then we
336
9
      // have undefined behavior.  Replace the unconditional branch with an
337
9
      // 'unreachable' instruction.
338
9
      if (TheOnlyDest) {
339
1
        BB->getTerminator()->eraseFromParent();
340
1
        new UnreachableInst(BB->getContext(), BB);
341
1
      }
342
9
343
9
      if (DTU)
344
5
        DTU->applyUpdatesPermissive(Updates);
345
9
      return true;
346
9
    }
347
13.3M
  }
348
13.3M
349
13.3M
  return false;
350
13.3M
}
351
352
//===----------------------------------------------------------------------===//
353
//  Local dead code elimination.
354
//
355
356
/// isInstructionTriviallyDead - Return true if the result produced by the
357
/// instruction is not used, and the instruction has no side effects.
358
///
359
bool llvm::isInstructionTriviallyDead(Instruction *I,
360
418M
                                      const TargetLibraryInfo *TLI) {
361
418M
  if (!I->use_empty())
362
281M
    return false;
363
137M
  return wouldInstructionBeTriviallyDead(I, TLI);
364
137M
}
365
366
bool llvm::wouldInstructionBeTriviallyDead(Instruction *I,
367
137M
                                           const TargetLibraryInfo *TLI) {
368
137M
  if (I->isTerminator())
369
70.8M
    return false;
370
66.8M
371
66.8M
  // We don't want the landingpad-like instructions removed by anything this
372
66.8M
  // general.
373
66.8M
  if (I->isEHPad())
374
552
    return false;
375
66.8M
376
66.8M
  // We don't want debug info removed by anything this general, unless
377
66.8M
  // debug info is empty.
378
66.8M
  if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(I)) {
379
88
    if (DDI->getAddress())
380
80
      return false;
381
8
    return true;
382
8
  }
383
66.8M
  if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(I)) {
384
1.79k
    if (DVI->getValue())
385
1.72k
      return false;
386
67
    return true;
387
67
  }
388
66.8M
  if (DbgLabelInst *DLI = dyn_cast<DbgLabelInst>(I)) {
389
20
    if (DLI->getLabel())
390
20
      return false;
391
0
    return true;
392
0
  }
393
66.8M
394
66.8M
  if (!I->mayHaveSideEffects())
395
7.12M
    return true;
396
59.7M
397
59.7M
  // Special case intrinsics that "may have side effects" but can be deleted
398
59.7M
  // when dead.
399
59.7M
  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
400
7.09M
    // Safe to delete llvm.stacksave and launder.invariant.group if dead.
401
7.09M
    if (II->getIntrinsicID() == Intrinsic::stacksave ||
402
7.09M
        
II->getIntrinsicID() == Intrinsic::launder_invariant_group7.09M
)
403
42
      return true;
404
7.09M
405
7.09M
    // Lifetime intrinsics are dead when their right-hand is undef.
406
7.09M
    if (II->isLifetimeStartOrEnd())
407
5.51M
      return isa<UndefValue>(II->getArgOperand(1));
408
1.57M
409
1.57M
    // Assumptions are dead if their condition is trivially true.  Guards on
410
1.57M
    // true are operationally no-ops.  In the future we can consider more
411
1.57M
    // sophisticated tradeoffs for guards considering potential for check
412
1.57M
    // widening, but for now we keep things simple.
413
1.57M
    if (II->getIntrinsicID() == Intrinsic::assume ||
414
1.57M
        
II->getIntrinsicID() == Intrinsic::experimental_guard1.57M
) {
415
1.35k
      if (ConstantInt *Cond = dyn_cast<ConstantInt>(II->getArgOperand(0)))
416
126
        return !Cond->isZero();
417
1.22k
418
1.22k
      return false;
419
1.22k
    }
420
1.57M
  }
421
54.2M
422
54.2M
  if (isAllocLikeFn(I, TLI))
423
19
    return true;
424
54.2M
425
54.2M
  if (CallInst *CI = isFreeCall(I, TLI))
426
847k
    if (Constant *C = dyn_cast<Constant>(CI->getArgOperand(0)))
427
507
      return C->isNullValue() || 
isa<UndefValue>(C)0
;
428
54.2M
429
54.2M
  if (auto *Call = dyn_cast<CallBase>(I))
430
22.3M
    if (isMathLibCallNoop(Call, TLI))
431
7
      return true;
432
54.2M
433
54.2M
  return false;
434
54.2M
}
435
436
/// RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a
437
/// trivially dead instruction, delete it.  If that makes any of its operands
438
/// trivially dead, delete them too, recursively.  Return true if any
439
/// instructions were deleted.
440
bool llvm::RecursivelyDeleteTriviallyDeadInstructions(
441
13.7M
    Value *V, const TargetLibraryInfo *TLI, MemorySSAUpdater *MSSAU) {
442
13.7M
  Instruction *I = dyn_cast<Instruction>(V);
443
13.7M
  if (!I || 
!isInstructionTriviallyDead(I, TLI)13.7M
)
444
13.2M
    return false;
445
467k
446
467k
  SmallVector<Instruction*, 16> DeadInsts;
447
467k
  DeadInsts.push_back(I);
448
467k
  RecursivelyDeleteTriviallyDeadInstructions(DeadInsts, TLI, MSSAU);
449
467k
450
467k
  return true;
451
467k
}
452
453
void llvm::RecursivelyDeleteTriviallyDeadInstructions(
454
    SmallVectorImpl<Instruction *> &DeadInsts, const TargetLibraryInfo *TLI,
455
467k
    MemorySSAUpdater *MSSAU) {
456
467k
  // Process the dead instruction list until empty.
457
1.09M
  while (!DeadInsts.empty()) {
458
625k
    Instruction &I = *DeadInsts.pop_back_val();
459
625k
    assert(I.use_empty() && "Instructions with uses are not dead.");
460
625k
    assert(isInstructionTriviallyDead(&I, TLI) &&
461
625k
           "Live instruction found in dead worklist!");
462
625k
463
625k
    // Don't lose the debug info while deleting the instructions.
464
625k
    salvageDebugInfo(I);
465
625k
466
625k
    // Null out all of the instruction's operands to see if any operand becomes
467
625k
    // dead as we go.
468
1.12M
    for (Use &OpU : I.operands()) {
469
1.12M
      Value *OpV = OpU.get();
470
1.12M
      OpU.set(nullptr);
471
1.12M
472
1.12M
      if (!OpV->use_empty())
473
919k
        continue;
474
205k
475
205k
      // If the operand is an instruction that became dead as we nulled out the
476
205k
      // operand, and if it is 'trivially' dead, delete it in a future loop
477
205k
      // iteration.
478
205k
      if (Instruction *OpI = dyn_cast<Instruction>(OpV))
479
158k
        if (isInstructionTriviallyDead(OpI, TLI))
480
158k
          DeadInsts.push_back(OpI);
481
205k
    }
482
625k
    if (MSSAU)
483
26
      MSSAU->removeMemoryAccess(&I);
484
625k
485
625k
    I.eraseFromParent();
486
625k
  }
487
467k
}
488
489
2.23M
bool llvm::replaceDbgUsesWithUndef(Instruction *I) {
490
2.23M
  SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
491
2.23M
  findDbgUsers(DbgUsers, I);
492
2.23M
  for (auto *DII : DbgUsers) {
493
11
    Value *Undef = UndefValue::get(I->getType());
494
11
    DII->setOperand(0, MetadataAsValue::get(DII->getContext(),
495
11
                                            ValueAsMetadata::get(Undef)));
496
11
  }
497
2.23M
  return !DbgUsers.empty();
498
2.23M
}
499
500
/// areAllUsesEqual - Check whether the uses of a value are all the same.
501
/// This is similar to Instruction::hasOneUse() except this will also return
502
/// true when there are no uses or multiple uses that all refer to the same
503
/// value.
504
1.10M
static bool areAllUsesEqual(Instruction *I) {
505
1.10M
  Value::user_iterator UI = I->user_begin();
506
1.10M
  Value::user_iterator UE = I->user_end();
507
1.10M
  if (UI == UE)
508
13.1k
    return true;
509
1.09M
510
1.09M
  User *TheUse = *UI;
511
1.11M
  for (++UI; UI != UE; 
++UI18.1k
) {
512
623k
    if (*UI != TheUse)
513
604k
      return false;
514
623k
  }
515
1.09M
  
return true491k
;
516
1.09M
}
517
518
/// RecursivelyDeleteDeadPHINode - If the specified value is an effectively
519
/// dead PHI node, due to being a def-use chain of single-use nodes that
520
/// either forms a cycle or is terminated by a trivially dead instruction,
521
/// delete it.  If that makes any of its operands trivially dead, delete them
522
/// too, recursively.  Return true if a change was made.
523
bool llvm::RecursivelyDeleteDeadPHINode(PHINode *PN,
524
703k
                                        const TargetLibraryInfo *TLI) {
525
703k
  SmallPtrSet<Instruction*, 4> Visited;
526
1.10M
  for (Instruction *I = PN; areAllUsesEqual(I) && 
!I->mayHaveSideEffects()504k
;
527
703k
       
I = cast<Instruction>(*I->user_begin())406k
) {
528
496k
    if (I->use_empty())
529
5.51k
      return RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
530
491k
531
491k
    // If we find an instruction more than once, we're on a cycle that
532
491k
    // won't prove fruitful.
533
491k
    if (!Visited.insert(I).second) {
534
84.9k
      // Break the cycle and delete the instruction and its operands.
535
84.9k
      I->replaceAllUsesWith(UndefValue::get(I->getType()));
536
84.9k
      (void)RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
537
84.9k
      return true;
538
84.9k
    }
539
491k
  }
540
703k
  
return false613k
;
541
703k
}
542
543
static bool
544
simplifyAndDCEInstruction(Instruction *I,
545
                          SmallSetVector<Instruction *, 16> &WorkList,
546
                          const DataLayout &DL,
547
142k
                          const TargetLibraryInfo *TLI) {
548
142k
  if (isInstructionTriviallyDead(I, TLI)) {
549
61.2k
    salvageDebugInfo(*I);
550
61.2k
551
61.2k
    // Null out all of the instruction's operands to see if any operand becomes
552
61.2k
    // dead as we go.
553
166k
    for (unsigned i = 0, e = I->getNumOperands(); i != e; 
++i105k
) {
554
105k
      Value *OpV = I->getOperand(i);
555
105k
      I->setOperand(i, nullptr);
556
105k
557
105k
      if (!OpV->use_empty() || 
I == OpV12.4k
)
558
92.8k
        continue;
559
12.4k
560
12.4k
      // If the operand is an instruction that became dead as we nulled out the
561
12.4k
      // operand, and if it is 'trivially' dead, delete it in a future loop
562
12.4k
      // iteration.
563
12.4k
      if (Instruction *OpI = dyn_cast<Instruction>(OpV))
564
11.6k
        if (isInstructionTriviallyDead(OpI, TLI))
565
11.6k
          WorkList.insert(OpI);
566
12.4k
    }
567
61.2k
568
61.2k
    I->eraseFromParent();
569
61.2k
570
61.2k
    return true;
571
61.2k
  }
572
81.6k
573
81.6k
  if (Value *SimpleV = SimplifyInstruction(I, DL)) {
574
41.3k
    // Add the users to the worklist. CAREFUL: an instruction can use itself,
575
41.3k
    // in the case of a phi node.
576
51.5k
    for (User *U : I->users()) {
577
51.5k
      if (U != I) {
578
51.5k
        WorkList.insert(cast<Instruction>(U));
579
51.5k
      }
580
51.5k
    }
581
41.3k
582
41.3k
    // Replace the instruction with its simplified value.
583
41.3k
    bool Changed = false;
584
41.3k
    if (!I->use_empty()) {
585
41.3k
      I->replaceAllUsesWith(SimpleV);
586
41.3k
      Changed = true;
587
41.3k
    }
588
41.3k
    if (isInstructionTriviallyDead(I, TLI)) {
589
41.3k
      I->eraseFromParent();
590
41.3k
      Changed = true;
591
41.3k
    }
592
41.3k
    return Changed;
593
41.3k
  }
594
40.3k
  return false;
595
40.3k
}
596
597
/// SimplifyInstructionsInBlock - Scan the specified basic block and try to
598
/// simplify any instructions in it and recursively delete dead instructions.
599
///
600
/// This returns true if it changed the code, note that it can delete
601
/// instructions in other blocks as well in this block.
602
bool llvm::SimplifyInstructionsInBlock(BasicBlock *BB,
603
49.9k
                                       const TargetLibraryInfo *TLI) {
604
49.9k
  bool MadeChange = false;
605
49.9k
  const DataLayout &DL = BB->getModule()->getDataLayout();
606
49.9k
607
#ifndef NDEBUG
608
  // In debug builds, ensure that the terminator of the block is never replaced
609
  // or deleted by these simplifications. The idea of simplification is that it
610
  // cannot introduce new instructions, and there is no way to replace the
611
  // terminator of a block without introducing a new instruction.
612
  AssertingVH<Instruction> TerminatorVH(&BB->back());
613
#endif
614
615
49.9k
  SmallSetVector<Instruction *, 16> WorkList;
616
49.9k
  // Iterate over the original function, only adding insts to the worklist
617
49.9k
  // if they actually need to be revisited. This avoids having to pre-init
618
49.9k
  // the worklist with the entire function's worth of instructions.
619
49.9k
  for (BasicBlock::iterator BI = BB->begin(), E = std::prev(BB->end());
620
160k
       BI != E;) {
621
110k
    assert(!BI->isTerminator());
622
110k
    Instruction *I = &*BI;
623
110k
    ++BI;
624
110k
625
110k
    // We're visiting this instruction now, so make sure it's not in the
626
110k
    // worklist from an earlier visit.
627
110k
    if (!WorkList.count(I))
628
82.8k
      MadeChange |= simplifyAndDCEInstruction(I, WorkList, DL, TLI);
629
110k
  }
630
49.9k
631
110k
  while (!WorkList.empty()) {
632
60.0k
    Instruction *I = WorkList.pop_back_val();
633
60.0k
    MadeChange |= simplifyAndDCEInstruction(I, WorkList, DL, TLI);
634
60.0k
  }
635
49.9k
  return MadeChange;
636
49.9k
}
637
638
//===----------------------------------------------------------------------===//
639
//  Control Flow Graph Restructuring.
640
//
641
642
/// RemovePredecessorAndSimplify - Like BasicBlock::removePredecessor, this
643
/// method is called when we're about to delete Pred as a predecessor of BB.  If
644
/// BB contains any PHI nodes, this drops the entries in the PHI nodes for Pred.
645
///
646
/// Unlike the removePredecessor method, this attempts to simplify uses of PHI
647
/// nodes that collapse into identity values.  For example, if we have:
648
///   x = phi(1, 0, 0, 0)
649
///   y = and x, z
650
///
651
/// .. and delete the predecessor corresponding to the '1', this will attempt to
652
/// recursively fold the and to 0.
653
void llvm::RemovePredecessorAndSimplify(BasicBlock *BB, BasicBlock *Pred,
654
0
                                        DomTreeUpdater *DTU) {
655
0
  // This only adjusts blocks with PHI nodes.
656
0
  if (!isa<PHINode>(BB->begin()))
657
0
    return;
658
0
659
0
  // Remove the entries for Pred from the PHI nodes in BB, but do not simplify
660
0
  // them down.  This will leave us with single entry phi nodes and other phis
661
0
  // that can be removed.
662
0
  BB->removePredecessor(Pred, true);
663
0
664
0
  WeakTrackingVH PhiIt = &BB->front();
665
0
  while (PHINode *PN = dyn_cast<PHINode>(PhiIt)) {
666
0
    PhiIt = &*++BasicBlock::iterator(cast<Instruction>(PhiIt));
667
0
    Value *OldPhiIt = PhiIt;
668
0
669
0
    if (!recursivelySimplifyInstruction(PN))
670
0
      continue;
671
0
672
0
    // If recursive simplification ended up deleting the next PHI node we would
673
0
    // iterate to, then our iterator is invalid, restart scanning from the top
674
0
    // of the block.
675
0
    if (PhiIt != OldPhiIt) PhiIt = &BB->front();
676
0
  }
677
0
  if (DTU)
678
0
    DTU->applyUpdatesPermissive({{DominatorTree::Delete, Pred, BB}});
679
0
}
680
681
/// MergeBasicBlockIntoOnlyPred - DestBB is a block with one predecessor and its
682
/// predecessor is known to have one successor (DestBB!). Eliminate the edge
683
/// between them, moving the instructions in the predecessor into DestBB and
684
/// deleting the predecessor block.
685
void llvm::MergeBasicBlockIntoOnlyPred(BasicBlock *DestBB,
686
73.0k
                                       DomTreeUpdater *DTU) {
687
73.0k
688
73.0k
  // If BB has single-entry PHI nodes, fold them.
689
108k
  while (PHINode *PN = dyn_cast<PHINode>(DestBB->begin())) {
690
35.6k
    Value *NewVal = PN->getIncomingValue(0);
691
35.6k
    // Replace self referencing PHI with undef, it must be dead.
692
35.6k
    if (NewVal == PN) 
NewVal = UndefValue::get(PN->getType())6
;
693
35.6k
    PN->replaceAllUsesWith(NewVal);
694
35.6k
    PN->eraseFromParent();
695
35.6k
  }
696
73.0k
697
73.0k
  BasicBlock *PredBB = DestBB->getSinglePredecessor();
698
73.0k
  assert(PredBB && "Block doesn't have a single predecessor!");
699
73.0k
700
73.0k
  bool ReplaceEntryBB = false;
701
73.0k
  if (PredBB == &DestBB->getParent()->getEntryBlock())
702
4.08k
    ReplaceEntryBB = true;
703
73.0k
704
73.0k
  // DTU updates: Collect all the edges that enter
705
73.0k
  // PredBB. These dominator edges will be redirected to DestBB.
706
73.0k
  SmallVector<DominatorTree::UpdateType, 32> Updates;
707
73.0k
708
73.0k
  if (DTU) {
709
73.0k
    Updates.push_back({DominatorTree::Delete, PredBB, DestBB});
710
211k
    for (auto I = pred_begin(PredBB), E = pred_end(PredBB); I != E; 
++I138k
) {
711
138k
      Updates.push_back({DominatorTree::Delete, *I, PredBB});
712
138k
      // This predecessor of PredBB may already have DestBB as a successor.
713
138k
      if (llvm::find(successors(*I), DestBB) == succ_end(*I))
714
138k
        Updates.push_back({DominatorTree::Insert, *I, DestBB});
715
138k
    }
716
73.0k
  }
717
73.0k
718
73.0k
  // Zap anything that took the address of DestBB.  Not doing this will give the
719
73.0k
  // address an invalid value.
720
73.0k
  if (DestBB->hasAddressTaken()) {
721
3
    BlockAddress *BA = BlockAddress::get(DestBB);
722
3
    Constant *Replacement =
723
3
      ConstantInt::get(Type::getInt32Ty(BA->getContext()), 1);
724
3
    BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(Replacement,
725
3
                                                     BA->getType()));
726
3
    BA->destroyConstant();
727
3
  }
728
73.0k
729
73.0k
  // Anything that branched to PredBB now branches to DestBB.
730
73.0k
  PredBB->replaceAllUsesWith(DestBB);
731
73.0k
732
73.0k
  // Splice all the instructions from PredBB to DestBB.
733
73.0k
  PredBB->getTerminator()->eraseFromParent();
734
73.0k
  DestBB->getInstList().splice(DestBB->begin(), PredBB->getInstList());
735
73.0k
  new UnreachableInst(PredBB->getContext(), PredBB);
736
73.0k
737
73.0k
  // If the PredBB is the entry block of the function, move DestBB up to
738
73.0k
  // become the entry block after we erase PredBB.
739
73.0k
  if (ReplaceEntryBB)
740
4.08k
    DestBB->moveAfter(PredBB);
741
73.0k
742
73.0k
  if (DTU) {
743
73.0k
    assert(PredBB->getInstList().size() == 1 &&
744
73.0k
           isa<UnreachableInst>(PredBB->getTerminator()) &&
745
73.0k
           "The successor list of PredBB isn't empty before "
746
73.0k
           "applying corresponding DTU updates.");
747
73.0k
    DTU->applyUpdatesPermissive(Updates);
748
73.0k
    DTU->deleteBB(PredBB);
749
73.0k
    // Recalculation of DomTree is needed when updating a forward DomTree and
750
73.0k
    // the Entry BB is replaced.
751
73.0k
    if (ReplaceEntryBB && 
DTU->hasDomTree()4.08k
) {
752
4.08k
      // The entry block was removed and there is no external interface for
753
4.08k
      // the dominator tree to be notified of this change. In this corner-case
754
4.08k
      // we recalculate the entire tree.
755
4.08k
      DTU->recalculate(*(DestBB->getParent()));
756
4.08k
    }
757
73.0k
  }
758
0
759
0
  else {
760
0
    PredBB->eraseFromParent(); // Nuke BB if DTU is nullptr.
761
0
  }
762
73.0k
}
763
764
/// CanMergeValues - Return true if we can choose one of these values to use
765
/// in place of the other. Note that we will always choose the non-undef
766
/// value to keep.
767
181k
static bool CanMergeValues(Value *First, Value *Second) {
768
181k
  return First == Second || 
isa<UndefValue>(First)155k
||
isa<UndefValue>(Second)155k
;
769
181k
}
770
771
/// CanPropagatePredecessorsForPHIs - Return true if we can fold BB, an
772
/// almost-empty BB ending in an unconditional branch to Succ, into Succ.
773
///
774
/// Assumption: Succ is the single successor for BB.
775
1.77M
static bool CanPropagatePredecessorsForPHIs(BasicBlock *BB, BasicBlock *Succ) {
776
1.77M
  assert(*succ_begin(BB) == Succ && "Succ is not successor of BB!");
777
1.77M
778
1.77M
  LLVM_DEBUG(dbgs() << "Looking to fold " << BB->getName() << " into "
779
1.77M
                    << Succ->getName() << "\n");
780
1.77M
  // Shortcut, if there is only a single predecessor it must be BB and merging
781
1.77M
  // is always safe
782
1.77M
  if (Succ->getSinglePredecessor()) 
return true106k
;
783
1.66M
784
1.66M
  // Make a list of the predecessors of BB
785
1.66M
  SmallPtrSet<BasicBlock*, 16> BBPreds(pred_begin(BB), pred_end(BB));
786
1.66M
787
1.66M
  // Look at all the phi nodes in Succ, to see if they present a conflict when
788
1.66M
  // merging these blocks
789
2.91M
  for (BasicBlock::iterator I = Succ->begin(); isa<PHINode>(I); 
++I1.25M
) {
790
1.40M
    PHINode *PN = cast<PHINode>(I);
791
1.40M
792
1.40M
    // If the incoming value from BB is again a PHINode in
793
1.40M
    // BB which has the same incoming value for *PI as PN does, we can
794
1.40M
    // merge the phi nodes and then the blocks can still be merged
795
1.40M
    PHINode *BBPN = dyn_cast<PHINode>(PN->getIncomingValueForBlock(BB));
796
1.40M
    if (BBPN && 
BBPN->getParent() == BB360k
) {
797
717k
      for (unsigned PI = 0, PE = PN->getNumIncomingValues(); PI != PE; 
++PI517k
) {
798
518k
        BasicBlock *IBB = PN->getIncomingBlock(PI);
799
518k
        if (BBPreds.count(IBB) &&
800
518k
            !CanMergeValues(BBPN->getIncomingValueForBlock(IBB),
801
3.79k
                            PN->getIncomingValue(PI))) {
802
1.46k
          LLVM_DEBUG(dbgs()
803
1.46k
                     << "Can't fold, phi node " << PN->getName() << " in "
804
1.46k
                     << Succ->getName() << " is conflicting with "
805
1.46k
                     << BBPN->getName() << " with regard to common predecessor "
806
1.46k
                     << IBB->getName() << "\n");
807
1.46k
          return false;
808
1.46k
        }
809
518k
      }
810
1.20M
    } else {
811
1.20M
      Value* Val = PN->getIncomingValueForBlock(BB);
812
6.36M
      for (unsigned PI = 0, PE = PN->getNumIncomingValues(); PI != PE; 
++PI5.15M
) {
813
5.30M
        // See if the incoming value for the common predecessor is equal to the
814
5.30M
        // one for BB, in which case this phi node will not prevent the merging
815
5.30M
        // of the block.
816
5.30M
        BasicBlock *IBB = PN->getIncomingBlock(PI);
817
5.30M
        if (BBPreds.count(IBB) &&
818
5.30M
            
!CanMergeValues(Val, PN->getIncomingValue(PI))177k
) {
819
153k
          LLVM_DEBUG(dbgs() << "Can't fold, phi node " << PN->getName()
820
153k
                            << " in " << Succ->getName()
821
153k
                            << " is conflicting with regard to common "
822
153k
                            << "predecessor " << IBB->getName() << "\n");
823
153k
          return false;
824
153k
        }
825
5.30M
      }
826
1.20M
    }
827
1.40M
  }
828
1.66M
829
1.66M
  
return true1.51M
;
830
1.66M
}
831
832
using PredBlockVector = SmallVector<BasicBlock *, 16>;
833
using IncomingValueMap = DenseMap<BasicBlock *, Value *>;
834
835
/// Determines the value to use as the phi node input for a block.
836
///
837
/// Select between \p OldVal any value that we know flows from \p BB
838
/// to a particular phi on the basis of which one (if either) is not
839
/// undef. Update IncomingValues based on the selected value.
840
///
841
/// \param OldVal The value we are considering selecting.
842
/// \param BB The block that the value flows in from.
843
/// \param IncomingValues A map from block-to-value for other phi inputs
844
/// that we have examined.
845
///
846
/// \returns the selected value.
847
static Value *selectIncomingValueForBlock(Value *OldVal, BasicBlock *BB,
848
1.38M
                                          IncomingValueMap &IncomingValues) {
849
1.38M
  if (!isa<UndefValue>(OldVal)) {
850
1.38M
    assert((!IncomingValues.count(BB) ||
851
1.38M
            IncomingValues.find(BB)->second == OldVal) &&
852
1.38M
           "Expected OldVal to match incoming value from BB!");
853
1.38M
854
1.38M
    IncomingValues.insert(std::make_pair(BB, OldVal));
855
1.38M
    return OldVal;
856
1.38M
  }
857
2.67k
858
2.67k
  IncomingValueMap::const_iterator It = IncomingValues.find(BB);
859
2.67k
  if (It != IncomingValues.end()) 
return It->second16
;
860
2.66k
861
2.66k
  return OldVal;
862
2.66k
}
863
864
/// Create a map from block to value for the operands of a
865
/// given phi.
866
///
867
/// Create a map from block to value for each non-undef value flowing
868
/// into \p PN.
869
///
870
/// \param PN The phi we are collecting the map for.
871
/// \param IncomingValues [out] The map from block to value for this phi.
872
static void gatherIncomingValuesToPhi(PHINode *PN,
873
1.18M
                                      IncomingValueMap &IncomingValues) {
874
3.88M
  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; 
++i2.69M
) {
875
2.69M
    BasicBlock *BB = PN->getIncomingBlock(i);
876
2.69M
    Value *V = PN->getIncomingValue(i);
877
2.69M
878
2.69M
    if (!isa<UndefValue>(V))
879
2.69M
      IncomingValues.insert(std::make_pair(BB, V));
880
2.69M
  }
881
1.18M
}
882
883
/// Replace the incoming undef values to a phi with the values
884
/// from a block-to-value map.
885
///
886
/// \param PN The phi we are replacing the undefs in.
887
/// \param IncomingValues A map from block to value.
888
static void replaceUndefValuesInPhi(PHINode *PN,
889
1.18M
                                    const IncomingValueMap &IncomingValues) {
890
5.27M
  for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; 
++i4.08M
) {
891
4.08M
    Value *V = PN->getIncomingValue(i);
892
4.08M
893
4.08M
    if (!isa<UndefValue>(V)) 
continue4.07M
;
894
8.91k
895
8.91k
    BasicBlock *BB = PN->getIncomingBlock(i);
896
8.91k
    IncomingValueMap::const_iterator It = IncomingValues.find(BB);
897
8.91k
    if (It == IncomingValues.end()) 
continue8.84k
;
898
67
899
67
    PN->setIncomingValue(i, It->second);
900
67
  }
901
1.18M
}
902
903
/// Replace a value flowing from a block to a phi with
904
/// potentially multiple instances of that value flowing from the
905
/// block's predecessors to the phi.
906
///
907
/// \param BB The block with the value flowing into the phi.
908
/// \param BBPreds The predecessors of BB.
909
/// \param PN The phi that we are updating.
910
static void redirectValuesFromPredecessorsToPhi(BasicBlock *BB,
911
                                                const PredBlockVector &BBPreds,
912
1.18M
                                                PHINode *PN) {
913
1.18M
  Value *OldVal = PN->removeIncomingValue(BB, false);
914
1.18M
  assert(OldVal && "No entry in PHI for Pred BB!");
915
1.18M
916
1.18M
  IncomingValueMap IncomingValues;
917
1.18M
918
1.18M
  // We are merging two blocks - BB, and the block containing PN - and
919
1.18M
  // as a result we need to redirect edges from the predecessors of BB
920
1.18M
  // to go to the block containing PN, and update PN
921
1.18M
  // accordingly. Since we allow merging blocks in the case where the
922
1.18M
  // predecessor and successor blocks both share some predecessors,
923
1.18M
  // and where some of those common predecessors might have undef
924
1.18M
  // values flowing into PN, we want to rewrite those values to be
925
1.18M
  // consistent with the non-undef values.
926
1.18M
927
1.18M
  gatherIncomingValuesToPhi(PN, IncomingValues);
928
1.18M
929
1.18M
  // If this incoming value is one of the PHI nodes in BB, the new entries
930
1.18M
  // in the PHI node are the entries from the old PHI.
931
1.18M
  if (isa<PHINode>(OldVal) && 
cast<PHINode>(OldVal)->getParent() == BB324k
) {
932
171k
    PHINode *OldValPN = cast<PHINode>(OldVal);
933
475k
    for (unsigned i = 0, e = OldValPN->getNumIncomingValues(); i != e; 
++i303k
) {
934
303k
      // Note that, since we are merging phi nodes and BB and Succ might
935
303k
      // have common predecessors, we could end up with a phi node with
936
303k
      // identical incoming branches. This will be cleaned up later (and
937
303k
      // will trigger asserts if we try to clean it up now, without also
938
303k
      // simplifying the corresponding conditional branch).
939
303k
      BasicBlock *PredBB = OldValPN->getIncomingBlock(i);
940
303k
      Value *PredVal = OldValPN->getIncomingValue(i);
941
303k
      Value *Selected = selectIncomingValueForBlock(PredVal, PredBB,
942
303k
                                                    IncomingValues);
943
303k
944
303k
      // And add a new incoming value for this predecessor for the
945
303k
      // newly retargeted branch.
946
303k
      PN->addIncoming(Selected, PredBB);
947
303k
    }
948
1.01M
  } else {
949
2.09M
    for (unsigned i = 0, e = BBPreds.size(); i != e; 
++i1.08M
) {
950
1.08M
      // Update existing incoming values in PN for this
951
1.08M
      // predecessor of BB.
952
1.08M
      BasicBlock *PredBB = BBPreds[i];
953
1.08M
      Value *Selected = selectIncomingValueForBlock(OldVal, PredBB,
954
1.08M
                                                    IncomingValues);
955
1.08M
956
1.08M
      // And add a new incoming value for this predecessor for the
957
1.08M
      // newly retargeted branch.
958
1.08M
      PN->addIncoming(Selected, PredBB);
959
1.08M
    }
960
1.01M
  }
961
1.18M
962
1.18M
  replaceUndefValuesInPhi(PN, IncomingValues);
963
1.18M
}
964
965
/// TryToSimplifyUncondBranchFromEmptyBlock - BB is known to contain an
966
/// unconditional branch, and contains no instructions other than PHI nodes,
967
/// potential side-effect free intrinsics and the branch.  If possible,
968
/// eliminate BB by rewriting all the predecessors to branch to the successor
969
/// block and return true.  If we can't transform, return false.
970
bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB,
971
1.77M
                                                   DomTreeUpdater *DTU) {
972
1.77M
  assert(BB != &BB->getParent()->getEntryBlock() &&
973
1.77M
         "TryToSimplifyUncondBranchFromEmptyBlock called on entry block!");
974
1.77M
975
1.77M
  // We can't eliminate infinite loops.
976
1.77M
  BasicBlock *Succ = cast<BranchInst>(BB->getTerminator())->getSuccessor(0);
977
1.77M
  if (BB == Succ) 
return false88
;
978
1.77M
979
1.77M
  // Check to see if merging these blocks would cause conflicts for any of the
980
1.77M
  // phi nodes in BB or Succ. If not, we can safely merge.
981
1.77M
  if (!CanPropagatePredecessorsForPHIs(BB, Succ)) 
return false155k
;
982
1.61M
983
1.61M
  // Check for cases where Succ has multiple predecessors and a PHI node in BB
984
1.61M
  // has uses which will not disappear when the PHI nodes are merged.  It is
985
1.61M
  // possible to handle such cases, but difficult: it requires checking whether
986
1.61M
  // BB dominates Succ, which is non-trivial to calculate in the case where
987
1.61M
  // Succ has multiple predecessors.  Also, it requires checking whether
988
1.61M
  // constructing the necessary self-referential PHI node doesn't introduce any
989
1.61M
  // conflicts; this isn't too difficult, but the previous code for doing this
990
1.61M
  // was incorrect.
991
1.61M
  //
992
1.61M
  // Note that if this check finds a live use, BB dominates Succ, so BB is
993
1.61M
  // something like a loop pre-header (or rarely, a part of an irreducible CFG);
994
1.61M
  // folding the branch isn't profitable in that case anyway.
995
1.61M
  if (!Succ->getSinglePredecessor()) {
996
1.51M
    BasicBlock::iterator BBI = BB->begin();
997
1.68M
    while (isa<PHINode>(*BBI)) {
998
209k
      for (Use &U : BBI->uses()) {
999
208k
        if (PHINode* PN = dyn_cast<PHINode>(U.getUser())) {
1000
176k
          if (PN->getIncomingBlock(U) != BB)
1001
2.80k
            return false;
1002
31.8k
        } else {
1003
31.8k
          return false;
1004
31.8k
        }
1005
208k
      }
1006
209k
      ++BBI;
1007
174k
    }
1008
1.51M
  }
1009
1.61M
1010
1.61M
  // We cannot fold the block if it's a branch to an already present callbr
1011
1.61M
  // successor because that creates duplicate successors.
1012
3.83M
  
for (auto I = pred_begin(BB), E = pred_end(BB); 1.58M
I != E;
++I2.24M
) {
1013
2.24M
    if (auto *CBI = dyn_cast<CallBrInst>((*I)->getTerminator())) {
1014
5
      if (Succ == CBI->getDefaultDest())
1015
0
        return false;
1016
7
      
for (unsigned i = 0, e = CBI->getNumIndirectDests(); 5
i != e;
++i2
)
1017
5
        if (Succ == CBI->getIndirectDest(i))
1018
3
          return false;
1019
5
    }
1020
2.24M
  }
1021
1.58M
1022
1.58M
  
LLVM_DEBUG1.58M
(dbgs() << "Killing Trivial BB: \n" << *BB);
1023
1.58M
1024
1.58M
  SmallVector<DominatorTree::UpdateType, 32> Updates;
1025
1.58M
  if (DTU) {
1026
232k
    Updates.push_back({DominatorTree::Delete, BB, Succ});
1027
232k
    // All predecessors of BB will be moved to Succ.
1028
561k
    for (auto I = pred_begin(BB), E = pred_end(BB); I != E; 
++I329k
) {
1029
329k
      Updates.push_back({DominatorTree::Delete, *I, BB});
1030
329k
      // This predecessor of BB may already have Succ as a successor.
1031
329k
      if (llvm::find(successors(*I), Succ) == succ_end(*I))
1032
328k
        Updates.push_back({DominatorTree::Insert, *I, Succ});
1033
329k
    }
1034
232k
  }
1035
1.58M
1036
1.58M
  if (isa<PHINode>(Succ->begin())) {
1037
880k
    // If there is more than one pred of succ, and there are PHI nodes in
1038
880k
    // the successor, then we need to add incoming edges for the PHI nodes
1039
880k
    //
1040
880k
    const PredBlockVector BBPreds(pred_begin(BB), pred_end(BB));
1041
880k
1042
880k
    // Loop over all of the PHI nodes in the successor of BB.
1043
2.06M
    for (BasicBlock::iterator I = Succ->begin(); isa<PHINode>(I); 
++I1.18M
) {
1044
1.18M
      PHINode *PN = cast<PHINode>(I);
1045
1.18M
1046
1.18M
      redirectValuesFromPredecessorsToPhi(BB, BBPreds, PN);
1047
1.18M
    }
1048
880k
  }
1049
1.58M
1050
1.58M
  if (Succ->getSinglePredecessor()) {
1051
106k
    // BB is the only predecessor of Succ, so Succ will end up with exactly
1052
106k
    // the same predecessors BB had.
1053
106k
1054
106k
    // Copy over any phi, debug or lifetime instruction.
1055
106k
    BB->getTerminator()->eraseFromParent();
1056
106k
    Succ->getInstList().splice(Succ->getFirstNonPHI()->getIterator(),
1057
106k
                               BB->getInstList());
1058
1.47M
  } else {
1059
1.65M
    while (PHINode *PN = dyn_cast<PHINode>(&BB->front())) {
1060
173k
      // We explicitly check for such uses in CanPropagatePredecessorsForPHIs.
1061
173k
      assert(PN->use_empty() && "There shouldn't be any uses here!");
1062
173k
      PN->eraseFromParent();
1063
173k
    }
1064
1.47M
  }
1065
1.58M
1066
1.58M
  // If the unconditional branch we replaced contains llvm.loop metadata, we
1067
1.58M
  // add the metadata to the branch instructions in the predecessors.
1068
1.58M
  unsigned LoopMDKind = BB->getContext().getMDKindID("llvm.loop");
1069
1.58M
  Instruction *TI = BB->getTerminator();
1070
1.58M
  if (TI)
1071
1.47M
    if (MDNode *LoopMD = TI->getMetadata(LoopMDKind))
1072
2.40k
      
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); 1.17k
PI != E;
++PI1.23k
) {
1073
1.23k
        BasicBlock *Pred = *PI;
1074
1.23k
        Pred->getTerminator()->setMetadata(LoopMDKind, LoopMD);
1075
1.23k
      }
1076
1.58M
1077
1.58M
  // Everything that jumped to BB now goes to Succ.
1078
1.58M
  BB->replaceAllUsesWith(Succ);
1079
1.58M
  if (!Succ->hasName()) 
Succ->takeName(BB)792k
;
1080
1.58M
1081
1.58M
  // Clear the successor list of BB to match updates applying to DTU later.
1082
1.58M
  if (BB->getTerminator())
1083
1.47M
    BB->getInstList().pop_back();
1084
1.58M
  new UnreachableInst(BB->getContext(), BB);
1085
1.58M
  assert(succ_empty(BB) && "The successor list of BB isn't empty before "
1086
1.58M
                           "applying corresponding DTU updates.");
1087
1.58M
1088
1.58M
  if (DTU) {
1089
232k
    DTU->applyUpdatesPermissive(Updates);
1090
232k
    DTU->deleteBB(BB);
1091
1.35M
  } else {
1092
1.35M
    BB->eraseFromParent(); // Delete the old basic block.
1093
1.35M
  }
1094
1.58M
  return true;
1095
1.58M
}
1096
1097
/// EliminateDuplicatePHINodes - Check for and eliminate duplicate PHI
1098
/// nodes in this block. This doesn't try to be clever about PHI nodes
1099
/// which differ only in the order of the incoming values, but instcombine
1100
/// orders them so it usually won't matter.
1101
32.9M
bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
1102
32.9M
  // This implementation doesn't currently consider undef operands
1103
32.9M
  // specially. Theoretically, two phis which are identical except for
1104
32.9M
  // one having an undef where the other doesn't could be collapsed.
1105
32.9M
1106
32.9M
  struct PHIDenseMapInfo {
1107
471M
    static PHINode *getEmptyKey() {
1108
471M
      return DenseMapInfo<PHINode *>::getEmptyKey();
1109
471M
    }
1110
32.9M
1111
47.9M
    static PHINode *getTombstoneKey() {
1112
47.9M
      return DenseMapInfo<PHINode *>::getTombstoneKey();
1113
47.9M
    }
1114
32.9M
1115
32.9M
    static unsigned getHashValue(PHINode *PN) {
1116
8.33M
      // Compute a hash value on the operands. Instcombine will likely have
1117
8.33M
      // sorted them, which helps expose duplicates, but we have to check all
1118
8.33M
      // the operands to be safe in case instcombine hasn't run.
1119
8.33M
      return static_cast<unsigned>(hash_combine(
1120
8.33M
          hash_combine_range(PN->value_op_begin(), PN->value_op_end()),
1121
8.33M
          hash_combine_range(PN->block_begin(), PN->block_end())));
1122
8.33M
    }
1123
32.9M
1124
417M
    static bool isEqual(PHINode *LHS, PHINode *RHS) {
1125
417M
      if (LHS == getEmptyKey() || 
LHS == getTombstoneKey()25.1M
||
1126
417M
          
RHS == getEmptyKey()25.1M
||
RHS == getTombstoneKey()8.43M
)
1127
417M
        return LHS == RHS;
1128
78.9k
      return LHS->isIdenticalTo(RHS);
1129
78.9k
    }
1130
32.9M
  };
1131
32.9M
1132
32.9M
  // Set of unique PHINodes.
1133
32.9M
  DenseSet<PHINode *, PHIDenseMapInfo> PHISet;
1134
32.9M
1135
32.9M
  // Examine each PHI.
1136
32.9M
  bool Changed = false;
1137
41.3M
  for (auto I = BB->begin(); PHINode *PN = dyn_cast<PHINode>(I++);) {
1138
8.33M
    auto Inserted = PHISet.insert(PN);
1139
8.33M
    if (!Inserted.second) {
1140
15.0k
      // A duplicate. Replace this PHI with its duplicate.
1141
15.0k
      PN->replaceAllUsesWith(*Inserted.first);
1142
15.0k
      PN->eraseFromParent();
1143
15.0k
      Changed = true;
1144
15.0k
1145
15.0k
      // The RAUW can change PHIs that we already visited. Start over from the
1146
15.0k
      // beginning.
1147
15.0k
      PHISet.clear();
1148
15.0k
      I = BB->begin();
1149
15.0k
    }
1150
8.33M
  }
1151
32.9M
1152
32.9M
  return Changed;
1153
32.9M
}
1154
1155
/// enforceKnownAlignment - If the specified pointer points to an object that
1156
/// we control, modify the object's alignment to PrefAlign. This isn't
1157
/// often possible though. If alignment is important, a more reliable approach
1158
/// is to simply align all global variables and allocation instructions to
1159
/// their preferred alignment from the beginning.
1160
static unsigned enforceKnownAlignment(Value *V, unsigned Align,
1161
                                      unsigned PrefAlign,
1162
18.8M
                                      const DataLayout &DL) {
1163
18.8M
  assert(PrefAlign > Align);
1164
18.8M
1165
18.8M
  V = V->stripPointerCasts();
1166
18.8M
1167
18.8M
  if (AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1168
3.32k
    // TODO: ideally, computeKnownBits ought to have used
1169
3.32k
    // AllocaInst::getAlignment() in its computation already, making
1170
3.32k
    // the below max redundant. But, as it turns out,
1171
3.32k
    // stripPointerCasts recurses through infinite layers of bitcasts,
1172
3.32k
    // while computeKnownBits is not allowed to traverse more than 6
1173
3.32k
    // levels.
1174
3.32k
    Align = std::max(AI->getAlignment(), Align);
1175
3.32k
    if (PrefAlign <= Align)
1176
4
      return Align;
1177
3.31k
1178
3.31k
    // If the preferred alignment is greater than the natural stack alignment
1179
3.31k
    // then don't round up. This avoids dynamic stack realignment.
1180
3.31k
    if (DL.exceedsNaturalStackAlignment(PrefAlign))
1181
679
      return Align;
1182
2.63k
    AI->setAlignment(PrefAlign);
1183
2.63k
    return PrefAlign;
1184
2.63k
  }
1185
18.8M
1186
18.8M
  if (auto *GO = dyn_cast<GlobalObject>(V)) {
1187
851
    // TODO: as above, this shouldn't be necessary.
1188
851
    Align = std::max(GO->getAlignment(), Align);
1189
851
    if (PrefAlign <= Align)
1190
230
      return Align;
1191
621
1192
621
    // If there is a large requested alignment and we can, bump up the alignment
1193
621
    // of the global.  If the memory we set aside for the global may not be the
1194
621
    // memory used by the final program then it is impossible for us to reliably
1195
621
    // enforce the preferred alignment.
1196
621
    if (!GO->canIncreaseAlignment())
1197
515
      return Align;
1198
106
1199
106
    GO->setAlignment(PrefAlign);
1200
106
    return PrefAlign;
1201
106
  }
1202
18.8M
1203
18.8M
  return Align;
1204
18.8M
}
1205
1206
unsigned llvm::getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign,
1207
                                          const DataLayout &DL,
1208
                                          const Instruction *CxtI,
1209
                                          AssumptionCache *AC,
1210
32.3M
                                          const DominatorTree *DT) {
1211
32.3M
  assert(V->getType()->isPointerTy() &&
1212
32.3M
         "getOrEnforceKnownAlignment expects a pointer!");
1213
32.3M
1214
32.3M
  KnownBits Known = computeKnownBits(V, DL, 0, AC, CxtI, DT);
1215
32.3M
  unsigned TrailZ = Known.countMinTrailingZeros();
1216
32.3M
1217
32.3M
  // Avoid trouble with ridiculously large TrailZ values, such as
1218
32.3M
  // those computed from a null pointer.
1219
32.3M
  TrailZ = std::min(TrailZ, unsigned(sizeof(unsigned) * CHAR_BIT - 1));
1220
32.3M
1221
32.3M
  unsigned Align = 1u << std::min(Known.getBitWidth() - 1, TrailZ);
1222
32.3M
1223
32.3M
  // LLVM doesn't support alignments larger than this currently.
1224
32.3M
  Align = std::min(Align, +Value::MaximumAlignment);
1225
32.3M
1226
32.3M
  if (PrefAlign > Align)
1227
18.8M
    Align = enforceKnownAlignment(V, Align, PrefAlign, DL);
1228
32.3M
1229
32.3M
  // We don't need to make any adjustment.
1230
32.3M
  return Align;
1231
32.3M
}
1232
1233
///===---------------------------------------------------------------------===//
1234
///  Dbg Intrinsic utilities
1235
///
1236
1237
/// See if there is a dbg.value intrinsic for DIVar before I.
1238
static bool LdStHasDebugValue(DILocalVariable *DIVar, DIExpression *DIExpr,
1239
200
                              Instruction *I) {
1240
200
  // Since we can't guarantee that the original dbg.declare instrinsic
1241
200
  // is removed by LowerDbgDeclare(), we need to make sure that we are
1242
200
  // not inserting the same dbg.value intrinsic over and over.
1243
200
  BasicBlock::InstListType::iterator PrevI(I);
1244
200
  if (PrevI != I->getParent()->getInstList().begin()) {
1245
168
    --PrevI;
1246
168
    if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(PrevI))
1247
21
      if (DVI->getValue() == I->getOperand(0) &&
1248
21
          
DVI->getVariable() == DIVar5
&&
1249
21
          
DVI->getExpression() == DIExpr5
)
1250
4
        return true;
1251
196
  }
1252
196
  return false;
1253
196
}
1254
1255
/// See if there is a dbg.value intrinsic for DIVar for the PHI node.
1256
static bool PhiHasDebugValue(DILocalVariable *DIVar,
1257
                             DIExpression *DIExpr,
1258
28
                             PHINode *APN) {
1259
28
  // Since we can't guarantee that the original dbg.declare instrinsic
1260
28
  // is removed by LowerDbgDeclare(), we need to make sure that we are
1261
28
  // not inserting the same dbg.value intrinsic over and over.
1262
28
  SmallVector<DbgValueInst *, 1> DbgValues;
1263
28
  findDbgValues(DbgValues, APN);
1264
28
  for (auto *DVI : DbgValues) {
1265
13
    assert(DVI->getValue() == APN);
1266
13
    if ((DVI->getVariable() == DIVar) && (DVI->getExpression() == DIExpr))
1267
13
      return true;
1268
13
  }
1269
28
  
return false15
;
1270
28
}
1271
1272
/// Check if the alloc size of \p ValTy is large enough to cover the variable
1273
/// (or fragment of the variable) described by \p DII.
1274
///
1275
/// This is primarily intended as a helper for the different
1276
/// ConvertDebugDeclareToDebugValue functions. The dbg.declare/dbg.addr that is
1277
/// converted describes an alloca'd variable, so we need to use the
1278
/// alloc size of the value when doing the comparison. E.g. an i1 value will be
1279
/// identified as covering an n-bit fragment, if the store size of i1 is at
1280
/// least n bits.
1281
215
static bool valueCoversEntireFragment(Type *ValTy, DbgVariableIntrinsic *DII) {
1282
215
  const DataLayout &DL = DII->getModule()->getDataLayout();
1283
215
  uint64_t ValueSize = DL.getTypeAllocSizeInBits(ValTy);
1284
215
  if (auto FragmentSize = DII->getFragmentSizeInBits())
1285
204
    return ValueSize >= *FragmentSize;
1286
11
  // We can't always calculate the size of the DI variable (e.g. if it is a
1287
11
  // VLA). Try to use the size of the alloca that the dbg intrinsic describes
1288
11
  // intead.
1289
11
  if (DII->isAddressOfVariable())
1290
11
    if (auto *AI = dyn_cast_or_null<AllocaInst>(DII->getVariableLocation()))
1291
11
      if (auto FragmentSize = AI->getAllocationSizeInBits(DL))
1292
8
        return ValueSize >= *FragmentSize;
1293
3
  // Could not determine size of variable. Conservatively return false.
1294
3
  return false;
1295
3
}
1296
1297
/// Produce a DebugLoc to use for each dbg.declare/inst pair that are promoted
1298
/// to a dbg.value. Because no machine insts can come from debug intrinsics,
1299
/// only the scope and inlinedAt is significant. Zero line numbers are used in
1300
/// case this DebugLoc leaks into any adjacent instructions.
1301
223
static DebugLoc getDebugValueLoc(DbgVariableIntrinsic *DII, Instruction *Src) {
1302
223
  // Original dbg.declare must have a location.
1303
223
  DebugLoc DeclareLoc = DII->getDebugLoc();
1304
223
  MDNode *Scope = DeclareLoc.getScope();
1305
223
  DILocation *InlinedAt = DeclareLoc.getInlinedAt();
1306
223
  // Produce an unknown location with the correct scope / inlinedAt fields.
1307
223
  return DebugLoc::get(0, 0, Scope, InlinedAt);
1308
223
}
1309
1310
/// Inserts a llvm.dbg.value intrinsic before a store to an alloca'd value
1311
/// that has an associated llvm.dbg.declare or llvm.dbg.addr intrinsic.
1312
void llvm::ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
1313
160
                                           StoreInst *SI, DIBuilder &Builder) {
1314
160
  assert(DII->isAddressOfVariable());
1315
160
  auto *DIVar = DII->getVariable();
1316
160
  assert(DIVar && "Missing variable");
1317
160
  auto *DIExpr = DII->getExpression();
1318
160
  Value *DV = SI->getValueOperand();
1319
160
1320
160
  DebugLoc NewLoc = getDebugValueLoc(DII, SI);
1321
160
1322
160
  if (!valueCoversEntireFragment(DV->getType(), DII)) {
1323
3
    // FIXME: If storing to a part of the variable described by the dbg.declare,
1324
3
    // then we want to insert a dbg.value for the corresponding fragment.
1325
3
    LLVM_DEBUG(dbgs() << "Failed to convert dbg.declare to dbg.value: "
1326
3
                      << *DII << '\n');
1327
3
    // For now, when there is a store to parts of the variable (but we do not
1328
3
    // know which part) we insert an dbg.value instrinsic to indicate that we
1329
3
    // know nothing about the variable's content.
1330
3
    DV = UndefValue::get(DV->getType());
1331
3
    if (!LdStHasDebugValue(DIVar, DIExpr, SI))
1332
3
      Builder.insertDbgValueIntrinsic(DV, DIVar, DIExpr, NewLoc, SI);
1333
3
    return;
1334
3
  }
1335
157
1336
157
  if (!LdStHasDebugValue(DIVar, DIExpr, SI))
1337
153
    Builder.insertDbgValueIntrinsic(DV, DIVar, DIExpr, NewLoc, SI);
1338
157
}
1339
1340
/// Inserts a llvm.dbg.value intrinsic before a load of an alloca'd value
1341
/// that has an associated llvm.dbg.declare or llvm.dbg.addr intrinsic.
1342
void llvm::ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
1343
40
                                           LoadInst *LI, DIBuilder &Builder) {
1344
40
  auto *DIVar = DII->getVariable();
1345
40
  auto *DIExpr = DII->getExpression();
1346
40
  assert(DIVar && "Missing variable");
1347
40
1348
40
  if (LdStHasDebugValue(DIVar, DIExpr, LI))
1349
0
    return;
1350
40
1351
40
  if (!valueCoversEntireFragment(LI->getType(), DII)) {
1352
0
    // FIXME: If only referring to a part of the variable described by the
1353
0
    // dbg.declare, then we want to insert a dbg.value for the corresponding
1354
0
    // fragment.
1355
0
    LLVM_DEBUG(dbgs() << "Failed to convert dbg.declare to dbg.value: "
1356
0
                      << *DII << '\n');
1357
0
    return;
1358
0
  }
1359
40
1360
40
  DebugLoc NewLoc = getDebugValueLoc(DII, nullptr);
1361
40
1362
40
  // We are now tracking the loaded value instead of the address. In the
1363
40
  // future if multi-location support is added to the IR, it might be
1364
40
  // preferable to keep tracking both the loaded value and the original
1365
40
  // address in case the alloca can not be elided.
1366
40
  Instruction *DbgValue = Builder.insertDbgValueIntrinsic(
1367
40
      LI, DIVar, DIExpr, NewLoc, (Instruction *)nullptr);
1368
40
  DbgValue->insertAfter(LI);
1369
40
}
1370
1371
/// Inserts a llvm.dbg.value intrinsic after a phi that has an associated
1372
/// llvm.dbg.declare or llvm.dbg.addr intrinsic.
1373
void llvm::ConvertDebugDeclareToDebugValue(DbgVariableIntrinsic *DII,
1374
28
                                           PHINode *APN, DIBuilder &Builder) {
1375
28
  auto *DIVar = DII->getVariable();
1376
28
  auto *DIExpr = DII->getExpression();
1377
28
  assert(DIVar && "Missing variable");
1378
28
1379
28
  if (PhiHasDebugValue(DIVar, DIExpr, APN))
1380
13
    return;
1381
15
1382
15
  if (!valueCoversEntireFragment(APN->getType(), DII)) {
1383
2
    // FIXME: If only referring to a part of the variable described by the
1384
2
    // dbg.declare, then we want to insert a dbg.value for the corresponding
1385
2
    // fragment.
1386
2
    LLVM_DEBUG(dbgs() << "Failed to convert dbg.declare to dbg.value: "
1387
2
                      << *DII << '\n');
1388
2
    return;
1389
2
  }
1390
13
1391
13
  BasicBlock *BB = APN->getParent();
1392
13
  auto InsertionPt = BB->getFirstInsertionPt();
1393
13
1394
13
  DebugLoc NewLoc = getDebugValueLoc(DII, nullptr);
1395
13
1396
13
  // The block may be a catchswitch block, which does not have a valid
1397
13
  // insertion point.
1398
13
  // FIXME: Insert dbg.value markers in the successors when appropriate.
1399
13
  if (InsertionPt != BB->end())
1400
13
    Builder.insertDbgValueIntrinsic(APN, DIVar, DIExpr, NewLoc, &*InsertionPt);
1401
13
}
1402
1403
/// Determine whether this alloca is either a VLA or an array.
1404
32
static bool isArray(AllocaInst *AI) {
1405
32
  return AI->isArrayAllocation() ||
1406
32
    
AI->getType()->getElementType()->isArrayTy()30
;
1407
32
}
1408
1409
/// LowerDbgDeclare - Lowers llvm.dbg.declare intrinsics into appropriate set
1410
/// of llvm.dbg.value intrinsics.
1411
3.18M
bool llvm::LowerDbgDeclare(Function &F) {
1412
3.18M
  DIBuilder DIB(*F.getParent(), /*AllowUnresolved*/ false);
1413
3.18M
  SmallVector<DbgDeclareInst *, 4> Dbgs;
1414
3.18M
  for (auto &FI : F)
1415
20.1M
    for (Instruction &BI : FI)
1416
110M
      if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
1417
46
        Dbgs.push_back(DDI);
1418
3.18M
1419
3.18M
  if (Dbgs.empty())
1420
3.18M
    return false;
1421
7
1422
46
  
for (auto &I : Dbgs)7
{
1423
46
    DbgDeclareInst *DDI = I;
1424
46
    AllocaInst *AI = dyn_cast_or_null<AllocaInst>(DDI->getAddress());
1425
46
    // If this is an alloca for a scalar variable, insert a dbg.value
1426
46
    // at each load and store to the alloca and erase the dbg.declare.
1427
46
    // The dbg.values allow tracking a variable even if it is not
1428
46
    // stored on the stack, while the dbg.declare can only describe
1429
46
    // the stack slot (and at a lexical-scope granularity). Later
1430
46
    // passes will attempt to elide the stack slot.
1431
46
    if (!AI || 
isArray(AI)32
)
1432
21
      continue;
1433
25
1434
25
    // A volatile load/store means that the alloca can't be elided anyway.
1435
86
    
if (25
llvm::any_of(AI->users(), [](User *U) -> bool 25
{
1436
86
          if (LoadInst *LI = dyn_cast<LoadInst>(U))
1437
41
            return LI->isVolatile();
1438
45
          if (StoreInst *SI = dyn_cast<StoreInst>(U))
1439
31
            return SI->isVolatile();
1440
14
          return false;
1441
14
        }))
1442
1
      continue;
1443
24
1444
85
    
for (auto &AIUse : AI->uses())24
{
1445
85
      User *U = AIUse.getUser();
1446
85
      if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
1447
31
        if (AIUse.getOperandNo() == 1)
1448
30
          ConvertDebugDeclareToDebugValue(DDI, SI, DIB);
1449
54
      } else if (LoadInst *LI = dyn_cast<LoadInst>(U)) {
1450
40
        ConvertDebugDeclareToDebugValue(DDI, LI, DIB);
1451
40
      } else 
if (CallInst *14
CI14
= dyn_cast<CallInst>(U)) {
1452
10
        // This is a call by-value or some other instruction that takes a
1453
10
        // pointer to the variable. Insert a *value* intrinsic that describes
1454
10
        // the variable by dereferencing the alloca.
1455
10
        DebugLoc NewLoc = getDebugValueLoc(DDI, nullptr);
1456
10
        auto *DerefExpr =
1457
10
            DIExpression::append(DDI->getExpression(), dwarf::DW_OP_deref);
1458
10
        DIB.insertDbgValueIntrinsic(AI, DDI->getVariable(), DerefExpr, NewLoc,
1459
10
                                    CI);
1460
10
      }
1461
85
    }
1462
24
    DDI->eraseFromParent();
1463
24
  }
1464
7
  return true;
1465
7
}
1466
1467
/// Propagate dbg.value intrinsics through the newly inserted PHIs.
1468
void llvm::insertDebugValuesForPHIs(BasicBlock *BB,
1469
698k
                                    SmallVectorImpl<PHINode *> &InsertedPHIs) {
1470
698k
  assert(BB && "No BasicBlock to clone dbg.value(s) from.");
1471
698k
  if (InsertedPHIs.size() == 0)
1472
1
    return;
1473
698k
1474
698k
  // Map existing PHI nodes to their dbg.values.
1475
698k
  ValueToValueMapTy DbgValueMap;
1476
6.65M
  for (auto &I : *BB) {
1477
6.65M
    if (auto DbgII = dyn_cast<DbgVariableIntrinsic>(&I)) {
1478
77
      if (auto *Loc = dyn_cast_or_null<PHINode>(DbgII->getVariableLocation()))
1479
36
        DbgValueMap.insert({Loc, DbgII});
1480
77
    }
1481
6.65M
  }
1482
698k
  if (DbgValueMap.size() == 0)
1483
698k
    return;
1484
21
1485
21
  // Then iterate through the new PHIs and look to see if they use one of the
1486
21
  // previously mapped PHIs. If so, insert a new dbg.value intrinsic that will
1487
21
  // propagate the info through the new PHI.
1488
21
  LLVMContext &C = BB->getContext();
1489
28
  for (auto PHI : InsertedPHIs) {
1490
28
    BasicBlock *Parent = PHI->getParent();
1491
28
    // Avoid inserting an intrinsic into an EH block.
1492
28
    if (Parent->getFirstNonPHI()->isEHPad())
1493
1
      continue;
1494
27
    auto PhiMAV = MetadataAsValue::get(C, ValueAsMetadata::get(PHI));
1495
39
    for (auto VI : PHI->operand_values()) {
1496
39
      auto V = DbgValueMap.find(VI);
1497
39
      if (V != DbgValueMap.end()) {
1498
17
        auto *DbgII = cast<DbgVariableIntrinsic>(V->second);
1499
17
        Instruction *NewDbgII = DbgII->clone();
1500
17
        NewDbgII->setOperand(0, PhiMAV);
1501
17
        auto InsertionPt = Parent->getFirstInsertionPt();
1502
17
        assert(InsertionPt != Parent->end() && "Ill-formed basic block");
1503
17
        NewDbgII->insertBefore(&*InsertionPt);
1504
17
      }
1505
39
    }
1506
27
  }
1507
21
}
1508
1509
/// Finds all intrinsics declaring local variables as living in the memory that
1510
/// 'V' points to. This may include a mix of dbg.declare and
1511
/// dbg.addr intrinsics.
1512
3.06M
TinyPtrVector<DbgVariableIntrinsic *> llvm::FindDbgAddrUses(Value *V) {
1513
3.06M
  // This function is hot. Check whether the value has any metadata to avoid a
1514
3.06M
  // DenseMap lookup.
1515
3.06M
  if (!V->isUsedByMetadata())
1516
3.06M
    return {};
1517
393
  auto *L = LocalAsMetadata::getIfExists(V);
1518
393
  if (!L)
1519
0
    return {};
1520
393
  auto *MDV = MetadataAsValue::getIfExists(V->getContext(), L);
1521
393
  if (!MDV)
1522
0
    return {};
1523
393
1524
393
  TinyPtrVector<DbgVariableIntrinsic *> Declares;
1525
393
  for (User *U : MDV->users()) {
1526
369
    if (auto *DII = dyn_cast<DbgVariableIntrinsic>(U))
1527
368
      if (DII->isAddressOfVariable())
1528
297
        Declares.push_back(DII);
1529
369
  }
1530
393
1531
393
  return Declares;
1532
393
}
1533
1534
1.02M
void llvm::findDbgValues(SmallVectorImpl<DbgValueInst *> &DbgValues, Value *V) {
1535
1.02M
  // This function is hot. Check whether the value has any metadata to avoid a
1536
1.02M
  // DenseMap lookup.
1537
1.02M
  if (!V->isUsedByMetadata())
1538
1.02M
    return;
1539
3.58k
  if (auto *L = LocalAsMetadata::getIfExists(V))
1540
3.58k
    if (auto *MDV = MetadataAsValue::getIfExists(V->getContext(), L))
1541
3.58k
      for (User *U : MDV->users())
1542
97
        if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(U))
1543
97
          DbgValues.push_back(DVI);
1544
3.58k
}
1545
1546
void llvm::findDbgUsers(SmallVectorImpl<DbgVariableIntrinsic *> &DbgUsers,
1547
8.52M
                        Value *V) {
1548
8.52M
  // This function is hot. Check whether the value has any metadata to avoid a
1549
8.52M
  // DenseMap lookup.
1550
8.52M
  if (!V->isUsedByMetadata())
1551
8.52M
    return;
1552
249
  if (auto *L = LocalAsMetadata::getIfExists(V))
1553
249
    if (auto *MDV = MetadataAsValue::getIfExists(V->getContext(), L))
1554
249
      for (User *U : MDV->users())
1555
194
        if (DbgVariableIntrinsic *DII = dyn_cast<DbgVariableIntrinsic>(U))
1556
194
          DbgUsers.push_back(DII);
1557
249
}
1558
1559
bool llvm::replaceDbgDeclare(Value *Address, Value *NewAddress,
1560
                             Instruction *InsertBefore, DIBuilder &Builder,
1561
104k
                             uint8_t DIExprFlags, int Offset) {
1562
104k
  auto DbgAddrs = FindDbgAddrUses(Address);
1563
104k
  for (DbgVariableIntrinsic *DII : DbgAddrs) {
1564
68
    DebugLoc Loc = DII->getDebugLoc();
1565
68
    auto *DIVar = DII->getVariable();
1566
68
    auto *DIExpr = DII->getExpression();
1567
68
    assert(DIVar && "Missing variable");
1568
68
    DIExpr = DIExpression::prepend(DIExpr, DIExprFlags, Offset);
1569
68
    // Insert llvm.dbg.declare immediately before InsertBefore, and remove old
1570
68
    // llvm.dbg.declare.
1571
68
    Builder.insertDeclare(NewAddress, DIVar, DIExpr, Loc, InsertBefore);
1572
68
    if (DII == InsertBefore)
1573
3
      InsertBefore = InsertBefore->getNextNode();
1574
68
    DII->eraseFromParent();
1575
68
  }
1576
104k
  return !DbgAddrs.empty();
1577
104k
}
1578
1579
bool llvm::replaceDbgDeclareForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
1580
                                      DIBuilder &Builder, uint8_t DIExprFlags,
1581
104k
                                      int Offset) {
1582
104k
  return replaceDbgDeclare(AI, NewAllocaAddress, AI->getNextNode(), Builder,
1583
104k
                           DIExprFlags, Offset);
1584
104k
}
1585
1586
static void replaceOneDbgValueForAlloca(DbgValueInst *DVI, Value *NewAddress,
1587
5
                                        DIBuilder &Builder, int Offset) {
1588
5
  DebugLoc Loc = DVI->getDebugLoc();
1589
5
  auto *DIVar = DVI->getVariable();
1590
5
  auto *DIExpr = DVI->getExpression();
1591
5
  assert(DIVar && "Missing variable");
1592
5
1593
5
  // This is an alloca-based llvm.dbg.value. The first thing it should do with
1594
5
  // the alloca pointer is dereference it. Otherwise we don't know how to handle
1595
5
  // it and give up.
1596
5
  if (!DIExpr || DIExpr->getNumElements() < 1 ||
1597
5
      
DIExpr->getElement(0) != dwarf::DW_OP_deref4
)
1598
2
    return;
1599
3
1600
3
  // Insert the offset before the first deref.
1601
3
  // We could just change the offset argument of dbg.value, but it's unsigned...
1602
3
  if (Offset)
1603
3
    DIExpr = DIExpression::prepend(DIExpr, 0, Offset);
1604
3
1605
3
  Builder.insertDbgValueIntrinsic(NewAddress, DIVar, DIExpr, Loc, DVI);
1606
3
  DVI->eraseFromParent();
1607
3
}
1608
1609
void llvm::replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
1610
232
                                    DIBuilder &Builder, int Offset) {
1611
232
  if (auto *L = LocalAsMetadata::getIfExists(AI))
1612
4
    if (auto *MDV = MetadataAsValue::getIfExists(AI->getContext(), L))
1613
10
      
for (auto UI = MDV->use_begin(), UE = MDV->use_end(); 4
UI != UE;) {
1614
6
        Use &U = *UI++;
1615
6
        if (auto *DVI = dyn_cast<DbgValueInst>(U.getUser()))
1616
5
          replaceOneDbgValueForAlloca(DVI, NewAllocaAddress, Builder, Offset);
1617
6
      }
1618
232
}
1619
1620
/// Wrap \p V in a ValueAsMetadata instance.
1621
72
static MetadataAsValue *wrapValueInMetadata(LLVMContext &C, Value *V) {
1622
72
  return MetadataAsValue::get(C, ValueAsMetadata::get(V));
1623
72
}
1624
1625
6.22M
bool llvm::salvageDebugInfo(Instruction &I) {
1626
6.22M
  SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
1627
6.22M
  findDbgUsers(DbgUsers, &I);
1628
6.22M
  if (DbgUsers.empty())
1629
6.22M
    return false;
1630
143
1631
143
  return salvageDebugInfoForDbgValues(I, DbgUsers);
1632
143
}
1633
1634
bool llvm::salvageDebugInfoForDbgValues(
1635
151
    Instruction &I, ArrayRef<DbgVariableIntrinsic *> DbgUsers) {
1636
151
  auto &Ctx = I.getContext();
1637
151
  auto wrapMD = [&](Value *V) 
{ return wrapValueInMetadata(Ctx, V); }57
;
1638
151
1639
154
  for (auto *DII : DbgUsers) {
1640
154
    // Do not add DW_OP_stack_value for DbgDeclare and DbgAddr, because they
1641
154
    // are implicitly pointing out the value as a DWARF memory location
1642
154
    // description.
1643
154
    bool StackValue = isa<DbgValueInst>(DII);
1644
154
1645
154
    DIExpression *DIExpr =
1646
154
        salvageDebugInfoImpl(I, DII->getExpression(), StackValue);
1647
154
1648
154
    // salvageDebugInfoImpl should fail on examining the first element of
1649
154
    // DbgUsers, or none of them.
1650
154
    if (!DIExpr)
1651
97
      return false;
1652
57
1653
57
    DII->setOperand(0, wrapMD(I.getOperand(0)));
1654
57
    DII->setOperand(2, MetadataAsValue::get(Ctx, DIExpr));
1655
57
    LLVM_DEBUG(dbgs() << "SALVAGE: " << *DII << '\n');
1656
57
  }
1657
151
1658
151
  
return true54
;
1659
151
}
1660
1661
DIExpression *llvm::salvageDebugInfoImpl(Instruction &I,
1662
                                         DIExpression *SrcDIExpr,
1663
156
                                         bool WithStackValue) {
1664
156
  auto &M = *I.getModule();
1665
156
  auto &DL = M.getDataLayout();
1666
156
1667
156
  // Apply a vector of opcodes to the source DIExpression.
1668
156
  auto doSalvage = [&](SmallVectorImpl<uint64_t> &Ops) -> DIExpression * {
1669
43
    DIExpression *DIExpr = SrcDIExpr;
1670
43
    if (!Ops.empty()) {
1671
41
      DIExpr = DIExpression::prependOpcodes(DIExpr, Ops, WithStackValue);
1672
41
    }
1673
43
    return DIExpr;
1674
43
  };
1675
156
1676
156
  // Apply the given offset to the source DIExpression.
1677
156
  auto applyOffset = [&](uint64_t Offset) -> DIExpression * {
1678
32
    SmallVector<uint64_t, 8> Ops;
1679
32
    DIExpression::appendOffset(Ops, Offset);
1680
32
    return doSalvage(Ops);
1681
32
  };
1682
156
1683
156
  // initializer-list helper for applying operators to the source DIExpression.
1684
156
  auto applyOps =
1685
156
      [&](std::initializer_list<uint64_t> Opcodes) -> DIExpression * {
1686
11
    SmallVector<uint64_t, 8> Ops(Opcodes);
1687
11
    return doSalvage(Ops);
1688
11
  };
1689
156
1690
156
  if (auto *CI = dyn_cast<CastInst>(&I)) {
1691
65
    // No-op casts and zexts are irrelevant for debug info.
1692
65
    if (CI->isNoopCast(DL) || 
isa<ZExtInst>(&I)53
)
1693
16
      return SrcDIExpr;
1694
49
    return nullptr;
1695
91
  } else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
1696
10
    unsigned BitWidth =
1697
10
        M.getDataLayout().getIndexSizeInBits(GEP->getPointerAddressSpace());
1698
10
    // Rewrite a constant GEP into a DIExpression.
1699
10
    APInt Offset(BitWidth, 0);
1700
10
    if (GEP->accumulateConstantOffset(M.getDataLayout(), Offset)) {
1701
8
      return applyOffset(Offset.getSExtValue());
1702
8
    } else {
1703
2
      return nullptr;
1704
2
    }
1705
81
  } else if (auto *BI = dyn_cast<BinaryOperator>(&I)) {
1706
38
    // Rewrite binary operations with constant integer operands.
1707
38
    auto *ConstInt = dyn_cast<ConstantInt>(I.getOperand(1));
1708
38
    if (!ConstInt || 
ConstInt->getBitWidth() > 6435
)
1709
3
      return nullptr;
1710
35
1711
35
    uint64_t Val = ConstInt->getSExtValue();
1712
35
    switch (BI->getOpcode()) {
1713
35
    case Instruction::Add:
1714
20
      return applyOffset(Val);
1715
35
    case Instruction::Sub:
1716
4
      return applyOffset(-int64_t(Val));
1717
35
    case Instruction::Mul:
1718
1
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_mul});
1719
35
    case Instruction::SDiv:
1720
1
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_div});
1721
35
    case Instruction::SRem:
1722
1
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_mod});
1723
35
    case Instruction::Or:
1724
1
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_or});
1725
35
    case Instruction::And:
1726
2
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_and});
1727
35
    case Instruction::Xor:
1728
1
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_xor});
1729
35
    case Instruction::Shl:
1730
1
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_shl});
1731
35
    case Instruction::LShr:
1732
2
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_shr});
1733
35
    case Instruction::AShr:
1734
1
      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_shra});
1735
35
    default:
1736
0
      // TODO: Salvage constants from each kind of binop we know about.
1737
0
      return nullptr;
1738
43
    }
1739
43
    // *Not* to do: we should not attempt to salvage load instructions,
1740
43
    // because the validity and lifetime of a dbg.value containing
1741
43
    // DW_OP_deref becomes difficult to analyze. See PR40628 for examples.
1742
43
  }
1743
43
  return nullptr;
1744
43
}
1745
1746
/// A replacement for a dbg.value expression.
1747
using DbgValReplacement = Optional<DIExpression *>;
1748
1749
/// Point debug users of \p From to \p To using exprs given by \p RewriteExpr,
1750
/// possibly moving/deleting users to prevent use-before-def. Returns true if
1751
/// changes are made.
1752
static bool rewriteDebugUsers(
1753
    Instruction &From, Value &To, Instruction &DomPoint, DominatorTree &DT,
1754
12
    function_ref<DbgValReplacement(DbgVariableIntrinsic &DII)> RewriteExpr) {
1755
12
  // Find debug users of From.
1756
12
  SmallVector<DbgVariableIntrinsic *, 1> Users;
1757
12
  findDbgUsers(Users, &From);
1758
12
  if (Users.empty())
1759
0
    return false;
1760
12
1761
12
  // Prevent use-before-def of To.
1762
12
  bool Changed = false;
1763
12
  SmallPtrSet<DbgVariableIntrinsic *, 1> DeleteOrSalvage;
1764
12
  if (isa<Instruction>(&To)) {
1765
12
    bool DomPointAfterFrom = From.getNextNonDebugInstruction() == &DomPoint;
1766
12
1767
18
    for (auto *DII : Users) {
1768
18
      // It's common to see a debug user between From and DomPoint. Move it
1769
18
      // after DomPoint to preserve the variable update without any reordering.
1770
18
      if (DomPointAfterFrom && 
DII->getNextNonDebugInstruction() == &DomPoint8
) {
1771
7
        LLVM_DEBUG(dbgs() << "MOVE:  " << *DII << '\n');
1772
7
        DII->moveAfter(&DomPoint);
1773
7
        Changed = true;
1774
7
1775
7
      // Users which otherwise aren't dominated by the replacement value must
1776
7
      // be salvaged or deleted.
1777
11
      } else if (!DT.dominates(&DomPoint, DII)) {
1778
3
        DeleteOrSalvage.insert(DII);
1779
3
      }
1780
18
    }
1781
12
  }
1782
12
1783
12
  // Update debug users without use-before-def risk.
1784
18
  for (auto *DII : Users) {
1785
18
    if (DeleteOrSalvage.count(DII))
1786
3
      continue;
1787
15
1788
15
    LLVMContext &Ctx = DII->getContext();
1789
15
    DbgValReplacement DVR = RewriteExpr(*DII);
1790
15
    if (!DVR)
1791
0
      continue;
1792
15
1793
15
    DII->setOperand(0, wrapValueInMetadata(Ctx, &To));
1794
15
    DII->setOperand(2, MetadataAsValue::get(Ctx, *DVR));
1795
15
    LLVM_DEBUG(dbgs() << "REWRITE:  " << *DII << '\n');
1796
15
    Changed = true;
1797
15
  }
1798
12
1799
12
  if (!DeleteOrSalvage.empty()) {
1800
3
    // Try to salvage the remaining debug users.
1801
3
    Changed |= salvageDebugInfo(From);
1802
3
1803
3
    // Delete the debug users which weren't salvaged.
1804
3
    for (auto *DII : DeleteOrSalvage) {
1805
3
      if (DII->getVariableLocation() == &From) {
1806
1
        LLVM_DEBUG(dbgs() << "Erased UseBeforeDef:  " << *DII << '\n');
1807
1
        DII->eraseFromParent();
1808
1
        Changed = true;
1809
1
      }
1810
3
    }
1811
3
  }
1812
12
1813
12
  return Changed;
1814
12
}
1815
1816
/// Check if a bitcast between a value of type \p FromTy to type \p ToTy would
1817
/// losslessly preserve the bits and semantics of the value. This predicate is
1818
/// symmetric, i.e swapping \p FromTy and \p ToTy should give the same result.
1819
///
1820
/// Note that Type::canLosslesslyBitCastTo is not suitable here because it
1821
/// allows semantically unequivalent bitcasts, such as <2 x i64> -> <4 x i32>,
1822
/// and also does not allow lossless pointer <-> integer conversions.
1823
static bool isBitCastSemanticsPreserving(const DataLayout &DL, Type *FromTy,
1824
36
                                         Type *ToTy) {
1825
36
  // Trivially compatible types.
1826
36
  if (FromTy == ToTy)
1827
2
    return true;
1828
34
1829
34
  // Handle compatible pointer <-> integer conversions.
1830
34
  if (FromTy->isIntOrPtrTy() && 
ToTy->isIntOrPtrTy()14
) {
1831
12
    bool SameSize = DL.getTypeSizeInBits(FromTy) == DL.getTypeSizeInBits(ToTy);
1832
12
    bool LosslessConversion = !DL.isNonIntegralPointerType(FromTy) &&
1833
12
                              !DL.isNonIntegralPointerType(ToTy);
1834
12
    return SameSize && 
LosslessConversion4
;
1835
12
  }
1836
22
1837
22
  // TODO: This is not exhaustive.
1838
22
  return false;
1839
22
}
1840
1841
bool llvm::replaceAllDbgUsesWith(Instruction &From, Value &To,
1842
65.0k
                                 Instruction &DomPoint, DominatorTree &DT) {
1843
65.0k
  // Exit early if From has no debug users.
1844
65.0k
  if (!From.isUsedByMetadata())
1845
64.9k
    return false;
1846
36
1847
36
  assert(&From != &To && "Can't replace something with itself");
1848
36
1849
36
  Type *FromTy = From.getType();
1850
36
  Type *ToTy = To.getType();
1851
36
1852
36
  auto Identity = [&](DbgVariableIntrinsic &DII) -> DbgValReplacement {
1853
8
    return DII.getExpression();
1854
8
  };
1855
36
1856
36
  // Handle no-op conversions.
1857
36
  Module &M = *From.getModule();
1858
36
  const DataLayout &DL = M.getDataLayout();
1859
36
  if (isBitCastSemanticsPreserving(DL, FromTy, ToTy))
1860
6
    return rewriteDebugUsers(From, To, DomPoint, DT, Identity);
1861
30
1862
30
  // Handle integer-to-integer widening and narrowing.
1863
30
  // FIXME: Use DW_OP_convert when it's available everywhere.
1864
30
  if (FromTy->isIntegerTy() && 
ToTy->isIntegerTy()9
) {
1865
6
    uint64_t FromBits = FromTy->getPrimitiveSizeInBits();
1866
6
    uint64_t ToBits = ToTy->getPrimitiveSizeInBits();
1867
6
    assert(FromBits != ToBits && "Unexpected no-op conversion");
1868
6
1869
6
    // When the width of the result grows, assume that a debugger will only
1870
6
    // access the low `FromBits` bits when inspecting the source variable.
1871
6
    if (FromBits < ToBits)
1872
4
      return rewriteDebugUsers(From, To, DomPoint, DT, Identity);
1873
2
1874
2
    // The width of the result has shrunk. Use sign/zero extension to describe
1875
2
    // the source variable's high bits.
1876
7
    
auto SignOrZeroExt = [&](DbgVariableIntrinsic &DII) -> DbgValReplacement 2
{
1877
7
      DILocalVariable *Var = DII.getVariable();
1878
7
1879
7
      // Without knowing signedness, sign/zero extension isn't possible.
1880
7
      auto Signedness = Var->getSignedness();
1881
7
      if (!Signedness)
1882
0
        return None;
1883
7
1884
7
      bool Signed = *Signedness == DIBasicType::Signedness::Signed;
1885
7
      dwarf::TypeKind TK = Signed ? dwarf::DW_ATE_signed : 
dwarf::DW_ATE_unsigned0
;
1886
7
      SmallVector<uint64_t, 8> Ops({dwarf::DW_OP_LLVM_convert, ToBits, TK,
1887
7
                                   dwarf::DW_OP_LLVM_convert, FromBits, TK});
1888
7
      return DIExpression::appendToStack(DII.getExpression(), Ops);
1889
7
    };
1890
2
    return rewriteDebugUsers(From, To, DomPoint, DT, SignOrZeroExt);
1891
2
  }
1892
24
1893
24
  // TODO: Floating-point conversions, vectors.
1894
24
  return false;
1895
24
}
1896
1897
93.0k
unsigned llvm::removeAllNonTerminatorAndEHPadInstructions(BasicBlock *BB) {
1898
93.0k
  unsigned NumDeadInst = 0;
1899
93.0k
  // Delete the instructions backwards, as it has a reduced likelihood of
1900
93.0k
  // having to update as many def-use and use-def chains.
1901
93.0k
  Instruction *EndInst = BB->getTerminator(); // Last not to be deleted.
1902
221k
  while (EndInst != &BB->front()) {
1903
128k
    // Delete the next to last instruction.
1904
128k
    Instruction *Inst = &*--EndInst->getIterator();
1905
128k
    if (!Inst->use_empty() && 
!Inst->getType()->isTokenTy()38.5k
)
1906
38.5k
      Inst->replaceAllUsesWith(UndefValue::get(Inst->getType()));
1907
128k
    if (Inst->isEHPad() || 
Inst->getType()->isTokenTy()127k
) {
1908
1.16k
      EndInst = Inst;
1909
1.16k
      continue;
1910
1.16k
    }
1911
127k
    if (!isa<DbgInfoIntrinsic>(Inst))
1912
127k
      ++NumDeadInst;
1913
127k
    Inst->eraseFromParent();
1914
127k
  }
1915
93.0k
  return NumDeadInst;
1916
93.0k
}
1917
1918
unsigned llvm::changeToUnreachable(Instruction *I, bool UseLLVMTrap,
1919
                                   bool PreserveLCSSA, DomTreeUpdater *DTU,
1920
13.6k
                                   MemorySSAUpdater *MSSAU) {
1921
13.6k
  BasicBlock *BB = I->getParent();
1922
13.6k
  std::vector <DominatorTree::UpdateType> Updates;
1923
13.6k
1924
13.6k
  if (MSSAU)
1925
0
    MSSAU->changeToUnreachable(I);
1926
13.6k
1927
13.6k
  // Loop over all of the successors, removing BB's entry from any PHI
1928
13.6k
  // nodes.
1929
13.6k
  if (DTU)
1930
13.2k
    Updates.reserve(BB->getTerminator()->getNumSuccessors());
1931
13.6k
  for (BasicBlock *Successor : successors(BB)) {
1932
11.2k
    Successor->removePredecessor(BB, PreserveLCSSA);
1933
11.2k
    if (DTU)
1934
11.0k
      Updates.push_back({DominatorTree::Delete, BB, Successor});
1935
11.2k
  }
1936
13.6k
  // Insert a call to llvm.trap right before this.  This turns the undefined
1937
13.6k
  // behavior into a hard fail instead of falling through into random code.
1938
13.6k
  if (UseLLVMTrap) {
1939
117
    Function *TrapFn =
1940
117
      Intrinsic::getDeclaration(BB->getParent()->getParent(), Intrinsic::trap);
1941
117
    CallInst *CallTrap = CallInst::Create(TrapFn, "", I);
1942
117
    CallTrap->setDebugLoc(I->getDebugLoc());
1943
117
  }
1944
13.6k
  auto *UI = new UnreachableInst(I->getContext(), I);
1945
13.6k
  UI->setDebugLoc(I->getDebugLoc());
1946
13.6k
1947
13.6k
  // All instructions after this are dead.
1948
13.6k
  unsigned NumInstrsRemoved = 0;
1949
13.6k
  BasicBlock::iterator BBI = I->getIterator(), BBE = BB->end();
1950
45.8k
  while (BBI != BBE) {
1951
32.1k
    if (!BBI->use_empty())
1952
5.24k
      BBI->replaceAllUsesWith(UndefValue::get(BBI->getType()));
1953
32.1k
    BB->getInstList().erase(BBI++);
1954
32.1k
    ++NumInstrsRemoved;
1955
32.1k
  }
1956
13.6k
  if (DTU)
1957
13.2k
    DTU->applyUpdatesPermissive(Updates);
1958
13.6k
  return NumInstrsRemoved;
1959
13.6k
}
1960
1961
/// changeToCall - Convert the specified invoke into a normal call.
1962
8.44k
void llvm::changeToCall(InvokeInst *II, DomTreeUpdater *DTU) {
1963
8.44k
  SmallVector<Value*, 8> Args(II->arg_begin(), II->arg_end());
1964
8.44k
  SmallVector<OperandBundleDef, 1> OpBundles;
1965
8.44k
  II->getOperandBundlesAsDefs(OpBundles);
1966
8.44k
  CallInst *NewCall = CallInst::Create(
1967
8.44k
      II->getFunctionType(), II->getCalledValue(), Args, OpBundles, "", II);
1968
8.44k
  NewCall->takeName(II);
1969
8.44k
  NewCall->setCallingConv(II->getCallingConv());
1970
8.44k
  NewCall->setAttributes(II->getAttributes());
1971
8.44k
  NewCall->setDebugLoc(II->getDebugLoc());
1972
8.44k
  NewCall->copyMetadata(*II);
1973
8.44k
  II->replaceAllUsesWith(NewCall);
1974
8.44k
1975
8.44k
  // Follow the call by a branch to the normal destination.
1976
8.44k
  BasicBlock *NormalDestBB = II->getNormalDest();
1977
8.44k
  BranchInst::Create(NormalDestBB, II);
1978
8.44k
1979
8.44k
  // Update PHI nodes in the unwind destination
1980
8.44k
  BasicBlock *BB = II->getParent();
1981
8.44k
  BasicBlock *UnwindDestBB = II->getUnwindDest();
1982
8.44k
  UnwindDestBB->removePredecessor(BB);
1983
8.44k
  II->eraseFromParent();
1984
8.44k
  if (DTU)
1985
0
    DTU->applyUpdatesPermissive({{DominatorTree::Delete, BB, UnwindDestBB}});
1986
8.44k
}
1987
1988
BasicBlock *llvm::changeToInvokeAndSplitBasicBlock(CallInst *CI,
1989
17.3k
                                                   BasicBlock *UnwindEdge) {
1990
17.3k
  BasicBlock *BB = CI->getParent();
1991
17.3k
1992
17.3k
  // Convert this function call into an invoke instruction.  First, split the
1993
17.3k
  // basic block.
1994
17.3k
  BasicBlock *Split =
1995
17.3k
      BB->splitBasicBlock(CI->getIterator(), CI->getName() + ".noexc");
1996
17.3k
1997
17.3k
  // Delete the unconditional branch inserted by splitBasicBlock
1998
17.3k
  BB->getInstList().pop_back();
1999
17.3k
2000
17.3k
  // Create the new invoke instruction.
2001
17.3k
  SmallVector<Value *, 8> InvokeArgs(CI->arg_begin(), CI->arg_end());
2002
17.3k
  SmallVector<OperandBundleDef, 1> OpBundles;
2003
17.3k
2004
17.3k
  CI->getOperandBundlesAsDefs(OpBundles);
2005
17.3k
2006
17.3k
  // Note: we're round tripping operand bundles through memory here, and that
2007
17.3k
  // can potentially be avoided with a cleverer API design that we do not have
2008
17.3k
  // as of this time.
2009
17.3k
2010
17.3k
  InvokeInst *II =
2011
17.3k
      InvokeInst::Create(CI->getFunctionType(), CI->getCalledValue(), Split,
2012
17.3k
                         UnwindEdge, InvokeArgs, OpBundles, CI->getName(), BB);
2013
17.3k
  II->setDebugLoc(CI->getDebugLoc());
2014
17.3k
  II->setCallingConv(CI->getCallingConv());
2015
17.3k
  II->setAttributes(CI->getAttributes());
2016
17.3k
2017
17.3k
  // Make sure that anything using the call now uses the invoke!  This also
2018
17.3k
  // updates the CallGraph if present, because it uses a WeakTrackingVH.
2019
17.3k
  CI->replaceAllUsesWith(II);
2020
17.3k
2021
17.3k
  // Delete the original call
2022
17.3k
  Split->getInstList().pop_front();
2023
17.3k
  return Split;
2024
17.3k
}
2025
2026
static bool markAliveBlocks(Function &F,
2027
                            SmallPtrSetImpl<BasicBlock *> &Reachable,
2028
5.67M
                            DomTreeUpdater *DTU = nullptr) {
2029
5.67M
  SmallVector<BasicBlock*, 128> Worklist;
2030
5.67M
  BasicBlock *BB = &F.front();
2031
5.67M
  Worklist.push_back(BB);
2032
5.67M
  Reachable.insert(BB);
2033
5.67M
  bool Changed = false;
2034
40.5M
  do {
2035
40.5M
    BB = Worklist.pop_back_val();
2036
40.5M
2037
40.5M
    // Do a quick scan of the basic block, turning any obviously unreachable
2038
40.5M
    // instructions into LLVM unreachable insts.  The instruction combining pass
2039
40.5M
    // canonicalizes unreachable insts into stores to null or undef.
2040
230M
    for (Instruction &I : *BB) {
2041
230M
      if (auto *CI = dyn_cast<CallInst>(&I)) {
2042
30.2M
        Value *Callee = CI->getCalledValue();
2043
30.2M
        // Handle intrinsic calls.
2044
30.2M
        if (Function *F = dyn_cast<Function>(Callee)) {
2045
29.1M
          auto IntrinsicID = F->getIntrinsicID();
2046
29.1M
          // Assumptions that are known to be false are equivalent to
2047
29.1M
          // unreachable. Also, if the condition is undefined, then we make the
2048
29.1M
          // choice most beneficial to the optimizer, and choose that to also be
2049
29.1M
          // unreachable.
2050
29.1M
          if (IntrinsicID == Intrinsic::assume) {
2051
463
            if (match(CI->getArgOperand(0), m_CombineOr(m_Zero(), m_Undef()))) {
2052
28
              // Don't insert a call to llvm.trap right before the unreachable.
2053
28
              changeToUnreachable(CI, false, false, DTU);
2054
28
              Changed = true;
2055
28
              break;
2056
28
            }
2057
29.1M
          } else if (IntrinsicID == Intrinsic::experimental_guard) {
2058
7
            // A call to the guard intrinsic bails out of the current
2059
7
            // compilation unit if the predicate passed to it is false. If the
2060
7
            // predicate is a constant false, then we know the guard will bail
2061
7
            // out of the current compile unconditionally, so all code following
2062
7
            // it is dead.
2063
7
            //
2064
7
            // Note: unlike in llvm.assume, it is not "obviously profitable" for
2065
7
            // guards to treat `undef` as `false` since a guard on `undef` can
2066
7
            // still be useful for widening.
2067
7
            if (match(CI->getArgOperand(0), m_Zero()))
2068
6
              if (!isa<UnreachableInst>(CI->getNextNode())) {
2069
3
                changeToUnreachable(CI->getNextNode(), /*UseLLVMTrap=*/false,
2070
3
                                    false, DTU);
2071
3
                Changed = true;
2072
3
                break;
2073
3
              }
2074
1.13M
          }
2075
1.13M
        } else if ((isa<ConstantPointerNull>(Callee) &&
2076
1.13M
                    
!NullPointerIsDefined(CI->getFunction())3
) ||
2077
1.13M
                   
isa<UndefValue>(Callee)1.13M
) {
2078
8
          changeToUnreachable(CI, /*UseLLVMTrap=*/false, false, DTU);
2079
8
          Changed = true;
2080
8
          break;
2081
8
        }
2082
30.2M
        if (CI->doesNotReturn() && 
!CI->isMustTailCall()535k
) {
2083
535k
          // If we found a call to a no-return function, insert an unreachable
2084
535k
          // instruction after it.  Make sure there isn't *already* one there
2085
535k
          // though.
2086
535k
          if (!isa<UnreachableInst>(CI->getNextNode())) {
2087
54
            // Don't insert a call to llvm.trap right before the unreachable.
2088
54
            changeToUnreachable(CI->getNextNode(), false, false, DTU);
2089
54
            Changed = true;
2090
54
          }
2091
535k
          break;
2092
535k
        }
2093
199M
      } else if (auto *SI = dyn_cast<StoreInst>(&I)) {
2094
18.7M
        // Store to undef and store to null are undefined and used to signal
2095
18.7M
        // that they should be changed to unreachable by passes that can't
2096
18.7M
        // modify the CFG.
2097
18.7M
2098
18.7M
        // Don't touch volatile stores.
2099
18.7M
        if (SI->isVolatile()) 
continue153k
;
2100
18.5M
2101
18.5M
        Value *Ptr = SI->getOperand(1);
2102
18.5M
2103
18.5M
        if (isa<UndefValue>(Ptr) ||
2104
18.5M
            
(18.5M
isa<ConstantPointerNull>(Ptr)18.5M
&&
2105
18.5M
             !NullPointerIsDefined(SI->getFunction(),
2106
114
                                   SI->getPointerAddressSpace()))) {
2107
114
          changeToUnreachable(SI, true, false, DTU);
2108
114
          Changed = true;
2109
114
          break;
2110
114
        }
2111
18.5M
      }
2112
230M
    }
2113
40.5M
2114
40.5M
    Instruction *Terminator = BB->getTerminator();
2115
40.5M
    if (auto *II = dyn_cast<InvokeInst>(Terminator)) {
2116
568k
      // Turn invokes that call 'nounwind' functions into ordinary calls.
2117
568k
      Value *Callee = II->getCalledValue();
2118
568k
      if ((isa<ConstantPointerNull>(Callee) &&
2119
568k
           
!NullPointerIsDefined(BB->getParent())2
) ||
2120
568k
          
isa<UndefValue>(Callee)568k
) {
2121
3
        changeToUnreachable(II, true, false, DTU);
2122
3
        Changed = true;
2123
568k
      } else if (II->doesNotThrow() && 
canSimplifyInvokeNoUnwind(&F)8.07k
) {
2124
8.05k
        if (II->use_empty() && 
II->onlyReadsMemory()2.57k
) {
2125
74
          // jump to the normal destination branch.
2126
74
          BasicBlock *NormalDestBB = II->getNormalDest();
2127
74
          BasicBlock *UnwindDestBB = II->getUnwindDest();
2128
74
          BranchInst::Create(NormalDestBB, II);
2129
74
          UnwindDestBB->removePredecessor(II->getParent());
2130
74
          II->eraseFromParent();
2131
74
          if (DTU)
2132
0
            DTU->applyUpdatesPermissive(
2133
0
                {{DominatorTree::Delete, BB, UnwindDestBB}});
2134
74
        } else
2135
7.97k
          changeToCall(II, DTU);
2136
8.05k
        Changed = true;
2137
8.05k
      }
2138
39.9M
    } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Terminator)) {
2139
427
      // Remove catchpads which cannot be reached.
2140
427
      struct CatchPadDenseMapInfo {
2141
7.02k
        static CatchPadInst *getEmptyKey() {
2142
7.02k
          return DenseMapInfo<CatchPadInst *>::getEmptyKey();
2143
7.02k
        }
2144
427
2145
2.80k
        static CatchPadInst *getTombstoneKey() {
2146
2.80k
          return DenseMapInfo<CatchPadInst *>::getTombstoneKey();
2147
2.80k
        }
2148
427
2149
462
        static unsigned getHashValue(CatchPadInst *CatchPad) {
2150
462
          return static_cast<unsigned>(hash_combine_range(
2151
462
              CatchPad->value_op_begin(), CatchPad->value_op_end()));
2152
462
        }
2153
427
2154
3.83k
        static bool isEqual(CatchPadInst *LHS, CatchPadInst *RHS) {
2155
3.83k
          if (LHS == getEmptyKey() || 
LHS == getTombstoneKey()1.41k
||
2156
3.83k
              
RHS == getEmptyKey()1.41k
||
RHS == getTombstoneKey()486
)
2157
3.82k
            return LHS == RHS;
2158
13
          return LHS->isIdenticalTo(RHS);
2159
13
        }
2160
427
      };
2161
427
2162
427
      // Set of unique CatchPads.
2163
427
      SmallDenseMap<CatchPadInst *, detail::DenseSetEmpty, 4,
2164
427
                    CatchPadDenseMapInfo, detail::DenseSetPair<CatchPadInst *>>
2165
427
          HandlerSet;
2166
427
      detail::DenseSetEmpty Empty;
2167
427
      for (CatchSwitchInst::handler_iterator I = CatchSwitch->handler_begin(),
2168
427
                                             E = CatchSwitch->handler_end();
2169
877
           I != E; 
++I450
) {
2170
450
        BasicBlock *HandlerBB = *I;
2171
450
        auto *CatchPad = cast<CatchPadInst>(HandlerBB->getFirstNonPHI());
2172
450
        if (!HandlerSet.insert({CatchPad, Empty}).second) {
2173
3
          CatchSwitch->removeHandler(I);
2174
3
          --I;
2175
3
          --E;
2176
3
          Changed = true;
2177
3
        }
2178
450
      }
2179
427
    }
2180
40.5M
2181
40.5M
    Changed |= ConstantFoldTerminator(BB, true, nullptr, DTU);
2182
40.5M
    for (BasicBlock *Successor : successors(BB))
2183
54.7M
      if (Reachable.insert(Successor).second)
2184
34.8M
        Worklist.push_back(Successor);
2185
40.5M
  } while (!Worklist.empty());
2186
5.67M
  return Changed;
2187
5.67M
}
2188
2189
475
void llvm::removeUnwindEdge(BasicBlock *BB, DomTreeUpdater *DTU) {
2190
475
  Instruction *TI = BB->getTerminator();
2191
475
2192
475
  if (auto *II = dyn_cast<InvokeInst>(TI)) {
2193
469
    changeToCall(II, DTU);
2194
469
    return;
2195
469
  }
2196
6
2197
6
  Instruction *NewTI;
2198
6
  BasicBlock *UnwindDest;
2199
6
2200
6
  if (auto *CRI = dyn_cast<CleanupReturnInst>(TI)) {
2201
1
    NewTI = CleanupReturnInst::Create(CRI->getCleanupPad(), nullptr, CRI);
2202
1
    UnwindDest = CRI->getUnwindDest();
2203
5
  } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(TI)) {
2204
5
    auto *NewCatchSwitch = CatchSwitchInst::Create(
2205
5
        CatchSwitch->getParentPad(), nullptr, CatchSwitch->getNumHandlers(),
2206
5
        CatchSwitch->getName(), CatchSwitch);
2207
5
    for (BasicBlock *PadBB : CatchSwitch->handlers())
2208
5
      NewCatchSwitch->addHandler(PadBB);
2209
5
2210
5
    NewTI = NewCatchSwitch;
2211
5
    UnwindDest = CatchSwitch->getUnwindDest();
2212
5
  } else {
2213
0
    llvm_unreachable("Could not find unwind successor");
2214
0
  }
2215
6
2216
6
  NewTI->takeName(TI);
2217
6
  NewTI->setDebugLoc(TI->getDebugLoc());
2218
6
  UnwindDest->removePredecessor(BB);
2219
6
  TI->replaceAllUsesWith(NewTI);
2220
6
  TI->eraseFromParent();
2221
6
  if (DTU)
2222
0
    DTU->applyUpdatesPermissive({{DominatorTree::Delete, BB, UnwindDest}});
2223
6
}
2224
2225
/// removeUnreachableBlocks - Remove blocks that are not reachable, even
2226
/// if they are in a dead cycle.  Return true if a change was made, false
2227
/// otherwise. If `LVI` is passed, this function preserves LazyValueInfo
2228
/// after modifying the CFG.
2229
bool llvm::removeUnreachableBlocks(Function &F, LazyValueInfo *LVI,
2230
                                   DomTreeUpdater *DTU,
2231
5.67M
                                   MemorySSAUpdater *MSSAU) {
2232
5.67M
  SmallPtrSet<BasicBlock*, 16> Reachable;
2233
5.67M
  bool Changed = markAliveBlocks(F, Reachable, DTU);
2234
5.67M
2235
5.67M
  // If there are unreachable blocks in the CFG...
2236
5.67M
  if (Reachable.size() == F.size())
2237
5.65M
    return Changed;
2238
18.7k
2239
18.7k
  assert(Reachable.size() < F.size());
2240
18.7k
  NumRemoved += F.size()-Reachable.size();
2241
18.7k
2242
18.7k
  SmallSetVector<BasicBlock *, 8> DeadBlockSet;
2243
717k
  for (Function::iterator I = ++F.begin(), E = F.end(); I != E; 
++I699k
) {
2244
699k
    auto *BB = &*I;
2245
699k
    if (Reachable.count(BB))
2246
618k
      continue;
2247
80.1k
    DeadBlockSet.insert(BB);
2248
80.1k
  }
2249
18.7k
2250
18.7k
  if (MSSAU)
2251
0
    MSSAU->removeBlocks(DeadBlockSet);
2252
18.7k
2253
18.7k
  // Loop over all of the basic blocks that are not reachable, dropping all of
2254
18.7k
  // their internal references. Update DTU and LVI if available.
2255
18.7k
  std::vector<DominatorTree::UpdateType> Updates;
2256
80.1k
  for (auto *BB : DeadBlockSet) {
2257
87.8k
    for (BasicBlock *Successor : successors(BB)) {
2258
87.8k
      if (!DeadBlockSet.count(Successor))
2259
23.9k
        Successor->removePredecessor(BB);
2260
87.8k
      if (DTU)
2261
10
        Updates.push_back({DominatorTree::Delete, BB, Successor});
2262
87.8k
    }
2263
80.1k
    if (LVI)
2264
0
      LVI->eraseBlock(BB);
2265
80.1k
    BB->dropAllReferences();
2266
80.1k
  }
2267
717k
  for (Function::iterator I = ++F.begin(); I != F.end();) {
2268
699k
    auto *BB = &*I;
2269
699k
    if (Reachable.count(BB)) {
2270
618k
      ++I;
2271
618k
      continue;
2272
618k
    }
2273
80.1k
    if (DTU) {
2274
19
      // Remove the terminator of BB to clear the successor list of BB.
2275
19
      if (BB->getTerminator())
2276
19
        BB->getInstList().pop_back();
2277
19
      new UnreachableInst(BB->getContext(), BB);
2278
19
      assert(succ_empty(BB) && "The successor list of BB isn't empty before "
2279
19
                               "applying corresponding DTU updates.");
2280
19
      ++I;
2281
80.1k
    } else {
2282
80.1k
      I = F.getBasicBlockList().erase(I);
2283
80.1k
    }
2284
80.1k
  }
2285
18.7k
2286
18.7k
  if (DTU) {
2287
12
    DTU->applyUpdatesPermissive(Updates);
2288
12
    bool Deleted = false;
2289
19
    for (auto *BB : DeadBlockSet) {
2290
19
      if (DTU->isBBPendingDeletion(BB))
2291
1
        --NumRemoved;
2292
18
      else
2293
18
        Deleted = true;
2294
19
      DTU->deleteBB(BB);
2295
19
    }
2296
12
    if (!Deleted)
2297
1
      return false;
2298
18.7k
  }
2299
18.7k
  return true;
2300
18.7k
}
2301
2302
void llvm::combineMetadata(Instruction *K, const Instruction *J,
2303
234k
                           ArrayRef<unsigned> KnownIDs, bool DoesKMove) {
2304
234k
  SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
2305
234k
  K->dropUnknownNonDebugMetadata(KnownIDs);
2306
234k
  K->getAllMetadataOtherThanDebugLoc(Metadata);
2307
234k
  for (const auto &MD : Metadata) {
2308
63.6k
    unsigned Kind = MD.first;
2309
63.6k
    MDNode *JMD = J->getMetadata(Kind);
2310
63.6k
    MDNode *KMD = MD.second;
2311
63.6k
2312
63.6k
    switch (Kind) {
2313
63.6k
      default:
2314
0
        K->setMetadata(Kind, nullptr); // Remove unknown metadata
2315
0
        break;
2316
63.6k
      case LLVMContext::MD_dbg:
2317
0
        llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
2318
63.6k
      case LLVMContext::MD_tbaa:
2319
61.6k
        K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
2320
61.6k
        break;
2321
63.6k
      case LLVMContext::MD_alias_scope:
2322
999
        K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
2323
999
        break;
2324
63.6k
      case LLVMContext::MD_noalias:
2325
536
      case LLVMContext::MD_mem_parallel_loop_access:
2326
536
        K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
2327
536
        break;
2328
536
      case LLVMContext::MD_access_group:
2329
15
        K->setMetadata(LLVMContext::MD_access_group,
2330
15
                       intersectAccessGroups(K, J));
2331
15
        break;
2332
536
      case LLVMContext::MD_range:
2333
441
2334
441
        // If K does move, use most generic range. Otherwise keep the range of
2335
441
        // K.
2336
441
        if (DoesKMove)
2337
407
          // FIXME: If K does move, we should drop the range info and nonnull.
2338
407
          //        Currently this function is used with DoesKMove in passes
2339
407
          //        doing hoisting/sinking and the current behavior of using the
2340
407
          //        most generic range is correct in those cases.
2341
407
          K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
2342
441
        break;
2343
536
      case LLVMContext::MD_fpmath:
2344
10
        K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
2345
10
        break;
2346
536
      case LLVMContext::MD_invariant_load:
2347
5
        // Only set the !invariant.load if it is present in both instructions.
2348
5
        K->setMetadata(Kind, JMD);
2349
5
        break;
2350
536
      case LLVMContext::MD_nonnull:
2351
9
        // If K does move, keep nonull if it is present in both instructions.
2352
9
        if (DoesKMove)
2353
4
          K->setMetadata(Kind, JMD);
2354
9
        break;
2355
536
      case LLVMContext::MD_invariant_group:
2356
5
        // Preserve !invariant.group in K.
2357
5
        break;
2358
536
      case LLVMContext::MD_align:
2359
3
        K->setMetadata(Kind,
2360
3
          MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
2361
3
        break;
2362
536
      case LLVMContext::MD_dereferenceable:
2363
6
      case LLVMContext::MD_dereferenceable_or_null:
2364
6
        K->setMetadata(Kind,
2365
6
          MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
2366
6
        break;
2367
63.6k
    }
2368
63.6k
  }
2369
234k
  // Set !invariant.group from J if J has it. If both instructions have it
2370
234k
  // then we will just pick it from J - even when they are different.
2371
234k
  // Also make sure that K is load or store - f.e. combining bitcast with load
2372
234k
  // could produce bitcast with invariant.group metadata, which is invalid.
2373
234k
  // FIXME: we should try to preserve both invariant.group md if they are
2374
234k
  // different, but right now instruction can only have one invariant.group.
2375
234k
  if (auto *JMD = J->getMetadata(LLVMContext::MD_invariant_group))
2376
12
    if (isa<LoadInst>(K) || 
isa<StoreInst>(K)6
)
2377
6
      K->setMetadata(LLVMContext::MD_invariant_group, JMD);
2378
234k
}
2379
2380
void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
2381
55.5k
                                 bool KDominatesJ) {
2382
55.5k
  unsigned KnownIDs[] = {
2383
55.5k
      LLVMContext::MD_tbaa,            LLVMContext::MD_alias_scope,
2384
55.5k
      LLVMContext::MD_noalias,         LLVMContext::MD_range,
2385
55.5k
      LLVMContext::MD_invariant_load,  LLVMContext::MD_nonnull,
2386
55.5k
      LLVMContext::MD_invariant_group, LLVMContext::MD_align,
2387
55.5k
      LLVMContext::MD_dereferenceable,
2388
55.5k
      LLVMContext::MD_dereferenceable_or_null,
2389
55.5k
      LLVMContext::MD_access_group};
2390
55.5k
  combineMetadata(K, J, KnownIDs, KDominatesJ);
2391
55.5k
}
2392
2393
116k
void llvm::patchReplacementInstruction(Instruction *I, Value *Repl) {
2394
116k
  auto *ReplInst = dyn_cast<Instruction>(Repl);
2395
116k
  if (!ReplInst)
2396
3.38k
    return;
2397
113k
2398
113k
  // Patch the replacement so that it is not more restrictive than the value
2399
113k
  // being replaced.
2400
113k
  // Note that if 'I' is a load being replaced by some operation,
2401
113k
  // for example, by an arithmetic operation, then andIRFlags()
2402
113k
  // would just erase all math flags from the original arithmetic
2403
113k
  // operation, which is clearly not wanted and not needed.
2404
113k
  if (!isa<LoadInst>(I))
2405
105k
    ReplInst->andIRFlags(I);
2406
113k
2407
113k
  // FIXME: If both the original and replacement value are part of the
2408
113k
  // same control-flow region (meaning that the execution of one
2409
113k
  // guarantees the execution of the other), then we can combine the
2410
113k
  // noalias scopes here and do better than the general conservative
2411
113k
  // answer used in combineMetadata().
2412
113k
2413
113k
  // In general, GVN unifies expressions over different control-flow
2414
113k
  // regions, and so we need a conservative combination of the noalias
2415
113k
  // scopes.
2416
113k
  static const unsigned KnownIDs[] = {
2417
113k
      LLVMContext::MD_tbaa,            LLVMContext::MD_alias_scope,
2418
113k
      LLVMContext::MD_noalias,         LLVMContext::MD_range,
2419
113k
      LLVMContext::MD_fpmath,          LLVMContext::MD_invariant_load,
2420
113k
      LLVMContext::MD_invariant_group, LLVMContext::MD_nonnull,
2421
113k
      LLVMContext::MD_access_group};
2422
113k
  combineMetadata(ReplInst, I, KnownIDs, false);
2423
113k
}
2424
2425
template <typename RootType, typename DominatesFn>
2426
static unsigned replaceDominatedUsesWith(Value *From, Value *To,
2427
                                         const RootType &Root,
2428
3.66M
                                         const DominatesFn &Dominates) {
2429
3.66M
  assert(From->getType() == To->getType());
2430
3.66M
2431
3.66M
  unsigned Count = 0;
2432
3.66M
  for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
2433
10.2M
       UI != UE;) {
2434
6.54M
    Use &U = *UI++;
2435
6.54M
    if (!Dominates(Root, U))
2436
6.51M
      continue;
2437
20.0k
    U.set(To);
2438
20.0k
    LLVM_DEBUG(dbgs() << "Replace dominated use of '" << From->getName()
2439
20.0k
                      << "' as " << *To << " in " << *U << "\n");
2440
20.0k
    ++Count;
2441
20.0k
  }
2442
3.66M
  return Count;
2443
3.66M
}
Local.cpp:unsigned int replaceDominatedUsesWith<llvm::BasicBlockEdge, llvm::replaceDominatedUsesWith(llvm::Value*, llvm::Value*, llvm::DominatorTree&, llvm::BasicBlockEdge const&)::$_7>(llvm::Value*, llvm::Value*, llvm::BasicBlockEdge const&, llvm::replaceDominatedUsesWith(llvm::Value*, llvm::Value*, llvm::DominatorTree&, llvm::BasicBlockEdge const&)::$_7 const&)
Line
Count
Source
2428
3.66M
                                         const DominatesFn &Dominates) {
2429
3.66M
  assert(From->getType() == To->getType());
2430
3.66M
2431
3.66M
  unsigned Count = 0;
2432
3.66M
  for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
2433
10.2M
       UI != UE;) {
2434
6.54M
    Use &U = *UI++;
2435
6.54M
    if (!Dominates(Root, U))
2436
6.51M
      continue;
2437
20.0k
    U.set(To);
2438
20.0k
    LLVM_DEBUG(dbgs() << "Replace dominated use of '" << From->getName()
2439
20.0k
                      << "' as " << *To << " in " << *U << "\n");
2440
20.0k
    ++Count;
2441
20.0k
  }
2442
3.66M
  return Count;
2443
3.66M
}
Local.cpp:unsigned int replaceDominatedUsesWith<llvm::BasicBlock const*, llvm::replaceDominatedUsesWith(llvm::Value*, llvm::Value*, llvm::DominatorTree&, llvm::BasicBlock const*)::$_8>(llvm::Value*, llvm::Value*, llvm::BasicBlock const* const&, llvm::replaceDominatedUsesWith(llvm::Value*, llvm::Value*, llvm::DominatorTree&, llvm::BasicBlock const*)::$_8 const&)
Line
Count
Source
2428
20
                                         const DominatesFn &Dominates) {
2429
20
  assert(From->getType() == To->getType());
2430
20
2431
20
  unsigned Count = 0;
2432
20
  for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
2433
65
       UI != UE;) {
2434
45
    Use &U = *UI++;
2435
45
    if (!Dominates(Root, U))
2436
34
      continue;
2437
11
    U.set(To);
2438
11
    LLVM_DEBUG(dbgs() << "Replace dominated use of '" << From->getName()
2439
11
                      << "' as " << *To << " in " << *U << "\n");
2440
11
    ++Count;
2441
11
  }
2442
20
  return Count;
2443
20
}
2444
2445
154
unsigned llvm::replaceNonLocalUsesWith(Instruction *From, Value *To) {
2446
154
   assert(From->getType() == To->getType());
2447
154
   auto *BB = From->getParent();
2448
154
   unsigned Count = 0;
2449
154
2450
154
  for (Value::use_iterator UI = From->use_begin(), UE = From->use_end();
2451
403
       UI != UE;) {
2452
249
    Use &U = *UI++;
2453
249
    auto *I = cast<Instruction>(U.getUser());
2454
249
    if (I->getParent() == BB)
2455
60
      continue;
2456
189
    U.set(To);
2457
189
    ++Count;
2458
189
  }
2459
154
  return Count;
2460
154
}
2461
2462
unsigned llvm::replaceDominatedUsesWith(Value *From, Value *To,
2463
                                        DominatorTree &DT,
2464
3.66M
                                        const BasicBlockEdge &Root) {
2465
6.54M
  auto Dominates = [&DT](const BasicBlockEdge &Root, const Use &U) {
2466
6.54M
    return DT.dominates(Root, U);
2467
6.54M
  };
2468
3.66M
  return ::replaceDominatedUsesWith(From, To, Root, Dominates);
2469
3.66M
}
2470
2471
unsigned llvm::replaceDominatedUsesWith(Value *From, Value *To,
2472
                                        DominatorTree &DT,
2473
20
                                        const BasicBlock *BB) {
2474
45
  auto ProperlyDominates = [&DT](const BasicBlock *BB, const Use &U) {
2475
45
    auto *I = cast<Instruction>(U.getUser())->getParent();
2476
45
    return DT.properlyDominates(BB, I);
2477
45
  };
2478
20
  return ::replaceDominatedUsesWith(From, To, BB, ProperlyDominates);
2479
20
}
2480
2481
bool llvm::callsGCLeafFunction(const CallBase *Call,
2482
494
                               const TargetLibraryInfo &TLI) {
2483
494
  // Check if the function is specifically marked as a gc leaf function.
2484
494
  if (Call->hasFnAttr("gc-leaf-function"))
2485
127
    return true;
2486
367
  if (const Function *F = Call->getCalledFunction()) {
2487
367
    if (F->hasFnAttribute("gc-leaf-function"))
2488
0
      return true;
2489
367
2490
367
    if (auto IID = F->getIntrinsicID())
2491
16
      // Most LLVM intrinsics do not take safepoints.
2492
16
      return IID != Intrinsic::experimental_gc_statepoint &&
2493
16
             IID != Intrinsic::experimental_deoptimize;
2494
351
  }
2495
351
2496
351
  // Lib calls can be materialized by some passes, and won't be
2497
351
  // marked as 'gc-leaf-function.' All available Libcalls are
2498
351
  // GC-leaf.
2499
351
  LibFunc LF;
2500
351
  if (TLI.getLibFunc(ImmutableCallSite(Call), LF)) {
2501
3
    return TLI.has(LF);
2502
3
  }
2503
348
2504
348
  return false;
2505
348
}
2506
2507
void llvm::copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
2508
8
                               LoadInst &NewLI) {
2509
8
  auto *NewTy = NewLI.getType();
2510
8
2511
8
  // This only directly applies if the new type is also a pointer.
2512
8
  if (NewTy->isPointerTy()) {
2513
4
    NewLI.setMetadata(LLVMContext::MD_nonnull, N);
2514
4
    return;
2515
4
  }
2516
4
2517
4
  // The only other translation we can do is to integral loads with !range
2518
4
  // metadata.
2519
4
  if (!NewTy->isIntegerTy())
2520
0
    return;
2521
4
2522
4
  MDBuilder MDB(NewLI.getContext());
2523
4
  const Value *Ptr = OldLI.getPointerOperand();
2524
4
  auto *ITy = cast<IntegerType>(NewTy);
2525
4
  auto *NullInt = ConstantExpr::getPtrToInt(
2526
4
      ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
2527
4
  auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
2528
4
  NewLI.setMetadata(LLVMContext::MD_range,
2529
4
                    MDB.createRange(NonNullInt, NullInt));
2530
4
}
2531
2532
void llvm::copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
2533
3
                             MDNode *N, LoadInst &NewLI) {
2534
3
  auto *NewTy = NewLI.getType();
2535
3
2536
3
  // Give up unless it is converted to a pointer where there is a single very
2537
3
  // valuable mapping we can do reliably.
2538
3
  // FIXME: It would be nice to propagate this in more ways, but the type
2539
3
  // conversions make it hard.
2540
3
  if (!NewTy->isPointerTy())
2541
1
    return;
2542
2
2543
2
  unsigned BitWidth = DL.getIndexTypeSizeInBits(NewTy);
2544
2
  if (!getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
2545
2
    MDNode *NN = MDNode::get(OldLI.getContext(), None);
2546
2
    NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
2547
2
  }
2548
2
}
2549
2550
4
void llvm::dropDebugUsers(Instruction &I) {
2551
4
  SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
2552
4
  findDbgUsers(DbgUsers, &I);
2553
4
  for (auto *DII : DbgUsers)
2554
4
    DII->eraseFromParent();
2555
4
}
2556
2557
void llvm::hoistAllInstructionsInto(BasicBlock *DomBlock, Instruction *InsertPt,
2558
26.3k
                                    BasicBlock *BB) {
2559
26.3k
  // Since we are moving the instructions out of its basic block, we do not
2560
26.3k
  // retain their original debug locations (DILocations) and debug intrinsic
2561
26.3k
  // instructions.
2562
26.3k
  //
2563
26.3k
  // Doing so would degrade the debugging experience and adversely affect the
2564
26.3k
  // accuracy of profiling information.
2565
26.3k
  //
2566
26.3k
  // Currently, when hoisting the instructions, we take the following actions:
2567
26.3k
  // - Remove their debug intrinsic instructions.
2568
26.3k
  // - Set their debug locations to the values from the insertion point.
2569
26.3k
  //
2570
26.3k
  // As per PR39141 (comment #8), the more fundamental reason why the dbg.values
2571
26.3k
  // need to be deleted, is because there will not be any instructions with a
2572
26.3k
  // DILocation in either branch left after performing the transformation. We
2573
26.3k
  // can only insert a dbg.value after the two branches are joined again.
2574
26.3k
  //
2575
26.3k
  // See PR38762, PR39243 for more details.
2576
26.3k
  //
2577
26.3k
  // TODO: Extend llvm.dbg.value to take more than one SSA Value (PR39141) to
2578
26.3k
  // encode predicated DIExpressions that yield different results on different
2579
26.3k
  // code paths.
2580
82.1k
  for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
2581
55.7k
    Instruction *I = &*II;
2582
55.7k
    I->dropUnknownNonDebugMetadata();
2583
55.7k
    if (I->isUsedByMetadata())
2584
4
      dropDebugUsers(*I);
2585
55.7k
    if (isa<DbgInfoIntrinsic>(I)) {
2586
3
      // Remove DbgInfo Intrinsics.
2587
3
      II = I->eraseFromParent();
2588
3
      continue;
2589
3
    }
2590
55.7k
    I->setDebugLoc(InsertPt->getDebugLoc());
2591
55.7k
    ++II;
2592
55.7k
  }
2593
26.3k
  DomBlock->getInstList().splice(InsertPt->getIterator(), BB->getInstList(),
2594
26.3k
                                 BB->begin(),
2595
26.3k
                                 BB->getTerminator()->getIterator());
2596
26.3k
}
2597
2598
namespace {
2599
2600
/// A potential constituent of a bitreverse or bswap expression. See
2601
/// collectBitParts for a fuller explanation.
2602
struct BitPart {
2603
636k
  BitPart(Value *P, unsigned BW) : Provider(P) {
2604
636k
    Provenance.resize(BW);
2605
636k
  }
2606
2607
  /// The Value that this is a bitreverse/bswap of.
2608
  Value *Provider;
2609
2610
  /// The "provenance" of each bit. Provenance[A] = B means that bit A
2611
  /// in Provider becomes bit B in the result of this expression.
2612
  SmallVector<int8_t, 32> Provenance; // int8_t means max size is i128.
2613
2614
  enum { Unset = -1 };
2615
};
2616
2617
} // end anonymous namespace
2618
2619
/// Analyze the specified subexpression and see if it is capable of providing
2620
/// pieces of a bswap or bitreverse. The subexpression provides a potential
2621
/// piece of a bswap or bitreverse if it can be proven that each non-zero bit in
2622
/// the output of the expression came from a corresponding bit in some other
2623
/// value. This function is recursive, and the end result is a mapping of
2624
/// bitnumber to bitnumber. It is the caller's responsibility to validate that
2625
/// the bitnumber to bitnumber mapping is correct for a bswap or bitreverse.
2626
///
2627
/// For example, if the current subexpression if "(shl i32 %X, 24)" then we know
2628
/// that the expression deposits the low byte of %X into the high byte of the
2629
/// result and that all other bits are zero. This expression is accepted and a
2630
/// BitPart is returned with Provider set to %X and Provenance[24-31] set to
2631
/// [0-7].
2632
///
2633
/// To avoid revisiting values, the BitPart results are memoized into the
2634
/// provided map. To avoid unnecessary copying of BitParts, BitParts are
2635
/// constructed in-place in the \c BPS map. Because of this \c BPS needs to
2636
/// store BitParts objects, not pointers. As we need the concept of a nullptr
2637
/// BitParts (Value has been analyzed and the analysis failed), we an Optional
2638
/// type instead to provide the same functionality.
2639
///
2640
/// Because we pass around references into \c BPS, we must use a container that
2641
/// does not invalidate internal references (std::map instead of DenseMap).
2642
static const Optional<BitPart> &
2643
collectBitParts(Value *V, bool MatchBSwaps, bool MatchBitReversals,
2644
1.48M
                std::map<Value *, Optional<BitPart>> &BPS, int Depth) {
2645
1.48M
  auto I = BPS.find(V);
2646
1.48M
  if (I != BPS.end())
2647
31.2k
    return I->second;
2648
1.45M
2649
1.45M
  auto &Result = BPS[V] = None;
2650
1.45M
  auto BitWidth = cast<IntegerType>(V->getType())->getBitWidth();
2651
1.45M
2652
1.45M
  // Prevent stack overflow by limiting the recursion depth
2653
1.45M
  if (Depth == BitPartRecursionMaxDepth) {
2654
1
    LLVM_DEBUG(dbgs() << "collectBitParts max recursion depth reached.\n");
2655
1
    return Result;
2656
1
  }
2657
1.45M
2658
1.45M
  if (Instruction *I = dyn_cast<Instruction>(V)) {
2659
1.41M
    // If this is an or instruction, it may be an inner node of the bswap.
2660
1.41M
    if (I->getOpcode() == Instruction::Or) {
2661
428k
      auto &A = collectBitParts(I->getOperand(0), MatchBSwaps,
2662
428k
                                MatchBitReversals, BPS, Depth + 1);
2663
428k
      auto &B = collectBitParts(I->getOperand(1), MatchBSwaps,
2664
428k
                                MatchBitReversals, BPS, Depth + 1);
2665
428k
      if (!A || 
!B243k
)
2666
257k
        return Result;
2667
171k
2668
171k
      // Try and merge the two together.
2669
171k
      if (!A->Provider || A->Provider != B->Provider)
2670
144k
        return Result;
2671
27.0k
2672
27.0k
      Result = BitPart(A->Provider, BitWidth);
2673
965k
      for (unsigned i = 0; i < A->Provenance.size(); 
++i938k
) {
2674
941k
        if (A->Provenance[i] != BitPart::Unset &&
2675
941k
            
B->Provenance[i] != BitPart::Unset475k
&&
2676
941k
            
A->Provenance[i] != B->Provenance[i]2.89k
)
2677
2.89k
          return Result = None;
2678
938k
2679
938k
        if (A->Provenance[i] == BitPart::Unset)
2680
466k
          Result->Provenance[i] = B->Provenance[i];
2681
472k
        else
2682
472k
          Result->Provenance[i] = A->Provenance[i];
2683
938k
      }
2684
27.0k
2685
27.0k
      
return Result24.1k
;
2686
985k
    }
2687
985k
2688
985k
    // If this is a logical shift by a constant, recurse then shift the result.
2689
985k
    if (I->isLogicalShift() && 
isa<ConstantInt>(I->getOperand(1))275k
) {
2690
220k
      unsigned BitShift =
2691
220k
          cast<ConstantInt>(I->getOperand(1))->getLimitedValue(~0U);
2692
220k
      // Ensure the shift amount is defined.
2693
220k
      if (BitShift > BitWidth)
2694
0
        return Result;
2695
220k
2696
220k
      auto &Res = collectBitParts(I->getOperand(0), MatchBSwaps,
2697
220k
                                  MatchBitReversals, BPS, Depth + 1);
2698
220k
      if (!Res)
2699
37.3k
        return Result;
2700
182k
      Result = Res;
2701
182k
2702
182k
      // Perform the "shift" on BitProvenance.
2703
182k
      auto &P = Result->Provenance;
2704
182k
      if (I->getOpcode() == Instruction::Shl) {
2705
145k
        P.erase(std::prev(P.end(), BitShift), P.end());
2706
145k
        P.insert(P.begin(), BitShift, BitPart::Unset);
2707
145k
      } else {
2708
37.7k
        P.erase(P.begin(), std::next(P.begin(), BitShift));
2709
37.7k
        P.insert(P.end(), BitShift, BitPart::Unset);
2710
37.7k
      }
2711
182k
2712
182k
      return Result;
2713
182k
    }
2714
764k
2715
764k
    // If this is a logical 'and' with a mask that clears bits, recurse then
2716
764k
    // unset the appropriate bits.
2717
764k
    if (I->getOpcode() == Instruction::And &&
2718
764k
        
isa<ConstantInt>(I->getOperand(1))238k
) {
2719
188k
      APInt Bit(I->getType()->getPrimitiveSizeInBits(), 1);
2720
188k
      const APInt &AndMask = cast<ConstantInt>(I->getOperand(1))->getValue();
2721
188k
2722
188k
      // Check that the mask allows a multiple of 8 bits for a bswap, for an
2723
188k
      // early exit.
2724
188k
      unsigned NumMaskedBits = AndMask.countPopulation();
2725
188k
      if (!MatchBitReversals && 
NumMaskedBits % 8 != 0161k
)
2726
149k
        return Result;
2727
38.8k
2728
38.8k
      auto &Res = collectBitParts(I->getOperand(0), MatchBSwaps,
2729
38.8k
                                  MatchBitReversals, BPS, Depth + 1);
2730
38.8k
      if (!Res)
2731
2.76k
        return Result;
2732
36.0k
      Result = Res;
2733
36.0k
2734
1.59M
      for (unsigned i = 0; i < BitWidth; 
++i, Bit <<= 11.55M
)
2735
1.55M
        // If the AndMask is zero for this bit, clear the bit.
2736
1.55M
        if ((AndMask & Bit) == 0)
2737
881k
          Result->Provenance[i] = BitPart::Unset;
2738
36.0k
      return Result;
2739
36.0k
    }
2740
576k
2741
576k
    // If this is a zext instruction zero extend the result.
2742
576k
    if (I->getOpcode() == Instruction::ZExt) {
2743
119k
      auto &Res = collectBitParts(I->getOperand(0), MatchBSwaps,
2744
119k
                                  MatchBitReversals, BPS, Depth + 1);
2745
119k
      if (!Res)
2746
5.68k
        return Result;
2747
113k
2748
113k
      Result = BitPart(Res->Provider, BitWidth);
2749
113k
      auto NarrowBitWidth =
2750
113k
          cast<IntegerType>(cast<ZExtInst>(I)->getSrcTy())->getBitWidth();
2751
1.26M
      for (unsigned i = 0; i < NarrowBitWidth; 
++i1.14M
)
2752
1.14M
        Result->Provenance[i] = Res->Provenance[i];
2753
3.88M
      for (unsigned i = NarrowBitWidth; i < BitWidth; 
++i3.77M
)
2754
3.77M
        Result->Provenance[i] = BitPart::Unset;
2755
113k
      return Result;
2756
113k
    }
2757
576k
  }
2758
495k
2759
495k
  // Okay, we got to something that isn't a shift, 'or' or 'and'.  This must be
2760
495k
  // the input value to the bswap/bitreverse.
2761
495k
  Result = BitPart(V, BitWidth);
2762
12.0M
  for (unsigned i = 0; i < BitWidth; 
++i11.5M
)
2763
11.5M
    Result->Provenance[i] = i;
2764
495k
  return Result;
2765
495k
}
2766
2767
static bool bitTransformIsCorrectForBSwap(unsigned From, unsigned To,
2768
795k
                                          unsigned BitWidth) {
2769
795k
  if (From % 8 != To % 8)
2770
691k
    return false;
2771
103k
  // Convert from bit indices to byte indices and check for a byte reversal.
2772
103k
  From >>= 3;
2773
103k
  To >>= 3;
2774
103k
  BitWidth >>= 3;
2775
103k
  return From == BitWidth - To - 1;
2776
103k
}
2777
2778
static bool bitTransformIsCorrectForBitReverse(unsigned From, unsigned To,
2779
795k
                                               unsigned BitWidth) {
2780
795k
  return From == BitWidth - To - 1;
2781
795k
}
2782
2783
bool llvm::recognizeBSwapOrBitReverseIdiom(
2784
    Instruction *I, bool MatchBSwaps, bool MatchBitReversals,
2785
7.93M
    SmallVectorImpl<Instruction *> &InsertedInsts) {
2786
7.93M
  if (Operator::getOpcode(I) != Instruction::Or)
2787
7.68M
    return false;
2788
251k
  if (!MatchBSwaps && 
!MatchBitReversals53.9k
)
2789
0
    return false;
2790
251k
  IntegerType *ITy = dyn_cast<IntegerType>(I->getType());
2791
251k
  if (!ITy || 
ITy->getBitWidth() > 128247k
)
2792
3.43k
    return false;   // Can't do vectors or integers > 128 bits.
2793
247k
  unsigned BW = ITy->getBitWidth();
2794
247k
2795
247k
  unsigned DemandedBW = BW;
2796
247k
  IntegerType *DemandedTy = ITy;
2797
247k
  if (I->hasOneUse()) {
2798
201k
    if (TruncInst *Trunc = dyn_cast<TruncInst>(I->user_back())) {
2799
3.05k
      DemandedTy = cast<IntegerType>(Trunc->getType());
2800
3.05k
      DemandedBW = DemandedTy->getBitWidth();
2801
3.05k
    }
2802
201k
  }
2803
247k
2804
247k
  // Try to find all the pieces corresponding to the bswap.
2805
247k
  std::map<Value *, Optional<BitPart>> BPS;
2806
247k
  auto Res = collectBitParts(I, MatchBSwaps, MatchBitReversals, BPS, 0);
2807
247k
  if (!Res)
2808
225k
    return false;
2809
21.6k
  auto &BitProvenance = Res->Provenance;
2810
21.6k
2811
21.6k
  // Now, is the bit permutation correct for a bswap or a bitreverse? We can
2812
21.6k
  // only byteswap values with an even number of bytes.
2813
21.6k
  bool OKForBSwap = DemandedBW % 16 == 0, OKForBitReverse = true;
2814
816k
  for (unsigned i = 0; i < DemandedBW; 
++i795k
) {
2815
795k
    OKForBSwap &=
2816
795k
        bitTransformIsCorrectForBSwap(BitProvenance[i], i, DemandedBW);
2817
795k
    OKForBitReverse &=
2818
795k
        bitTransformIsCorrectForBitReverse(BitProvenance[i], i, DemandedBW);
2819
795k
  }
2820
21.6k
2821
21.6k
  Intrinsic::ID Intrin;
2822
21.6k
  if (OKForBSwap && 
MatchBSwaps97
)
2823
97
    Intrin = Intrinsic::bswap;
2824
21.5k
  else if (OKForBitReverse && 
MatchBitReversals36
)
2825
9
    Intrin = Intrinsic::bitreverse;
2826
21.5k
  else
2827
21.5k
    return false;
2828
106
2829
106
  if (ITy != DemandedTy) {
2830
26
    Function *F = Intrinsic::getDeclaration(I->getModule(), Intrin, DemandedTy);
2831
26
    Value *Provider = Res->Provider;
2832
26
    IntegerType *ProviderTy = cast<IntegerType>(Provider->getType());
2833
26
    // We may need to truncate the provider.
2834
26
    if (DemandedTy != ProviderTy) {
2835
4
      auto *Trunc = CastInst::Create(Instruction::Trunc, Provider, DemandedTy,
2836
4
                                     "trunc", I);
2837
4
      InsertedInsts.push_back(Trunc);
2838
4
      Provider = Trunc;
2839
4
    }
2840
26
    auto *CI = CallInst::Create(F, Provider, "rev", I);
2841
26
    InsertedInsts.push_back(CI);
2842
26
    auto *ExtInst = CastInst::Create(Instruction::ZExt, CI, ITy, "zext", I);
2843
26
    InsertedInsts.push_back(ExtInst);
2844
26
    return true;
2845
26
  }
2846
80
2847
80
  Function *F = Intrinsic::getDeclaration(I->getModule(), Intrin, ITy);
2848
80
  InsertedInsts.push_back(CallInst::Create(F, Res->Provider, "rev", I));
2849
80
  return true;
2850
80
}
2851
2852
// CodeGen has special handling for some string functions that may replace
2853
// them with target-specific intrinsics.  Since that'd skip our interceptors
2854
// in ASan/MSan/TSan/DFSan, and thus make us miss some memory accesses,
2855
// we mark affected calls as NoBuiltin, which will disable optimization
2856
// in CodeGen.
2857
void llvm::maybeMarkSanitizerLibraryCallNoBuiltin(
2858
1.04k
    CallInst *CI, const TargetLibraryInfo *TLI) {
2859
1.04k
  Function *F = CI->getCalledFunction();
2860
1.04k
  LibFunc Func;
2861
1.04k
  if (F && 
!F->hasLocalLinkage()1.02k
&&
F->hasName()930
&&
2862
1.04k
      
TLI->getLibFunc(F->getName(), Func)930
&&
TLI->hasOptimizedCodeGen(Func)110
&&
2863
1.04k
      
!F->doesNotAccessMemory()28
)
2864
28
    CI->addAttribute(AttributeList::FunctionIndex, Attribute::NoBuiltin);
2865
1.04k
}
2866
2867
28.9M
bool llvm::canReplaceOperandWithVariable(const Instruction *I, unsigned OpIdx) {
2868
28.9M
  // We can't have a PHI with a metadata type.
2869
28.9M
  if (I->getOperand(OpIdx)->getType()->isMetadataTy())
2870
18.9k
    return false;
2871
28.9M
2872
28.9M
  // Early exit.
2873
28.9M
  if (!isa<Constant>(I->getOperand(OpIdx)))
2874
18.3M
    return true;
2875
10.5M
2876
10.5M
  switch (I->getOpcode()) {
2877
10.5M
  default:
2878
3.22M
    return true;
2879
10.5M
  case Instruction::Call:
2880
3.29M
  case Instruction::Invoke:
2881
3.29M
    // Can't handle inline asm. Skip it.
2882
3.29M
    if (isa<InlineAsm>(ImmutableCallSite(I).getCalledValue()))
2883
2.65k
      return false;
2884
3.29M
    // Many arithmetic intrinsics have no issue taking a
2885
3.29M
    // variable, however it's hard to distingish these from
2886
3.29M
    // specials such as @llvm.frameaddress that require a constant.
2887
3.29M
    if (isa<IntrinsicInst>(I))
2888
776k
      return false;
2889
2.51M
2890
2.51M
    // Constant bundle operands may need to retain their constant-ness for
2891
2.51M
    // correctness.
2892
2.51M
    if (ImmutableCallSite(I).isBundleOperand(OpIdx))
2893
4
      return false;
2894
2.51M
    return true;
2895
2.51M
  case Instruction::ShuffleVector:
2896
127k
    // Shufflevector masks are constant.
2897
127k
    return OpIdx != 2;
2898
2.51M
  case Instruction::Switch:
2899
61.3k
  case Instruction::ExtractValue:
2900
61.3k
    // All operands apart from the first are constant.
2901
61.3k
    return OpIdx == 0;
2902
61.3k
  case Instruction::InsertValue:
2903
3.21k
    // All operands apart from the first and the second are constant.
2904
3.21k
    return OpIdx < 2;
2905
84.1k
  case Instruction::Alloca:
2906
84.1k
    // Static allocas (constant size in the entry block) are handled by
2907
84.1k
    // prologue/epilogue insertion so they're free anyway. We definitely don't
2908
84.1k
    // want to make them non-constant.
2909
84.1k
    return !cast<AllocaInst>(I)->isStaticAlloca();
2910
3.77M
  case Instruction::GetElementPtr:
2911
3.77M
    if (OpIdx == 0)
2912
35.0k
      return true;
2913
3.74M
    gep_type_iterator It = gep_type_begin(I);
2914
7.64M
    for (auto E = std::next(It, OpIdx); It != E; 
++It3.89M
)
2915
5.77M
      if (It.isStruct())
2916
1.87M
        return false;
2917
3.74M
    
return true1.86M
;
2918
10.5M
  }
2919
10.5M
}
2920
2921
using AllocaForValueMapTy = DenseMap<Value *, AllocaInst *>;
2922
AllocaInst *llvm::findAllocaForValue(Value *V,
2923
237
                                     AllocaForValueMapTy &AllocaForValue) {
2924
237
  if (AllocaInst *AI = dyn_cast<AllocaInst>(V))
2925
96
    return AI;
2926
141
  // See if we've already calculated (or started to calculate) alloca for a
2927
141
  // given value.
2928
141
  AllocaForValueMapTy::iterator I = AllocaForValue.find(V);
2929
141
  if (I != AllocaForValue.end())
2930
34
    return I->second;
2931
107
  // Store 0 while we're calculating alloca for value V to avoid
2932
107
  // infinite recursion if the value references itself.
2933
107
  AllocaForValue[V] = nullptr;
2934
107
  AllocaInst *Res = nullptr;
2935
107
  if (CastInst *CI = dyn_cast<CastInst>(V))
2936
84
    Res = findAllocaForValue(CI->getOperand(0), AllocaForValue);
2937
23
  else if (PHINode *PN = dyn_cast<PHINode>(V)) {
2938
4
    for (Value *IncValue : PN->incoming_values()) {
2939
4
      // Allow self-referencing phi-nodes.
2940
4
      if (IncValue == PN)
2941
0
        continue;
2942
4
      AllocaInst *IncValueAI = findAllocaForValue(IncValue, AllocaForValue);
2943
4
      // AI for incoming values should exist and should all be equal.
2944
4
      if (IncValueAI == nullptr || (Res != nullptr && 
IncValueAI != Res2
))
2945
0
        return nullptr;
2946
4
      Res = IncValueAI;
2947
4
    }
2948
21
  } else if (GetElementPtrInst *EP = dyn_cast<GetElementPtrInst>(V)) {
2949
8
    Res = findAllocaForValue(EP->getPointerOperand(), AllocaForValue);
2950
13
  } else {
2951
13
    LLVM_DEBUG(dbgs() << "Alloca search cancelled on unknown instruction: "
2952
13
                      << *V << "\n");
2953
13
  }
2954
107
  if (Res)
2955
94
    AllocaForValue[V] = Res;
2956
107
  return Res;
2957
107
}