Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/StackColoring.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- StackColoring.cpp --------------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This pass implements the stack-coloring optimization that looks for
10
// lifetime markers machine instructions (LIFESTART_BEGIN and LIFESTART_END),
11
// which represent the possible lifetime of stack slots. It attempts to
12
// merge disjoint stack slots and reduce the used stack space.
13
// NOTE: This pass is not StackSlotColoring, which optimizes spill slots.
14
//
15
// TODO: In the future we plan to improve stack coloring in the following ways:
16
// 1. Allow merging multiple small slots into a single larger slot at different
17
//    offsets.
18
// 2. Merge this pass with StackSlotColoring and allow merging of allocas with
19
//    spill slots.
20
//
21
//===----------------------------------------------------------------------===//
22
23
#include "llvm/ADT/BitVector.h"
24
#include "llvm/ADT/DenseMap.h"
25
#include "llvm/ADT/DepthFirstIterator.h"
26
#include "llvm/ADT/SmallPtrSet.h"
27
#include "llvm/ADT/SmallVector.h"
28
#include "llvm/ADT/Statistic.h"
29
#include "llvm/Analysis/ValueTracking.h"
30
#include "llvm/CodeGen/LiveInterval.h"
31
#include "llvm/CodeGen/MachineBasicBlock.h"
32
#include "llvm/CodeGen/MachineFrameInfo.h"
33
#include "llvm/CodeGen/MachineFunction.h"
34
#include "llvm/CodeGen/MachineFunctionPass.h"
35
#include "llvm/CodeGen/MachineInstr.h"
36
#include "llvm/CodeGen/MachineMemOperand.h"
37
#include "llvm/CodeGen/MachineOperand.h"
38
#include "llvm/CodeGen/Passes.h"
39
#include "llvm/CodeGen/SelectionDAGNodes.h"
40
#include "llvm/CodeGen/SlotIndexes.h"
41
#include "llvm/CodeGen/TargetOpcodes.h"
42
#include "llvm/CodeGen/WinEHFuncInfo.h"
43
#include "llvm/Config/llvm-config.h"
44
#include "llvm/IR/Constants.h"
45
#include "llvm/IR/DebugInfoMetadata.h"
46
#include "llvm/IR/Function.h"
47
#include "llvm/IR/Instructions.h"
48
#include "llvm/IR/Metadata.h"
49
#include "llvm/IR/Use.h"
50
#include "llvm/IR/Value.h"
51
#include "llvm/Pass.h"
52
#include "llvm/Support/Casting.h"
53
#include "llvm/Support/CommandLine.h"
54
#include "llvm/Support/Compiler.h"
55
#include "llvm/Support/Debug.h"
56
#include "llvm/Support/raw_ostream.h"
57
#include <algorithm>
58
#include <cassert>
59
#include <limits>
60
#include <memory>
61
#include <utility>
62
63
using namespace llvm;
64
65
#define DEBUG_TYPE "stack-coloring"
66
67
static cl::opt<bool>
68
DisableColoring("no-stack-coloring",
69
        cl::init(false), cl::Hidden,
70
        cl::desc("Disable stack coloring"));
71
72
/// The user may write code that uses allocas outside of the declared lifetime
73
/// zone. This can happen when the user returns a reference to a local
74
/// data-structure. We can detect these cases and decide not to optimize the
75
/// code. If this flag is enabled, we try to save the user. This option
76
/// is treated as overriding LifetimeStartOnFirstUse below.
77
static cl::opt<bool>
78
ProtectFromEscapedAllocas("protect-from-escaped-allocas",
79
                          cl::init(false), cl::Hidden,
80
                          cl::desc("Do not optimize lifetime zones that "
81
                                   "are broken"));
82
83
/// Enable enhanced dataflow scheme for lifetime analysis (treat first
84
/// use of stack slot as start of slot lifetime, as opposed to looking
85
/// for LIFETIME_START marker). See "Implementation notes" below for
86
/// more info.
87
static cl::opt<bool>
88
LifetimeStartOnFirstUse("stackcoloring-lifetime-start-on-first-use",
89
        cl::init(true), cl::Hidden,
90
        cl::desc("Treat stack lifetimes as starting on first use, not on START marker."));
91
92
93
STATISTIC(NumMarkerSeen,  "Number of lifetime markers found.");
94
STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
95
STATISTIC(StackSlotMerged, "Number of stack slot merged.");
96
STATISTIC(EscapedAllocas, "Number of allocas that escaped the lifetime region");
97
98
//===----------------------------------------------------------------------===//
99
//                           StackColoring Pass
100
//===----------------------------------------------------------------------===//
101
//
102
// Stack Coloring reduces stack usage by merging stack slots when they
103
// can't be used together. For example, consider the following C program:
104
//
105
//     void bar(char *, int);
106
//     void foo(bool var) {
107
//         A: {
108
//             char z[4096];
109
//             bar(z, 0);
110
//         }
111
//
112
//         char *p;
113
//         char x[4096];
114
//         char y[4096];
115
//         if (var) {
116
//             p = x;
117
//         } else {
118
//             bar(y, 1);
119
//             p = y + 1024;
120
//         }
121
//     B:
122
//         bar(p, 2);
123
//     }
124
//
125
// Naively-compiled, this program would use 12k of stack space. However, the
126
// stack slot corresponding to `z` is always destroyed before either of the
127
// stack slots for `x` or `y` are used, and then `x` is only used if `var`
128
// is true, while `y` is only used if `var` is false. So in no time are 2
129
// of the stack slots used together, and therefore we can merge them,
130
// compiling the function using only a single 4k alloca:
131
//
132
//     void foo(bool var) { // equivalent
133
//         char x[4096];
134
//         char *p;
135
//         bar(x, 0);
136
//         if (var) {
137
//             p = x;
138
//         } else {
139
//             bar(x, 1);
140
//             p = x + 1024;
141
//         }
142
//         bar(p, 2);
143
//     }
144
//
145
// This is an important optimization if we want stack space to be under
146
// control in large functions, both open-coded ones and ones created by
147
// inlining.
148
//
149
// Implementation Notes:
150
// ---------------------
151
//
152
// An important part of the above reasoning is that `z` can't be accessed
153
// while the latter 2 calls to `bar` are running. This is justified because
154
// `z`'s lifetime is over after we exit from block `A:`, so any further
155
// accesses to it would be UB. The way we represent this information
156
// in LLVM is by having frontends delimit blocks with `lifetime.start`
157
// and `lifetime.end` intrinsics.
158
//
159
// The effect of these intrinsics seems to be as follows (maybe I should
160
// specify this in the reference?):
161
//
162
//   L1) at start, each stack-slot is marked as *out-of-scope*, unless no
163
//   lifetime intrinsic refers to that stack slot, in which case
164
//   it is marked as *in-scope*.
165
//   L2) on a `lifetime.start`, a stack slot is marked as *in-scope* and
166
//   the stack slot is overwritten with `undef`.
167
//   L3) on a `lifetime.end`, a stack slot is marked as *out-of-scope*.
168
//   L4) on function exit, all stack slots are marked as *out-of-scope*.
169
//   L5) `lifetime.end` is a no-op when called on a slot that is already
170
//   *out-of-scope*.
171
//   L6) memory accesses to *out-of-scope* stack slots are UB.
172
//   L7) when a stack-slot is marked as *out-of-scope*, all pointers to it
173
//   are invalidated, unless the slot is "degenerate". This is used to
174
//   justify not marking slots as in-use until the pointer to them is
175
//   used, but feels a bit hacky in the presence of things like LICM. See
176
//   the "Degenerate Slots" section for more details.
177
//
178
// Now, let's ground stack coloring on these rules. We'll define a slot
179
// as *in-use* at a (dynamic) point in execution if it either can be
180
// written to at that point, or if it has a live and non-undef content
181
// at that point.
182
//
183
// Obviously, slots that are never *in-use* together can be merged, and
184
// in our example `foo`, the slots for `x`, `y` and `z` are never
185
// in-use together (of course, sometimes slots that *are* in-use together
186
// might still be mergable, but we don't care about that here).
187
//
188
// In this implementation, we successively merge pairs of slots that are
189
// not *in-use* together. We could be smarter - for example, we could merge
190
// a single large slot with 2 small slots, or we could construct the
191
// interference graph and run a "smart" graph coloring algorithm, but with
192
// that aside, how do we find out whether a pair of slots might be *in-use*
193
// together?
194
//
195
// From our rules, we see that *out-of-scope* slots are never *in-use*,
196
// and from (L7) we see that "non-degenerate" slots remain non-*in-use*
197
// until their address is taken. Therefore, we can approximate slot activity
198
// using dataflow.
199
//
200
// A subtle point: naively, we might try to figure out which pairs of
201
// stack-slots interfere by propagating `S in-use` through the CFG for every
202
// stack-slot `S`, and having `S` and `T` interfere if there is a CFG point in
203
// which they are both *in-use*.
204
//
205
// That is sound, but overly conservative in some cases: in our (artificial)
206
// example `foo`, either `x` or `y` might be in use at the label `B:`, but
207
// as `x` is only in use if we came in from the `var` edge and `y` only
208
// if we came from the `!var` edge, they still can't be in use together.
209
// See PR32488 for an important real-life case.
210
//
211
// If we wanted to find all points of interference precisely, we could
212
// propagate `S in-use` and `S&T in-use` predicates through the CFG. That
213
// would be precise, but requires propagating `O(n^2)` dataflow facts.
214
//
215
// However, we aren't interested in the *set* of points of interference
216
// between 2 stack slots, only *whether* there *is* such a point. So we
217
// can rely on a little trick: for `S` and `T` to be in-use together,
218
// one of them needs to become in-use while the other is in-use (or
219
// they might both become in use simultaneously). We can check this
220
// by also keeping track of the points at which a stack slot might *start*
221
// being in-use.
222
//
223
// Exact first use:
224
// ----------------
225
//
226
// Consider the following motivating example:
227
//
228
//     int foo() {
229
//       char b1[1024], b2[1024];
230
//       if (...) {
231
//         char b3[1024];
232
//         <uses of b1, b3>;
233
//         return x;
234
//       } else {
235
//         char b4[1024], b5[1024];
236
//         <uses of b2, b4, b5>;
237
//         return y;
238
//       }
239
//     }
240
//
241
// In the code above, "b3" and "b4" are declared in distinct lexical
242
// scopes, meaning that it is easy to prove that they can share the
243
// same stack slot. Variables "b1" and "b2" are declared in the same
244
// scope, meaning that from a lexical point of view, their lifetimes
245
// overlap. From a control flow pointer of view, however, the two
246
// variables are accessed in disjoint regions of the CFG, thus it
247
// should be possible for them to share the same stack slot. An ideal
248
// stack allocation for the function above would look like:
249
//
250
//     slot 0: b1, b2
251
//     slot 1: b3, b4
252
//     slot 2: b5
253
//
254
// Achieving this allocation is tricky, however, due to the way
255
// lifetime markers are inserted. Here is a simplified view of the
256
// control flow graph for the code above:
257
//
258
//                +------  block 0 -------+
259
//               0| LIFETIME_START b1, b2 |
260
//               1| <test 'if' condition> |
261
//                +-----------------------+
262
//                   ./              \.
263
//   +------  block 1 -------+   +------  block 2 -------+
264
//  2| LIFETIME_START b3     |  5| LIFETIME_START b4, b5 |
265
//  3| <uses of b1, b3>      |  6| <uses of b2, b4, b5>  |
266
//  4| LIFETIME_END b3       |  7| LIFETIME_END b4, b5   |
267
//   +-----------------------+   +-----------------------+
268
//                   \.              /.
269
//                +------  block 3 -------+
270
//               8| <cleanupcode>         |
271
//               9| LIFETIME_END b1, b2   |
272
//              10| return                |
273
//                +-----------------------+
274
//
275
// If we create live intervals for the variables above strictly based
276
// on the lifetime markers, we'll get the set of intervals on the
277
// left. If we ignore the lifetime start markers and instead treat a
278
// variable's lifetime as beginning with the first reference to the
279
// var, then we get the intervals on the right.
280
//
281
//            LIFETIME_START      First Use
282
//     b1:    [0,9]               [3,4] [8,9]
283
//     b2:    [0,9]               [6,9]
284
//     b3:    [2,4]               [3,4]
285
//     b4:    [5,7]               [6,7]
286
//     b5:    [5,7]               [6,7]
287
//
288
// For the intervals on the left, the best we can do is overlap two
289
// variables (b3 and b4, for example); this gives us a stack size of
290
// 4*1024 bytes, not ideal. When treating first-use as the start of a
291
// lifetime, we can additionally overlap b1 and b5, giving us a 3*1024
292
// byte stack (better).
293
//
294
// Degenerate Slots:
295
// -----------------
296
//
297
// Relying entirely on first-use of stack slots is problematic,
298
// however, due to the fact that optimizations can sometimes migrate
299
// uses of a variable outside of its lifetime start/end region. Here
300
// is an example:
301
//
302
//     int bar() {
303
//       char b1[1024], b2[1024];
304
//       if (...) {
305
//         <uses of b2>
306
//         return y;
307
//       } else {
308
//         <uses of b1>
309
//         while (...) {
310
//           char b3[1024];
311
//           <uses of b3>
312
//         }
313
//       }
314
//     }
315
//
316
// Before optimization, the control flow graph for the code above
317
// might look like the following:
318
//
319
//                +------  block 0 -------+
320
//               0| LIFETIME_START b1, b2 |
321
//               1| <test 'if' condition> |
322
//                +-----------------------+
323
//                   ./              \.
324
//   +------  block 1 -------+    +------- block 2 -------+
325
//  2| <uses of b2>          |   3| <uses of b1>          |
326
//   +-----------------------+    +-----------------------+
327
//              |                            |
328
//              |                 +------- block 3 -------+ <-\.
329
//              |                4| <while condition>     |    |
330
//              |                 +-----------------------+    |
331
//              |               /          |                   |
332
//              |              /  +------- block 4 -------+
333
//              \             /  5| LIFETIME_START b3     |    |
334
//               \           /   6| <uses of b3>          |    |
335
//                \         /    7| LIFETIME_END b3       |    |
336
//                 \        |    +------------------------+    |
337
//                  \       |                 \                /
338
//                +------  block 5 -----+      \---------------
339
//               8| <cleanupcode>       |
340
//               9| LIFETIME_END b1, b2 |
341
//              10| return              |
342
//                +---------------------+
343
//
344
// During optimization, however, it can happen that an instruction
345
// computing an address in "b3" (for example, a loop-invariant GEP) is
346
// hoisted up out of the loop from block 4 to block 2.  [Note that
347
// this is not an actual load from the stack, only an instruction that
348
// computes the address to be loaded]. If this happens, there is now a
349
// path leading from the first use of b3 to the return instruction
350
// that does not encounter the b3 LIFETIME_END, hence b3's lifetime is
351
// now larger than if we were computing live intervals strictly based
352
// on lifetime markers. In the example above, this lengthened lifetime
353
// would mean that it would appear illegal to overlap b3 with b2.
354
//
355
// To deal with this such cases, the code in ::collectMarkers() below
356
// tries to identify "degenerate" slots -- those slots where on a single
357
// forward pass through the CFG we encounter a first reference to slot
358
// K before we hit the slot K lifetime start marker. For such slots,
359
// we fall back on using the lifetime start marker as the beginning of
360
// the variable's lifetime.  NB: with this implementation, slots can
361
// appear degenerate in cases where there is unstructured control flow:
362
//
363
//    if (q) goto mid;
364
//    if (x > 9) {
365
//         int b[100];
366
//         memcpy(&b[0], ...);
367
//    mid: b[k] = ...;
368
//         abc(&b);
369
//    }
370
//
371
// If in RPO ordering chosen to walk the CFG  we happen to visit the b[k]
372
// before visiting the memcpy block (which will contain the lifetime start
373
// for "b" then it will appear that 'b' has a degenerate lifetime.
374
//
375
376
namespace {
377
378
/// StackColoring - A machine pass for merging disjoint stack allocations,
379
/// marked by the LIFETIME_START and LIFETIME_END pseudo instructions.
380
class StackColoring : public MachineFunctionPass {
381
  MachineFrameInfo *MFI;
382
  MachineFunction *MF;
383
384
  /// A class representing liveness information for a single basic block.
385
  /// Each bit in the BitVector represents the liveness property
386
  /// for a different stack slot.
387
  struct BlockLifetimeInfo {
388
    /// Which slots BEGINs in each basic block.
389
    BitVector Begin;
390
391
    /// Which slots ENDs in each basic block.
392
    BitVector End;
393
394
    /// Which slots are marked as LIVE_IN, coming into each basic block.
395
    BitVector LiveIn;
396
397
    /// Which slots are marked as LIVE_OUT, coming out of each basic block.
398
    BitVector LiveOut;
399
  };
400
401
  /// Maps active slots (per bit) for each basic block.
402
  using LivenessMap = DenseMap<const MachineBasicBlock *, BlockLifetimeInfo>;
403
  LivenessMap BlockLiveness;
404
405
  /// Maps serial numbers to basic blocks.
406
  DenseMap<const MachineBasicBlock *, int> BasicBlocks;
407
408
  /// Maps basic blocks to a serial number.
409
  SmallVector<const MachineBasicBlock *, 8> BasicBlockNumbering;
410
411
  /// Maps slots to their use interval. Outside of this interval, slots
412
  /// values are either dead or `undef` and they will not be written to.
413
  SmallVector<std::unique_ptr<LiveInterval>, 16> Intervals;
414
415
  /// Maps slots to the points where they can become in-use.
416
  SmallVector<SmallVector<SlotIndex, 4>, 16> LiveStarts;
417
418
  /// VNInfo is used for the construction of LiveIntervals.
419
  VNInfo::Allocator VNInfoAllocator;
420
421
  /// SlotIndex analysis object.
422
  SlotIndexes *Indexes;
423
424
  /// The list of lifetime markers found. These markers are to be removed
425
  /// once the coloring is done.
426
  SmallVector<MachineInstr*, 8> Markers;
427
428
  /// Record the FI slots for which we have seen some sort of
429
  /// lifetime marker (either start or end).
430
  BitVector InterestingSlots;
431
432
  /// FI slots that need to be handled conservatively (for these
433
  /// slots lifetime-start-on-first-use is disabled).
434
  BitVector ConservativeSlots;
435
436
  /// Number of iterations taken during data flow analysis.
437
  unsigned NumIterations;
438
439
public:
440
  static char ID;
441
442
34.5k
  StackColoring() : MachineFunctionPass(ID) {
443
34.5k
    initializeStackColoringPass(*PassRegistry::getPassRegistry());
444
34.5k
  }
445
446
  void getAnalysisUsage(AnalysisUsage &AU) const override;
447
  bool runOnMachineFunction(MachineFunction &Func) override;
448
449
private:
450
  /// Used in collectMarkers
451
  using BlockBitVecMap = DenseMap<const MachineBasicBlock *, BitVector>;
452
453
  /// Debug.
454
  void dump() const;
455
  void dumpIntervals() const;
456
  void dumpBB(MachineBasicBlock *MBB) const;
457
  void dumpBV(const char *tag, const BitVector &BV) const;
458
459
  /// Removes all of the lifetime marker instructions from the function.
460
  /// \returns true if any markers were removed.
461
  bool removeAllMarkers();
462
463
  /// Scan the machine function and find all of the lifetime markers.
464
  /// Record the findings in the BEGIN and END vectors.
465
  /// \returns the number of markers found.
466
  unsigned collectMarkers(unsigned NumSlot);
467
468
  /// Perform the dataflow calculation and calculate the lifetime for each of
469
  /// the slots, based on the BEGIN/END vectors. Set the LifetimeLIVE_IN and
470
  /// LifetimeLIVE_OUT maps that represent which stack slots are live coming
471
  /// in and out blocks.
472
  void calculateLocalLiveness();
473
474
  /// Returns TRUE if we're using the first-use-begins-lifetime method for
475
  /// this slot (if FALSE, then the start marker is treated as start of lifetime).
476
533k
  bool applyFirstUse(int Slot) {
477
533k
    if (!LifetimeStartOnFirstUse || 
ProtectFromEscapedAllocas533k
)
478
110
      return false;
479
533k
    if (ConservativeSlots.test(Slot))
480
65.4k
      return false;
481
467k
    return true;
482
467k
  }
483
484
  /// Examines the specified instruction and returns TRUE if the instruction
485
  /// represents the start or end of an interesting lifetime. The slot or slots
486
  /// starting or ending are added to the vector "slots" and "isStart" is set
487
  /// accordingly.
488
  /// \returns True if inst contains a lifetime start or end
489
  bool isLifetimeStartOrEnd(const MachineInstr &MI,
490
                            SmallVector<int, 4> &slots,
491
                            bool &isStart);
492
493
  /// Construct the LiveIntervals for the slots.
494
  void calculateLiveIntervals(unsigned NumSlots);
495
496
  /// Go over the machine function and change instructions which use stack
497
  /// slots to use the joint slots.
498
  void remapInstructions(DenseMap<int, int> &SlotRemap);
499
500
  /// The input program may contain instructions which are not inside lifetime
501
  /// markers. This can happen due to a bug in the compiler or due to a bug in
502
  /// user code (for example, returning a reference to a local variable).
503
  /// This procedure checks all of the instructions in the function and
504
  /// invalidates lifetime ranges which do not contain all of the instructions
505
  /// which access that frame slot.
506
  void removeInvalidSlotRanges();
507
508
  /// Map entries which point to other entries to their destination.
509
  ///   A->B->C becomes A->C.
510
  void expungeSlotMap(DenseMap<int, int> &SlotRemap, unsigned NumSlots);
511
};
512
513
} // end anonymous namespace
514
515
char StackColoring::ID = 0;
516
517
char &llvm::StackColoringID = StackColoring::ID;
518
519
42.3k
INITIALIZE_PASS_BEGIN(StackColoring, DEBUG_TYPE,
520
42.3k
                      "Merge disjoint stack slots", false, false)
521
42.3k
INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
522
42.3k
INITIALIZE_PASS_END(StackColoring, DEBUG_TYPE,
523
                    "Merge disjoint stack slots", false, false)
524
525
34.2k
void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const {
526
34.2k
  AU.addRequired<SlotIndexes>();
527
34.2k
  MachineFunctionPass::getAnalysisUsage(AU);
528
34.2k
}
529
530
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
531
LLVM_DUMP_METHOD void StackColoring::dumpBV(const char *tag,
532
                                            const BitVector &BV) const {
533
  dbgs() << tag << " : { ";
534
  for (unsigned I = 0, E = BV.size(); I != E; ++I)
535
    dbgs() << BV.test(I) << " ";
536
  dbgs() << "}\n";
537
}
538
539
LLVM_DUMP_METHOD void StackColoring::dumpBB(MachineBasicBlock *MBB) const {
540
  LivenessMap::const_iterator BI = BlockLiveness.find(MBB);
541
  assert(BI != BlockLiveness.end() && "Block not found");
542
  const BlockLifetimeInfo &BlockInfo = BI->second;
543
544
  dumpBV("BEGIN", BlockInfo.Begin);
545
  dumpBV("END", BlockInfo.End);
546
  dumpBV("LIVE_IN", BlockInfo.LiveIn);
547
  dumpBV("LIVE_OUT", BlockInfo.LiveOut);
548
}
549
550
LLVM_DUMP_METHOD void StackColoring::dump() const {
551
  for (MachineBasicBlock *MBB : depth_first(MF)) {
552
    dbgs() << "Inspecting block #" << MBB->getNumber() << " ["
553
           << MBB->getName() << "]\n";
554
    dumpBB(MBB);
555
  }
556
}
557
558
LLVM_DUMP_METHOD void StackColoring::dumpIntervals() const {
559
  for (unsigned I = 0, E = Intervals.size(); I != E; ++I) {
560
    dbgs() << "Interval[" << I << "]:\n";
561
    Intervals[I]->dump();
562
  }
563
}
564
#endif
565
566
static inline int getStartOrEndSlot(const MachineInstr &MI)
567
197k
{
568
197k
  assert((MI.getOpcode() == TargetOpcode::LIFETIME_START ||
569
197k
          MI.getOpcode() == TargetOpcode::LIFETIME_END) &&
570
197k
         "Expected LIFETIME_START or LIFETIME_END op");
571
197k
  const MachineOperand &MO = MI.getOperand(0);
572
197k
  int Slot = MO.getIndex();
573
197k
  if (Slot >= 0)
574
197k
    return Slot;
575
2
  return -1;
576
2
}
577
578
// At the moment the only way to end a variable lifetime is with
579
// a VARIABLE_LIFETIME op (which can't contain a start). If things
580
// change and the IR allows for a single inst that both begins
581
// and ends lifetime(s), this interface will need to be reworked.
582
bool StackColoring::isLifetimeStartOrEnd(const MachineInstr &MI,
583
                                         SmallVector<int, 4> &slots,
584
7.94M
                                         bool &isStart) {
585
7.94M
  if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
586
7.94M
      
MI.getOpcode() == TargetOpcode::LIFETIME_END7.88M
) {
587
129k
    int Slot = getStartOrEndSlot(MI);
588
129k
    if (Slot < 0)
589
1
      return false;
590
129k
    if (!InterestingSlots.test(Slot))
591
0
      return false;
592
129k
    slots.push_back(Slot);
593
129k
    if (MI.getOpcode() == TargetOpcode::LIFETIME_END) {
594
65.9k
      isStart = false;
595
65.9k
      return true;
596
65.9k
    }
597
63.5k
    if (!applyFirstUse(Slot)) {
598
7.78k
      isStart = true;
599
7.78k
      return true;
600
7.78k
    }
601
7.81M
  } else if (LifetimeStartOnFirstUse && 
!ProtectFromEscapedAllocas7.81M
) {
602
7.81M
    if (!MI.isDebugInstr()) {
603
7.81M
      bool found = false;
604
28.0M
      for (const MachineOperand &MO : MI.operands()) {
605
28.0M
        if (!MO.isFI())
606
27.5M
          continue;
607
502k
        int Slot = MO.getIndex();
608
502k
        if (Slot<0)
609
11.6k
          continue;
610
490k
        if (InterestingSlots.test(Slot) && 
applyFirstUse(Slot)469k
) {
611
411k
          slots.push_back(Slot);
612
411k
          found = true;
613
411k
        }
614
490k
      }
615
7.81M
      if (found) {
616
411k
        isStart = true;
617
411k
        return true;
618
411k
      }
619
7.46M
    }
620
7.81M
  }
621
7.46M
  return false;
622
7.46M
}
623
624
45.4k
unsigned StackColoring::collectMarkers(unsigned NumSlot) {
625
45.4k
  unsigned MarkersFound = 0;
626
45.4k
  BlockBitVecMap SeenStartMap;
627
45.4k
  InterestingSlots.clear();
628
45.4k
  InterestingSlots.resize(NumSlot);
629
45.4k
  ConservativeSlots.clear();
630
45.4k
  ConservativeSlots.resize(NumSlot);
631
45.4k
632
45.4k
  // number of start and end lifetime ops for each slot
633
45.4k
  SmallVector<int, 8> NumStartLifetimes(NumSlot, 0);
634
45.4k
  SmallVector<int, 8> NumEndLifetimes(NumSlot, 0);
635
45.4k
636
45.4k
  // Step 1: collect markers and populate the "InterestingSlots"
637
45.4k
  // and "ConservativeSlots" sets.
638
907k
  for (MachineBasicBlock *MBB : depth_first(MF)) {
639
907k
    // Compute the set of slots for which we've seen a START marker but have
640
907k
    // not yet seen an END marker at this point in the walk (e.g. on entry
641
907k
    // to this bb).
642
907k
    BitVector BetweenStartEnd;
643
907k
    BetweenStartEnd.resize(NumSlot);
644
907k
    for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
645
2.27M
             PE = MBB->pred_end(); PI != PE; 
++PI1.37M
) {
646
1.37M
      BlockBitVecMap::const_iterator I = SeenStartMap.find(*PI);
647
1.37M
      if (I != SeenStartMap.end()) {
648
957k
        BetweenStartEnd |= I->second;
649
957k
      }
650
1.37M
    }
651
907k
652
907k
    // Walk the instructions in the block to look for start/end ops.
653
8.86M
    for (MachineInstr &MI : *MBB) {
654
8.86M
      if (MI.getOpcode() == TargetOpcode::LIFETIME_START ||
655
8.86M
          
MI.getOpcode() == TargetOpcode::LIFETIME_END8.83M
) {
656
68.2k
        int Slot = getStartOrEndSlot(MI);
657
68.2k
        if (Slot < 0)
658
1
          continue;
659
68.2k
        InterestingSlots.set(Slot);
660
68.2k
        if (MI.getOpcode() == TargetOpcode::LIFETIME_START) {
661
33.5k
          BetweenStartEnd.set(Slot);
662
33.5k
          NumStartLifetimes[Slot] += 1;
663
34.6k
        } else {
664
34.6k
          BetweenStartEnd.reset(Slot);
665
34.6k
          NumEndLifetimes[Slot] += 1;
666
34.6k
        }
667
68.2k
        const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
668
68.2k
        if (Allocation) {
669
68.2k
          LLVM_DEBUG(dbgs() << "Found a lifetime ");
670
68.2k
          LLVM_DEBUG(dbgs() << (MI.getOpcode() == TargetOpcode::LIFETIME_START
671
68.2k
                                    ? "start"
672
68.2k
                                    : "end"));
673
68.2k
          LLVM_DEBUG(dbgs() << " marker for slot #" << Slot);
674
68.2k
          LLVM_DEBUG(dbgs()
675
68.2k
                     << " with allocation: " << Allocation->getName() << "\n");
676
68.2k
        }
677
68.2k
        Markers.push_back(&MI);
678
68.2k
        MarkersFound += 1;
679
8.80M
      } else {
680
30.5M
        for (const MachineOperand &MO : MI.operands()) {
681
30.5M
          if (!MO.isFI())
682
30.0M
            continue;
683
453k
          int Slot = MO.getIndex();
684
453k
          if (Slot < 0)
685
15.0k
            continue;
686
438k
          if (! BetweenStartEnd.test(Slot)) {
687
200k
            ConservativeSlots.set(Slot);
688
200k
          }
689
438k
        }
690
8.80M
      }
691
8.86M
    }
692
907k
    BitVector &SeenStart = SeenStartMap[MBB];
693
907k
    SeenStart |= BetweenStartEnd;
694
907k
  }
695
45.4k
  if (!MarkersFound) {
696
30.7k
    return 0;
697
30.7k
  }
698
14.7k
699
14.7k
  // PR27903: slots with multiple start or end lifetime ops are not
700
14.7k
  // safe to enable for "lifetime-start-on-first-use".
701
49.2k
  
for (unsigned slot = 0; 14.7k
slot < NumSlot;
++slot34.5k
)
702
34.5k
    if (NumStartLifetimes[slot] > 1 || 
NumEndLifetimes[slot] > 133.9k
)
703
1.88k
      ConservativeSlots.set(slot);
704
14.7k
  LLVM_DEBUG(dumpBV("Conservative slots", ConservativeSlots));
705
14.7k
706
14.7k
  // Step 2: compute begin/end sets for each block
707
14.7k
708
14.7k
  // NOTE: We use a depth-first iteration to ensure that we obtain a
709
14.7k
  // deterministic numbering.
710
453k
  for (MachineBasicBlock *MBB : depth_first(MF)) {
711
453k
    // Assign a serial number to this basic block.
712
453k
    BasicBlocks[MBB] = BasicBlockNumbering.size();
713
453k
    BasicBlockNumbering.push_back(MBB);
714
453k
715
453k
    // Keep a reference to avoid repeated lookups.
716
453k
    BlockLifetimeInfo &BlockInfo = BlockLiveness[MBB];
717
453k
718
453k
    BlockInfo.Begin.resize(NumSlot);
719
453k
    BlockInfo.End.resize(NumSlot);
720
453k
721
453k
    SmallVector<int, 4> slots;
722
4.17M
    for (MachineInstr &MI : *MBB) {
723
4.17M
      bool isStart = false;
724
4.17M
      slots.clear();
725
4.17M
      if (isLifetimeStartOrEnd(MI, slots, isStart)) {
726
248k
        if (!isStart) {
727
34.6k
          assert(slots.size() == 1 && "unexpected: MI ends multiple slots");
728
34.6k
          int Slot = slots[0];
729
34.6k
          if (BlockInfo.Begin.test(Slot)) {
730
18.7k
            BlockInfo.Begin.reset(Slot);
731
18.7k
          }
732
34.6k
          BlockInfo.End.set(Slot);
733
213k
        } else {
734
213k
          for (auto Slot : slots) {
735
213k
            LLVM_DEBUG(dbgs() << "Found a use of slot #" << Slot);
736
213k
            LLVM_DEBUG(dbgs()
737
213k
                       << " at " << printMBBReference(*MBB) << " index ");
738
213k
            LLVM_DEBUG(Indexes->getInstructionIndex(MI).print(dbgs()));
739
213k
            const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
740
213k
            if (Allocation) {
741
213k
              LLVM_DEBUG(dbgs()
742
213k
                         << " with allocation: " << Allocation->getName());
743
213k
            }
744
213k
            LLVM_DEBUG(dbgs() << "\n");
745
213k
            if (BlockInfo.End.test(Slot)) {
746
572
              BlockInfo.End.reset(Slot);
747
572
            }
748
213k
            BlockInfo.Begin.set(Slot);
749
213k
          }
750
213k
        }
751
248k
      }
752
4.17M
    }
753
453k
  }
754
14.7k
755
14.7k
  // Update statistics.
756
14.7k
  NumMarkerSeen += MarkersFound;
757
14.7k
  return MarkersFound;
758
14.7k
}
759
760
11.6k
void StackColoring::calculateLocalLiveness() {
761
11.6k
  unsigned NumIters = 0;
762
11.6k
  bool changed = true;
763
32.5k
  while (changed) {
764
20.8k
    changed = false;
765
20.8k
    ++NumIters;
766
20.8k
767
848k
    for (const MachineBasicBlock *BB : BasicBlockNumbering) {
768
848k
      // Use an iterator to avoid repeated lookups.
769
848k
      LivenessMap::iterator BI = BlockLiveness.find(BB);
770
848k
      assert(BI != BlockLiveness.end() && "Block not found");
771
848k
      BlockLifetimeInfo &BlockInfo = BI->second;
772
848k
773
848k
      // Compute LiveIn by unioning together the LiveOut sets of all preds.
774
848k
      BitVector LocalLiveIn;
775
848k
      for (MachineBasicBlock::const_pred_iterator PI = BB->pred_begin(),
776
2.18M
           PE = BB->pred_end(); PI != PE; 
++PI1.34M
) {
777
1.34M
        LivenessMap::const_iterator I = BlockLiveness.find(*PI);
778
1.34M
        // PR37130: transformations prior to stack coloring can
779
1.34M
        // sometimes leave behind statically unreachable blocks; these
780
1.34M
        // can be safely skipped here.
781
1.34M
        if (I != BlockLiveness.end())
782
1.34M
          LocalLiveIn |= I->second.LiveOut;
783
1.34M
      }
784
848k
785
848k
      // Compute LiveOut by subtracting out lifetimes that end in this
786
848k
      // block, then adding in lifetimes that begin in this block.  If
787
848k
      // we have both BEGIN and END markers in the same basic block
788
848k
      // then we know that the BEGIN marker comes after the END,
789
848k
      // because we already handle the case where the BEGIN comes
790
848k
      // before the END when collecting the markers (and building the
791
848k
      // BEGIN/END vectors).
792
848k
      BitVector LocalLiveOut = LocalLiveIn;
793
848k
      LocalLiveOut.reset(BlockInfo.End);
794
848k
      LocalLiveOut |= BlockInfo.Begin;
795
848k
796
848k
      // Update block LiveIn set, noting whether it has changed.
797
848k
      if (LocalLiveIn.test(BlockInfo.LiveIn)) {
798
243k
        changed = true;
799
243k
        BlockInfo.LiveIn |= LocalLiveIn;
800
243k
      }
801
848k
802
848k
      // Update block LiveOut set, noting whether it has changed.
803
848k
      if (LocalLiveOut.test(BlockInfo.LiveOut)) {
804
243k
        changed = true;
805
243k
        BlockInfo.LiveOut |= LocalLiveOut;
806
243k
      }
807
848k
    }
808
20.8k
  } // while changed.
809
11.6k
810
11.6k
  NumIterations = NumIters;
811
11.6k
}
812
813
11.6k
void StackColoring::calculateLiveIntervals(unsigned NumSlots) {
814
11.6k
  SmallVector<SlotIndex, 16> Starts;
815
11.6k
  SmallVector<bool, 16> DefinitelyInUse;
816
11.6k
817
11.6k
  // For each block, find which slots are active within this block
818
11.6k
  // and update the live intervals.
819
411k
  for (const MachineBasicBlock &MBB : *MF) {
820
411k
    Starts.clear();
821
411k
    Starts.resize(NumSlots);
822
411k
    DefinitelyInUse.clear();
823
411k
    DefinitelyInUse.resize(NumSlots);
824
411k
825
411k
    // Start the interval of the slots that we previously found to be 'in-use'.
826
411k
    BlockLifetimeInfo &MBBLiveness = BlockLiveness[&MBB];
827
873k
    for (int pos = MBBLiveness.LiveIn.find_first(); pos != -1;
828
461k
         pos = MBBLiveness.LiveIn.find_next(pos)) {
829
461k
      Starts[pos] = Indexes->getMBBStartIdx(&MBB);
830
461k
    }
831
411k
832
411k
    // Create the interval for the basic blocks containing lifetime begin/end.
833
3.76M
    for (const MachineInstr &MI : MBB) {
834
3.76M
      SmallVector<int, 4> slots;
835
3.76M
      bool IsStart = false;
836
3.76M
      if (!isLifetimeStartOrEnd(MI, slots, IsStart))
837
3.52M
        continue;
838
237k
      SlotIndex ThisIndex = Indexes->getInstructionIndex(MI);
839
237k
      for (auto Slot : slots) {
840
237k
        if (IsStart) {
841
205k
          // If a slot is already definitely in use, we don't have to emit
842
205k
          // a new start marker because there is already a pre-existing
843
205k
          // one.
844
205k
          if (!DefinitelyInUse[Slot]) {
845
124k
            LiveStarts[Slot].push_back(ThisIndex);
846
124k
            DefinitelyInUse[Slot] = true;
847
124k
          }
848
205k
          if (!Starts[Slot].isValid())
849
29.6k
            Starts[Slot] = ThisIndex;
850
205k
        } else {
851
31.2k
          if (Starts[Slot].isValid()) {
852
30.9k
            VNInfo *VNI = Intervals[Slot]->getValNumInfo(0);
853
30.9k
            Intervals[Slot]->addSegment(
854
30.9k
                LiveInterval::Segment(Starts[Slot], ThisIndex, VNI));
855
30.9k
            Starts[Slot] = SlotIndex(); // Invalidate the start index
856
30.9k
            DefinitelyInUse[Slot] = false;
857
30.9k
          }
858
31.2k
        }
859
237k
      }
860
237k
    }
861
411k
862
411k
    // Finish up started segments
863
3.36M
    for (unsigned i = 0; i < NumSlots; 
++i2.95M
) {
864
2.95M
      if (!Starts[i].isValid())
865
2.49M
        continue;
866
460k
867
460k
      SlotIndex EndIdx = Indexes->getMBBEndIdx(&MBB);
868
460k
      VNInfo *VNI = Intervals[i]->getValNumInfo(0);
869
460k
      Intervals[i]->addSegment(LiveInterval::Segment(Starts[i], EndIdx, VNI));
870
460k
    }
871
411k
  }
872
11.6k
}
873
874
45.4k
bool StackColoring::removeAllMarkers() {
875
45.4k
  unsigned Count = 0;
876
68.2k
  for (MachineInstr *MI : Markers) {
877
68.2k
    MI->eraseFromParent();
878
68.2k
    Count++;
879
68.2k
  }
880
45.4k
  Markers.clear();
881
45.4k
882
45.4k
  LLVM_DEBUG(dbgs() << "Removed " << Count << " markers.\n");
883
45.4k
  return Count;
884
45.4k
}
885
886
11.6k
void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
887
11.6k
  unsigned FixedInstr = 0;
888
11.6k
  unsigned FixedMemOp = 0;
889
11.6k
  unsigned FixedDbg = 0;
890
11.6k
891
11.6k
  // Remap debug information that refers to stack slots.
892
11.6k
  for (auto &VI : MF->getVariableDbgInfo()) {
893
18
    if (!VI.Var)
894
0
      continue;
895
18
    if (SlotRemap.count(VI.Slot)) {
896
0
      LLVM_DEBUG(dbgs() << "Remapping debug info for ["
897
0
                        << cast<DILocalVariable>(VI.Var)->getName() << "].\n");
898
0
      VI.Slot = SlotRemap[VI.Slot];
899
0
      FixedDbg++;
900
0
    }
901
18
  }
902
11.6k
903
11.6k
  // Keep a list of *allocas* which need to be remapped.
904
11.6k
  DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
905
11.6k
906
11.6k
  // Keep a list of allocas which has been affected by the remap.
907
11.6k
  SmallPtrSet<const AllocaInst*, 32> MergedAllocas;
908
11.6k
909
11.6k
  for (const std::pair<int, int> &SI : SlotRemap) {
910
10.9k
    const AllocaInst *From = MFI->getObjectAllocation(SI.first);
911
10.9k
    const AllocaInst *To = MFI->getObjectAllocation(SI.second);
912
10.9k
    assert(To && From && "Invalid allocation object");
913
10.9k
    Allocas[From] = To;
914
10.9k
915
10.9k
    // AA might be used later for instruction scheduling, and we need it to be
916
10.9k
    // able to deduce the correct aliasing releationships between pointers
917
10.9k
    // derived from the alloca being remapped and the target of that remapping.
918
10.9k
    // The only safe way, without directly informing AA about the remapping
919
10.9k
    // somehow, is to directly update the IR to reflect the change being made
920
10.9k
    // here.
921
10.9k
    Instruction *Inst = const_cast<AllocaInst *>(To);
922
10.9k
    if (From->getType() != To->getType()) {
923
4.61k
      BitCastInst *Cast = new BitCastInst(Inst, From->getType());
924
4.61k
      Cast->insertAfter(Inst);
925
4.61k
      Inst = Cast;
926
4.61k
    }
927
10.9k
928
10.9k
    // We keep both slots to maintain AliasAnalysis metadata later.
929
10.9k
    MergedAllocas.insert(From);
930
10.9k
    MergedAllocas.insert(To);
931
10.9k
932
10.9k
    // Transfer the stack protector layout tag, but make sure that SSPLK_AddrOf
933
10.9k
    // does not overwrite SSPLK_SmallArray or SSPLK_LargeArray, and make sure
934
10.9k
    // that SSPLK_SmallArray does not overwrite SSPLK_LargeArray.
935
10.9k
    MachineFrameInfo::SSPLayoutKind FromKind
936
10.9k
        = MFI->getObjectSSPLayout(SI.first);
937
10.9k
    MachineFrameInfo::SSPLayoutKind ToKind = MFI->getObjectSSPLayout(SI.second);
938
10.9k
    if (FromKind != MachineFrameInfo::SSPLK_None &&
939
10.9k
        
(87
ToKind == MachineFrameInfo::SSPLK_None87
||
940
87
         
(80
ToKind != MachineFrameInfo::SSPLK_LargeArray80
&&
941
80
          
FromKind != MachineFrameInfo::SSPLK_AddrOf0
)))
942
7
      MFI->setObjectSSPLayout(SI.second, FromKind);
943
10.9k
944
10.9k
    // The new alloca might not be valid in a llvm.dbg.declare for this
945
10.9k
    // variable, so undef out the use to make the verifier happy.
946
10.9k
    AllocaInst *FromAI = const_cast<AllocaInst *>(From);
947
10.9k
    if (FromAI->isUsedByMetadata())
948
0
      ValueAsMetadata::handleRAUW(FromAI, UndefValue::get(FromAI->getType()));
949
69.1k
    for (auto &Use : FromAI->uses()) {
950
69.1k
      if (BitCastInst *BCI = dyn_cast<BitCastInst>(Use.get()))
951
0
        if (BCI->isUsedByMetadata())
952
0
          ValueAsMetadata::handleRAUW(BCI, UndefValue::get(BCI->getType()));
953
69.1k
    }
954
10.9k
955
10.9k
    // Note that this will not replace uses in MMOs (which we'll update below),
956
10.9k
    // or anywhere else (which is why we won't delete the original
957
10.9k
    // instruction).
958
10.9k
    FromAI->replaceAllUsesWith(Inst);
959
10.9k
  }
960
11.6k
961
11.6k
  // Remap all instructions to the new stack slots.
962
11.6k
  for (MachineBasicBlock &BB : *MF)
963
3.76M
    
for (MachineInstr &I : BB)411k
{
964
3.76M
      // Skip lifetime markers. We'll remove them soon.
965
3.76M
      if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
966
3.76M
          
I.getOpcode() == TargetOpcode::LIFETIME_END3.73M
)
967
61.2k
        continue;
968
3.70M
969
3.70M
      // Update the MachineMemOperand to use the new alloca.
970
3.70M
      for (MachineMemOperand *MMO : I.memoperands()) {
971
581k
        // We've replaced IR-level uses of the remapped allocas, so we only
972
581k
        // need to replace direct uses here.
973
581k
        const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
974
581k
        if (!AI)
975
573k
          continue;
976
8.39k
977
8.39k
        if (!Allocas.count(AI))
978
6.67k
          continue;
979
1.71k
980
1.71k
        MMO->setValue(Allocas[AI]);
981
1.71k
        FixedMemOp++;
982
1.71k
      }
983
3.70M
984
3.70M
      // Update all of the machine instruction operands.
985
13.3M
      for (MachineOperand &MO : I.operands()) {
986
13.3M
        if (!MO.isFI())
987
13.0M
          continue;
988
245k
        int FromSlot = MO.getIndex();
989
245k
990
245k
        // Don't touch arguments.
991
245k
        if (FromSlot<0)
992
5.04k
          continue;
993
240k
994
240k
        // Only look at mapped slots.
995
240k
        if (!SlotRemap.count(FromSlot))
996
185k
          continue;
997
55.4k
998
55.4k
        // In a debug build, check that the instruction that we are modifying is
999
55.4k
        // inside the expected live range. If the instruction is not inside
1000
55.4k
        // the calculated range then it means that the alloca usage moved
1001
55.4k
        // outside of the lifetime markers, or that the user has a bug.
1002
55.4k
        // NOTE: Alloca address calculations which happen outside the lifetime
1003
55.4k
        // zone are okay, despite the fact that we don't have a good way
1004
55.4k
        // for validating all of the usages of the calculation.
1005
#ifndef NDEBUG
1006
        bool TouchesMemory = I.mayLoad() || I.mayStore();
1007
        // If we *don't* protect the user from escaped allocas, don't bother
1008
        // validating the instructions.
1009
        if (!I.isDebugInstr() && TouchesMemory && ProtectFromEscapedAllocas) {
1010
          SlotIndex Index = Indexes->getInstructionIndex(I);
1011
          const LiveInterval *Interval = &*Intervals[FromSlot];
1012
          assert(Interval->find(Index) != Interval->end() &&
1013
                 "Found instruction usage outside of live range.");
1014
        }
1015
#endif
1016
1017
55.4k
        // Fix the machine instructions.
1018
55.4k
        int ToSlot = SlotRemap[FromSlot];
1019
55.4k
        MO.setIndex(ToSlot);
1020
55.4k
        FixedInstr++;
1021
55.4k
      }
1022
3.70M
1023
3.70M
      // We adjust AliasAnalysis information for merged stack slots.
1024
3.70M
      SmallVector<MachineMemOperand *, 2> NewMMOs;
1025
3.70M
      bool ReplaceMemOps = false;
1026
3.70M
      for (MachineMemOperand *MMO : I.memoperands()) {
1027
581k
        // If this memory location can be a slot remapped here,
1028
581k
        // we remove AA information.
1029
581k
        bool MayHaveConflictingAAMD = false;
1030
581k
        if (MMO->getAAInfo()) {
1031
438k
          if (const Value *MMOV = MMO->getValue()) {
1032
438k
            SmallVector<Value *, 4> Objs;
1033
438k
            getUnderlyingObjectsForCodeGen(MMOV, Objs, MF->getDataLayout());
1034
438k
1035
438k
            if (Objs.empty())
1036
229k
              MayHaveConflictingAAMD = true;
1037
208k
            else
1038
210k
              
for (Value *V : Objs)208k
{
1039
210k
                // If this memory location comes from a known stack slot
1040
210k
                // that is not remapped, we continue checking.
1041
210k
                // Otherwise, we need to invalidate AA infomation.
1042
210k
                const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(V);
1043
210k
                if (AI && 
MergedAllocas.count(AI)165k
) {
1044
46.4k
                  MayHaveConflictingAAMD = true;
1045
46.4k
                  break;
1046
46.4k
                }
1047
210k
              }
1048
438k
          }
1049
438k
        }
1050
581k
        if (MayHaveConflictingAAMD) {
1051
275k
          NewMMOs.push_back(MF->getMachineMemOperand(MMO, AAMDNodes()));
1052
275k
          ReplaceMemOps = true;
1053
306k
        } else {
1054
306k
          NewMMOs.push_back(MMO);
1055
306k
        }
1056
581k
      }
1057
3.70M
1058
3.70M
      // If any memory operand is updated, set memory references of
1059
3.70M
      // this instruction.
1060
3.70M
      if (ReplaceMemOps)
1061
273k
        I.setMemRefs(*MF, NewMMOs);
1062
3.70M
    }
1063
11.6k
1064
11.6k
  // Update the location of C++ catch objects for the MSVC personality routine.
1065
11.6k
  if (WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo())
1066
1
    for (WinEHTryBlockMapEntry &TBME : EHInfo->TryBlockMap)
1067
1
      for (WinEHHandlerType &H : TBME.HandlerArray)
1068
1
        if (H.CatchObj.FrameIndex != std::numeric_limits<int>::max() &&
1069
1
            
SlotRemap.count(H.CatchObj.FrameIndex)0
)
1070
0
          H.CatchObj.FrameIndex = SlotRemap[H.CatchObj.FrameIndex];
1071
11.6k
1072
11.6k
  LLVM_DEBUG(dbgs() << "Fixed " << FixedMemOp << " machine memory operands.\n");
1073
11.6k
  LLVM_DEBUG(dbgs() << "Fixed " << FixedDbg << " debug locations.\n");
1074
11.6k
  LLVM_DEBUG(dbgs() << "Fixed " << FixedInstr << " machine instructions.\n");
1075
11.6k
}
1076
1077
0
void StackColoring::removeInvalidSlotRanges() {
1078
0
  for (MachineBasicBlock &BB : *MF)
1079
0
    for (MachineInstr &I : BB) {
1080
0
      if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
1081
0
          I.getOpcode() == TargetOpcode::LIFETIME_END || I.isDebugInstr())
1082
0
        continue;
1083
0
1084
0
      // Some intervals are suspicious! In some cases we find address
1085
0
      // calculations outside of the lifetime zone, but not actual memory
1086
0
      // read or write. Memory accesses outside of the lifetime zone are a clear
1087
0
      // violation, but address calculations are okay. This can happen when
1088
0
      // GEPs are hoisted outside of the lifetime zone.
1089
0
      // So, in here we only check instructions which can read or write memory.
1090
0
      if (!I.mayLoad() && !I.mayStore())
1091
0
        continue;
1092
0
1093
0
      // Check all of the machine operands.
1094
0
      for (const MachineOperand &MO : I.operands()) {
1095
0
        if (!MO.isFI())
1096
0
          continue;
1097
0
1098
0
        int Slot = MO.getIndex();
1099
0
1100
0
        if (Slot<0)
1101
0
          continue;
1102
0
1103
0
        if (Intervals[Slot]->empty())
1104
0
          continue;
1105
0
1106
0
        // Check that the used slot is inside the calculated lifetime range.
1107
0
        // If it is not, warn about it and invalidate the range.
1108
0
        LiveInterval *Interval = &*Intervals[Slot];
1109
0
        SlotIndex Index = Indexes->getInstructionIndex(I);
1110
0
        if (Interval->find(Index) == Interval->end()) {
1111
0
          Interval->clear();
1112
0
          LLVM_DEBUG(dbgs() << "Invalidating range #" << Slot << "\n");
1113
0
          EscapedAllocas++;
1114
0
        }
1115
0
      }
1116
0
    }
1117
0
}
1118
1119
void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
1120
11.6k
                                   unsigned NumSlots) {
1121
11.6k
  // Expunge slot remap map.
1122
42.5k
  for (unsigned i=0; i < NumSlots; 
++i30.8k
) {
1123
30.8k
    // If we are remapping i
1124
30.8k
    if (SlotRemap.count(i)) {
1125
10.9k
      int Target = SlotRemap[i];
1126
10.9k
      // As long as our target is mapped to something else, follow it.
1127
10.9k
      while (SlotRemap.count(Target)) {
1128
0
        Target = SlotRemap[Target];
1129
0
        SlotRemap[i] = Target;
1130
0
      }
1131
10.9k
    }
1132
30.8k
  }
1133
11.6k
}
1134
1135
489k
bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
1136
489k
  LLVM_DEBUG(dbgs() << "********** Stack Coloring **********\n"
1137
489k
                    << "********** Function: " << Func.getName() << '\n');
1138
489k
  MF = &Func;
1139
489k
  MFI = &MF->getFrameInfo();
1140
489k
  Indexes = &getAnalysis<SlotIndexes>();
1141
489k
  BlockLiveness.clear();
1142
489k
  BasicBlocks.clear();
1143
489k
  BasicBlockNumbering.clear();
1144
489k
  Markers.clear();
1145
489k
  Intervals.clear();
1146
489k
  LiveStarts.clear();
1147
489k
  VNInfoAllocator.Reset();
1148
489k
1149
489k
  unsigned NumSlots = MFI->getObjectIndexEnd();
1150
489k
1151
489k
  // If there are no stack slots then there are no markers to remove.
1152
489k
  if (!NumSlots)
1153
444k
    return false;
1154
45.4k
1155
45.4k
  SmallVector<int, 8> SortedSlots;
1156
45.4k
  SortedSlots.reserve(NumSlots);
1157
45.4k
  Intervals.reserve(NumSlots);
1158
45.4k
  LiveStarts.resize(NumSlots);
1159
45.4k
1160
45.4k
  unsigned NumMarkers = collectMarkers(NumSlots);
1161
45.4k
1162
45.4k
  unsigned TotalSize = 0;
1163
45.4k
  LLVM_DEBUG(dbgs() << "Found " << NumMarkers << " markers and " << NumSlots
1164
45.4k
                    << " slots\n");
1165
45.4k
  LLVM_DEBUG(dbgs() << "Slot structure:\n");
1166
45.4k
1167
144k
  for (int i=0; i < MFI->getObjectIndexEnd(); 
++i99.2k
) {
1168
99.2k
    LLVM_DEBUG(dbgs() << "Slot #" << i << " - " << MFI->getObjectSize(i)
1169
99.2k
                      << " bytes.\n");
1170
99.2k
    TotalSize += MFI->getObjectSize(i);
1171
99.2k
  }
1172
45.4k
1173
45.4k
  LLVM_DEBUG(dbgs() << "Total Stack size: " << TotalSize << " bytes\n\n");
1174
45.4k
1175
45.4k
  // Don't continue because there are not enough lifetime markers, or the
1176
45.4k
  // stack is too small, or we are told not to optimize the slots.
1177
45.4k
  if (NumMarkers < 2 || 
TotalSize < 1614.5k
||
DisableColoring11.7k
||
1178
45.4k
      
skipFunction(Func.getFunction())11.6k
) {
1179
33.7k
    LLVM_DEBUG(dbgs() << "Will not try to merge slots.\n");
1180
33.7k
    return removeAllMarkers();
1181
33.7k
  }
1182
11.6k
1183
42.5k
  
for (unsigned i=0; 11.6k
i < NumSlots;
++i30.8k
) {
1184
30.8k
    std::unique_ptr<LiveInterval> LI(new LiveInterval(i, 0));
1185
30.8k
    LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
1186
30.8k
    Intervals.push_back(std::move(LI));
1187
30.8k
    SortedSlots.push_back(i);
1188
30.8k
  }
1189
11.6k
1190
11.6k
  // Calculate the liveness of each block.
1191
11.6k
  calculateLocalLiveness();
1192
11.6k
  LLVM_DEBUG(dbgs() << "Dataflow iterations: " << NumIterations << "\n");
1193
11.6k
  LLVM_DEBUG(dump());
1194
11.6k
1195
11.6k
  // Propagate the liveness information.
1196
11.6k
  calculateLiveIntervals(NumSlots);
1197
11.6k
  LLVM_DEBUG(dumpIntervals());
1198
11.6k
1199
11.6k
  // Search for allocas which are used outside of the declared lifetime
1200
11.6k
  // markers.
1201
11.6k
  if (ProtectFromEscapedAllocas)
1202
0
    removeInvalidSlotRanges();
1203
11.6k
1204
11.6k
  // Maps old slots to new slots.
1205
11.6k
  DenseMap<int, int> SlotRemap;
1206
11.6k
  unsigned RemovedSlots = 0;
1207
11.6k
  unsigned ReducedSize = 0;
1208
11.6k
1209
11.6k
  // Do not bother looking at empty intervals.
1210
42.5k
  for (unsigned I = 0; I < NumSlots; 
++I30.8k
) {
1211
30.8k
    if (Intervals[SortedSlots[I]]->empty())
1212
2.26k
      SortedSlots[I] = -1;
1213
30.8k
  }
1214
11.6k
1215
11.6k
  // This is a simple greedy algorithm for merging allocas. First, sort the
1216
11.6k
  // slots, placing the largest slots first. Next, perform an n^2 scan and look
1217
11.6k
  // for disjoint slots. When you find disjoint slots, merge the samller one
1218
11.6k
  // into the bigger one and update the live interval. Remove the small alloca
1219
11.6k
  // and continue.
1220
11.6k
1221
11.6k
  // Sort the slots according to their size. Place unused slots at the end.
1222
11.6k
  // Use stable sort to guarantee deterministic code generation.
1223
41.9k
  llvm::stable_sort(SortedSlots, [this](int LHS, int RHS) {
1224
41.9k
    // We use -1 to denote a uninteresting slot. Place these slots at the end.
1225
41.9k
    if (LHS == -1)
1226
1.46k
      return false;
1227
40.5k
    if (RHS == -1)
1228
4.56k
      return true;
1229
35.9k
    // Sort according to size.
1230
35.9k
    return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
1231
35.9k
  });
1232
11.6k
1233
11.6k
  for (auto &s : LiveStarts)
1234
30.8k
    llvm::sort(s);
1235
11.6k
1236
11.6k
  bool Changed = true;
1237
26.0k
  while (Changed) {
1238
14.3k
    Changed = false;
1239
61.6k
    for (unsigned I = 0; I < NumSlots; 
++I47.3k
) {
1240
47.3k
      if (SortedSlots[I] == -1)
1241
24.7k
        continue;
1242
22.5k
1243
91.8k
      
for (unsigned J=I+1; 22.5k
J < NumSlots;
++J69.2k
) {
1244
69.2k
        if (SortedSlots[J] == -1)
1245
33.9k
          continue;
1246
35.3k
1247
35.3k
        int FirstSlot = SortedSlots[I];
1248
35.3k
        int SecondSlot = SortedSlots[J];
1249
35.3k
        LiveInterval *First = &*Intervals[FirstSlot];
1250
35.3k
        LiveInterval *Second = &*Intervals[SecondSlot];
1251
35.3k
        auto &FirstS = LiveStarts[FirstSlot];
1252
35.3k
        auto &SecondS = LiveStarts[SecondSlot];
1253
35.3k
        assert(!First->empty() && !Second->empty() && "Found an empty range");
1254
35.3k
1255
35.3k
        // Merge disjoint slots. This is a little bit tricky - see the
1256
35.3k
        // Implementation Notes section for an explanation.
1257
35.3k
        if (!First->isLiveAtIndexes(SecondS) &&
1258
35.3k
            
!Second->isLiveAtIndexes(FirstS)16.2k
) {
1259
10.9k
          Changed = true;
1260
10.9k
          First->MergeSegmentsInAsValue(*Second, First->getValNumInfo(0));
1261
10.9k
1262
10.9k
          int OldSize = FirstS.size();
1263
10.9k
          FirstS.append(SecondS.begin(), SecondS.end());
1264
10.9k
          auto Mid = FirstS.begin() + OldSize;
1265
10.9k
          std::inplace_merge(FirstS.begin(), Mid, FirstS.end());
1266
10.9k
1267
10.9k
          SlotRemap[SecondSlot] = FirstSlot;
1268
10.9k
          SortedSlots[J] = -1;
1269
10.9k
          LLVM_DEBUG(dbgs() << "Merging #" << FirstSlot << " and slots #"
1270
10.9k
                            << SecondSlot << " together.\n");
1271
10.9k
          unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
1272
10.9k
                                           MFI->getObjectAlignment(SecondSlot));
1273
10.9k
1274
10.9k
          assert(MFI->getObjectSize(FirstSlot) >=
1275
10.9k
                 MFI->getObjectSize(SecondSlot) &&
1276
10.9k
                 "Merging a small object into a larger one");
1277
10.9k
1278
10.9k
          RemovedSlots+=1;
1279
10.9k
          ReducedSize += MFI->getObjectSize(SecondSlot);
1280
10.9k
          MFI->setObjectAlignment(FirstSlot, MaxAlignment);
1281
10.9k
          MFI->RemoveStackObject(SecondSlot);
1282
10.9k
        }
1283
35.3k
      }
1284
22.5k
    }
1285
14.3k
  }// While changed.
1286
11.6k
1287
11.6k
  // Record statistics.
1288
11.6k
  StackSpaceSaved += ReducedSize;
1289
11.6k
  StackSlotMerged += RemovedSlots;
1290
11.6k
  LLVM_DEBUG(dbgs() << "Merge " << RemovedSlots << " slots. Saved "
1291
11.6k
                    << ReducedSize << " bytes\n");
1292
11.6k
1293
11.6k
  // Scan the entire function and update all machine operands that use frame
1294
11.6k
  // indices to use the remapped frame index.
1295
11.6k
  expungeSlotMap(SlotRemap, NumSlots);
1296
11.6k
  remapInstructions(SlotRemap);
1297
11.6k
1298
11.6k
  return removeAllMarkers();
1299
11.6k
}