Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/RegisterPressure.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- RegisterPressure.cpp - Dynamic Register Pressure -------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements the RegisterPressure class which can be used to track
10
// MachineInstr level register pressure.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "llvm/CodeGen/RegisterPressure.h"
15
#include "llvm/ADT/ArrayRef.h"
16
#include "llvm/ADT/STLExtras.h"
17
#include "llvm/ADT/SmallVector.h"
18
#include "llvm/CodeGen/LiveInterval.h"
19
#include "llvm/CodeGen/LiveIntervals.h"
20
#include "llvm/CodeGen/MachineBasicBlock.h"
21
#include "llvm/CodeGen/MachineFunction.h"
22
#include "llvm/CodeGen/MachineInstr.h"
23
#include "llvm/CodeGen/MachineInstrBundle.h"
24
#include "llvm/CodeGen/MachineOperand.h"
25
#include "llvm/CodeGen/MachineRegisterInfo.h"
26
#include "llvm/CodeGen/RegisterClassInfo.h"
27
#include "llvm/CodeGen/SlotIndexes.h"
28
#include "llvm/CodeGen/TargetRegisterInfo.h"
29
#include "llvm/CodeGen/TargetSubtargetInfo.h"
30
#include "llvm/Config/llvm-config.h"
31
#include "llvm/MC/LaneBitmask.h"
32
#include "llvm/MC/MCRegisterInfo.h"
33
#include "llvm/Support/Compiler.h"
34
#include "llvm/Support/Debug.h"
35
#include "llvm/Support/ErrorHandling.h"
36
#include "llvm/Support/raw_ostream.h"
37
#include <algorithm>
38
#include <cassert>
39
#include <cstdint>
40
#include <cstdlib>
41
#include <cstring>
42
#include <iterator>
43
#include <limits>
44
#include <utility>
45
#include <vector>
46
47
using namespace llvm;
48
49
/// Increase pressure for each pressure set provided by TargetRegisterInfo.
50
static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure,
51
                                const MachineRegisterInfo &MRI, unsigned Reg,
52
1.08M
                                LaneBitmask PrevMask, LaneBitmask NewMask) {
53
1.08M
  assert((PrevMask & ~NewMask).none() && "Must not remove bits");
54
1.08M
  if (PrevMask.any() || 
NewMask.none()1.08M
)
55
4.31k
    return;
56
1.08M
57
1.08M
  PSetIterator PSetI = MRI.getPressureSets(Reg);
58
1.08M
  unsigned Weight = PSetI.getWeight();
59
3.61M
  for (; PSetI.isValid(); 
++PSetI2.53M
)
60
2.53M
    CurrSetPressure[*PSetI] += Weight;
61
1.08M
}
62
63
/// Decrease pressure for each pressure set provided by TargetRegisterInfo.
64
static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure,
65
                                const MachineRegisterInfo &MRI, unsigned Reg,
66
8.52M
                                LaneBitmask PrevMask, LaneBitmask NewMask) {
67
8.52M
  //assert((NewMask & !PrevMask) == 0 && "Must not add bits");
68
8.52M
  if (NewMask.any() || 
PrevMask.none()7.36M
)
69
1.15M
    return;
70
7.36M
71
7.36M
  PSetIterator PSetI = MRI.getPressureSets(Reg);
72
7.36M
  unsigned Weight = PSetI.getWeight();
73
67.6M
  for (; PSetI.isValid(); 
++PSetI60.2M
) {
74
60.2M
    assert(CurrSetPressure[*PSetI] >= Weight && "register pressure underflow");
75
60.2M
    CurrSetPressure[*PSetI] -= Weight;
76
60.2M
  }
77
7.36M
}
78
79
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
80
LLVM_DUMP_METHOD
81
void llvm::dumpRegSetPressure(ArrayRef<unsigned> SetPressure,
82
                              const TargetRegisterInfo *TRI) {
83
  bool Empty = true;
84
  for (unsigned i = 0, e = SetPressure.size(); i < e; ++i) {
85
    if (SetPressure[i] != 0) {
86
      dbgs() << TRI->getRegPressureSetName(i) << "=" << SetPressure[i] << '\n';
87
      Empty = false;
88
    }
89
  }
90
  if (Empty)
91
    dbgs() << "\n";
92
}
93
94
LLVM_DUMP_METHOD
95
void RegisterPressure::dump(const TargetRegisterInfo *TRI) const {
96
  dbgs() << "Max Pressure: ";
97
  dumpRegSetPressure(MaxSetPressure, TRI);
98
  dbgs() << "Live In: ";
99
  for (const RegisterMaskPair &P : LiveInRegs) {
100
    dbgs() << printVRegOrUnit(P.RegUnit, TRI);
101
    if (!P.LaneMask.all())
102
      dbgs() << ':' << PrintLaneMask(P.LaneMask);
103
    dbgs() << ' ';
104
  }
105
  dbgs() << '\n';
106
  dbgs() << "Live Out: ";
107
  for (const RegisterMaskPair &P : LiveOutRegs) {
108
    dbgs() << printVRegOrUnit(P.RegUnit, TRI);
109
    if (!P.LaneMask.all())
110
      dbgs() << ':' << PrintLaneMask(P.LaneMask);
111
    dbgs() << ' ';
112
  }
113
  dbgs() << '\n';
114
}
115
116
LLVM_DUMP_METHOD
117
void RegPressureTracker::dump() const {
118
  if (!isTopClosed() || !isBottomClosed()) {
119
    dbgs() << "Curr Pressure: ";
120
    dumpRegSetPressure(CurrSetPressure, TRI);
121
  }
122
  P.dump(TRI);
123
}
124
125
LLVM_DUMP_METHOD
126
void PressureDiff::dump(const TargetRegisterInfo &TRI) const {
127
  const char *sep = "";
128
  for (const PressureChange &Change : *this) {
129
    if (!Change.isValid())
130
      break;
131
    dbgs() << sep << TRI.getRegPressureSetName(Change.getPSet())
132
           << " " << Change.getUnitInc();
133
    sep = "    ";
134
  }
135
  dbgs() << '\n';
136
}
137
#endif
138
139
void RegPressureTracker::increaseRegPressure(unsigned RegUnit,
140
                                             LaneBitmask PreviousMask,
141
18.1M
                                             LaneBitmask NewMask) {
142
18.1M
  if (PreviousMask.any() || 
NewMask.none()16.8M
)
143
1.32M
    return;
144
16.8M
145
16.8M
  PSetIterator PSetI = MRI->getPressureSets(RegUnit);
146
16.8M
  unsigned Weight = PSetI.getWeight();
147
87.4M
  for (; PSetI.isValid(); 
++PSetI70.6M
) {
148
70.6M
    CurrSetPressure[*PSetI] += Weight;
149
70.6M
    P.MaxSetPressure[*PSetI] =
150
70.6M
        std::max(P.MaxSetPressure[*PSetI], CurrSetPressure[*PSetI]);
151
70.6M
  }
152
16.8M
}
153
154
void RegPressureTracker::decreaseRegPressure(unsigned RegUnit,
155
                                             LaneBitmask PreviousMask,
156
8.52M
                                             LaneBitmask NewMask) {
157
8.52M
  decreaseSetPressure(CurrSetPressure, *MRI, RegUnit, PreviousMask, NewMask);
158
8.52M
}
159
160
/// Clear the result so it can be used for another round of pressure tracking.
161
2.97M
void IntervalPressure::reset() {
162
2.97M
  TopIdx = BottomIdx = SlotIndex();
163
2.97M
  MaxSetPressure.clear();
164
2.97M
  LiveInRegs.clear();
165
2.97M
  LiveOutRegs.clear();
166
2.97M
}
167
168
/// Clear the result so it can be used for another round of pressure tracking.
169
0
void RegionPressure::reset() {
170
0
  TopPos = BottomPos = MachineBasicBlock::const_iterator();
171
0
  MaxSetPressure.clear();
172
0
  LiveInRegs.clear();
173
0
  LiveOutRegs.clear();
174
0
}
175
176
/// If the current top is not less than or equal to the next index, open it.
177
/// We happen to need the SlotIndex for the next top for pressure update.
178
0
void IntervalPressure::openTop(SlotIndex NextTop) {
179
0
  if (TopIdx <= NextTop)
180
0
    return;
181
0
  TopIdx = SlotIndex();
182
0
  LiveInRegs.clear();
183
0
}
184
185
/// If the current top is the previous instruction (before receding), open it.
186
0
void RegionPressure::openTop(MachineBasicBlock::const_iterator PrevTop) {
187
0
  if (TopPos != PrevTop)
188
0
    return;
189
0
  TopPos = MachineBasicBlock::const_iterator();
190
0
  LiveInRegs.clear();
191
0
}
192
193
/// If the current bottom is not greater than the previous index, open it.
194
0
void IntervalPressure::openBottom(SlotIndex PrevBottom) {
195
0
  if (BottomIdx > PrevBottom)
196
0
    return;
197
0
  BottomIdx = SlotIndex();
198
0
  LiveInRegs.clear();
199
0
}
200
201
/// If the current bottom is the previous instr (before advancing), open it.
202
0
void RegionPressure::openBottom(MachineBasicBlock::const_iterator PrevBottom) {
203
0
  if (BottomPos != PrevBottom)
204
0
    return;
205
0
  BottomPos = MachineBasicBlock::const_iterator();
206
0
  LiveInRegs.clear();
207
0
}
208
209
474k
void LiveRegSet::init(const MachineRegisterInfo &MRI) {
210
474k
  const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
211
474k
  unsigned NumRegUnits = TRI.getNumRegs();
212
474k
  unsigned NumVirtRegs = MRI.getNumVirtRegs();
213
474k
  Regs.setUniverse(NumRegUnits + NumVirtRegs);
214
474k
  this->NumRegUnits = NumRegUnits;
215
474k
}
216
217
2.97M
void LiveRegSet::clear() {
218
2.97M
  Regs.clear();
219
2.97M
}
220
221
2.12M
static const LiveRange *getLiveRange(const LiveIntervals &LIS, unsigned Reg) {
222
2.12M
  if (TargetRegisterInfo::isVirtualRegister(Reg))
223
1.96M
    return &LIS.getInterval(Reg);
224
157k
  return LIS.getCachedRegUnit(Reg);
225
157k
}
226
227
2.97M
void RegPressureTracker::reset() {
228
2.97M
  MBB = nullptr;
229
2.97M
  LIS = nullptr;
230
2.97M
231
2.97M
  CurrSetPressure.clear();
232
2.97M
  LiveThruPressure.clear();
233
2.97M
  P.MaxSetPressure.clear();
234
2.97M
235
2.97M
  if (RequireIntervals)
236
2.97M
    static_cast<IntervalPressure&>(P).reset();
237
18.4E
  else
238
18.4E
    static_cast<RegionPressure&>(P).reset();
239
2.97M
240
2.97M
  LiveRegs.clear();
241
2.97M
  UntiedDefs.clear();
242
2.97M
}
243
244
/// Setup the RegPressureTracker.
245
///
246
/// TODO: Add support for pressure without LiveIntervals.
247
void RegPressureTracker::init(const MachineFunction *mf,
248
                              const RegisterClassInfo *rci,
249
                              const LiveIntervals *lis,
250
                              const MachineBasicBlock *mbb,
251
                              MachineBasicBlock::const_iterator pos,
252
474k
                              bool TrackLaneMasks, bool TrackUntiedDefs) {
253
474k
  reset();
254
474k
255
474k
  MF = mf;
256
474k
  TRI = MF->getSubtarget().getRegisterInfo();
257
474k
  RCI = rci;
258
474k
  MRI = &MF->getRegInfo();
259
474k
  MBB = mbb;
260
474k
  this->TrackUntiedDefs = TrackUntiedDefs;
261
474k
  this->TrackLaneMasks = TrackLaneMasks;
262
474k
263
474k
  if (RequireIntervals) {
264
474k
    assert(lis && "IntervalPressure requires LiveIntervals");
265
474k
    LIS = lis;
266
474k
  }
267
474k
268
474k
  CurrPos = pos;
269
474k
  CurrSetPressure.assign(TRI->getNumRegPressureSets(), 0);
270
474k
271
474k
  P.MaxSetPressure = CurrSetPressure;
272
474k
273
474k
  LiveRegs.init(*MRI);
274
474k
  if (TrackUntiedDefs)
275
158k
    UntiedDefs.setUniverse(MRI->getNumVirtRegs());
276
474k
}
277
278
/// Does this pressure result have a valid top position and live ins.
279
6.24M
bool RegPressureTracker::isTopClosed() const {
280
6.24M
  if (RequireIntervals)
281
6.24M
    return static_cast<IntervalPressure&>(P).TopIdx.isValid();
282
0
  return (static_cast<RegionPressure&>(P).TopPos ==
283
0
          MachineBasicBlock::const_iterator());
284
0
}
285
286
/// Does this pressure result have a valid bottom position and live outs.
287
6.14M
bool RegPressureTracker::isBottomClosed() const {
288
6.14M
  if (RequireIntervals)
289
6.14M
    return static_cast<IntervalPressure&>(P).BottomIdx.isValid();
290
0
  return (static_cast<RegionPressure&>(P).BottomPos ==
291
0
          MachineBasicBlock::const_iterator());
292
0
}
293
294
1.81M
SlotIndex RegPressureTracker::getCurrSlot() const {
295
1.81M
  MachineBasicBlock::const_iterator IdxPos =
296
1.81M
    skipDebugInstructionsForward(CurrPos, MBB->end());
297
1.81M
  if (IdxPos == MBB->end())
298
215k
    return LIS->getMBBEndIdx(MBB);
299
1.60M
  return LIS->getInstructionIndex(*IdxPos).getRegSlot();
300
1.60M
}
301
302
/// Set the boundary for the top of the region and summarize live ins.
303
316k
void RegPressureTracker::closeTop() {
304
316k
  if (RequireIntervals)
305
316k
    static_cast<IntervalPressure&>(P).TopIdx = getCurrSlot();
306
0
  else
307
0
    static_cast<RegionPressure&>(P).TopPos = CurrPos;
308
316k
309
316k
  assert(P.LiveInRegs.empty() && "inconsistent max pressure result");
310
316k
  P.LiveInRegs.reserve(LiveRegs.size());
311
316k
  LiveRegs.appendTo(P.LiveInRegs);
312
316k
}
313
314
/// Set the boundary for the bottom of the region and summarize live outs.
315
316k
void RegPressureTracker::closeBottom() {
316
316k
  if (RequireIntervals)
317
316k
    static_cast<IntervalPressure&>(P).BottomIdx = getCurrSlot();
318
0
  else
319
0
    static_cast<RegionPressure&>(P).BottomPos = CurrPos;
320
316k
321
316k
  assert(P.LiveOutRegs.empty() && "inconsistent max pressure result");
322
316k
  P.LiveOutRegs.reserve(LiveRegs.size());
323
316k
  LiveRegs.appendTo(P.LiveOutRegs);
324
316k
}
325
326
/// Finalize the region boundaries and record live ins and live outs.
327
158k
void RegPressureTracker::closeRegion() {
328
158k
  if (!isTopClosed() && 
!isBottomClosed()158k
) {
329
0
    assert(LiveRegs.size() == 0 && "no region boundary");
330
0
    return;
331
0
  }
332
158k
  if (!isBottomClosed())
333
21
    closeBottom();
334
158k
  else if (!isTopClosed())
335
158k
    closeTop();
336
158k
  // If both top and bottom are closed, do nothing.
337
158k
}
338
339
/// The register tracker is unaware of global liveness so ignores normal
340
/// live-thru ranges. However, two-address or coalesced chains can also lead
341
/// to live ranges with no holes. Count these to inform heuristics that we
342
/// can never drop below this pressure.
343
158k
void RegPressureTracker::initLiveThru(const RegPressureTracker &RPTracker) {
344
158k
  LiveThruPressure.assign(TRI->getNumRegPressureSets(), 0);
345
158k
  assert(isBottomClosed() && "need bottom-up tracking to intialize.");
346
562k
  for (const RegisterMaskPair &Pair : P.LiveOutRegs) {
347
562k
    unsigned RegUnit = Pair.RegUnit;
348
562k
    if (TargetRegisterInfo::isVirtualRegister(RegUnit)
349
562k
        && 
!RPTracker.hasUntiedDef(RegUnit)556k
)
350
197k
      increaseSetPressure(LiveThruPressure, *MRI, RegUnit,
351
197k
                          LaneBitmask::getNone(), Pair.LaneMask);
352
562k
  }
353
158k
}
354
355
static LaneBitmask getRegLanes(ArrayRef<RegisterMaskPair> RegUnits,
356
2.61M
                               unsigned RegUnit) {
357
2.61M
  auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
358
2.57M
    return Other.RegUnit == RegUnit;
359
2.57M
  });
360
2.61M
  if (I == RegUnits.end())
361
2.54M
    return LaneBitmask::getNone();
362
67.7k
  return I->LaneMask;
363
67.7k
}
364
365
static void addRegLanes(SmallVectorImpl<RegisterMaskPair> &RegUnits,
366
43.1M
                        RegisterMaskPair Pair) {
367
43.1M
  unsigned RegUnit = Pair.RegUnit;
368
43.1M
  assert(Pair.LaneMask.any());
369
43.1M
  auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
370
11.8M
    return Other.RegUnit == RegUnit;
371
11.8M
  });
372
43.1M
  if (I == RegUnits.end()) {
373
43.0M
    RegUnits.push_back(Pair);
374
43.0M
  } else {
375
125k
    I->LaneMask |= Pair.LaneMask;
376
125k
  }
377
43.1M
}
378
379
static void setRegZero(SmallVectorImpl<RegisterMaskPair> &RegUnits,
380
197k
                       unsigned RegUnit) {
381
197k
  auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
382
9.78k
    return Other.RegUnit == RegUnit;
383
9.78k
  });
384
197k
  if (I == RegUnits.end()) {
385
197k
    RegUnits.push_back(RegisterMaskPair(RegUnit, LaneBitmask::getNone()));
386
197k
  } else {
387
0
    I->LaneMask = LaneBitmask::getNone();
388
0
  }
389
197k
}
390
391
static void removeRegLanes(SmallVectorImpl<RegisterMaskPair> &RegUnits,
392
16.1M
                           RegisterMaskPair Pair) {
393
16.1M
  unsigned RegUnit = Pair.RegUnit;
394
16.1M
  assert(Pair.LaneMask.any());
395
16.1M
  auto I = llvm::find_if(RegUnits, [RegUnit](const RegisterMaskPair Other) {
396
77.0k
    return Other.RegUnit == RegUnit;
397
77.0k
  });
398
16.1M
  if (I != RegUnits.end()) {
399
7.47k
    I->LaneMask &= ~Pair.LaneMask;
400
7.47k
    if (I->LaneMask.none())
401
7.47k
      RegUnits.erase(I);
402
7.47k
  }
403
16.1M
}
404
405
static LaneBitmask getLanesWithProperty(const LiveIntervals &LIS,
406
    const MachineRegisterInfo &MRI, bool TrackLaneMasks, unsigned RegUnit,
407
    SlotIndex Pos, LaneBitmask SafeDefault,
408
27.0M
    bool(*Property)(const LiveRange &LR, SlotIndex Pos)) {
409
27.0M
  if (TargetRegisterInfo::isVirtualRegister(RegUnit)) {
410
24.9M
    const LiveInterval &LI = LIS.getInterval(RegUnit);
411
24.9M
    LaneBitmask Result;
412
24.9M
    if (TrackLaneMasks && 
LI.hasSubRanges()8.80M
) {
413
16.8M
        for (const LiveInterval::SubRange &SR : LI.subranges()) {
414
16.8M
          if (Property(SR, Pos))
415
11.3M
            Result |= SR.LaneMask;
416
16.8M
        }
417
21.1M
    } else if (Property(LI, Pos)) {
418
5.14M
      Result = TrackLaneMasks ? 
MRI.getMaxLaneMaskForVReg(RegUnit)4.51M
419
5.14M
                              : 
LaneBitmask::getAll()628k
;
420
5.14M
    }
421
24.9M
422
24.9M
    return Result;
423
24.9M
  } else {
424
2.10M
    const LiveRange *LR = LIS.getCachedRegUnit(RegUnit);
425
2.10M
    // Be prepared for missing liveranges: We usually do not compute liveranges
426
2.10M
    // for physical registers on targets with many registers (GPUs).
427
2.10M
    if (LR == nullptr)
428
290k
      return SafeDefault;
429
1.81M
    return Property(*LR, Pos) ? 
LaneBitmask::getAll()1.23M
:
LaneBitmask::getNone()578k
;
430
1.81M
  }
431
27.0M
}
432
433
static LaneBitmask getLiveLanesAt(const LiveIntervals &LIS,
434
                                  const MachineRegisterInfo &MRI,
435
                                  bool TrackLaneMasks, unsigned RegUnit,
436
8.76M
                                  SlotIndex Pos) {
437
8.76M
  return getLanesWithProperty(LIS, MRI, TrackLaneMasks, RegUnit, Pos,
438
8.76M
                              LaneBitmask::getAll(),
439
20.9M
                              [](const LiveRange &LR, SlotIndex Pos) {
440
20.9M
                                return LR.liveAt(Pos);
441
20.9M
                              });
442
8.76M
}
443
444
445
namespace {
446
447
/// Collect this instruction's unique uses and defs into SmallVectors for
448
/// processing defs and uses in order.
449
///
450
/// FIXME: always ignore tied opers
451
class RegisterOperandsCollector {
452
  friend class llvm::RegisterOperands;
453
454
  RegisterOperands &RegOpers;
455
  const TargetRegisterInfo &TRI;
456
  const MachineRegisterInfo &MRI;
457
  bool IgnoreDead;
458
459
  RegisterOperandsCollector(RegisterOperands &RegOpers,
460
                            const TargetRegisterInfo &TRI,
461
                            const MachineRegisterInfo &MRI, bool IgnoreDead)
462
23.2M
    : RegOpers(RegOpers), TRI(TRI), MRI(MRI), IgnoreDead(IgnoreDead) {}
463
464
18.7M
  void collectInstr(const MachineInstr &MI) const {
465
73.7M
    for (ConstMIBundleOperands OperI(MI); OperI.isValid(); 
++OperI55.0M
)
466
55.0M
      collectOperand(*OperI);
467
18.7M
468
18.7M
    // Remove redundant physreg dead defs.
469
18.7M
    for (const RegisterMaskPair &P : RegOpers.Defs)
470
12.1M
      removeRegLanes(RegOpers.DeadDefs, P);
471
18.7M
  }
472
473
4.55M
  void collectInstrLanes(const MachineInstr &MI) const {
474
23.4M
    for (ConstMIBundleOperands OperI(MI); OperI.isValid(); 
++OperI18.9M
)
475
18.9M
      collectOperandLanes(*OperI);
476
4.55M
477
4.55M
    // Remove redundant physreg dead defs.
478
4.55M
    for (const RegisterMaskPair &P : RegOpers.Defs)
479
4.03M
      removeRegLanes(RegOpers.DeadDefs, P);
480
4.55M
  }
481
482
  /// Push this operand's register onto the correct vectors.
483
55.0M
  void collectOperand(const MachineOperand &MO) const {
484
55.0M
    if (!MO.isReg() || 
!MO.getReg()34.5M
)
485
21.4M
      return;
486
33.6M
    unsigned Reg = MO.getReg();
487
33.6M
    if (MO.isUse()) {
488
20.6M
      if (!MO.isUndef() && 
!MO.isInternalRead()20.6M
)
489
20.6M
        pushReg(Reg, RegOpers.Uses);
490
20.6M
    } else {
491
13.0M
      assert(MO.isDef());
492
13.0M
      // Subregister definitions may imply a register read.
493
13.0M
      if (MO.readsReg())
494
49.1k
        pushReg(Reg, RegOpers.Uses);
495
13.0M
496
13.0M
      if (MO.isDead()) {
497
366k
        if (!IgnoreDead)
498
365k
          pushReg(Reg, RegOpers.DeadDefs);
499
366k
      } else
500
12.6M
        pushReg(Reg, RegOpers.Defs);
501
13.0M
    }
502
33.6M
  }
503
504
  void pushReg(unsigned Reg,
505
33.6M
               SmallVectorImpl<RegisterMaskPair> &RegUnits) const {
506
33.6M
    if (TargetRegisterInfo::isVirtualRegister(Reg)) {
507
30.8M
      addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneBitmask::getAll()));
508
30.8M
    } else 
if (2.81M
MRI.isAllocatable(Reg)2.81M
) {
509
2.07M
      for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); 
++Units1.21M
)
510
1.21M
        addRegLanes(RegUnits, RegisterMaskPair(*Units, LaneBitmask::getAll()));
511
860k
    }
512
33.6M
  }
513
514
18.9M
  void collectOperandLanes(const MachineOperand &MO) const {
515
18.9M
    if (!MO.isReg() || 
!MO.getReg()12.1M
)
516
6.76M
      return;
517
12.1M
    unsigned Reg = MO.getReg();
518
12.1M
    unsigned SubRegIdx = MO.getSubReg();
519
12.1M
    if (MO.isUse()) {
520
7.67M
      if (!MO.isUndef() && 
!MO.isInternalRead()7.66M
)
521
7.66M
        pushRegLanes(Reg, SubRegIdx, RegOpers.Uses);
522
7.67M
    } else {
523
4.49M
      assert(MO.isDef());
524
4.49M
      // Treat read-undef subreg defs as definitions of the whole register.
525
4.49M
      if (MO.isUndef())
526
47.3k
        SubRegIdx = 0;
527
4.49M
528
4.49M
      if (MO.isDead()) {
529
422k
        if (!IgnoreDead)
530
104k
          pushRegLanes(Reg, SubRegIdx, RegOpers.DeadDefs);
531
422k
      } else
532
4.06M
        pushRegLanes(Reg, SubRegIdx, RegOpers.Defs);
533
4.49M
    }
534
12.1M
  }
535
536
  void pushRegLanes(unsigned Reg, unsigned SubRegIdx,
537
11.8M
                    SmallVectorImpl<RegisterMaskPair> &RegUnits) const {
538
11.8M
    if (TargetRegisterInfo::isVirtualRegister(Reg)) {
539
8.05M
      LaneBitmask LaneMask = SubRegIdx != 0
540
8.05M
                             ? 
TRI.getSubRegIndexLaneMask(SubRegIdx)2.97M
541
8.05M
                             : 
MRI.getMaxLaneMaskForVReg(Reg)5.08M
;
542
8.05M
      addRegLanes(RegUnits, RegisterMaskPair(Reg, LaneMask));
543
8.05M
    } else 
if (3.78M
MRI.isAllocatable(Reg)3.78M
) {
544
1.47M
      for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); 
++Units779k
)
545
779k
        addRegLanes(RegUnits, RegisterMaskPair(*Units, LaneBitmask::getAll()));
546
690k
    }
547
11.8M
  }
548
};
549
550
} // end anonymous namespace
551
552
void RegisterOperands::collect(const MachineInstr &MI,
553
                               const TargetRegisterInfo &TRI,
554
                               const MachineRegisterInfo &MRI,
555
23.2M
                               bool TrackLaneMasks, bool IgnoreDead) {
556
23.2M
  RegisterOperandsCollector Collector(*this, TRI, MRI, IgnoreDead);
557
23.2M
  if (TrackLaneMasks)
558
4.55M
    Collector.collectInstrLanes(MI);
559
18.7M
  else
560
18.7M
    Collector.collectInstr(MI);
561
23.2M
}
562
563
void RegisterOperands::detectDeadDefs(const MachineInstr &MI,
564
2.89M
                                      const LiveIntervals &LIS) {
565
2.89M
  SlotIndex SlotIdx = LIS.getInstructionIndex(MI);
566
5.02M
  for (auto RI = Defs.begin(); RI != Defs.end(); /*empty*/) {
567
2.12M
    unsigned Reg = RI->RegUnit;
568
2.12M
    const LiveRange *LR = getLiveRange(LIS, Reg);
569
2.12M
    if (LR != nullptr) {
570
2.05M
      LiveQueryResult LRQ = LR->Query(SlotIdx);
571
2.05M
      if (LRQ.isDeadDef()) {
572
0
        // LiveIntervals knows this is a dead even though it's MachineOperand is
573
0
        // not flagged as such.
574
0
        DeadDefs.push_back(*RI);
575
0
        RI = Defs.erase(RI);
576
0
        continue;
577
0
      }
578
2.12M
    }
579
2.12M
    ++RI;
580
2.12M
  }
581
2.89M
}
582
583
void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS,
584
                                          const MachineRegisterInfo &MRI,
585
                                          SlotIndex Pos,
586
4.55M
                                          MachineInstr *AddFlagsMI) {
587
8.58M
  for (auto I = Defs.begin(); I != Defs.end(); ) {
588
4.03M
    LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
589
4.03M
                                           Pos.getDeadSlot());
590
4.03M
    // If the def is all that is live after the instruction, then in case
591
4.03M
    // of a subregister def we need a read-undef flag.
592
4.03M
    unsigned RegUnit = I->RegUnit;
593
4.03M
    if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
594
4.03M
        
AddFlagsMI != nullptr3.89M
&&
(LiveAfter & ~I->LaneMask).none()353k
)
595
258k
      AddFlagsMI->setRegisterDefReadUndef(RegUnit);
596
4.03M
597
4.03M
    LaneBitmask ActualDef = I->LaneMask & LiveAfter;
598
4.03M
    if (ActualDef.none()) {
599
261
      I = Defs.erase(I);
600
4.02M
    } else {
601
4.02M
      I->LaneMask = ActualDef;
602
4.02M
      ++I;
603
4.02M
    }
604
4.03M
  }
605
9.27M
  for (auto I = Uses.begin(); I != Uses.end(); ) {
606
4.72M
    LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
607
4.72M
                                            Pos.getBaseIndex());
608
4.72M
    LaneBitmask LaneMask = I->LaneMask & LiveBefore;
609
4.72M
    if (LaneMask.none()) {
610
0
      I = Uses.erase(I);
611
4.72M
    } else {
612
4.72M
      I->LaneMask = LaneMask;
613
4.72M
      ++I;
614
4.72M
    }
615
4.72M
  }
616
4.55M
  if (AddFlagsMI != nullptr) {
617
410k
    for (const RegisterMaskPair &P : DeadDefs) {
618
16.5k
      unsigned RegUnit = P.RegUnit;
619
16.5k
      if (!TargetRegisterInfo::isVirtualRegister(RegUnit))
620
8.55k
        continue;
621
7.98k
      LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, RegUnit,
622
7.98k
                                             Pos.getDeadSlot());
623
7.98k
      if (LiveAfter.none())
624
7.98k
        AddFlagsMI->setRegisterDefReadUndef(RegUnit);
625
7.98k
    }
626
410k
  }
627
4.55M
}
628
629
/// Initialize an array of N PressureDiffs.
630
158k
void PressureDiffs::init(unsigned N) {
631
158k
  Size = N;
632
158k
  if (N <= Max) {
633
36.6k
    memset(PDiffArray, 0, N * sizeof(PressureDiff));
634
36.6k
    return;
635
36.6k
  }
636
121k
  Max = Size;
637
121k
  free(PDiffArray);
638
121k
  PDiffArray = static_cast<PressureDiff*>(safe_calloc(N, sizeof(PressureDiff)));
639
121k
}
640
641
void PressureDiffs::addInstruction(unsigned Idx,
642
                                   const RegisterOperands &RegOpers,
643
3.01M
                                   const MachineRegisterInfo &MRI) {
644
3.01M
  PressureDiff &PDiff = (*this)[Idx];
645
3.01M
  assert(!PDiff.begin()->isValid() && "stale PDiff");
646
3.01M
  for (const RegisterMaskPair &P : RegOpers.Defs)
647
2.45M
    PDiff.addPressureChange(P.RegUnit, true, &MRI);
648
3.01M
649
3.01M
  for (const RegisterMaskPair &P : RegOpers.Uses)
650
3.92M
    PDiff.addPressureChange(P.RegUnit, false, &MRI);
651
3.01M
}
652
653
/// Add a change in pressure to the pressure diff of a given instruction.
654
void PressureDiff::addPressureChange(unsigned RegUnit, bool IsDec,
655
9.91M
                                     const MachineRegisterInfo *MRI) {
656
9.91M
  PSetIterator PSetI = MRI->getPressureSets(RegUnit);
657
9.91M
  int Weight = IsDec ? 
-PSetI.getWeight()5.98M
:
PSetI.getWeight()3.92M
;
658
40.0M
  for (; PSetI.isValid(); 
++PSetI30.1M
) {
659
30.3M
    // Find an existing entry in the pressure diff for this PSet.
660
30.3M
    PressureDiff::iterator I = nonconst_begin(), E = nonconst_end();
661
107M
    for (; I != E && 
I->isValid()106M
;
++I76.7M
) {
662
90.7M
      if (I->getPSet() >= *PSetI)
663
13.9M
        break;
664
90.7M
    }
665
30.3M
    // If all pressure sets are more constrained, skip the remaining PSets.
666
30.3M
    if (I == E)
667
217k
      break;
668
30.1M
    // Insert this PressureChange.
669
30.1M
    if (!I->isValid() || 
I->getPSet() != *PSetI13.9M
) {
670
18.2M
      PressureChange PTmp = PressureChange(*PSetI);
671
43.1M
      for (PressureDiff::iterator J = I; J != E && 
PTmp.isValid()42.7M
;
++J24.8M
)
672
24.8M
        std::swap(*J, PTmp);
673
18.2M
    }
674
30.1M
    // Update the units for this pressure set.
675
30.1M
    unsigned NewUnitInc = I->getUnitInc() + Weight;
676
30.1M
    if (NewUnitInc != 0) {
677
20.0M
      I->setUnitInc(NewUnitInc);
678
20.0M
    } else {
679
10.0M
      // Remove entry
680
10.0M
      PressureDiff::iterator J;
681
41.9M
      for (J = std::next(I); J != E && 
J->isValid()41.9M
;
++J, ++I31.8M
)
682
31.8M
        *I = *J;
683
10.0M
      *I = PressureChange();
684
10.0M
    }
685
30.1M
  }
686
9.91M
}
687
688
/// Force liveness of registers.
689
316k
void RegPressureTracker::addLiveRegs(ArrayRef<RegisterMaskPair> Regs) {
690
1.16M
  for (const RegisterMaskPair &P : Regs) {
691
1.16M
    LaneBitmask PrevMask = LiveRegs.insert(P);
692
1.16M
    LaneBitmask NewMask = PrevMask | P.LaneMask;
693
1.16M
    increaseRegPressure(P.RegUnit, PrevMask, NewMask);
694
1.16M
  }
695
316k
}
696
697
void RegPressureTracker::discoverLiveInOrOut(RegisterMaskPair Pair,
698
567k
    SmallVectorImpl<RegisterMaskPair> &LiveInOrOut) {
699
567k
  assert(Pair.LaneMask.any());
700
567k
701
567k
  unsigned RegUnit = Pair.RegUnit;
702
6.11M
  auto I = llvm::find_if(LiveInOrOut, [RegUnit](const RegisterMaskPair &Other) {
703
6.11M
    return Other.RegUnit == RegUnit;
704
6.11M
  });
705
567k
  LaneBitmask PrevMask;
706
567k
  LaneBitmask NewMask;
707
567k
  if (I == LiveInOrOut.end()) {
708
562k
    PrevMask = LaneBitmask::getNone();
709
562k
    NewMask = Pair.LaneMask;
710
562k
    LiveInOrOut.push_back(Pair);
711
562k
  } else {
712
4.31k
    PrevMask = I->LaneMask;
713
4.31k
    NewMask = PrevMask | Pair.LaneMask;
714
4.31k
    I->LaneMask = NewMask;
715
4.31k
  }
716
567k
  increaseSetPressure(P.MaxSetPressure, *MRI, RegUnit, PrevMask, NewMask);
717
567k
}
718
719
49
void RegPressureTracker::discoverLiveIn(RegisterMaskPair Pair) {
720
49
  discoverLiveInOrOut(Pair, P.LiveInRegs);
721
49
}
722
723
567k
void RegPressureTracker::discoverLiveOut(RegisterMaskPair Pair) {
724
567k
  discoverLiveInOrOut(Pair, P.LiveOutRegs);
725
567k
}
726
727
23.2M
void RegPressureTracker::bumpDeadDefs(ArrayRef<RegisterMaskPair> DeadDefs) {
728
23.2M
  for (const RegisterMaskPair &P : DeadDefs) {
729
93.3k
    unsigned Reg = P.RegUnit;
730
93.3k
    LaneBitmask LiveMask = LiveRegs.contains(Reg);
731
93.3k
    LaneBitmask BumpedMask = LiveMask | P.LaneMask;
732
93.3k
    increaseRegPressure(Reg, LiveMask, BumpedMask);
733
93.3k
  }
734
23.2M
  for (const RegisterMaskPair &P : DeadDefs) {
735
93.3k
    unsigned Reg = P.RegUnit;
736
93.3k
    LaneBitmask LiveMask = LiveRegs.contains(Reg);
737
93.3k
    LaneBitmask BumpedMask = LiveMask | P.LaneMask;
738
93.3k
    decreaseRegPressure(Reg, BumpedMask, LiveMask);
739
93.3k
  }
740
23.2M
}
741
742
/// Recede across the previous instruction. If LiveUses is provided, record any
743
/// RegUnits that are made live by the current instruction's uses. This includes
744
/// registers that are both defined and used by the instruction.  If a pressure
745
/// difference pointer is provided record the changes is pressure caused by this
746
/// instruction independent of liveness.
747
void RegPressureTracker::recede(const RegisterOperands &RegOpers,
748
6.13M
                                SmallVectorImpl<RegisterMaskPair> *LiveUses) {
749
6.13M
  assert(!CurrPos->isDebugInstr());
750
6.13M
751
6.13M
  // Boost pressure for all dead defs together.
752
6.13M
  bumpDeadDefs(RegOpers.DeadDefs);
753
6.13M
754
6.13M
  // Kill liveness at live defs.
755
6.13M
  // TODO: consider earlyclobbers?
756
6.13M
  for (const RegisterMaskPair &Def : RegOpers.Defs) {
757
4.74M
    unsigned Reg = Def.RegUnit;
758
4.74M
759
4.74M
    LaneBitmask PreviousMask = LiveRegs.erase(Def);
760
4.74M
    LaneBitmask NewMask = PreviousMask & ~Def.LaneMask;
761
4.74M
762
4.74M
    LaneBitmask LiveOut = Def.LaneMask & ~PreviousMask;
763
4.74M
    if (LiveOut.any()) {
764
324k
      discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
765
324k
      // Retroactively model effects on pressure of the live out lanes.
766
324k
      increaseSetPressure(CurrSetPressure, *MRI, Reg, LaneBitmask::getNone(),
767
324k
                          LiveOut);
768
324k
      PreviousMask = LiveOut;
769
324k
    }
770
4.74M
771
4.74M
    if (NewMask.none()) {
772
4.58M
      // Add a 0 entry to LiveUses as a marker that the complete vreg has become
773
4.58M
      // dead.
774
4.58M
      if (TrackLaneMasks && 
LiveUses != nullptr474k
)
775
197k
        setRegZero(*LiveUses, Reg);
776
4.58M
    }
777
4.74M
778
4.74M
    decreaseRegPressure(Reg, PreviousMask, NewMask);
779
4.74M
  }
780
6.13M
781
6.13M
  SlotIndex SlotIdx;
782
6.13M
  if (RequireIntervals)
783
6.13M
    SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
784
6.13M
785
6.13M
  // Generate liveness for uses.
786
7.90M
  for (const RegisterMaskPair &Use : RegOpers.Uses) {
787
7.90M
    unsigned Reg = Use.RegUnit;
788
7.90M
    assert(Use.LaneMask.any());
789
7.90M
    LaneBitmask PreviousMask = LiveRegs.insert(Use);
790
7.90M
    LaneBitmask NewMask = PreviousMask | Use.LaneMask;
791
7.90M
    if (NewMask == PreviousMask)
792
2.92M
      continue;
793
4.97M
794
4.97M
    // Did the register just become live?
795
4.97M
    if (PreviousMask.none()) {
796
4.88M
      if (LiveUses != nullptr) {
797
2.23M
        if (!TrackLaneMasks) {
798
1.99M
          addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
799
1.99M
        } else {
800
242k
          auto I =
801
301k
              llvm::find_if(*LiveUses, [Reg](const RegisterMaskPair Other) {
802
301k
                return Other.RegUnit == Reg;
803
301k
              });
804
242k
          bool IsRedef = I != LiveUses->end();
805
242k
          if (IsRedef) {
806
3.02k
            // ignore re-defs here...
807
3.02k
            assert(I->LaneMask.none());
808
3.02k
            removeRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
809
239k
          } else {
810
239k
            addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
811
239k
          }
812
242k
        }
813
2.23M
      }
814
4.88M
815
4.88M
      // Discover live outs if this may be the first occurance of this register.
816
4.88M
      if (RequireIntervals) {
817
4.88M
        LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx);
818
4.88M
        if (LiveOut.any())
819
242k
          discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
820
4.88M
      }
821
4.88M
    }
822
4.97M
823
4.97M
    increaseRegPressure(Reg, PreviousMask, NewMask);
824
4.97M
  }
825
6.13M
  if (TrackUntiedDefs) {
826
3.16M
    for (const RegisterMaskPair &Def : RegOpers.Defs) {
827
2.46M
      unsigned RegUnit = Def.RegUnit;
828
2.46M
      if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
829
2.46M
          
(LiveRegs.contains(RegUnit) & Def.LaneMask).none()2.29M
)
830
2.03M
        UntiedDefs.insert(RegUnit);
831
2.46M
    }
832
3.16M
  }
833
6.13M
}
834
835
5.64M
void RegPressureTracker::recedeSkipDebugValues() {
836
5.64M
  assert(CurrPos != MBB->begin());
837
5.64M
  if (!isBottomClosed())
838
158k
    closeBottom();
839
5.64M
840
5.64M
  // Open the top of the region using block iterators.
841
5.64M
  if (!RequireIntervals && 
isTopClosed()0
)
842
0
    static_cast<RegionPressure&>(P).openTop(CurrPos);
843
5.64M
844
5.64M
  // Find the previous instruction.
845
5.64M
  CurrPos = skipDebugInstructionsBackward(std::prev(CurrPos), MBB->begin());
846
5.64M
847
5.64M
  SlotIndex SlotIdx;
848
5.64M
  if (RequireIntervals && !CurrPos->isDebugInstr())
849
5.64M
    SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
850
5.64M
851
5.64M
  // Open the top of the region using slot indexes.
852
5.64M
  if (RequireIntervals && isTopClosed())
853
0
    static_cast<IntervalPressure&>(P).openTop(SlotIdx);
854
5.64M
}
855
856
292k
void RegPressureTracker::recede(SmallVectorImpl<RegisterMaskPair> *LiveUses) {
857
292k
  recedeSkipDebugValues();
858
292k
  if (CurrPos->isDebugValue()) {
859
0
    // It's possible to only have debug_value instructions and hit the start of
860
0
    // the block.
861
0
    assert(CurrPos == MBB->begin());
862
0
    return;
863
0
  }
864
292k
865
292k
  const MachineInstr &MI = *CurrPos;
866
292k
  RegisterOperands RegOpers;
867
292k
  RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false);
868
292k
  if (TrackLaneMasks) {
869
52.4k
    SlotIndex SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
870
52.4k
    RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
871
239k
  } else if (RequireIntervals) {
872
239k
    RegOpers.detectDeadDefs(MI, *LIS);
873
239k
  }
874
292k
875
292k
  recede(RegOpers, LiveUses);
876
292k
}
877
878
/// Advance across the current instruction.
879
184k
void RegPressureTracker::advance(const RegisterOperands &RegOpers) {
880
184k
  assert(!TrackUntiedDefs && "unsupported mode");
881
184k
  assert(CurrPos != MBB->end());
882
184k
  if (!isTopClosed())
883
21
    closeTop();
884
184k
885
184k
  SlotIndex SlotIdx;
886
184k
  if (RequireIntervals)
887
184k
    SlotIdx = getCurrSlot();
888
184k
889
184k
  // Open the bottom of the region using slot indexes.
890
184k
  if (isBottomClosed()) {
891
0
    if (RequireIntervals)
892
0
      static_cast<IntervalPressure&>(P).openBottom(SlotIdx);
893
0
    else
894
0
      static_cast<RegionPressure&>(P).openBottom(CurrPos);
895
0
  }
896
184k
897
212k
  for (const RegisterMaskPair &Use : RegOpers.Uses) {
898
212k
    unsigned Reg = Use.RegUnit;
899
212k
    LaneBitmask LiveMask = LiveRegs.contains(Reg);
900
212k
    LaneBitmask LiveIn = Use.LaneMask & ~LiveMask;
901
212k
    if (LiveIn.any()) {
902
49
      discoverLiveIn(RegisterMaskPair(Reg, LiveIn));
903
49
      increaseRegPressure(Reg, LiveMask, LiveMask | LiveIn);
904
49
      LiveRegs.insert(RegisterMaskPair(Reg, LiveIn));
905
49
    }
906
212k
    // Kill liveness at last uses.
907
212k
    if (RequireIntervals) {
908
212k
      LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
909
212k
      if (LastUseMask.any()) {
910
176k
        LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask));
911
176k
        decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask);
912
176k
      }
913
212k
    }
914
212k
  }
915
184k
916
184k
  // Generate liveness for defs.
917
184k
  for (const RegisterMaskPair &Def : RegOpers.Defs) {
918
179k
    LaneBitmask PreviousMask = LiveRegs.insert(Def);
919
179k
    LaneBitmask NewMask = PreviousMask | Def.LaneMask;
920
179k
    increaseRegPressure(Def.RegUnit, PreviousMask, NewMask);
921
179k
  }
922
184k
923
184k
  // Boost pressure for all dead defs together.
924
184k
  bumpDeadDefs(RegOpers.DeadDefs);
925
184k
926
184k
  // Find the next instruction.
927
184k
  CurrPos = skipDebugInstructionsForward(std::next(CurrPos), MBB->end());
928
184k
}
929
930
126
void RegPressureTracker::advance() {
931
126
  const MachineInstr &MI = *CurrPos;
932
126
  RegisterOperands RegOpers;
933
126
  RegOpers.collect(MI, *TRI, *MRI, TrackLaneMasks, false);
934
126
  if (TrackLaneMasks) {
935
0
    SlotIndex SlotIdx = getCurrSlot();
936
0
    RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
937
0
  }
938
126
  advance(RegOpers);
939
126
}
940
941
/// Find the max change in excess pressure across all sets.
942
static void computeExcessPressureDelta(ArrayRef<unsigned> OldPressureVec,
943
                                       ArrayRef<unsigned> NewPressureVec,
944
                                       RegPressureDelta &Delta,
945
                                       const RegisterClassInfo *RCI,
946
13.2M
                                       ArrayRef<unsigned> LiveThruPressureVec) {
947
13.2M
  Delta.Excess = PressureChange();
948
433M
  for (unsigned i = 0, e = OldPressureVec.size(); i < e; 
++i420M
) {
949
420M
    unsigned POld = OldPressureVec[i];
950
420M
    unsigned PNew = NewPressureVec[i];
951
420M
    int PDiff = (int)PNew - (int)POld;
952
420M
    if (!PDiff) // No change in this set in the common case.
953
404M
      continue;
954
16.4M
    // Only consider change beyond the limit.
955
16.4M
    unsigned Limit = RCI->getRegPressureSetLimit(i);
956
16.4M
    if (!LiveThruPressureVec.empty())
957
16.4M
      Limit += LiveThruPressureVec[i];
958
16.4M
959
16.4M
    if (Limit > POld) {
960
16.4M
      if (Limit > PNew)
961
16.4M
        PDiff = 0;            // Under the limit
962
340
      else
963
340
        PDiff = PNew - Limit; // Just exceeded limit.
964
16.4M
    } else 
if (23.4k
Limit > PNew23.4k
)
965
4.51k
      PDiff = Limit - POld;   // Just obeyed limit.
966
16.4M
967
16.4M
    if (PDiff) {
968
18.9k
      Delta.Excess = PressureChange(i);
969
18.9k
      Delta.Excess.setUnitInc(PDiff);
970
18.9k
      break;
971
18.9k
    }
972
16.4M
  }
973
13.2M
}
974
975
/// Find the max change in max pressure that either surpasses a critical PSet
976
/// limit or exceeds the current MaxPressureLimit.
977
///
978
/// FIXME: comparing each element of the old and new MaxPressure vectors here is
979
/// silly. It's done now to demonstrate the concept but will go away with a
980
/// RegPressureTracker API change to work with pressure differences.
981
static void computeMaxPressureDelta(ArrayRef<unsigned> OldMaxPressureVec,
982
                                    ArrayRef<unsigned> NewMaxPressureVec,
983
                                    ArrayRef<PressureChange> CriticalPSets,
984
                                    ArrayRef<unsigned> MaxPressureLimit,
985
13.2M
                                    RegPressureDelta &Delta) {
986
13.2M
  Delta.CriticalMax = PressureChange();
987
13.2M
  Delta.CurrentMax = PressureChange();
988
13.2M
989
13.2M
  unsigned CritIdx = 0, CritEnd = CriticalPSets.size();
990
433M
  for (unsigned i = 0, e = OldMaxPressureVec.size(); i < e; 
++i420M
) {
991
420M
    unsigned POld = OldMaxPressureVec[i];
992
420M
    unsigned PNew = NewMaxPressureVec[i];
993
420M
    if (PNew == POld) // No change in this set in the common case.
994
404M
      continue;
995
15.7M
996
15.7M
    if (!Delta.CriticalMax.isValid()) {
997
16.4M
      while (CritIdx != CritEnd && 
CriticalPSets[CritIdx].getPSet() < i6.30M
)
998
844k
        ++CritIdx;
999
15.6M
1000
15.6M
      if (CritIdx != CritEnd && 
CriticalPSets[CritIdx].getPSet() == i5.46M
) {
1001
4.94M
        int PDiff = (int)PNew - (int)CriticalPSets[CritIdx].getUnitInc();
1002
4.94M
        if (PDiff > 0) {
1003
124k
          Delta.CriticalMax = PressureChange(i);
1004
124k
          Delta.CriticalMax.setUnitInc(PDiff);
1005
124k
        }
1006
4.94M
      }
1007
15.6M
    }
1008
15.7M
    // Find the first increase above MaxPressureLimit.
1009
15.7M
    // (Ignores negative MDiff).
1010
15.7M
    if (!Delta.CurrentMax.isValid() && 
PNew > MaxPressureLimit[i]15.7M
) {
1011
58.0k
      Delta.CurrentMax = PressureChange(i);
1012
58.0k
      Delta.CurrentMax.setUnitInc(PNew - POld);
1013
58.0k
      if (CritIdx == CritEnd || 
Delta.CriticalMax.isValid()1.67k
)
1014
56.4k
        break;
1015
58.0k
    }
1016
15.7M
  }
1017
13.2M
}
1018
1019
/// Record the upward impact of a single instruction on current register
1020
/// pressure. Unlike the advance/recede pressure tracking interface, this does
1021
/// not discover live in/outs.
1022
///
1023
/// This is intended for speculative queries. It leaves pressure inconsistent
1024
/// with the current position, so must be restored by the caller.
1025
3.00M
void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
1026
3.00M
  assert(!MI->isDebugInstr() && "Expect a nondebug instruction.");
1027
3.00M
1028
3.00M
  SlotIndex SlotIdx;
1029
3.00M
  if (RequireIntervals)
1030
3.00M
    SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
1031
3.00M
1032
3.00M
  // Account for register pressure similar to RegPressureTracker::recede().
1033
3.00M
  RegisterOperands RegOpers;
1034
3.00M
  RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, /*IgnoreDead=*/true);
1035
3.00M
  assert(RegOpers.DeadDefs.size() == 0);
1036
3.00M
  if (TrackLaneMasks)
1037
2.96M
    RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
1038
48.6k
  else if (RequireIntervals)
1039
48.6k
    RegOpers.detectDeadDefs(*MI, *LIS);
1040
3.00M
1041
3.00M
  // Boost max pressure for all dead defs together.
1042
3.00M
  // Since CurrSetPressure and MaxSetPressure
1043
3.00M
  bumpDeadDefs(RegOpers.DeadDefs);
1044
3.00M
1045
3.00M
  // Kill liveness at live defs.
1046
3.00M
  for (const RegisterMaskPair &P : RegOpers.Defs) {
1047
2.61M
    unsigned Reg = P.RegUnit;
1048
2.61M
    LaneBitmask LiveLanes = LiveRegs.contains(Reg);
1049
2.61M
    LaneBitmask UseLanes = getRegLanes(RegOpers.Uses, Reg);
1050
2.61M
    LaneBitmask DefLanes = P.LaneMask;
1051
2.61M
    LaneBitmask LiveAfter = (LiveLanes & ~DefLanes) | UseLanes;
1052
2.61M
    decreaseRegPressure(Reg, LiveLanes, LiveAfter);
1053
2.61M
  }
1054
3.00M
  // Generate liveness for uses.
1055
3.18M
  for (const RegisterMaskPair &P : RegOpers.Uses) {
1056
3.18M
    unsigned Reg = P.RegUnit;
1057
3.18M
    LaneBitmask LiveLanes = LiveRegs.contains(Reg);
1058
3.18M
    LaneBitmask LiveAfter = LiveLanes | P.LaneMask;
1059
3.18M
    increaseRegPressure(Reg, LiveLanes, LiveAfter);
1060
3.18M
  }
1061
3.00M
}
1062
1063
/// Consider the pressure increase caused by traversing this instruction
1064
/// bottom-up. Find the pressure set with the most change beyond its pressure
1065
/// limit based on the tracker's current pressure, and return the change in
1066
/// number of register units of that pressure set introduced by this
1067
/// instruction.
1068
///
1069
/// This assumes that the current LiveOut set is sufficient.
1070
///
1071
/// This is expensive for an on-the-fly query because it calls
1072
/// bumpUpwardPressure to recompute the pressure sets based on current
1073
/// liveness. This mainly exists to verify correctness, e.g. with
1074
/// -verify-misched. getUpwardPressureDelta is the fast version of this query
1075
/// that uses the per-SUnit cache of the PressureDiff.
1076
void RegPressureTracker::
1077
getMaxUpwardPressureDelta(const MachineInstr *MI, PressureDiff *PDiff,
1078
                          RegPressureDelta &Delta,
1079
                          ArrayRef<PressureChange> CriticalPSets,
1080
48.6k
                          ArrayRef<unsigned> MaxPressureLimit) {
1081
48.6k
  // Snapshot Pressure.
1082
48.6k
  // FIXME: The snapshot heap space should persist. But I'm planning to
1083
48.6k
  // summarize the pressure effect so we don't need to snapshot at all.
1084
48.6k
  std::vector<unsigned> SavedPressure = CurrSetPressure;
1085
48.6k
  std::vector<unsigned> SavedMaxPressure = P.MaxSetPressure;
1086
48.6k
1087
48.6k
  bumpUpwardPressure(MI);
1088
48.6k
1089
48.6k
  computeExcessPressureDelta(SavedPressure, CurrSetPressure, Delta, RCI,
1090
48.6k
                             LiveThruPressure);
1091
48.6k
  computeMaxPressureDelta(SavedMaxPressure, P.MaxSetPressure, CriticalPSets,
1092
48.6k
                          MaxPressureLimit, Delta);
1093
48.6k
  assert(Delta.CriticalMax.getUnitInc() >= 0 &&
1094
48.6k
         Delta.CurrentMax.getUnitInc() >= 0 && "cannot decrease max pressure");
1095
48.6k
1096
48.6k
  // Restore the tracker's state.
1097
48.6k
  P.MaxSetPressure.swap(SavedMaxPressure);
1098
48.6k
  CurrSetPressure.swap(SavedPressure);
1099
48.6k
1100
#ifndef NDEBUG
1101
  if (!PDiff)
1102
    return;
1103
1104
  // Check if the alternate algorithm yields the same result.
1105
  RegPressureDelta Delta2;
1106
  getUpwardPressureDelta(MI, *PDiff, Delta2, CriticalPSets, MaxPressureLimit);
1107
  if (Delta != Delta2) {
1108
    dbgs() << "PDiff: ";
1109
    PDiff->dump(*TRI);
1110
    dbgs() << "DELTA: " << *MI;
1111
    if (Delta.Excess.isValid())
1112
      dbgs() << "Excess1 " << TRI->getRegPressureSetName(Delta.Excess.getPSet())
1113
             << " " << Delta.Excess.getUnitInc() << "\n";
1114
    if (Delta.CriticalMax.isValid())
1115
      dbgs() << "Critic1 " << TRI->getRegPressureSetName(Delta.CriticalMax.getPSet())
1116
             << " " << Delta.CriticalMax.getUnitInc() << "\n";
1117
    if (Delta.CurrentMax.isValid())
1118
      dbgs() << "CurrMx1 " << TRI->getRegPressureSetName(Delta.CurrentMax.getPSet())
1119
             << " " << Delta.CurrentMax.getUnitInc() << "\n";
1120
    if (Delta2.Excess.isValid())
1121
      dbgs() << "Excess2 " << TRI->getRegPressureSetName(Delta2.Excess.getPSet())
1122
             << " " << Delta2.Excess.getUnitInc() << "\n";
1123
    if (Delta2.CriticalMax.isValid())
1124
      dbgs() << "Critic2 " << TRI->getRegPressureSetName(Delta2.CriticalMax.getPSet())
1125
             << " " << Delta2.CriticalMax.getUnitInc() << "\n";
1126
    if (Delta2.CurrentMax.isValid())
1127
      dbgs() << "CurrMx2 " << TRI->getRegPressureSetName(Delta2.CurrentMax.getPSet())
1128
             << " " << Delta2.CurrentMax.getUnitInc() << "\n";
1129
    llvm_unreachable("RegP Delta Mismatch");
1130
  }
1131
#endif
1132
}
1133
1134
/// This is the fast version of querying register pressure that does not
1135
/// directly depend on current liveness.
1136
///
1137
/// @param Delta captures information needed for heuristics.
1138
///
1139
/// @param CriticalPSets Are the pressure sets that are known to exceed some
1140
/// limit within the region, not necessarily at the current position.
1141
///
1142
/// @param MaxPressureLimit Is the max pressure within the region, not
1143
/// necessarily at the current position.
1144
void RegPressureTracker::
1145
getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff,
1146
                       RegPressureDelta &Delta,
1147
                       ArrayRef<PressureChange> CriticalPSets,
1148
29.1M
                       ArrayRef<unsigned> MaxPressureLimit) const {
1149
29.1M
  unsigned CritIdx = 0, CritEnd = CriticalPSets.size();
1150
29.1M
  for (PressureDiff::const_iterator
1151
29.1M
         PDiffI = PDiff.begin(), PDiffE = PDiff.end();
1152
127M
       PDiffI != PDiffE && 
PDiffI->isValid()126M
;
++PDiffI97.9M
) {
1153
97.9M
1154
97.9M
    unsigned PSetID = PDiffI->getPSet();
1155
97.9M
    unsigned Limit = RCI->getRegPressureSetLimit(PSetID);
1156
97.9M
    if (!LiveThruPressure.empty())
1157
97.9M
      Limit += LiveThruPressure[PSetID];
1158
97.9M
1159
97.9M
    unsigned POld = CurrSetPressure[PSetID];
1160
97.9M
    unsigned MOld = P.MaxSetPressure[PSetID];
1161
97.9M
    unsigned MNew = MOld;
1162
97.9M
    // Ignore DeadDefs here because they aren't captured by PressureChange.
1163
97.9M
    unsigned PNew = POld + PDiffI->getUnitInc();
1164
97.9M
    assert((PDiffI->getUnitInc() >= 0) == (PNew >= POld)
1165
97.9M
           && "PSet overflow/underflow");
1166
97.9M
    if (PNew > MOld)
1167
22.7M
      MNew = PNew;
1168
97.9M
    // Check if current pressure has exceeded the limit.
1169
97.9M
    if (!Delta.Excess.isValid()) {
1170
36.5M
      unsigned ExcessInc = 0;
1171
36.5M
      if (PNew > Limit)
1172
7.96M
        ExcessInc = POld > Limit ? 
PNew - POld7.08M
:
PNew - Limit874k
;
1173
28.5M
      else if (POld > Limit)
1174
18.1k
        ExcessInc = Limit - POld;
1175
36.5M
      if (ExcessInc) {
1176
7.97M
        Delta.Excess = PressureChange(PSetID);
1177
7.97M
        Delta.Excess.setUnitInc(ExcessInc);
1178
7.97M
      }
1179
36.5M
    }
1180
97.9M
    // Check if max pressure has exceeded a critical pressure set max.
1181
97.9M
    if (MNew == MOld)
1182
75.1M
      continue;
1183
22.7M
    if (!Delta.CriticalMax.isValid()) {
1184
14.5M
      while (CritIdx != CritEnd && 
CriticalPSets[CritIdx].getPSet() < PSetID7.14M
)
1185
1.39M
        ++CritIdx;
1186
13.1M
1187
13.1M
      if (CritIdx != CritEnd && 
CriticalPSets[CritIdx].getPSet() == PSetID5.75M
) {
1188
4.26M
        int CritInc = (int)MNew - (int)CriticalPSets[CritIdx].getUnitInc();
1189
4.26M
        if (CritInc > 0 && 
CritInc <= std::numeric_limits<int16_t>::max()4.24M
) {
1190
4.24M
          Delta.CriticalMax = PressureChange(PSetID);
1191
4.24M
          Delta.CriticalMax.setUnitInc(CritInc);
1192
4.24M
        }
1193
4.26M
      }
1194
13.1M
    }
1195
22.7M
    // Check if max pressure has exceeded the current max.
1196
22.7M
    if (!Delta.CurrentMax.isValid() && 
MNew > MaxPressureLimit[PSetID]22.1M
) {
1197
1.54M
      Delta.CurrentMax = PressureChange(PSetID);
1198
1.54M
      Delta.CurrentMax.setUnitInc(MNew - MOld);
1199
1.54M
    }
1200
22.7M
  }
1201
29.1M
}
1202
1203
/// Helper to find a vreg use between two indices [PriorUseIdx, NextUseIdx).
1204
/// The query starts with a lane bitmask which gets lanes/bits removed for every
1205
/// use we find.
1206
static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
1207
                                  SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
1208
                                  const MachineRegisterInfo &MRI,
1209
999k
                                  const LiveIntervals *LIS) {
1210
999k
  const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
1211
1.36M
  for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
1212
1.36M
    if (MO.isUndef())
1213
89
      continue;
1214
1.36M
    const MachineInstr *MI = MO.getParent();
1215
1.36M
    SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
1216
1.36M
    if (InstSlot >= PriorUseIdx && 
InstSlot < NextUseIdx1.07M
) {
1217
197k
      unsigned SubRegIdx = MO.getSubReg();
1218
197k
      LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
1219
197k
      LastUseMask &= ~UseMask;
1220
197k
      if (LastUseMask.none())
1221
102k
        return LaneBitmask::getNone();
1222
197k
    }
1223
1.36M
  }
1224
999k
  
return LastUseMask897k
;
1225
999k
}
1226
1227
LaneBitmask RegPressureTracker::getLiveLanesAt(unsigned RegUnit,
1228
0
                                               SlotIndex Pos) const {
1229
0
  assert(RequireIntervals);
1230
0
  return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos,
1231
0
                              LaneBitmask::getAll(),
1232
0
      [](const LiveRange &LR, SlotIndex Pos) {
1233
0
        return LR.liveAt(Pos);
1234
0
      });
1235
0
}
1236
1237
LaneBitmask RegPressureTracker::getLastUsedLanes(unsigned RegUnit,
1238
13.4M
                                                 SlotIndex Pos) const {
1239
13.4M
  assert(RequireIntervals);
1240
13.4M
  return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit,
1241
13.4M
                              Pos.getBaseIndex(), LaneBitmask::getNone(),
1242
13.8M
      [](const LiveRange &LR, SlotIndex Pos) {
1243
13.8M
        const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
1244
13.8M
        return S != nullptr && 
S->end == Pos.getRegSlot()13.6M
;
1245
13.8M
      });
1246
13.4M
}
1247
1248
LaneBitmask RegPressureTracker::getLiveThroughAt(unsigned RegUnit,
1249
4.88M
                                                 SlotIndex Pos) const {
1250
4.88M
  assert(RequireIntervals);
1251
4.88M
  return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit, Pos,
1252
4.88M
                              LaneBitmask::getNone(),
1253
4.97M
      [](const LiveRange &LR, SlotIndex Pos) {
1254
4.97M
        const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
1255
4.97M
        return S != nullptr && 
S->start < Pos.getRegSlot(true)787k
&&
1256
4.97M
               
S->end != Pos.getDeadSlot()244k
;
1257
4.97M
      });
1258
4.88M
}
1259
1260
/// Record the downward impact of a single instruction on current register
1261
/// pressure. Unlike the advance/recede pressure tracking interface, this does
1262
/// not discover live in/outs.
1263
///
1264
/// This is intended for speculative queries. It leaves pressure inconsistent
1265
/// with the current position, so must be restored by the caller.
1266
13.9M
void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
1267
13.9M
  assert(!MI->isDebugInstr() && "Expect a nondebug instruction.");
1268
13.9M
1269
13.9M
  SlotIndex SlotIdx;
1270
13.9M
  if (RequireIntervals)
1271
13.9M
    SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
1272
13.9M
1273
13.9M
  // Account for register pressure similar to RegPressureTracker::recede().
1274
13.9M
  RegisterOperands RegOpers;
1275
13.9M
  RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, false);
1276
13.9M
  if (TrackLaneMasks)
1277
721k
    RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
1278
13.9M
1279
13.9M
  if (RequireIntervals) {
1280
13.9M
    for (const RegisterMaskPair &Use : RegOpers.Uses) {
1281
13.2M
      unsigned Reg = Use.RegUnit;
1282
13.2M
      LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
1283
13.2M
      if (LastUseMask.none())
1284
12.2M
        continue;
1285
999k
      // The LastUseMask is queried from the liveness information of instruction
1286
999k
      // which may be further down the schedule. Some lanes may actually not be
1287
999k
      // last uses for the current position.
1288
999k
      // FIXME: allow the caller to pass in the list of vreg uses that remain
1289
999k
      // to be bottom-scheduled to avoid searching uses at each query.
1290
999k
      SlotIndex CurrIdx = getCurrSlot();
1291
999k
      LastUseMask
1292
999k
        = findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS);
1293
999k
      if (LastUseMask.none())
1294
102k
        continue;
1295
897k
1296
897k
      LaneBitmask LiveMask = LiveRegs.contains(Reg);
1297
897k
      LaneBitmask NewMask = LiveMask & ~LastUseMask;
1298
897k
      decreaseRegPressure(Reg, LiveMask, NewMask);
1299
897k
    }
1300
13.9M
  }
1301
13.9M
1302
13.9M
  // Generate liveness for defs.
1303
13.9M
  for (const RegisterMaskPair &Def : RegOpers.Defs) {
1304
8.59M
    unsigned Reg = Def.RegUnit;
1305
8.59M
    LaneBitmask LiveMask = LiveRegs.contains(Reg);
1306
8.59M
    LaneBitmask NewMask = LiveMask | Def.LaneMask;
1307
8.59M
    increaseRegPressure(Reg, LiveMask, NewMask);
1308
8.59M
  }
1309
13.9M
1310
13.9M
  // Boost pressure for all dead defs together.
1311
13.9M
  bumpDeadDefs(RegOpers.DeadDefs);
1312
13.9M
}
1313
1314
/// Consider the pressure increase caused by traversing this instruction
1315
/// top-down. Find the register class with the most change in its pressure limit
1316
/// based on the tracker's current pressure, and return the number of excess
1317
/// register units of that pressure set introduced by this instruction.
1318
///
1319
/// This assumes that the current LiveIn set is sufficient.
1320
///
1321
/// This is expensive for an on-the-fly query because it calls
1322
/// bumpDownwardPressure to recompute the pressure sets based on current
1323
/// liveness. We don't yet have a fast version of downward pressure tracking
1324
/// analogous to getUpwardPressureDelta.
1325
void RegPressureTracker::
1326
getMaxDownwardPressureDelta(const MachineInstr *MI, RegPressureDelta &Delta,
1327
                            ArrayRef<PressureChange> CriticalPSets,
1328
13.2M
                            ArrayRef<unsigned> MaxPressureLimit) {
1329
13.2M
  // Snapshot Pressure.
1330
13.2M
  std::vector<unsigned> SavedPressure = CurrSetPressure;
1331
13.2M
  std::vector<unsigned> SavedMaxPressure = P.MaxSetPressure;
1332
13.2M
1333
13.2M
  bumpDownwardPressure(MI);
1334
13.2M
1335
13.2M
  computeExcessPressureDelta(SavedPressure, CurrSetPressure, Delta, RCI,
1336
13.2M
                             LiveThruPressure);
1337
13.2M
  computeMaxPressureDelta(SavedMaxPressure, P.MaxSetPressure, CriticalPSets,
1338
13.2M
                          MaxPressureLimit, Delta);
1339
13.2M
  assert(Delta.CriticalMax.getUnitInc() >= 0 &&
1340
13.2M
         Delta.CurrentMax.getUnitInc() >= 0 && "cannot decrease max pressure");
1341
13.2M
1342
13.2M
  // Restore the tracker's state.
1343
13.2M
  P.MaxSetPressure.swap(SavedMaxPressure);
1344
13.2M
  CurrSetPressure.swap(SavedPressure);
1345
13.2M
}
1346
1347
/// Get the pressure of each PSet after traversing this instruction bottom-up.
1348
void RegPressureTracker::
1349
getUpwardPressure(const MachineInstr *MI,
1350
                  std::vector<unsigned> &PressureResult,
1351
2.96M
                  std::vector<unsigned> &MaxPressureResult) {
1352
2.96M
  // Snapshot pressure.
1353
2.96M
  PressureResult = CurrSetPressure;
1354
2.96M
  MaxPressureResult = P.MaxSetPressure;
1355
2.96M
1356
2.96M
  bumpUpwardPressure(MI);
1357
2.96M
1358
2.96M
  // Current pressure becomes the result. Restore current pressure.
1359
2.96M
  P.MaxSetPressure.swap(MaxPressureResult);
1360
2.96M
  CurrSetPressure.swap(PressureResult);
1361
2.96M
}
1362
1363
/// Get the pressure of each PSet after traversing this instruction top-down.
1364
void RegPressureTracker::
1365
getDownwardPressure(const MachineInstr *MI,
1366
                    std::vector<unsigned> &PressureResult,
1367
721k
                    std::vector<unsigned> &MaxPressureResult) {
1368
721k
  // Snapshot pressure.
1369
721k
  PressureResult = CurrSetPressure;
1370
721k
  MaxPressureResult = P.MaxSetPressure;
1371
721k
1372
721k
  bumpDownwardPressure(MI);
1373
721k
1374
721k
  // Current pressure becomes the result. Restore current pressure.
1375
721k
  P.MaxSetPressure.swap(MaxPressureResult);
1376
721k
  CurrSetPressure.swap(PressureResult);
1377
721k
}