Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- SIFormMemoryClauses.cpp -------------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// This pass creates bundles of SMEM and VMEM instructions forming memory
11
/// clauses if XNACK is enabled. Def operands of clauses are marked as early
12
/// clobber to make sure we will not override any source within a clause.
13
///
14
//===----------------------------------------------------------------------===//
15
16
#include "AMDGPU.h"
17
#include "AMDGPUSubtarget.h"
18
#include "GCNRegPressure.h"
19
#include "SIInstrInfo.h"
20
#include "SIMachineFunctionInfo.h"
21
#include "SIRegisterInfo.h"
22
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
23
#include "llvm/ADT/DenseMap.h"
24
#include "llvm/CodeGen/LiveIntervals.h"
25
#include "llvm/CodeGen/MachineFunctionPass.h"
26
27
using namespace llvm;
28
29
#define DEBUG_TYPE "si-form-memory-clauses"
30
31
// Clauses longer then 15 instructions would overflow one of the counters
32
// and stall. They can stall even earlier if there are outstanding counters.
33
static cl::opt<unsigned>
34
MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
35
          cl::desc("Maximum length of a memory clause, instructions"));
36
37
namespace {
38
39
class SIFormMemoryClauses : public MachineFunctionPass {
40
  typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse;
41
42
public:
43
  static char ID;
44
45
public:
46
2.37k
  SIFormMemoryClauses() : MachineFunctionPass(ID) {
47
2.37k
    initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry());
48
2.37k
  }
49
50
  bool runOnMachineFunction(MachineFunction &MF) override;
51
52
27.6k
  StringRef getPassName() const override {
53
27.6k
    return "SI Form memory clauses";
54
27.6k
  }
55
56
2.37k
  void getAnalysisUsage(AnalysisUsage &AU) const override {
57
2.37k
    AU.addRequired<LiveIntervals>();
58
2.37k
    AU.setPreservesAll();
59
2.37k
    MachineFunctionPass::getAnalysisUsage(AU);
60
2.37k
  }
61
62
private:
63
  template <typename Callable>
64
  void forAllLanes(unsigned Reg, LaneBitmask LaneMask, Callable Func) const;
65
66
  bool canBundle(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
67
  bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
68
  void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
69
  bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
70
                      GCNDownwardRPTracker &RPT);
71
72
  const GCNSubtarget *ST;
73
  const SIRegisterInfo *TRI;
74
  const MachineRegisterInfo *MRI;
75
  SIMachineFunctionInfo *MFI;
76
77
  unsigned LastRecordedOccupancy;
78
  unsigned MaxVGPRs;
79
  unsigned MaxSGPRs;
80
};
81
82
} // End anonymous namespace.
83
84
101k
INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
85
101k
                      "SI Form memory clauses", false, false)
86
101k
INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
87
101k
INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
88
                    "SI Form memory clauses", false, false)
89
90
91
char SIFormMemoryClauses::ID = 0;
92
93
char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;
94
95
0
FunctionPass *llvm::createSIFormMemoryClausesPass() {
96
0
  return new SIFormMemoryClauses();
97
0
}
98
99
78.7k
static bool isVMEMClauseInst(const MachineInstr &MI) {
100
78.7k
  return SIInstrInfo::isFLAT(MI) || 
SIInstrInfo::isVMEM(MI)76.8k
;
101
78.7k
}
102
103
6.99k
static bool isSMEMClauseInst(const MachineInstr &MI) {
104
6.99k
  return SIInstrInfo::isSMRD(MI);
105
6.99k
}
106
107
// There no sense to create store clauses, they do not define anything,
108
// thus there is nothing to set early-clobber.
109
84.6k
static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
110
84.6k
  if (MI.isDebugValue() || 
MI.isBundled()84.6k
)
111
9
    return false;
112
84.6k
  if (!MI.mayLoad() || 
MI.mayStore()10.3k
)
113
74.7k
    return false;
114
9.86k
  if (AMDGPU::getAtomicNoRetOp(MI.getOpcode()) != -1 ||
115
9.86k
      AMDGPU::getAtomicRetOp(MI.getOpcode()) != -1)
116
0
    return false;
117
9.86k
  if (IsVMEMClause && 
!isVMEMClauseInst(MI)2.87k
)
118
35
    return false;
119
9.83k
  if (!IsVMEMClause && 
!isSMEMClauseInst(MI)6.99k
)
120
720
    return false;
121
9.11k
  // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
122
9.11k
  for (const MachineOperand &ResMO : MI.defs()) {
123
9.11k
    unsigned ResReg = ResMO.getReg();
124
48.1k
    for (const MachineOperand &MO : MI.uses()) {
125
48.1k
      if (!MO.isReg() || 
MO.isDef()13.5k
)
126
34.5k
        continue;
127
13.5k
      if (MO.getReg() == ResReg)
128
9
        return false;
129
13.5k
    }
130
9.11k
    
break9.10k
; // Only check the first def.
131
9.11k
  }
132
9.11k
  
return true9.10k
;
133
9.11k
}
134
135
21.8k
static unsigned getMopState(const MachineOperand &MO) {
136
21.8k
  unsigned S = 0;
137
21.8k
  if (MO.isImplicit())
138
3.05k
    S |= RegState::Implicit;
139
21.8k
  if (MO.isDead())
140
41
    S |= RegState::Dead;
141
21.8k
  if (MO.isUndef())
142
1.76k
    S |= RegState::Undef;
143
21.8k
  if (MO.isKill())
144
0
    S |= RegState::Kill;
145
21.8k
  if (MO.isEarlyClobber())
146
0
    S |= RegState::EarlyClobber;
147
21.8k
  if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()) && 
MO.isRenamable()3.44k
)
148
0
    S |= RegState::Renamable;
149
21.8k
  return S;
150
21.8k
}
151
152
template <typename Callable>
153
void SIFormMemoryClauses::forAllLanes(unsigned Reg, LaneBitmask LaneMask,
154
8.20k
                                      Callable Func) const {
155
8.20k
  if (LaneMask.all() || 
TargetRegisterInfo::isPhysicalRegister(Reg)1.38k
||
156
8.20k
      
LaneMask == MRI->getMaxLaneMaskForVReg(Reg)1.38k
) {
157
6.85k
    Func(0);
158
6.85k
    return;
159
6.85k
  }
160
1.34k
161
1.34k
  const TargetRegisterClass *RC = MRI->getRegClass(Reg);
162
1.34k
  unsigned E = TRI->getNumSubRegIndices();
163
1.34k
  SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs;
164
50.9k
  for (unsigned Idx = 1; Idx < E; 
++Idx49.6k
) {
165
50.9k
    // Is this index even compatible with the given class?
166
50.9k
    if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
167
39.7k
      continue;
168
11.1k
    LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
169
11.1k
    // Early exit if we found a perfect match.
170
11.1k
    if (SubRegMask == LaneMask) {
171
1.33k
      Func(Idx);
172
1.33k
      return;
173
1.33k
    }
174
9.86k
175
9.86k
    if ((SubRegMask & ~LaneMask).any() || 
(SubRegMask & LaneMask).none()2.87k
)
176
6.98k
      continue;
177
2.87k
178
2.87k
    CoveringSubregs.push_back(Idx);
179
2.87k
  }
180
1.34k
181
1.34k
  
llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) 7
{
182
209
    LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
183
209
    LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
184
209
    unsigned NA = MaskA.getNumLanes();
185
209
    unsigned NB = MaskB.getNumLanes();
186
209
    if (NA != NB)
187
127
      return NA > NB;
188
82
    return MaskA.getHighestLane() > MaskB.getHighestLane();
189
82
  });
SIFormMemoryClauses.cpp:void (anonymous namespace)::SIFormMemoryClauses::forAllLanes<(anonymous namespace)::SIFormMemoryClauses::runOnMachineFunction(llvm::MachineFunction&)::$_1>(unsigned int, llvm::LaneBitmask, (anonymous namespace)::SIFormMemoryClauses::runOnMachineFunction(llvm::MachineFunction&)::$_1) const::'lambda'(unsigned int, unsigned int)::operator()(unsigned int, unsigned int) const
Line
Count
Source
181
5
  llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) {
182
5
    LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
183
5
    LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
184
5
    unsigned NA = MaskA.getNumLanes();
185
5
    unsigned NB = MaskB.getNumLanes();
186
5
    if (NA != NB)
187
3
      return NA > NB;
188
2
    return MaskA.getHighestLane() > MaskB.getHighestLane();
189
2
  });
SIFormMemoryClauses.cpp:void (anonymous namespace)::SIFormMemoryClauses::forAllLanes<(anonymous namespace)::SIFormMemoryClauses::runOnMachineFunction(llvm::MachineFunction&)::$_2>(unsigned int, llvm::LaneBitmask, (anonymous namespace)::SIFormMemoryClauses::runOnMachineFunction(llvm::MachineFunction&)::$_2) const::'lambda'(unsigned int, unsigned int)::operator()(unsigned int, unsigned int) const
Line
Count
Source
181
204
  llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) {
182
204
    LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
183
204
    LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
184
204
    unsigned NA = MaskA.getNumLanes();
185
204
    unsigned NB = MaskB.getNumLanes();
186
204
    if (NA != NB)
187
124
      return NA > NB;
188
80
    return MaskA.getHighestLane() > MaskB.getHighestLane();
189
80
  });
190
7
191
22
  for (unsigned Idx : CoveringSubregs) {
192
22
    LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
193
22
    if ((SubRegMask & ~LaneMask).any() || 
(SubRegMask & LaneMask).none()14
)
194
8
      continue;
195
14
196
14
    Func(Idx);
197
14
    LaneMask &= ~SubRegMask;
198
14
    if (LaneMask.none())
199
7
      return;
200
14
  }
201
7
202
7
  
llvm_unreachable0
("Failed to find all subregs to cover lane mask");
203
7
}
SIFormMemoryClauses.cpp:void (anonymous namespace)::SIFormMemoryClauses::forAllLanes<(anonymous namespace)::SIFormMemoryClauses::runOnMachineFunction(llvm::MachineFunction&)::$_1>(unsigned int, llvm::LaneBitmask, (anonymous namespace)::SIFormMemoryClauses::runOnMachineFunction(llvm::MachineFunction&)::$_1) const
Line
Count
Source
154
5.06k
                                      Callable Func) const {
155
5.06k
  if (LaneMask.all() || 
TargetRegisterInfo::isPhysicalRegister(Reg)1.32k
||
156
5.06k
      
LaneMask == MRI->getMaxLaneMaskForVReg(Reg)1.32k
) {
157
3.77k
    Func(0);
158
3.77k
    return;
159
3.77k
  }
160
1.29k
161
1.29k
  const TargetRegisterClass *RC = MRI->getRegClass(Reg);
162
1.29k
  unsigned E = TRI->getNumSubRegIndices();
163
1.29k
  SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs;
164
47.8k
  for (unsigned Idx = 1; Idx < E; 
++Idx46.5k
) {
165
47.8k
    // Is this index even compatible with the given class?
166
47.8k
    if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
167
37.1k
      continue;
168
10.7k
    LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
169
10.7k
    // Early exit if we found a perfect match.
170
10.7k
    if (SubRegMask == LaneMask) {
171
1.29k
      Func(Idx);
172
1.29k
      return;
173
1.29k
    }
174
9.41k
175
9.41k
    if ((SubRegMask & ~LaneMask).any() || 
(SubRegMask & LaneMask).none()2.68k
)
176
6.73k
      continue;
177
2.68k
178
2.68k
    CoveringSubregs.push_back(Idx);
179
2.68k
  }
180
1.29k
181
1.29k
  llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) {
182
1
    LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
183
1
    LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
184
1
    unsigned NA = MaskA.getNumLanes();
185
1
    unsigned NB = MaskB.getNumLanes();
186
1
    if (NA != NB)
187
1
      return NA > NB;
188
1
    return MaskA.getHighestLane() > MaskB.getHighestLane();
189
1
  });
190
1
191
2
  for (unsigned Idx : CoveringSubregs) {
192
2
    LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
193
2
    if ((SubRegMask & ~LaneMask).any() || (SubRegMask & LaneMask).none())
194
0
      continue;
195
2
196
2
    Func(Idx);
197
2
    LaneMask &= ~SubRegMask;
198
2
    if (LaneMask.none())
199
1
      return;
200
2
  }
201
1
202
1
  
llvm_unreachable0
("Failed to find all subregs to cover lane mask");
203
}
SIFormMemoryClauses.cpp:void (anonymous namespace)::SIFormMemoryClauses::forAllLanes<(anonymous namespace)::SIFormMemoryClauses::runOnMachineFunction(llvm::MachineFunction&)::$_2>(unsigned int, llvm::LaneBitmask, (anonymous namespace)::SIFormMemoryClauses::runOnMachineFunction(llvm::MachineFunction&)::$_2) const
Line
Count
Source
154
3.13k
                                      Callable Func) const {
155
3.13k
  if (LaneMask.all() || 
TargetRegisterInfo::isPhysicalRegister(Reg)58
||
156
3.13k
      
LaneMask == MRI->getMaxLaneMaskForVReg(Reg)58
) {
157
3.08k
    Func(0);
158
3.08k
    return;
159
3.08k
  }
160
51
161
51
  const TargetRegisterClass *RC = MRI->getRegClass(Reg);
162
51
  unsigned E = TRI->getNumSubRegIndices();
163
51
  SmallVector<unsigned, AMDGPU::NUM_TARGET_SUBREGS> CoveringSubregs;
164
3.06k
  for (unsigned Idx = 1; Idx < E; 
++Idx3.01k
) {
165
3.05k
    // Is this index even compatible with the given class?
166
3.05k
    if (TRI->getSubClassWithSubReg(RC, Idx) != RC)
167
2.57k
      continue;
168
487
    LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
169
487
    // Early exit if we found a perfect match.
170
487
    if (SubRegMask == LaneMask) {
171
45
      Func(Idx);
172
45
      return;
173
45
    }
174
442
175
442
    if ((SubRegMask & ~LaneMask).any() || 
(SubRegMask & LaneMask).none()188
)
176
254
      continue;
177
188
178
188
    CoveringSubregs.push_back(Idx);
179
188
  }
180
51
181
51
  llvm::sort(CoveringSubregs, [this](unsigned A, unsigned B) {
182
6
    LaneBitmask MaskA = TRI->getSubRegIndexLaneMask(A);
183
6
    LaneBitmask MaskB = TRI->getSubRegIndexLaneMask(B);
184
6
    unsigned NA = MaskA.getNumLanes();
185
6
    unsigned NB = MaskB.getNumLanes();
186
6
    if (NA != NB)
187
6
      return NA > NB;
188
6
    return MaskA.getHighestLane() > MaskB.getHighestLane();
189
6
  });
190
6
191
20
  for (unsigned Idx : CoveringSubregs) {
192
20
    LaneBitmask SubRegMask = TRI->getSubRegIndexLaneMask(Idx);
193
20
    if ((SubRegMask & ~LaneMask).any() || 
(SubRegMask & LaneMask).none()12
)
194
8
      continue;
195
12
196
12
    Func(Idx);
197
12
    LaneMask &= ~SubRegMask;
198
12
    if (LaneMask.none())
199
6
      return;
200
12
  }
201
6
202
6
  
llvm_unreachable0
("Failed to find all subregs to cover lane mask");
203
}
204
205
// Returns false if there is a use of a def already in the map.
206
// In this case we must break the clause.
207
bool SIFormMemoryClauses::canBundle(const MachineInstr &MI,
208
9.10k
                                    RegUse &Defs, RegUse &Uses) const {
209
9.10k
  // Check interference with defs.
210
56.2k
  for (const MachineOperand &MO : MI.operands()) {
211
56.2k
    // TODO: Prologue/Epilogue Insertion pass does not process bundled
212
56.2k
    //       instructions.
213
56.2k
    if (MO.isFI())
214
15
      return false;
215
56.2k
216
56.2k
    if (!MO.isReg())
217
33.6k
      continue;
218
22.5k
219
22.5k
    unsigned Reg = MO.getReg();
220
22.5k
221
22.5k
    // If it is tied we will need to write same register as we read.
222
22.5k
    if (MO.isTied())
223
0
      return false;
224
22.5k
225
22.5k
    RegUse &Map = MO.isDef() ? 
Uses9.10k
:
Defs13.4k
;
226
22.5k
    auto Conflict = Map.find(Reg);
227
22.5k
    if (Conflict == Map.end())
228
22.3k
      continue;
229
272
230
272
    if (TargetRegisterInfo::isPhysicalRegister(Reg))
231
0
      return false;
232
272
233
272
    LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
234
272
    if ((Conflict->second.second & Mask).any())
235
272
      return false;
236
272
  }
237
9.10k
238
9.10k
  
return true8.81k
;
239
9.10k
}
240
241
// Since all defs in the clause are early clobber we can run out of registers.
242
// Function returns false if pressure would hit the limit if instruction is
243
// bundled into a memory clause.
244
bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
245
8.81k
                                        GCNDownwardRPTracker &RPT) {
246
8.81k
  // NB: skip advanceBeforeNext() call. Since all defs will be marked
247
8.81k
  // early-clobber they will all stay alive at least to the end of the
248
8.81k
  // clause. Therefor we should not decrease pressure even if load
249
8.81k
  // pointer becomes dead and could otherwise be reused for destination.
250
8.81k
  RPT.advanceToNext();
251
8.81k
  GCNRegPressure MaxPressure = RPT.moveMaxPressure();
252
8.81k
  unsigned Occupancy = MaxPressure.getOccupancy(*ST);
253
8.81k
  if (Occupancy >= MFI->getMinAllowedOccupancy() &&
254
8.81k
      
MaxPressure.getVGPRNum() <= MaxVGPRs8.81k
&&
255
8.81k
      
MaxPressure.getSGPRNum() <= MaxSGPRs8.77k
) {
256
8.75k
    LastRecordedOccupancy = Occupancy;
257
8.75k
    return true;
258
8.75k
  }
259
64
  return false;
260
64
}
261
262
// Collect register defs and uses along with their lane masks and states.
263
void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
264
8.75k
                                         RegUse &Defs, RegUse &Uses) const {
265
55.1k
  for (const MachineOperand &MO : MI.operands()) {
266
55.1k
    if (!MO.isReg())
267
33.3k
      continue;
268
21.8k
    unsigned Reg = MO.getReg();
269
21.8k
    if (!Reg)
270
0
      continue;
271
21.8k
272
21.8k
    LaneBitmask Mask = TargetRegisterInfo::isVirtualRegister(Reg) ?
273
18.3k
                         TRI->getSubRegIndexLaneMask(MO.getSubReg()) :
274
21.8k
                         
LaneBitmask::getAll()3.44k
;
275
21.8k
    RegUse &Map = MO.isDef() ? 
Defs8.75k
:
Uses13.0k
;
276
21.8k
277
21.8k
    auto Loc = Map.find(Reg);
278
21.8k
    unsigned State = getMopState(MO);
279
21.8k
    if (Loc == Map.end()) {
280
17.4k
      Map[Reg] = std::make_pair(State, Mask);
281
17.4k
    } else {
282
4.35k
      Loc->second.first |= State;
283
4.35k
      Loc->second.second |= Mask;
284
4.35k
    }
285
21.8k
  }
286
8.75k
}
287
288
// Check register def/use conflicts, occupancy limits and collect def/use maps.
289
// Return true if instruction can be bundled with previous. It it cannot
290
// def/use maps are not updated.
291
bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
292
                                         RegUse &Defs, RegUse &Uses,
293
9.10k
                                         GCNDownwardRPTracker &RPT) {
294
9.10k
  if (!canBundle(MI, Defs, Uses))
295
287
    return false;
296
8.81k
297
8.81k
  if (!checkPressure(MI, RPT))
298
64
    return false;
299
8.75k
300
8.75k
  collectRegUses(MI, Defs, Uses);
301
8.75k
  return true;
302
8.75k
}
303
304
25.2k
bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
305
25.2k
  if (skipFunction(MF.getFunction()))
306
8
    return false;
307
25.2k
308
25.2k
  ST = &MF.getSubtarget<GCNSubtarget>();
309
25.2k
  if (!ST->isXNACKEnabled())
310
20.9k
    return false;
311
4.24k
312
4.24k
  const SIInstrInfo *TII = ST->getInstrInfo();
313
4.24k
  TRI = ST->getRegisterInfo();
314
4.24k
  MRI = &MF.getRegInfo();
315
4.24k
  MFI = MF.getInfo<SIMachineFunctionInfo>();
316
4.24k
  LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
317
4.24k
  SlotIndexes *Ind = LIS->getSlotIndexes();
318
4.24k
  bool Changed = false;
319
4.24k
320
4.24k
  MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
321
4.24k
  MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
322
4.24k
  unsigned FuncMaxClause = AMDGPU::getIntegerAttribute(
323
4.24k
      MF.getFunction(), "amdgpu-max-memory-clause", MaxClause);
324
4.24k
325
5.04k
  for (MachineBasicBlock &MBB : MF) {
326
5.04k
    MachineBasicBlock::instr_iterator Next;
327
80.9k
    for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; 
I = Next75.9k
) {
328
75.9k
      MachineInstr &MI = *I;
329
75.9k
      Next = std::next(I);
330
75.9k
331
75.9k
      bool IsVMEM = isVMEMClauseInst(MI);
332
75.9k
333
75.9k
      if (!isValidClauseInst(MI, IsVMEM))
334
70.1k
        continue;
335
5.74k
336
5.74k
      RegUse Defs, Uses;
337
5.74k
      GCNDownwardRPTracker RPT(*LIS);
338
5.74k
      RPT.reset(MI);
339
5.74k
340
5.74k
      if (!processRegUses(MI, Defs, Uses, RPT))
341
70
        continue;
342
5.67k
343
5.67k
      unsigned Length = 1;
344
8.75k
      for ( ; Next != E && 
Length < FuncMaxClause8.73k
;
++Next3.07k
) {
345
8.71k
        if (!isValidClauseInst(*Next, IsVMEM))
346
5.35k
          break;
347
3.35k
348
3.35k
        // A load from pointer which was loaded inside the same bundle is an
349
3.35k
        // impossible clause because we will need to write and read the same
350
3.35k
        // register inside. In this case processRegUses will return false.
351
3.35k
        if (!processRegUses(*Next, Defs, Uses, RPT))
352
281
          break;
353
3.07k
354
3.07k
        ++Length;
355
3.07k
      }
356
5.67k
      if (Length < 2)
357
3.59k
        continue;
358
2.07k
359
2.07k
      Changed = true;
360
2.07k
      MFI->limitOccupancy(LastRecordedOccupancy);
361
2.07k
362
2.07k
      auto B = BuildMI(MBB, I, DebugLoc(), TII->get(TargetOpcode::BUNDLE));
363
2.07k
      Ind->insertMachineInstrInMaps(*B);
364
2.07k
365
7.23k
      for (auto BI = I; BI != Next; 
++BI5.15k
) {
366
5.15k
        BI->bundleWithPred();
367
5.15k
        Ind->removeSingleMachineInstrFromMaps(*BI);
368
5.15k
369
5.15k
        for (MachineOperand &MO : BI->defs())
370
5.15k
          if (MO.readsReg())
371
110
            MO.setIsInternalRead(true);
372
5.15k
      }
373
2.07k
374
5.06k
      for (auto &&R : Defs) {
375
5.06k
        forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
376
5.06k
          unsigned S = R.second.first | RegState::EarlyClobber;
377
5.06k
          if (!SubReg)
378
3.77k
            S &= ~(RegState::Undef | RegState::Dead);
379
5.06k
          B.addDef(R.first, S, SubReg);
380
5.06k
        });
381
5.06k
      }
382
2.07k
383
3.13k
      for (auto &&R : Uses) {
384
3.14k
        forAllLanes(R.first, R.second.second, [&R, &B](unsigned SubReg) {
385
3.14k
          B.addUse(R.first, R.second.first & ~RegState::Kill, SubReg);
386
3.14k
        });
387
3.13k
      }
388
2.07k
389
5.06k
      for (auto &&R : Defs) {
390
5.06k
        unsigned Reg = R.first;
391
5.06k
        Uses.erase(Reg);
392
5.06k
        if (TargetRegisterInfo::isPhysicalRegister(Reg))
393
0
          continue;
394
5.06k
        LIS->removeInterval(Reg);
395
5.06k
        LIS->createAndComputeVirtRegInterval(Reg);
396
5.06k
      }
397
2.07k
398
3.13k
      for (auto &&R : Uses) {
399
3.13k
        unsigned Reg = R.first;
400
3.13k
        if (TargetRegisterInfo::isPhysicalRegister(Reg))
401
498
          continue;
402
2.63k
        LIS->removeInterval(Reg);
403
2.63k
        LIS->createAndComputeVirtRegInterval(Reg);
404
2.63k
      }
405
2.07k
    }
406
5.04k
  }
407
4.24k
408
4.24k
  return Changed;
409
4.24k
}