Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- SIModeRegister.cpp - Mode Register --------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
/// \file
9
/// This pass inserts changes to the Mode register settings as required.
10
/// Note that currently it only deals with the Double Precision Floating Point
11
/// rounding mode setting, but is intended to be generic enough to be easily
12
/// expanded.
13
///
14
//===----------------------------------------------------------------------===//
15
//
16
#include "AMDGPU.h"
17
#include "AMDGPUInstrInfo.h"
18
#include "AMDGPUSubtarget.h"
19
#include "SIInstrInfo.h"
20
#include "SIMachineFunctionInfo.h"
21
#include "llvm/ADT/Statistic.h"
22
#include "llvm/CodeGen/MachineFunctionPass.h"
23
#include "llvm/CodeGen/MachineInstrBuilder.h"
24
#include "llvm/CodeGen/MachineRegisterInfo.h"
25
#include "llvm/IR/Constants.h"
26
#include "llvm/IR/Function.h"
27
#include "llvm/IR/LLVMContext.h"
28
#include "llvm/Support/Debug.h"
29
#include "llvm/Support/raw_ostream.h"
30
#include "llvm/Target/TargetMachine.h"
31
#include <queue>
32
33
#define DEBUG_TYPE "si-mode-register"
34
35
STATISTIC(NumSetregInserted, "Number of setreg of mode register inserted.");
36
37
using namespace llvm;
38
39
struct Status {
40
  // Mask is a bitmask where a '1' indicates the corresponding Mode bit has a
41
  // known value
42
  unsigned Mask;
43
  unsigned Mode;
44
45
573k
  Status() : Mask(0), Mode(0){};
46
47
40.7k
  Status(unsigned NewMask, unsigned NewMode) : Mask(NewMask), Mode(NewMode) {
48
40.7k
    Mode &= Mask;
49
40.7k
  };
50
51
  // merge two status values such that only values that don't conflict are
52
  // preserved
53
34.1k
  Status merge(const Status &S) const {
54
34.1k
    return Status((Mask | S.Mask), ((Mode & ~S.Mask) | (S.Mode & S.Mask)));
55
34.1k
  }
56
57
  // merge an unknown value by using the unknown value's mask to remove bits
58
  // from the result
59
76
  Status mergeUnknown(unsigned newMask) {
60
76
    return Status(Mask & ~newMask, Mode & ~newMask);
61
76
  }
62
63
  // intersect two Status values to produce a mode and mask that is a subset
64
  // of both values
65
3.92k
  Status intersect(const Status &S) const {
66
3.92k
    unsigned NewMask = (Mask & S.Mask) & (Mode ^ ~S.Mode);
67
3.92k
    unsigned NewMode = (Mode & NewMask);
68
3.92k
    return Status(NewMask, NewMode);
69
3.92k
  }
70
71
  // produce the delta required to change the Mode to the required Mode
72
59
  Status delta(const Status &S) const {
73
59
    return Status((S.Mask & (Mode ^ S.Mode)) | (~Mask & S.Mask), S.Mode);
74
59
  }
75
76
32.4k
  bool operator==(const Status &S) const {
77
32.4k
    return (Mask == S.Mask) && 
(Mode == S.Mode)6.37k
;
78
32.4k
  }
79
80
32.4k
  bool operator!=(const Status &S) const { return !(*this == S); }
81
82
461k
  bool isCompatible(Status &S) {
83
461k
    return ((Mask & S.Mask) == S.Mask) && 
((Mode & S.Mask) == S.Mode)459k
;
84
461k
  }
85
86
16
  bool isCombinable(Status &S) {
87
16
    return !(Mask & S.Mask) || isCompatible(S);
88
16
  }
89
};
90
91
class BlockData {
92
public:
93
  // The Status that represents the mode register settings required by the
94
  // FirstInsertionPoint (if any) in this block. Calculated in Phase 1.
95
  Status Require;
96
97
  // The Status that represents the net changes to the Mode register made by
98
  // this block, Calculated in Phase 1.
99
  Status Change;
100
101
  // The Status that represents the mode register settings on exit from this
102
  // block. Calculated in Phase 2.
103
  Status Exit;
104
105
  // The Status that represents the intersection of exit Mode register settings
106
  // from all predecessor blocks. Calculated in Phase 2, and used by Phase 3.
107
  Status Pred;
108
109
  // In Phase 1 we record the first instruction that has a mode requirement,
110
  // which is used in Phase 3 if we need to insert a mode change.
111
  MachineInstr *FirstInsertionPoint;
112
113
28.8k
  BlockData() : FirstInsertionPoint(nullptr) {};
114
};
115
116
namespace {
117
118
class SIModeRegister : public MachineFunctionPass {
119
public:
120
  static char ID;
121
122
  std::vector<std::unique_ptr<BlockData>> BlockInfo;
123
  std::queue<MachineBasicBlock *> Phase2List;
124
125
  // The default mode register setting currently only caters for the floating
126
  // point double precision rounding mode.
127
  // We currently assume the default rounding mode is Round to Nearest
128
  // NOTE: this should come from a per function rounding mode setting once such
129
  // a setting exists.
130
  unsigned DefaultMode = FP_ROUND_ROUND_TO_NEAREST;
131
  Status DefaultStatus =
132
      Status(FP_ROUND_MODE_DP(0x3), FP_ROUND_MODE_DP(DefaultMode));
133
134
public:
135
2.44k
  SIModeRegister() : MachineFunctionPass(ID) {}
136
137
  bool runOnMachineFunction(MachineFunction &MF) override;
138
139
2.41k
  void getAnalysisUsage(AnalysisUsage &AU) const override {
140
2.41k
    AU.setPreservesCFG();
141
2.41k
    MachineFunctionPass::getAnalysisUsage(AU);
142
2.41k
  }
143
144
  void processBlockPhase1(MachineBasicBlock &MBB, const SIInstrInfo *TII);
145
146
  void processBlockPhase2(MachineBasicBlock &MBB, const SIInstrInfo *TII);
147
148
  void processBlockPhase3(MachineBasicBlock &MBB, const SIInstrInfo *TII);
149
150
  Status getInstructionMode(MachineInstr &MI, const SIInstrInfo *TII);
151
152
  void insertSetreg(MachineBasicBlock &MBB, MachineInstr *I,
153
                    const SIInstrInfo *TII, Status InstrMode);
154
};
155
} // End anonymous namespace.
156
157
INITIALIZE_PASS(SIModeRegister, DEBUG_TYPE,
158
                "Insert required mode register values", false, false)
159
160
char SIModeRegister::ID = 0;
161
162
char &llvm::SIModeRegisterID = SIModeRegister::ID;
163
164
2.44k
FunctionPass *llvm::createSIModeRegisterPass() { return new SIModeRegister(); }
165
166
// Determine the Mode register setting required for this instruction.
167
// Instructions which don't use the Mode register return a null Status.
168
// Note this currently only deals with instructions that use the floating point
169
// double precision setting.
170
Status SIModeRegister::getInstructionMode(MachineInstr &MI,
171
432k
                                          const SIInstrInfo *TII) {
172
432k
  if (TII->usesFPDPRounding(MI)) {
173
3.66k
    switch (MI.getOpcode()) {
174
3.66k
    case AMDGPU::V_INTERP_P1LL_F16:
175
49
    case AMDGPU::V_INTERP_P1LV_F16:
176
49
    case AMDGPU::V_INTERP_P2_F16:
177
49
      // f16 interpolation instructions need double precision round to zero
178
49
      return Status(FP_ROUND_MODE_DP(3),
179
49
                    FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_ZERO));
180
3.61k
    default:
181
3.61k
      return DefaultStatus;
182
429k
    }
183
429k
  }
184
429k
  return Status();
185
429k
}
186
187
// Insert a setreg instruction to update the Mode register.
188
// It is possible (though unlikely) for an instruction to require a change to
189
// the value of disjoint parts of the Mode register when we don't know the
190
// value of the intervening bits. In that case we need to use more than one
191
// setreg instruction.
192
void SIModeRegister::insertSetreg(MachineBasicBlock &MBB, MachineInstr *MI,
193
43
                                  const SIInstrInfo *TII, Status InstrMode) {
194
86
  while (InstrMode.Mask) {
195
43
    unsigned Offset = countTrailingZeros<unsigned>(InstrMode.Mask);
196
43
    unsigned Width = countTrailingOnes<unsigned>(InstrMode.Mask >> Offset);
197
43
    unsigned Value = (InstrMode.Mode >> Offset) & ((1 << Width) - 1);
198
43
    BuildMI(MBB, MI, 0, TII->get(AMDGPU::S_SETREG_IMM32_B32))
199
43
        .addImm(Value)
200
43
        .addImm(((Width - 1) << AMDGPU::Hwreg::WIDTH_M1_SHIFT_) |
201
43
                (Offset << AMDGPU::Hwreg::OFFSET_SHIFT_) |
202
43
                (AMDGPU::Hwreg::ID_MODE << AMDGPU::Hwreg::ID_SHIFT_));
203
43
    ++NumSetregInserted;
204
43
    InstrMode.Mask &= ~(((1 << Width) - 1) << Offset);
205
43
  }
206
43
}
207
208
// In Phase 1 we iterate through the instructions of the block and for each
209
// instruction we get its mode usage. If the instruction uses the Mode register
210
// we:
211
// - update the Change status, which tracks the changes to the Mode register
212
//   made by this block
213
// - if this instruction's requirements are compatible with the current setting
214
//   of the Mode register we merge the modes
215
// - if it isn't compatible and an InsertionPoint isn't set, then we set the
216
//   InsertionPoint to the current instruction, and we remember the current
217
//   mode
218
// - if it isn't compatible and InsertionPoint is set we insert a seteg before
219
//   that instruction (unless this instruction forms part of the block's
220
//   entry requirements in which case the insertion is deferred until Phase 3
221
//   when predecessor exit values are known), and move the insertion point to
222
//   this instruction
223
// - if this is a setreg instruction we treat it as an incompatible instruction.
224
//   This is sub-optimal but avoids some nasty corner cases, and is expected to
225
//   occur very rarely.
226
// - on exit we have set the Require, Change, and initial Exit modes.
227
void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
228
28.8k
                                        const SIInstrInfo *TII) {
229
28.8k
  auto NewInfo = llvm::make_unique<BlockData>();
230
28.8k
  MachineInstr *InsertionPoint = nullptr;
231
28.8k
  // RequirePending is used to indicate whether we are collecting the initial
232
28.8k
  // requirements for the block, and need to defer the first InsertionPoint to
233
28.8k
  // Phase 3. It is set to false once we have set FirstInsertionPoint, or when
234
28.8k
  // we discover an explict setreg that means this block doesn't have any
235
28.8k
  // initial requirements.
236
28.8k
  bool RequirePending = true;
237
28.8k
  Status IPChange;
238
432k
  for (MachineInstr &MI : MBB) {
239
432k
    Status InstrMode = getInstructionMode(MI, TII);
240
432k
    if ((MI.getOpcode() == AMDGPU::S_SETREG_B32) ||
241
432k
        
(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32)432k
) {
242
235
      // We preserve any explicit mode register setreg instruction we encounter,
243
235
      // as we assume it has been inserted by a higher authority (this is
244
235
      // likely to be a very rare occurrence).
245
235
      unsigned Dst = TII->getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
246
235
      if (((Dst & AMDGPU::Hwreg::ID_MASK_) >> AMDGPU::Hwreg::ID_SHIFT_) !=
247
235
          AMDGPU::Hwreg::ID_MODE)
248
99
        continue;
249
136
250
136
      unsigned Width = ((Dst & AMDGPU::Hwreg::WIDTH_M1_MASK_) >>
251
136
                        AMDGPU::Hwreg::WIDTH_M1_SHIFT_) +
252
136
                       1;
253
136
      unsigned Offset =
254
136
          (Dst & AMDGPU::Hwreg::OFFSET_MASK_) >> AMDGPU::Hwreg::OFFSET_SHIFT_;
255
136
      unsigned Mask = ((1 << Width) - 1) << Offset;
256
136
257
136
      // If an InsertionPoint is set we will insert a setreg there.
258
136
      if (InsertionPoint) {
259
3
        insertSetreg(MBB, InsertionPoint, TII, IPChange.delta(NewInfo->Change));
260
3
        InsertionPoint = nullptr;
261
3
      }
262
136
      // If this is an immediate then we know the value being set, but if it is
263
136
      // not an immediate then we treat the modified bits of the mode register
264
136
      // as unknown.
265
136
      if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32) {
266
60
        unsigned Val = TII->getNamedOperand(MI, AMDGPU::OpName::imm)->getImm();
267
60
        unsigned Mode = (Val << Offset) & Mask;
268
60
        Status Setreg = Status(Mask, Mode);
269
60
        // If we haven't already set the initial requirements for the block we
270
60
        // don't need to as the requirements start from this explicit setreg.
271
60
        RequirePending = false;
272
60
        NewInfo->Change = NewInfo->Change.merge(Setreg);
273
76
      } else {
274
76
        NewInfo->Change = NewInfo->Change.mergeUnknown(Mask);
275
76
      }
276
432k
    } else if (!NewInfo->Change.isCompatible(InstrMode)) {
277
1.67k
      // This instruction uses the Mode register and its requirements aren't
278
1.67k
      // compatible with the current mode.
279
1.67k
      if (InsertionPoint) {
280
16
        // If the required mode change cannot be included in the current
281
16
        // InsertionPoint changes, we need a setreg and start a new
282
16
        // InsertionPoint.
283
16
        if (!IPChange.delta(NewInfo->Change).isCombinable(InstrMode)) {
284
16
          if (RequirePending) {
285
12
            // This is the first insertionPoint in the block so we will defer
286
12
            // the insertion of the setreg to Phase 3 where we know whether or
287
12
            // not it is actually needed.
288
12
            NewInfo->FirstInsertionPoint = InsertionPoint;
289
12
            NewInfo->Require = NewInfo->Change;
290
12
            RequirePending = false;
291
12
          } else {
292
4
            insertSetreg(MBB, InsertionPoint, TII,
293
4
                         IPChange.delta(NewInfo->Change));
294
4
            IPChange = NewInfo->Change;
295
4
          }
296
16
          // Set the new InsertionPoint
297
16
          InsertionPoint = &MI;
298
16
        }
299
16
        NewInfo->Change = NewInfo->Change.merge(InstrMode);
300
1.65k
      } else {
301
1.65k
        // No InsertionPoint is currently set - this is either the first in
302
1.65k
        // the block or we have previously seen an explicit setreg.
303
1.65k
        InsertionPoint = &MI;
304
1.65k
        IPChange = NewInfo->Change;
305
1.65k
        NewInfo->Change = NewInfo->Change.merge(InstrMode);
306
1.65k
      }
307
1.67k
    }
308
432k
  }
309
28.8k
  if (RequirePending) {
310
28.7k
    // If we haven't yet set the initial requirements for the block we set them
311
28.7k
    // now.
312
28.7k
    NewInfo->FirstInsertionPoint = InsertionPoint;
313
28.7k
    NewInfo->Require = NewInfo->Change;
314
28.7k
  } else 
if (47
InsertionPoint47
) {
315
16
    // We need to insert a setreg at the InsertionPoint
316
16
    insertSetreg(MBB, InsertionPoint, TII, IPChange.delta(NewInfo->Change));
317
16
  }
318
28.8k
  NewInfo->Exit = NewInfo->Change;
319
28.8k
  BlockInfo[MBB.getNumber()] = std::move(NewInfo);
320
28.8k
}
321
322
// In Phase 2 we revisit each block and calculate the common Mode register
323
// value provided by all predecessor blocks. If the Exit value for the block
324
// is changed, then we add the successor blocks to the worklist so that the
325
// exit value is propagated.
326
void SIModeRegister::processBlockPhase2(MachineBasicBlock &MBB,
327
32.4k
                                        const SIInstrInfo *TII) {
328
32.4k
//  BlockData *BI = BlockInfo[MBB.getNumber()];
329
32.4k
  unsigned ThisBlock = MBB.getNumber();
330
32.4k
  if (MBB.pred_empty()) {
331
25.4k
    // There are no predecessors, so use the default starting status.
332
25.4k
    BlockInfo[ThisBlock]->Pred = DefaultStatus;
333
25.4k
  } else {
334
7.01k
    // Build a status that is common to all the predecessors by intersecting
335
7.01k
    // all the predecessor exit status values.
336
7.01k
    MachineBasicBlock::pred_iterator P = MBB.pred_begin(), E = MBB.pred_end();
337
7.01k
    MachineBasicBlock &PB = *(*P);
338
7.01k
    BlockInfo[ThisBlock]->Pred = BlockInfo[PB.getNumber()]->Exit;
339
7.01k
340
10.9k
    for (P = std::next(P); P != E; 
P = std::next(P)3.92k
) {
341
3.92k
      MachineBasicBlock *Pred = *P;
342
3.92k
      BlockInfo[ThisBlock]->Pred = BlockInfo[ThisBlock]->Pred.intersect(BlockInfo[Pred->getNumber()]->Exit);
343
3.92k
    }
344
7.01k
  }
345
32.4k
  Status TmpStatus = BlockInfo[ThisBlock]->Pred.merge(BlockInfo[ThisBlock]->Change);
346
32.4k
  if (BlockInfo[ThisBlock]->Exit != TmpStatus) {
347
26.0k
    BlockInfo[ThisBlock]->Exit = TmpStatus;
348
26.0k
    // Add the successors to the work list so we can propagate the changed exit
349
26.0k
    // status.
350
26.0k
    for (MachineBasicBlock::succ_iterator S = MBB.succ_begin(),
351
26.0k
                                          E = MBB.succ_end();
352
29.6k
         S != E; 
S = std::next(S)3.61k
) {
353
3.61k
      MachineBasicBlock &B = *(*S);
354
3.61k
      Phase2List.push(&B);
355
3.61k
    }
356
26.0k
  }
357
32.4k
}
358
359
// In Phase 3 we revisit each block and if it has an insertion point defined we
360
// check whether the predecessor mode meets the block's entry requirements. If
361
// not we insert an appropriate setreg instruction to modify the Mode register.
362
void SIModeRegister::processBlockPhase3(MachineBasicBlock &MBB,
363
28.8k
                                        const SIInstrInfo *TII) {
364
28.8k
//  BlockData *BI = BlockInfo[MBB.getNumber()];
365
28.8k
  unsigned ThisBlock = MBB.getNumber();
366
28.8k
  if (!BlockInfo[ThisBlock]->Pred.isCompatible(BlockInfo[ThisBlock]->Require)) {
367
20
    Status Delta = BlockInfo[ThisBlock]->Pred.delta(BlockInfo[ThisBlock]->Require);
368
20
    if (BlockInfo[ThisBlock]->FirstInsertionPoint)
369
20
      insertSetreg(MBB, BlockInfo[ThisBlock]->FirstInsertionPoint, TII, Delta);
370
0
    else
371
0
      insertSetreg(MBB, &MBB.instr_front(), TII, Delta);
372
20
  }
373
28.8k
}
374
375
25.4k
bool SIModeRegister::runOnMachineFunction(MachineFunction &MF) {
376
25.4k
  BlockInfo.resize(MF.getNumBlockIDs());
377
25.4k
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
378
25.4k
  const SIInstrInfo *TII = ST.getInstrInfo();
379
25.4k
380
25.4k
  // Processing is performed in a number of phases
381
25.4k
382
25.4k
  // Phase 1 - determine the initial mode required by each block, and add setreg
383
25.4k
  // instructions for intra block requirements.
384
25.4k
  for (MachineBasicBlock &BB : MF)
385
28.8k
    processBlockPhase1(BB, TII);
386
25.4k
387
25.4k
  // Phase 2 - determine the exit mode from each block. We add all blocks to the
388
25.4k
  // list here, but will also add any that need to be revisited during Phase 2
389
25.4k
  // processing.
390
25.4k
  for (MachineBasicBlock &BB : MF)
391
28.8k
    Phase2List.push(&BB);
392
57.8k
  while (!Phase2List.empty()) {
393
32.4k
    processBlockPhase2(*Phase2List.front(), TII);
394
32.4k
    Phase2List.pop();
395
32.4k
  }
396
25.4k
397
25.4k
  // Phase 3 - add an initial setreg to each block where the required entry mode
398
25.4k
  // is not satisfied by the exit mode of all its predecessors.
399
25.4k
  for (MachineBasicBlock &BB : MF)
400
28.8k
    processBlockPhase3(BB, TII);
401
25.4k
402
25.4k
  BlockInfo.clear();
403
25.4k
404
25.4k
  return NumSetregInserted > 0;
405
25.4k
}