Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- R600ExpandSpecialInstrs.cpp - Expand special instructions ----------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// Vector, Reduction, and Cube instructions need to fill the entire instruction
11
/// group to work correctly.  This pass expands these individual instructions
12
/// into several instructions that will completely fill the instruction group.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#include "AMDGPU.h"
17
#include "AMDGPUSubtarget.h"
18
#include "R600Defines.h"
19
#include "R600InstrInfo.h"
20
#include "R600RegisterInfo.h"
21
#include "llvm/CodeGen/MachineBasicBlock.h"
22
#include "llvm/CodeGen/MachineFunction.h"
23
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
24
#include "llvm/CodeGen/MachineFunctionPass.h"
25
#include "llvm/CodeGen/MachineInstr.h"
26
#include "llvm/CodeGen/MachineInstrBuilder.h"
27
#include "llvm/CodeGen/MachineOperand.h"
28
#include "llvm/Pass.h"
29
#include <cassert>
30
#include <cstdint>
31
#include <iterator>
32
33
using namespace llvm;
34
35
#define DEBUG_TYPE "r600-expand-special-instrs"
36
37
namespace {
38
39
class R600ExpandSpecialInstrsPass : public MachineFunctionPass {
40
private:
41
  const R600InstrInfo *TII = nullptr;
42
43
  void SetFlagInNewMI(MachineInstr *NewMI, const MachineInstr *OldMI,
44
      unsigned Op);
45
46
public:
47
  static char ID;
48
49
280
  R600ExpandSpecialInstrsPass() : MachineFunctionPass(ID) {}
50
51
  bool runOnMachineFunction(MachineFunction &MF) override;
52
53
2.57k
  StringRef getPassName() const override {
54
2.57k
    return "R600 Expand special instructions pass";
55
2.57k
  }
56
};
57
58
} // end anonymous namespace
59
60
101k
INITIALIZE_PASS_BEGIN(R600ExpandSpecialInstrsPass, DEBUG_TYPE,
61
101k
                     "R600 Expand Special Instrs", false, false)
62
101k
INITIALIZE_PASS_END(R600ExpandSpecialInstrsPass, DEBUG_TYPE,
63
                    "R600ExpandSpecialInstrs", false, false)
64
65
char R600ExpandSpecialInstrsPass::ID = 0;
66
67
char &llvm::R600ExpandSpecialInstrsPassID = R600ExpandSpecialInstrsPass::ID;
68
69
280
FunctionPass *llvm::createR600ExpandSpecialInstrsPass() {
70
280
  return new R600ExpandSpecialInstrsPass();
71
280
}
72
73
void R600ExpandSpecialInstrsPass::SetFlagInNewMI(MachineInstr *NewMI,
74
1.29k
    const MachineInstr *OldMI, unsigned Op) {
75
1.29k
  int OpIdx = TII->getOperandIdx(*OldMI, Op);
76
1.29k
  if (OpIdx > -1) {
77
936
    uint64_t Val = OldMI->getOperand(OpIdx).getImm();
78
936
    TII->setImmOperand(*NewMI, Op, Val);
79
936
  }
80
1.29k
}
81
82
2.29k
bool R600ExpandSpecialInstrsPass::runOnMachineFunction(MachineFunction &MF) {
83
2.29k
  const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>();
84
2.29k
  TII = ST.getInstrInfo();
85
2.29k
86
2.29k
  const R600RegisterInfo &TRI = TII->getRegisterInfo();
87
2.29k
88
2.29k
  for (MachineFunction::iterator BB = MF.begin(), BB_E = MF.end();
89
4.58k
                                                  BB != BB_E; 
++BB2.29k
) {
90
2.29k
    MachineBasicBlock &MBB = *BB;
91
2.29k
    MachineBasicBlock::iterator I = MBB.begin();
92
63.3k
    while (I != MBB.end()) {
93
61.0k
      MachineInstr &MI = *I;
94
61.0k
      I = std::next(I);
95
61.0k
96
61.0k
      // Expand LDS_*_RET instructions
97
61.0k
      if (TII->isLDSRetInstr(MI.getOpcode())) {
98
886
        int DstIdx = TII->getOperandIdx(MI.getOpcode(), R600::OpName::dst);
99
886
        assert(DstIdx != -1);
100
886
        MachineOperand &DstOp = MI.getOperand(DstIdx);
101
886
        MachineInstr *Mov = TII->buildMovInstr(&MBB, I,
102
886
                                               DstOp.getReg(), R600::OQAP);
103
886
        DstOp.setReg(R600::OQAP);
104
886
        int LDSPredSelIdx = TII->getOperandIdx(MI.getOpcode(),
105
886
                                           R600::OpName::pred_sel);
106
886
        int MovPredSelIdx = TII->getOperandIdx(Mov->getOpcode(),
107
886
                                           R600::OpName::pred_sel);
108
886
        // Copy the pred_sel bit
109
886
        Mov->getOperand(MovPredSelIdx).setReg(
110
886
            MI.getOperand(LDSPredSelIdx).getReg());
111
886
      }
112
61.0k
113
61.0k
      switch (MI.getOpcode()) {
114
61.0k
      
default: break60.9k
;
115
61.0k
      // Expand PRED_X to one of the PRED_SET instructions.
116
61.0k
      case R600::PRED_X: {
117
88
        uint64_t Flags = MI.getOperand(3).getImm();
118
88
        // The native opcode used by PRED_X is stored as an immediate in the
119
88
        // third operand.
120
88
        MachineInstr *PredSet = TII->buildDefaultInstruction(MBB, I,
121
88
                                            MI.getOperand(2).getImm(), // opcode
122
88
                                            MI.getOperand(0).getReg(), // dst
123
88
                                            MI.getOperand(1).getReg(), // src0
124
88
                                            R600::ZERO);             // src1
125
88
        TII->addFlag(*PredSet, 0, MO_FLAG_MASK);
126
88
        if (Flags & MO_FLAG_PUSH) {
127
64
          TII->setImmOperand(*PredSet, R600::OpName::update_exec_mask, 1);
128
64
        } else {
129
24
          TII->setImmOperand(*PredSet, R600::OpName::update_pred, 1);
130
24
        }
131
88
        MI.eraseFromParent();
132
88
        continue;
133
61.0k
        }
134
61.0k
      case R600::DOT_4: {
135
32
136
32
        const R600RegisterInfo &TRI = TII->getRegisterInfo();
137
32
138
32
        unsigned DstReg = MI.getOperand(0).getReg();
139
32
        unsigned DstBase = TRI.getEncodingValue(DstReg) & HW_REG_MASK;
140
32
141
160
        for (unsigned Chan = 0; Chan < 4; 
++Chan128
) {
142
128
          bool Mask = (Chan != TRI.getHWRegChan(DstReg));
143
128
          unsigned SubDstReg =
144
128
              R600::R600_TReg32RegClass.getRegister((DstBase * 4) + Chan);
145
128
          MachineInstr *BMI =
146
128
              TII->buildSlotOfVectorInstruction(MBB, &MI, Chan, SubDstReg);
147
128
          if (Chan > 0) {
148
96
            BMI->bundleWithPred();
149
96
          }
150
128
          if (Mask) {
151
96
            TII->addFlag(*BMI, 0, MO_FLAG_MASK);
152
96
          }
153
128
          if (Chan != 3)
154
96
            TII->addFlag(*BMI, 0, MO_FLAG_NOT_LAST);
155
128
          unsigned Opcode = BMI->getOpcode();
156
128
          // While not strictly necessary from hw point of view, we force
157
128
          // all src operands of a dot4 inst to belong to the same slot.
158
128
          unsigned Src0 = BMI->getOperand(
159
128
              TII->getOperandIdx(Opcode, R600::OpName::src0))
160
128
              .getReg();
161
128
          unsigned Src1 = BMI->getOperand(
162
128
              TII->getOperandIdx(Opcode, R600::OpName::src1))
163
128
              .getReg();
164
128
          (void) Src0;
165
128
          (void) Src1;
166
128
          if ((TRI.getEncodingValue(Src0) & 0xff) < 127 &&
167
128
              
(TRI.getEncodingValue(Src1) & 0xff) < 12796
)
168
128
            assert(TRI.getHWRegChan(Src0) == TRI.getHWRegChan(Src1));
169
128
        }
170
32
        MI.eraseFromParent();
171
32
        continue;
172
60.9k
      }
173
60.9k
      }
174
60.9k
175
60.9k
      bool IsReduction = TII->isReductionOp(MI.getOpcode());
176
60.9k
      bool IsVector = TII->isVector(MI);
177
60.9k
      bool IsCube = TII->isCubeOp(MI.getOpcode());
178
60.9k
      if (!IsReduction && !IsVector && 
!IsCube60.8k
) {
179
60.8k
        continue;
180
60.8k
      }
181
54
182
54
      // Expand the instruction
183
54
      //
184
54
      // Reduction instructions:
185
54
      // T0_X = DP4 T1_XYZW, T2_XYZW
186
54
      // becomes:
187
54
      // TO_X = DP4 T1_X, T2_X
188
54
      // TO_Y (write masked) = DP4 T1_Y, T2_Y
189
54
      // TO_Z (write masked) = DP4 T1_Z, T2_Z
190
54
      // TO_W (write masked) = DP4 T1_W, T2_W
191
54
      //
192
54
      // Vector instructions:
193
54
      // T0_X = MULLO_INT T1_X, T2_X
194
54
      // becomes:
195
54
      // T0_X = MULLO_INT T1_X, T2_X
196
54
      // T0_Y (write masked) = MULLO_INT T1_X, T2_X
197
54
      // T0_Z (write masked) = MULLO_INT T1_X, T2_X
198
54
      // T0_W (write masked) = MULLO_INT T1_X, T2_X
199
54
      //
200
54
      // Cube instructions:
201
54
      // T0_XYZW = CUBE T1_XYZW
202
54
      // becomes:
203
54
      // TO_X = CUBE T1_Z, T1_Y
204
54
      // T0_Y = CUBE T1_Z, T1_X
205
54
      // T0_Z = CUBE T1_X, T1_Z
206
54
      // T0_W = CUBE T1_Y, T1_Z
207
270
      
for (unsigned Chan = 0; 54
Chan < 4;
Chan++216
) {
208
216
        unsigned DstReg = MI.getOperand(
209
216
                            TII->getOperandIdx(MI, R600::OpName::dst)).getReg();
210
216
        unsigned Src0 = MI.getOperand(
211
216
                           TII->getOperandIdx(MI, R600::OpName::src0)).getReg();
212
216
        unsigned Src1 = 0;
213
216
214
216
        // Determine the correct source registers
215
216
        if (!IsCube) {
216
208
          int Src1Idx = TII->getOperandIdx(MI, R600::OpName::src1);
217
208
          if (Src1Idx != -1) {
218
52
            Src1 = MI.getOperand(Src1Idx).getReg();
219
52
          }
220
208
        }
221
216
        if (IsReduction) {
222
0
          unsigned SubRegIndex = AMDGPURegisterInfo::getSubRegFromChannel(Chan);
223
0
          Src0 = TRI.getSubReg(Src0, SubRegIndex);
224
0
          Src1 = TRI.getSubReg(Src1, SubRegIndex);
225
216
        } else if (IsCube) {
226
8
          static const int CubeSrcSwz[] = {2, 2, 0, 1};
227
8
          unsigned SubRegIndex0 = AMDGPURegisterInfo::getSubRegFromChannel(CubeSrcSwz[Chan]);
228
8
          unsigned SubRegIndex1 = AMDGPURegisterInfo::getSubRegFromChannel(CubeSrcSwz[3 - Chan]);
229
8
          Src1 = TRI.getSubReg(Src0, SubRegIndex1);
230
8
          Src0 = TRI.getSubReg(Src0, SubRegIndex0);
231
8
        }
232
216
233
216
        // Determine the correct destination registers;
234
216
        bool Mask = false;
235
216
        bool NotLast = true;
236
216
        if (IsCube) {
237
8
          unsigned SubRegIndex = AMDGPURegisterInfo::getSubRegFromChannel(Chan);
238
8
          DstReg = TRI.getSubReg(DstReg, SubRegIndex);
239
208
        } else {
240
208
          // Mask the write if the original instruction does not write to
241
208
          // the current Channel.
242
208
          Mask = (Chan != TRI.getHWRegChan(DstReg));
243
208
          unsigned DstBase = TRI.getEncodingValue(DstReg) & HW_REG_MASK;
244
208
          DstReg = R600::R600_TReg32RegClass.getRegister((DstBase * 4) + Chan);
245
208
        }
246
216
247
216
        // Set the IsLast bit
248
216
        NotLast = (Chan != 3 );
249
216
250
216
        // Add the new instruction
251
216
        unsigned Opcode = MI.getOpcode();
252
216
        switch (Opcode) {
253
216
        case R600::CUBE_r600_pseudo:
254
0
          Opcode = R600::CUBE_r600_real;
255
0
          break;
256
216
        case R600::CUBE_eg_pseudo:
257
8
          Opcode = R600::CUBE_eg_real;
258
8
          break;
259
216
        default:
260
208
          break;
261
216
        }
262
216
263
216
        MachineInstr *NewMI =
264
216
          TII->buildDefaultInstruction(MBB, I, Opcode, DstReg, Src0, Src1);
265
216
266
216
        if (Chan != 0)
267
162
          NewMI->bundleWithPred();
268
216
        if (Mask) {
269
156
          TII->addFlag(*NewMI, 0, MO_FLAG_MASK);
270
156
        }
271
216
        if (NotLast) {
272
162
          TII->addFlag(*NewMI, 0, MO_FLAG_NOT_LAST);
273
162
        }
274
216
        SetFlagInNewMI(NewMI, &MI, R600::OpName::clamp);
275
216
        SetFlagInNewMI(NewMI, &MI, R600::OpName::literal);
276
216
        SetFlagInNewMI(NewMI, &MI, R600::OpName::src0_abs);
277
216
        SetFlagInNewMI(NewMI, &MI, R600::OpName::src1_abs);
278
216
        SetFlagInNewMI(NewMI, &MI, R600::OpName::src0_neg);
279
216
        SetFlagInNewMI(NewMI, &MI, R600::OpName::src1_neg);
280
216
      }
281
54
      MI.eraseFromParent();
282
54
    }
283
2.29k
  }
284
2.29k
  return false;
285
2.29k
}