Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- SIInstrInfo.cpp - SI Instruction Information  ----------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// SI Implementation of TargetInstrInfo.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "SIInstrInfo.h"
15
#include "AMDGPU.h"
16
#include "AMDGPUSubtarget.h"
17
#include "GCNHazardRecognizer.h"
18
#include "SIDefines.h"
19
#include "SIMachineFunctionInfo.h"
20
#include "SIRegisterInfo.h"
21
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22
#include "Utils/AMDGPUBaseInfo.h"
23
#include "llvm/ADT/APInt.h"
24
#include "llvm/ADT/ArrayRef.h"
25
#include "llvm/ADT/SmallVector.h"
26
#include "llvm/ADT/StringRef.h"
27
#include "llvm/ADT/iterator_range.h"
28
#include "llvm/Analysis/AliasAnalysis.h"
29
#include "llvm/Analysis/MemoryLocation.h"
30
#include "llvm/Analysis/ValueTracking.h"
31
#include "llvm/CodeGen/MachineBasicBlock.h"
32
#include "llvm/CodeGen/MachineDominators.h"
33
#include "llvm/CodeGen/MachineFrameInfo.h"
34
#include "llvm/CodeGen/MachineFunction.h"
35
#include "llvm/CodeGen/MachineInstr.h"
36
#include "llvm/CodeGen/MachineInstrBuilder.h"
37
#include "llvm/CodeGen/MachineInstrBundle.h"
38
#include "llvm/CodeGen/MachineMemOperand.h"
39
#include "llvm/CodeGen/MachineOperand.h"
40
#include "llvm/CodeGen/MachineRegisterInfo.h"
41
#include "llvm/CodeGen/RegisterScavenging.h"
42
#include "llvm/CodeGen/ScheduleDAG.h"
43
#include "llvm/CodeGen/SelectionDAGNodes.h"
44
#include "llvm/CodeGen/TargetOpcodes.h"
45
#include "llvm/CodeGen/TargetRegisterInfo.h"
46
#include "llvm/IR/DebugLoc.h"
47
#include "llvm/IR/DiagnosticInfo.h"
48
#include "llvm/IR/Function.h"
49
#include "llvm/IR/InlineAsm.h"
50
#include "llvm/IR/LLVMContext.h"
51
#include "llvm/MC/MCInstrDesc.h"
52
#include "llvm/Support/Casting.h"
53
#include "llvm/Support/CommandLine.h"
54
#include "llvm/Support/Compiler.h"
55
#include "llvm/Support/ErrorHandling.h"
56
#include "llvm/Support/MachineValueType.h"
57
#include "llvm/Support/MathExtras.h"
58
#include "llvm/Target/TargetMachine.h"
59
#include <cassert>
60
#include <cstdint>
61
#include <iterator>
62
#include <utility>
63
64
using namespace llvm;
65
66
#define GET_INSTRINFO_CTOR_DTOR
67
#include "AMDGPUGenInstrInfo.inc"
68
69
namespace llvm {
70
namespace AMDGPU {
71
#define GET_D16ImageDimIntrinsics_IMPL
72
#define GET_ImageDimIntrinsicTable_IMPL
73
#define GET_RsrcIntrinsics_IMPL
74
#include "AMDGPUGenSearchableTables.inc"
75
}
76
}
77
78
79
// Must be at least 4 to be able to branch over minimum unconditional branch
80
// code. This is only for making it possible to write reasonably small tests for
81
// long branches.
82
static cl::opt<unsigned>
83
BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
84
                 cl::desc("Restrict range of branch instructions (DEBUG)"));
85
86
SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST)
87
  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
88
3.64k
    RI(ST), ST(ST) {}
89
90
//===----------------------------------------------------------------------===//
91
// TargetInstrInfo callbacks
92
//===----------------------------------------------------------------------===//
93
94
4.58k
static unsigned getNumOperandsNoGlue(SDNode *Node) {
95
4.58k
  unsigned N = Node->getNumOperands();
96
5.88k
  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
97
1.30k
    --N;
98
4.58k
  return N;
99
4.58k
}
100
101
/// Returns true if both nodes have the same value for the given
102
///        operand \p Op, or if both nodes do not have this operand.
103
624k
static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
104
624k
  unsigned Opc0 = N0->getMachineOpcode();
105
624k
  unsigned Opc1 = N1->getMachineOpcode();
106
624k
107
624k
  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
108
624k
  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
109
624k
110
624k
  if (Op0Idx == -1 && 
Op1Idx == -14.93k
)
111
4.92k
    return true;
112
619k
113
619k
114
619k
  if ((Op0Idx == -1 && 
Op1Idx != -110
) ||
115
619k
      
(619k
Op1Idx == -1619k
&&
Op0Idx != -110
))
116
20
    return false;
117
619k
118
619k
  // getNamedOperandIdx returns the index for the MachineInstr's operands,
119
619k
  // which includes the result as the first operand. We are indexing into the
120
619k
  // MachineSDNode's operands, so we need to skip the result operand to get
121
619k
  // the real index.
122
619k
  --Op0Idx;
123
619k
  --Op1Idx;
124
619k
125
619k
  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
126
619k
}
127
128
bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
129
15.6k
                                                    AliasAnalysis *AA) const {
130
15.6k
  // TODO: The generic check fails for VALU instructions that should be
131
15.6k
  // rematerializable due to implicit reads of exec. We really want all of the
132
15.6k
  // generic logic for this except for this.
133
15.6k
  switch (MI.getOpcode()) {
134
15.6k
  case AMDGPU::V_MOV_B32_e32:
135
7.87k
  case AMDGPU::V_MOV_B32_e64:
136
7.87k
  case AMDGPU::V_MOV_B64_PSEUDO:
137
7.87k
    // No implicit operands.
138
7.87k
    return MI.getNumOperands() == MI.getDesc().getNumOperands();
139
7.87k
  default:
140
7.78k
    return false;
141
15.6k
  }
142
15.6k
}
143
144
bool SIInstrInfo::areLoadsFromSameBasePtr(SDNode *Load0, SDNode *Load1,
145
                                          int64_t &Offset0,
146
433k
                                          int64_t &Offset1) const {
147
433k
  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
148
81.8k
    return false;
149
351k
150
351k
  unsigned Opc0 = Load0->getMachineOpcode();
151
351k
  unsigned Opc1 = Load1->getMachineOpcode();
152
351k
153
351k
  // Make sure both are actually loads.
154
351k
  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
155
27.7k
    return false;
156
323k
157
323k
  if (isDS(Opc0) && 
isDS(Opc1)3.29k
) {
158
2.29k
159
2.29k
    // FIXME: Handle this case:
160
2.29k
    if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
161
24
      return false;
162
2.27k
163
2.27k
    // Check base reg.
164
2.27k
    if (Load0->getOperand(0) != Load1->getOperand(0))
165
41
      return false;
166
2.22k
167
2.22k
    // Skip read2 / write2 variants for simplicity.
168
2.22k
    // TODO: We should report true if the used offsets are adjacent (excluded
169
2.22k
    // st64 versions).
170
2.22k
    int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
171
2.22k
    int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
172
2.22k
    if (Offset0Idx == -1 || 
Offset1Idx == -12.20k
)
173
20
      return false;
174
2.20k
175
2.20k
    // XXX - be careful of datalesss loads
176
2.20k
    // getNamedOperandIdx returns the index for MachineInstrs.  Since they
177
2.20k
    // include the output in the operand list, but SDNodes don't, we need to
178
2.20k
    // subtract the index by one.
179
2.20k
    Offset0Idx -= get(Opc0).NumDefs;
180
2.20k
    Offset1Idx -= get(Opc1).NumDefs;
181
2.20k
    Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
182
2.20k
    Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
183
2.20k
    return true;
184
2.20k
  }
185
321k
186
321k
  if (isSMRD(Opc0) && 
isSMRD(Opc1)43.5k
) {
187
30.1k
    // Skip time and cache invalidation instructions.
188
30.1k
    if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
189
30.1k
        
AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -130.1k
)
190
18
      return false;
191
30.1k
192
30.1k
    assert(getNumOperandsNoGlue(Load0) == getNumOperandsNoGlue(Load1));
193
30.1k
194
30.1k
    // Check base reg.
195
30.1k
    if (Load0->getOperand(0) != Load1->getOperand(0))
196
9.25k
      return false;
197
20.8k
198
20.8k
    const ConstantSDNode *Load0Offset =
199
20.8k
        dyn_cast<ConstantSDNode>(Load0->getOperand(1));
200
20.8k
    const ConstantSDNode *Load1Offset =
201
20.8k
        dyn_cast<ConstantSDNode>(Load1->getOperand(1));
202
20.8k
203
20.8k
    if (!Load0Offset || 
!Load1Offset20.8k
)
204
12
      return false;
205
20.8k
206
20.8k
    Offset0 = Load0Offset->getZExtValue();
207
20.8k
    Offset1 = Load1Offset->getZExtValue();
208
20.8k
    return true;
209
20.8k
  }
210
291k
211
291k
  // MUBUF and MTBUF can access the same addresses.
212
291k
  if ((isMUBUF(Opc0) || 
isMTBUF(Opc0)66.5k
) &&
(224k
isMUBUF(Opc1)224k
||
isMTBUF(Opc1)6.24k
)) {
213
218k
214
218k
    // MUBUF and MTBUF have vaddr at different indices.
215
218k
    if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
216
218k
        
!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr)218k
||
217
218k
        
!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc)187k
)
218
33.1k
      return false;
219
185k
220
185k
    int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
221
185k
    int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
222
185k
223
185k
    if (OffIdx0 == -1 || OffIdx1 == -1)
224
0
      return false;
225
185k
226
185k
    // getNamedOperandIdx returns the index for MachineInstrs.  Since they
227
185k
    // include the output in the operand list, but SDNodes don't, we need to
228
185k
    // subtract the index by one.
229
185k
    OffIdx0 -= get(Opc0).NumDefs;
230
185k
    OffIdx1 -= get(Opc1).NumDefs;
231
185k
232
185k
    SDValue Off0 = Load0->getOperand(OffIdx0);
233
185k
    SDValue Off1 = Load1->getOperand(OffIdx1);
234
185k
235
185k
    // The offset might be a FrameIndexSDNode.
236
185k
    if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
237
0
      return false;
238
185k
239
185k
    Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
240
185k
    Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
241
185k
    return true;
242
185k
  }
243
72.5k
244
72.5k
  return false;
245
72.5k
}
246
247
57.1k
static bool isStride64(unsigned Opc) {
248
57.1k
  switch (Opc) {
249
57.1k
  case AMDGPU::DS_READ2ST64_B32:
250
5
  case AMDGPU::DS_READ2ST64_B64:
251
5
  case AMDGPU::DS_WRITE2ST64_B32:
252
5
  case AMDGPU::DS_WRITE2ST64_B64:
253
5
    return true;
254
57.1k
  default:
255
57.1k
    return false;
256
57.1k
  }
257
57.1k
}
258
259
bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt,
260
                                          const MachineOperand *&BaseOp,
261
                                          int64_t &Offset,
262
1.20M
                                          const TargetRegisterInfo *TRI) const {
263
1.20M
  unsigned Opc = LdSt.getOpcode();
264
1.20M
265
1.20M
  if (isDS(LdSt)) {
266
135k
    const MachineOperand *OffsetImm =
267
135k
        getNamedOperand(LdSt, AMDGPU::OpName::offset);
268
135k
    if (OffsetImm) {
269
65.0k
      // Normal, single offset LDS instruction.
270
65.0k
      BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
271
65.0k
      // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
272
65.0k
      // report that here?
273
65.0k
      if (!BaseOp)
274
144
        return false;
275
64.9k
276
64.9k
      Offset = OffsetImm->getImm();
277
64.9k
      assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
278
64.9k
                                "operands of type register.");
279
64.9k
      return true;
280
64.9k
    }
281
70.8k
282
70.8k
    // The 2 offset instructions use offset0 and offset1 instead. We can treat
283
70.8k
    // these as a load with a single offset if the 2 offsets are consecutive. We
284
70.8k
    // will use this for some partially aligned loads.
285
70.8k
    const MachineOperand *Offset0Imm =
286
70.8k
        getNamedOperand(LdSt, AMDGPU::OpName::offset0);
287
70.8k
    const MachineOperand *Offset1Imm =
288
70.8k
        getNamedOperand(LdSt, AMDGPU::OpName::offset1);
289
70.8k
290
70.8k
    uint8_t Offset0 = Offset0Imm->getImm();
291
70.8k
    uint8_t Offset1 = Offset1Imm->getImm();
292
70.8k
293
70.8k
    if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
294
57.1k
      // Each of these offsets is in element sized units, so we need to convert
295
57.1k
      // to bytes of the individual reads.
296
57.1k
297
57.1k
      unsigned EltSize;
298
57.1k
      if (LdSt.mayLoad())
299
15.6k
        EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
300
41.5k
      else {
301
41.5k
        assert(LdSt.mayStore());
302
41.5k
        int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
303
41.5k
        EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
304
41.5k
      }
305
57.1k
306
57.1k
      if (isStride64(Opc))
307
5
        EltSize *= 64;
308
57.1k
309
57.1k
      BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
310
57.1k
      Offset = EltSize * Offset0;
311
57.1k
      assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
312
57.1k
                                "operands of type register.");
313
57.1k
      return true;
314
57.1k
    }
315
13.7k
316
13.7k
    return false;
317
13.7k
  }
318
1.07M
319
1.07M
  if (isMUBUF(LdSt) || 
isMTBUF(LdSt)136k
) {
320
934k
    const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
321
934k
    if (SOffset && 
SOffset->isReg()934k
)
322
811k
      return false;
323
123k
324
123k
    const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
325
123k
    if (!AddrReg)
326
116k
      return false;
327
6.90k
328
6.90k
    const MachineOperand *OffsetImm =
329
6.90k
        getNamedOperand(LdSt, AMDGPU::OpName::offset);
330
6.90k
    BaseOp = AddrReg;
331
6.90k
    Offset = OffsetImm->getImm();
332
6.90k
333
6.90k
    if (SOffset) // soffset can be an inline immediate.
334
6.90k
      Offset += SOffset->getImm();
335
6.90k
336
6.90k
    assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
337
6.90k
                              "operands of type register.");
338
6.90k
    return true;
339
6.90k
  }
340
135k
341
135k
  if (isSMRD(LdSt)) {
342
32.8k
    const MachineOperand *OffsetImm =
343
32.8k
        getNamedOperand(LdSt, AMDGPU::OpName::offset);
344
32.8k
    if (!OffsetImm)
345
139
      return false;
346
32.7k
347
32.7k
    const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
348
32.7k
    BaseOp = SBaseReg;
349
32.7k
    Offset = OffsetImm->getImm();
350
32.7k
    assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
351
32.7k
                              "operands of type register.");
352
32.7k
    return true;
353
32.7k
  }
354
102k
355
102k
  if (isFLAT(LdSt)) {
356
99.6k
    const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
357
99.6k
    if (VAddr) {
358
99.6k
      // Can't analyze 2 offsets.
359
99.6k
      if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
360
236
        return false;
361
99.4k
362
99.4k
      BaseOp = VAddr;
363
99.4k
    } else {
364
0
      // scratch instructions have either vaddr or saddr.
365
0
      BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
366
0
    }
367
99.6k
368
99.6k
    Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
369
99.4k
    assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
370
99.4k
                              "operands of type register.");
371
99.4k
    return true;
372
3.13k
  }
373
3.13k
374
3.13k
  return false;
375
3.13k
}
376
377
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
378
                                  const MachineOperand &BaseOp1,
379
                                  const MachineInstr &MI2,
380
28.6k
                                  const MachineOperand &BaseOp2) {
381
28.6k
  // Support only base operands with base registers.
382
28.6k
  // Note: this could be extended to support FI operands.
383
28.6k
  if (!BaseOp1.isReg() || !BaseOp2.isReg())
384
0
    return false;
385
28.6k
386
28.6k
  if (BaseOp1.isIdenticalTo(BaseOp2))
387
15.7k
    return true;
388
12.9k
389
12.9k
  if (!MI1.hasOneMemOperand() || 
!MI2.hasOneMemOperand()12.9k
)
390
291
    return false;
391
12.6k
392
12.6k
  auto MO1 = *MI1.memoperands_begin();
393
12.6k
  auto MO2 = *MI2.memoperands_begin();
394
12.6k
  if (MO1->getAddrSpace() != MO2->getAddrSpace())
395
8.42k
    return false;
396
4.21k
397
4.21k
  auto Base1 = MO1->getValue();
398
4.21k
  auto Base2 = MO2->getValue();
399
4.21k
  if (!Base1 || 
!Base24.17k
)
400
92
    return false;
401
4.12k
  const MachineFunction &MF = *MI1.getParent()->getParent();
402
4.12k
  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
403
4.12k
  Base1 = GetUnderlyingObject(Base1, DL);
404
4.12k
  Base2 = GetUnderlyingObject(Base1, DL);
405
4.12k
406
4.12k
  if (isa<UndefValue>(Base1) || 
isa<UndefValue>(Base2)4.09k
)
407
30
    return false;
408
4.09k
409
4.09k
  return Base1 == Base2;
410
4.09k
}
411
412
bool SIInstrInfo::shouldClusterMemOps(const MachineOperand &BaseOp1,
413
                                      const MachineOperand &BaseOp2,
414
28.6k
                                      unsigned NumLoads) const {
415
28.6k
  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
416
28.6k
  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
417
28.6k
418
28.6k
  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
419
8.83k
    return false;
420
19.8k
421
19.8k
  const MachineOperand *FirstDst = nullptr;
422
19.8k
  const MachineOperand *SecondDst = nullptr;
423
19.8k
424
19.8k
  if ((isMUBUF(FirstLdSt) && 
isMUBUF(SecondLdSt)343
) ||
425
19.8k
      
(19.5k
isMTBUF(FirstLdSt)19.5k
&&
isMTBUF(SecondLdSt)9
) ||
426
19.8k
      
(19.4k
isFLAT(FirstLdSt)19.4k
&&
isFLAT(SecondLdSt)2.89k
)) {
427
3.24k
    const unsigned MaxGlobalLoadCluster = 6;
428
3.24k
    if (NumLoads > MaxGlobalLoadCluster)
429
0
      return false;
430
3.24k
431
3.24k
    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
432
3.24k
    if (!FirstDst)
433
1.23k
      FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
434
3.24k
    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
435
3.24k
    if (!SecondDst)
436
1.23k
      SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
437
16.6k
  } else if (isSMRD(FirstLdSt) && 
isSMRD(SecondLdSt)13.2k
) {
438
13.0k
    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
439
13.0k
    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
440
13.0k
  } else 
if (3.57k
isDS(FirstLdSt)3.57k
&&
isDS(SecondLdSt)3.37k
) {
441
3.37k
    FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
442
3.37k
    SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
443
3.37k
  }
444
19.8k
445
19.8k
  if (!FirstDst || 
!SecondDst18.2k
)
446
1.62k
    return false;
447
18.2k
448
18.2k
  // Try to limit clustering based on the total number of bytes loaded
449
18.2k
  // rather than the number of instructions.  This is done to help reduce
450
18.2k
  // register pressure.  The method used is somewhat inexact, though,
451
18.2k
  // because it assumes that all loads in the cluster will load the
452
18.2k
  // same number of bytes as FirstLdSt.
453
18.2k
454
18.2k
  // The unit of this value is bytes.
455
18.2k
  // FIXME: This needs finer tuning.
456
18.2k
  unsigned LoadClusterThreshold = 16;
457
18.2k
458
18.2k
  const MachineRegisterInfo &MRI =
459
18.2k
      FirstLdSt.getParent()->getParent()->getRegInfo();
460
18.2k
461
18.2k
  const unsigned Reg = FirstDst->getReg();
462
18.2k
463
18.2k
  const TargetRegisterClass *DstRC = TargetRegisterInfo::isVirtualRegister(Reg)
464
18.2k
                                         ? 
MRI.getRegClass(Reg)18.2k
465
18.2k
                                         : 
RI.getPhysRegClass(Reg)1
;
466
18.2k
467
18.2k
  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
468
18.2k
}
469
470
// FIXME: This behaves strangely. If, for example, you have 32 load + stores,
471
// the first 16 loads will be interleaved with the stores, and the next 16 will
472
// be clustered as expected. It should really split into 2 16 store batches.
473
//
474
// Loads are clustered until this returns false, rather than trying to schedule
475
// groups of stores. This also means we have to deal with saying different
476
// address space loads should be clustered, and ones which might cause bank
477
// conflicts.
478
//
479
// This might be deprecated so it might not be worth that much effort to fix.
480
bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1,
481
                                          int64_t Offset0, int64_t Offset1,
482
35.0k
                                          unsigned NumLoads) const {
483
35.0k
  assert(Offset1 > Offset0 &&
484
35.0k
         "Second offset should be larger than first offset!");
485
35.0k
  // If we have less than 16 loads in a row, and the offsets are within 64
486
35.0k
  // bytes, then schedule together.
487
35.0k
488
35.0k
  // A cacheline is 64 bytes (for global memory).
489
35.0k
  return (NumLoads <= 16 && 
(Offset1 - Offset0) < 6434.9k
);
490
35.0k
}
491
492
static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB,
493
                              MachineBasicBlock::iterator MI,
494
                              const DebugLoc &DL, unsigned DestReg,
495
14
                              unsigned SrcReg, bool KillSrc) {
496
14
  MachineFunction *MF = MBB.getParent();
497
14
  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
498
14
                                        "illegal SGPR to VGPR copy",
499
14
                                        DL, DS_Error);
500
14
  LLVMContext &C = MF->getFunction().getContext();
501
14
  C.diagnose(IllegalCopy);
502
14
503
14
  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
504
14
    .addReg(SrcReg, getKillRegState(KillSrc));
505
14
}
506
507
void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
508
                              MachineBasicBlock::iterator MI,
509
                              const DebugLoc &DL, unsigned DestReg,
510
68.6k
                              unsigned SrcReg, bool KillSrc) const {
511
68.6k
  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
512
68.6k
513
68.6k
  if (RC == &AMDGPU::VGPR_32RegClass) {
514
38.7k
    assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
515
38.7k
           AMDGPU::SReg_32RegClass.contains(SrcReg) ||
516
38.7k
           AMDGPU::AGPR_32RegClass.contains(SrcReg));
517
38.7k
    unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
518
38.7k
                     
AMDGPU::V_ACCVGPR_READ_B3215
: AMDGPU::V_MOV_B32_e32;
519
38.7k
    BuildMI(MBB, MI, DL, get(Opc), DestReg)
520
38.7k
      .addReg(SrcReg, getKillRegState(KillSrc));
521
38.7k
    return;
522
38.7k
  }
523
29.9k
524
29.9k
  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
525
29.9k
      RC == &AMDGPU::SReg_32RegClass) {
526
19.6k
    if (SrcReg == AMDGPU::SCC) {
527
2
      BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
528
2
          .addImm(-1)
529
2
          .addImm(0);
530
2
      return;
531
2
    }
532
19.6k
533
19.6k
    if (DestReg == AMDGPU::VCC_LO) {
534
1
      if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
535
1
        BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
536
1
          .addReg(SrcReg, getKillRegState(KillSrc));
537
1
      } else {
538
0
        // FIXME: Hack until VReg_1 removed.
539
0
        assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
540
0
        BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
541
0
          .addImm(0)
542
0
          .addReg(SrcReg, getKillRegState(KillSrc));
543
0
      }
544
1
545
1
      return;
546
1
    }
547
19.6k
548
19.6k
    if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
549
4
      reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
550
4
      return;
551
4
    }
552
19.6k
553
19.6k
    BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
554
19.6k
            .addReg(SrcReg, getKillRegState(KillSrc));
555
19.6k
    return;
556
19.6k
  }
557
10.2k
558
10.2k
  if (RC == &AMDGPU::SReg_64RegClass) {
559
2.17k
    if (DestReg == AMDGPU::VCC) {
560
25
      if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
561
25
        BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
562
25
          .addReg(SrcReg, getKillRegState(KillSrc));
563
25
      } else {
564
0
        // FIXME: Hack until VReg_1 removed.
565
0
        assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
566
0
        BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
567
0
          .addImm(0)
568
0
          .addReg(SrcReg, getKillRegState(KillSrc));
569
0
      }
570
25
571
25
      return;
572
25
    }
573
2.14k
574
2.14k
    if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
575
4
      reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
576
4
      return;
577
4
    }
578
2.14k
579
2.14k
    BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
580
2.14k
            .addReg(SrcReg, getKillRegState(KillSrc));
581
2.14k
    return;
582
2.14k
  }
583
8.11k
584
8.11k
  if (DestReg == AMDGPU::SCC) {
585
0
    assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
586
0
    BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
587
0
      .addReg(SrcReg, getKillRegState(KillSrc))
588
0
      .addImm(0);
589
0
    return;
590
0
  }
591
8.11k
592
8.11k
  if (RC == &AMDGPU::AGPR_32RegClass) {
593
309
    assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
594
309
           AMDGPU::SReg_32RegClass.contains(SrcReg) ||
595
309
           AMDGPU::AGPR_32RegClass.contains(SrcReg));
596
309
    if (!AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
597
308
      // First try to find defining accvgpr_write to avoid temporary registers.
598
4.25k
      for (auto Def = MI, E = MBB.begin(); Def != E; ) {
599
4.25k
        --Def;
600
4.25k
        if (!Def->definesRegister(SrcReg, &RI))
601
3.94k
          continue;
602
305
        if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32)
603
252
          break;
604
53
605
53
        MachineOperand &DefOp = Def->getOperand(1);
606
53
        assert(DefOp.isReg() || DefOp.isImm());
607
53
608
53
        if (DefOp.isReg()) {
609
5
          // Check that register source operand if not clobbered before MI.
610
5
          // Immediate operands are always safe to propagate.
611
5
          bool SafeToPropagate = true;
612
18
          for (auto I = Def; I != MI && 
SafeToPropagate13
;
++I13
)
613
13
            if (I->modifiesRegister(DefOp.getReg(), &RI))
614
0
              SafeToPropagate = false;
615
5
616
5
          if (!SafeToPropagate)
617
0
            break;
618
5
619
5
          DefOp.setIsKill(false);
620
5
        }
621
53
622
53
        BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
623
53
          .add(DefOp);
624
53
        return;
625
53
      }
626
308
627
308
      RegScavenger RS;
628
255
      RS.enterBasicBlock(MBB);
629
255
      RS.forward(MI);
630
255
631
255
      // Ideally we want to have three registers for a long reg_sequence copy
632
255
      // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
633
255
      unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
634
255
                                                 *MBB.getParent());
635
255
636
255
      // Registers in the sequence are allocated contiguously so we can just
637
255
      // use register number to pick one of three round-robin temps.
638
255
      unsigned RegNo = DestReg % 3;
639
255
      unsigned Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
640
255
      if (!Tmp)
641
0
        report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
642
255
      RS.setRegUsed(Tmp);
643
255
      // Only loop through if there are any free registers left, otherwise
644
255
      // scavenger may report a fatal error without emergency spill slot
645
255
      // or spill with the slot.
646
510
      while (RegNo-- && 
RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)256
) {
647
255
        unsigned Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
648
255
        if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
649
0
          break;
650
255
        Tmp = Tmp2;
651
255
        RS.setRegUsed(Tmp);
652
255
      }
653
255
      copyPhysReg(MBB, MI, DL, Tmp, SrcReg, KillSrc);
654
255
      BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
655
255
        .addReg(Tmp, RegState::Kill);
656
255
      return;
657
255
    }
658
1
659
1
    BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
660
1
      .addReg(SrcReg, getKillRegState(KillSrc));
661
1
    return;
662
1
  }
663
7.80k
664
7.80k
  unsigned EltSize = 4;
665
7.80k
  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
666
7.80k
  if (RI.isSGPRClass(RC)) {
667
254
    // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
668
254
    if (!(RI.getRegSizeInBits(*RC) % 64)) {
669
254
      Opcode =  AMDGPU::S_MOV_B64;
670
254
      EltSize = 8;
671
254
    } else {
672
0
      Opcode = AMDGPU::S_MOV_B32;
673
0
      EltSize = 4;
674
0
    }
675
254
676
254
    if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
677
6
      reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
678
6
      return;
679
6
    }
680
7.54k
  } else if (RI.hasAGPRs(RC)) {
681
31
    Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ?
682
28
      
AMDGPU::V_ACCVGPR_WRITE_B323
: AMDGPU::COPY;
683
7.51k
  } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) {
684
21
    Opcode = AMDGPU::V_ACCVGPR_READ_B32;
685
21
  }
686
7.80k
687
7.80k
  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
688
7.79k
  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
689
7.79k
690
24.2k
  for (unsigned Idx = 0; Idx < SubIndices.size(); 
++Idx16.4k
) {
691
16.4k
    unsigned SubIdx;
692
16.4k
    if (Forward)
693
11.9k
      SubIdx = SubIndices[Idx];
694
4.47k
    else
695
4.47k
      SubIdx = SubIndices[SubIndices.size() - Idx - 1];
696
16.4k
697
16.4k
    if (Opcode == TargetOpcode::COPY) {
698
252
      copyPhysReg(MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
699
252
                  RI.getSubReg(SrcReg, SubIdx), KillSrc);
700
252
      continue;
701
252
    }
702
16.2k
703
16.2k
    MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
704
16.2k
      get(Opcode), RI.getSubReg(DestReg, SubIdx));
705
16.2k
706
16.2k
    Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
707
16.2k
708
16.2k
    if (Idx == 0)
709
7.76k
      Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
710
16.2k
711
16.2k
    bool UseKill = KillSrc && 
Idx == SubIndices.size() - 18.52k
;
712
16.2k
    Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
713
16.2k
  }
714
7.79k
}
715
716
376k
int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
717
376k
  int NewOpc;
718
376k
719
376k
  // Try to map original to commuted opcode
720
376k
  NewOpc = AMDGPU::getCommuteRev(Opcode);
721
376k
  if (NewOpc != -1)
722
19.8k
    // Check if the commuted (REV) opcode exists on the target.
723
19.8k
    return pseudoToMCOpcode(NewOpc) != -1 ? 
NewOpc19.8k
:
-12
;
724
356k
725
356k
  // Try to map commuted to original opcode
726
356k
  NewOpc = AMDGPU::getCommuteOrig(Opcode);
727
356k
  if (NewOpc != -1)
728
47.8k
    // Check if the original (non-REV) opcode exists on the target.
729
47.8k
    return pseudoToMCOpcode(NewOpc) != -1 ? 
NewOpc24.3k
:
-123.4k
;
730
309k
731
309k
  return Opcode;
732
309k
}
733
734
void SIInstrInfo::materializeImmediate(MachineBasicBlock &MBB,
735
                                       MachineBasicBlock::iterator MI,
736
                                       const DebugLoc &DL, unsigned DestReg,
737
0
                                       int64_t Value) const {
738
0
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
739
0
  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
740
0
  if (RegClass == &AMDGPU::SReg_32RegClass ||
741
0
      RegClass == &AMDGPU::SGPR_32RegClass ||
742
0
      RegClass == &AMDGPU::SReg_32_XM0RegClass ||
743
0
      RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
744
0
    BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
745
0
      .addImm(Value);
746
0
    return;
747
0
  }
748
0
749
0
  if (RegClass == &AMDGPU::SReg_64RegClass ||
750
0
      RegClass == &AMDGPU::SGPR_64RegClass ||
751
0
      RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
752
0
    BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
753
0
      .addImm(Value);
754
0
    return;
755
0
  }
756
0
757
0
  if (RegClass == &AMDGPU::VGPR_32RegClass) {
758
0
    BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
759
0
      .addImm(Value);
760
0
    return;
761
0
  }
762
0
  if (RegClass == &AMDGPU::VReg_64RegClass) {
763
0
    BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
764
0
      .addImm(Value);
765
0
    return;
766
0
  }
767
0
768
0
  unsigned EltSize = 4;
769
0
  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
770
0
  if (RI.isSGPRClass(RegClass)) {
771
0
    if (RI.getRegSizeInBits(*RegClass) > 32) {
772
0
      Opcode =  AMDGPU::S_MOV_B64;
773
0
      EltSize = 8;
774
0
    } else {
775
0
      Opcode = AMDGPU::S_MOV_B32;
776
0
      EltSize = 4;
777
0
    }
778
0
  }
779
0
780
0
  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
781
0
  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
782
0
    int64_t IdxValue = Idx == 0 ? Value : 0;
783
0
784
0
    MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
785
0
      get(Opcode), RI.getSubReg(DestReg, Idx));
786
0
    Builder.addImm(IdxValue);
787
0
  }
788
0
}
789
790
const TargetRegisterClass *
791
0
SIInstrInfo::getPreferredSelectRegClass(unsigned Size) const {
792
0
  return &AMDGPU::VGPR_32RegClass;
793
0
}
794
795
void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB,
796
                                     MachineBasicBlock::iterator I,
797
                                     const DebugLoc &DL, unsigned DstReg,
798
                                     ArrayRef<MachineOperand> Cond,
799
                                     unsigned TrueReg,
800
0
                                     unsigned FalseReg) const {
801
0
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
802
0
  MachineFunction *MF = MBB.getParent();
803
0
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
804
0
  const TargetRegisterClass *BoolXExecRC =
805
0
    RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
806
0
  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
807
0
         "Not a VGPR32 reg");
808
0
809
0
  if (Cond.size() == 1) {
810
0
    unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
811
0
    BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
812
0
      .add(Cond[0]);
813
0
    BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
814
0
      .addImm(0)
815
0
      .addReg(FalseReg)
816
0
      .addImm(0)
817
0
      .addReg(TrueReg)
818
0
      .addReg(SReg);
819
0
  } else if (Cond.size() == 2) {
820
0
    assert(Cond[0].isImm() && "Cond[0] is not an immediate");
821
0
    switch (Cond[0].getImm()) {
822
0
    case SIInstrInfo::SCC_TRUE: {
823
0
      unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
824
0
      BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
825
0
                                            : AMDGPU::S_CSELECT_B64), SReg)
826
0
        .addImm(-1)
827
0
        .addImm(0);
828
0
      BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
829
0
        .addImm(0)
830
0
        .addReg(FalseReg)
831
0
        .addImm(0)
832
0
        .addReg(TrueReg)
833
0
        .addReg(SReg);
834
0
      break;
835
0
    }
836
0
    case SIInstrInfo::SCC_FALSE: {
837
0
      unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
838
0
      BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
839
0
                                            : AMDGPU::S_CSELECT_B64), SReg)
840
0
        .addImm(0)
841
0
        .addImm(-1);
842
0
      BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
843
0
        .addImm(0)
844
0
        .addReg(FalseReg)
845
0
        .addImm(0)
846
0
        .addReg(TrueReg)
847
0
        .addReg(SReg);
848
0
      break;
849
0
    }
850
0
    case SIInstrInfo::VCCNZ: {
851
0
      MachineOperand RegOp = Cond[1];
852
0
      RegOp.setImplicit(false);
853
0
      unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
854
0
      BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
855
0
        .add(RegOp);
856
0
      BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
857
0
          .addImm(0)
858
0
          .addReg(FalseReg)
859
0
          .addImm(0)
860
0
          .addReg(TrueReg)
861
0
          .addReg(SReg);
862
0
      break;
863
0
    }
864
0
    case SIInstrInfo::VCCZ: {
865
0
      MachineOperand RegOp = Cond[1];
866
0
      RegOp.setImplicit(false);
867
0
      unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
868
0
      BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
869
0
        .add(RegOp);
870
0
      BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
871
0
          .addImm(0)
872
0
          .addReg(TrueReg)
873
0
          .addImm(0)
874
0
          .addReg(FalseReg)
875
0
          .addReg(SReg);
876
0
      break;
877
0
    }
878
0
    case SIInstrInfo::EXECNZ: {
879
0
      unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
880
0
      unsigned SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
881
0
      BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
882
0
                                            : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
883
0
        .addImm(0);
884
0
      BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
885
0
                                            : AMDGPU::S_CSELECT_B64), SReg)
886
0
        .addImm(-1)
887
0
        .addImm(0);
888
0
      BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
889
0
        .addImm(0)
890
0
        .addReg(FalseReg)
891
0
        .addImm(0)
892
0
        .addReg(TrueReg)
893
0
        .addReg(SReg);
894
0
      break;
895
0
    }
896
0
    case SIInstrInfo::EXECZ: {
897
0
      unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
898
0
      unsigned SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
899
0
      BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
900
0
                                            : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
901
0
        .addImm(0);
902
0
      BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
903
0
                                            : AMDGPU::S_CSELECT_B64), SReg)
904
0
        .addImm(0)
905
0
        .addImm(-1);
906
0
      BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
907
0
        .addImm(0)
908
0
        .addReg(FalseReg)
909
0
        .addImm(0)
910
0
        .addReg(TrueReg)
911
0
        .addReg(SReg);
912
0
      llvm_unreachable("Unhandled branch predicate EXECZ");
913
0
      break;
914
0
    }
915
0
    default:
916
0
      llvm_unreachable("invalid branch predicate");
917
0
    }
918
0
  } else {
919
0
    llvm_unreachable("Can only handle Cond size 1 or 2");
920
0
  }
921
0
}
922
923
unsigned SIInstrInfo::insertEQ(MachineBasicBlock *MBB,
924
                               MachineBasicBlock::iterator I,
925
                               const DebugLoc &DL,
926
0
                               unsigned SrcReg, int Value) const {
927
0
  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
928
0
  unsigned Reg = MRI.createVirtualRegister(RI.getBoolRC());
929
0
  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
930
0
    .addImm(Value)
931
0
    .addReg(SrcReg);
932
0
933
0
  return Reg;
934
0
}
935
936
unsigned SIInstrInfo::insertNE(MachineBasicBlock *MBB,
937
                               MachineBasicBlock::iterator I,
938
                               const DebugLoc &DL,
939
0
                               unsigned SrcReg, int Value) const {
940
0
  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
941
0
  unsigned Reg = MRI.createVirtualRegister(RI.getBoolRC());
942
0
  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
943
0
    .addImm(Value)
944
0
    .addReg(SrcReg);
945
0
946
0
  return Reg;
947
0
}
948
949
14.5k
unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
950
14.5k
951
14.5k
  if (RI.hasAGPRs(DstRC))
952
0
    return AMDGPU::COPY;
953
14.5k
  if (RI.getRegSizeInBits(*DstRC) == 32) {
954
14.0k
    return RI.isSGPRClass(DstRC) ? 
AMDGPU::S_MOV_B326.24k
:
AMDGPU::V_MOV_B32_e327.79k
;
955
14.0k
  } else 
if (484
RI.getRegSizeInBits(*DstRC) == 64484
&&
RI.isSGPRClass(DstRC)484
) {
956
10
    return AMDGPU::S_MOV_B64;
957
474
  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
958
474
    return  AMDGPU::V_MOV_B64_PSEUDO;
959
474
  }
960
0
  return AMDGPU::COPY;
961
0
}
962
963
895
static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
964
895
  switch (Size) {
965
895
  case 4:
966
516
    return AMDGPU::SI_SPILL_S32_SAVE;
967
895
  case 8:
968
241
    return AMDGPU::SI_SPILL_S64_SAVE;
969
895
  case 12:
970
3
    return AMDGPU::SI_SPILL_S96_SAVE;
971
895
  case 16:
972
91
    return AMDGPU::SI_SPILL_S128_SAVE;
973
895
  case 20:
974
3
    return AMDGPU::SI_SPILL_S160_SAVE;
975
895
  case 32:
976
33
    return AMDGPU::SI_SPILL_S256_SAVE;
977
895
  case 64:
978
8
    return AMDGPU::SI_SPILL_S512_SAVE;
979
895
  case 128:
980
0
    return AMDGPU::SI_SPILL_S1024_SAVE;
981
895
  default:
982
0
    llvm_unreachable("unknown register size");
983
895
  }
984
895
}
985
986
1.33k
static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
987
1.33k
  switch (Size) {
988
1.33k
  case 4:
989
577
    return AMDGPU::SI_SPILL_V32_SAVE;
990
1.33k
  case 8:
991
42
    return AMDGPU::SI_SPILL_V64_SAVE;
992
1.33k
  case 12:
993
1
    return AMDGPU::SI_SPILL_V96_SAVE;
994
1.33k
  case 16:
995
693
    return AMDGPU::SI_SPILL_V128_SAVE;
996
1.33k
  case 20:
997
0
    return AMDGPU::SI_SPILL_V160_SAVE;
998
1.33k
  case 32:
999
0
    return AMDGPU::SI_SPILL_V256_SAVE;
1000
1.33k
  case 64:
1001
10
    return AMDGPU::SI_SPILL_V512_SAVE;
1002
1.33k
  case 128:
1003
9
    return AMDGPU::SI_SPILL_V1024_SAVE;
1004
1.33k
  default:
1005
0
    llvm_unreachable("unknown register size");
1006
1.33k
  }
1007
1.33k
}
1008
1009
14
static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1010
14
  switch (Size) {
1011
14
  case 4:
1012
12
    return AMDGPU::SI_SPILL_A32_SAVE;
1013
14
  case 8:
1014
0
    return AMDGPU::SI_SPILL_A64_SAVE;
1015
14
  case 16:
1016
0
    return AMDGPU::SI_SPILL_A128_SAVE;
1017
14
  case 64:
1018
2
    return AMDGPU::SI_SPILL_A512_SAVE;
1019
14
  case 128:
1020
0
    return AMDGPU::SI_SPILL_A1024_SAVE;
1021
14
  default:
1022
0
    llvm_unreachable("unknown register size");
1023
14
  }
1024
14
}
1025
1026
void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
1027
                                      MachineBasicBlock::iterator MI,
1028
                                      unsigned SrcReg, bool isKill,
1029
                                      int FrameIndex,
1030
                                      const TargetRegisterClass *RC,
1031
2.24k
                                      const TargetRegisterInfo *TRI) const {
1032
2.24k
  MachineFunction *MF = MBB.getParent();
1033
2.24k
  SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1034
2.24k
  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1035
2.24k
  const DebugLoc &DL = MBB.findDebugLoc(MI);
1036
2.24k
1037
2.24k
  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1038
2.24k
  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1039
2.24k
  MachinePointerInfo PtrInfo
1040
2.24k
    = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1041
2.24k
  MachineMemOperand *MMO
1042
2.24k
    = MF->getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
1043
2.24k
                               Size, Align);
1044
2.24k
  unsigned SpillSize = TRI->getSpillSize(*RC);
1045
2.24k
1046
2.24k
  if (RI.isSGPRClass(RC)) {
1047
895
    MFI->setHasSpilledSGPRs();
1048
895
1049
895
    // We are only allowed to create one new instruction when spilling
1050
895
    // registers, so we need to use pseudo instruction for spilling SGPRs.
1051
895
    const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1052
895
1053
895
    // The SGPR spill/restore instructions only work on number sgprs, so we need
1054
895
    // to make sure we are using the correct register class.
1055
895
    if (TargetRegisterInfo::isVirtualRegister(SrcReg) && 
SpillSize == 4140
) {
1056
25
      MachineRegisterInfo &MRI = MF->getRegInfo();
1057
25
      MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
1058
25
    }
1059
895
1060
895
    MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
1061
895
      .addReg(SrcReg, getKillRegState(isKill)) // data
1062
895
      .addFrameIndex(FrameIndex)               // addr
1063
895
      .addMemOperand(MMO)
1064
895
      .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
1065
895
      .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1066
895
    // Add the scratch resource registers as implicit uses because we may end up
1067
895
    // needing them, and need to ensure that the reserved registers are
1068
895
    // correctly handled.
1069
895
    if (RI.spillSGPRToVGPR())
1070
846
      FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1071
895
    if (ST.hasScalarStores()) {
1072
479
      // m0 is used for offset to scalar stores if used to spill.
1073
479
      Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1074
479
    }
1075
895
1076
895
    return;
1077
895
  }
1078
1.34k
1079
1.34k
  unsigned Opcode = RI.hasAGPRs(RC) ? 
getAGPRSpillSaveOpcode(SpillSize)14
1080
1.34k
                                    : 
getVGPRSpillSaveOpcode(SpillSize)1.33k
;
1081
1.34k
  MFI->setHasSpilledVGPRs();
1082
1.34k
1083
1.34k
  auto MIB = BuildMI(MBB, MI, DL, get(Opcode));
1084
1.34k
  if (RI.hasAGPRs(RC)) {
1085
14
    MachineRegisterInfo &MRI = MF->getRegInfo();
1086
14
    unsigned Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1087
14
    MIB.addReg(Tmp, RegState::Define);
1088
14
  }
1089
1.34k
  MIB.addReg(SrcReg, getKillRegState(isKill)) // data
1090
1.34k
     .addFrameIndex(FrameIndex)               // addr
1091
1.34k
     .addReg(MFI->getScratchRSrcReg())        // scratch_rsrc
1092
1.34k
     .addReg(MFI->getStackPtrOffsetReg())     // scratch_offset
1093
1.34k
     .addImm(0)                               // offset
1094
1.34k
     .addMemOperand(MMO);
1095
1.34k
}
1096
1097
889
static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1098
889
  switch (Size) {
1099
889
  case 4:
1100
511
    return AMDGPU::SI_SPILL_S32_RESTORE;
1101
889
  case 8:
1102
240
    return AMDGPU::SI_SPILL_S64_RESTORE;
1103
889
  case 12:
1104
3
    return AMDGPU::SI_SPILL_S96_RESTORE;
1105
889
  case 16:
1106
91
    return AMDGPU::SI_SPILL_S128_RESTORE;
1107
889
  case 20:
1108
3
    return AMDGPU::SI_SPILL_S160_RESTORE;
1109
889
  case 32:
1110
33
    return AMDGPU::SI_SPILL_S256_RESTORE;
1111
889
  case 64:
1112
8
    return AMDGPU::SI_SPILL_S512_RESTORE;
1113
889
  case 128:
1114
0
    return AMDGPU::SI_SPILL_S1024_RESTORE;
1115
889
  default:
1116
0
    llvm_unreachable("unknown register size");
1117
889
  }
1118
889
}
1119
1120
1.28k
static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1121
1.28k
  switch (Size) {
1122
1.28k
  case 4:
1123
519
    return AMDGPU::SI_SPILL_V32_RESTORE;
1124
1.28k
  case 8:
1125
52
    return AMDGPU::SI_SPILL_V64_RESTORE;
1126
1.28k
  case 12:
1127
2
    return AMDGPU::SI_SPILL_V96_RESTORE;
1128
1.28k
  case 16:
1129
695
    return AMDGPU::SI_SPILL_V128_RESTORE;
1130
1.28k
  case 20:
1131
0
    return AMDGPU::SI_SPILL_V160_RESTORE;
1132
1.28k
  case 32:
1133
0
    return AMDGPU::SI_SPILL_V256_RESTORE;
1134
1.28k
  case 64:
1135
10
    return AMDGPU::SI_SPILL_V512_RESTORE;
1136
1.28k
  case 128:
1137
9
    return AMDGPU::SI_SPILL_V1024_RESTORE;
1138
1.28k
  default:
1139
0
    llvm_unreachable("unknown register size");
1140
1.28k
  }
1141
1.28k
}
1142
1143
14
static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1144
14
  switch (Size) {
1145
14
  case 4:
1146
12
    return AMDGPU::SI_SPILL_A32_RESTORE;
1147
14
  case 8:
1148
0
    return AMDGPU::SI_SPILL_A64_RESTORE;
1149
14
  case 16:
1150
0
    return AMDGPU::SI_SPILL_A128_RESTORE;
1151
14
  case 64:
1152
2
    return AMDGPU::SI_SPILL_A512_RESTORE;
1153
14
  case 128:
1154
0
    return AMDGPU::SI_SPILL_A1024_RESTORE;
1155
14
  default:
1156
0
    llvm_unreachable("unknown register size");
1157
14
  }
1158
14
}
1159
1160
void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
1161
                                       MachineBasicBlock::iterator MI,
1162
                                       unsigned DestReg, int FrameIndex,
1163
                                       const TargetRegisterClass *RC,
1164
2.19k
                                       const TargetRegisterInfo *TRI) const {
1165
2.19k
  MachineFunction *MF = MBB.getParent();
1166
2.19k
  SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1167
2.19k
  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1168
2.19k
  const DebugLoc &DL = MBB.findDebugLoc(MI);
1169
2.19k
  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1170
2.19k
  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1171
2.19k
  unsigned SpillSize = TRI->getSpillSize(*RC);
1172
2.19k
1173
2.19k
  MachinePointerInfo PtrInfo
1174
2.19k
    = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1175
2.19k
1176
2.19k
  MachineMemOperand *MMO = MF->getMachineMemOperand(
1177
2.19k
    PtrInfo, MachineMemOperand::MOLoad, Size, Align);
1178
2.19k
1179
2.19k
  if (RI.isSGPRClass(RC)) {
1180
889
    MFI->setHasSpilledSGPRs();
1181
889
1182
889
    // FIXME: Maybe this should not include a memoperand because it will be
1183
889
    // lowered to non-memory instructions.
1184
889
    const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1185
889
    if (TargetRegisterInfo::isVirtualRegister(DestReg) && 
SpillSize == 4251
) {
1186
25
      MachineRegisterInfo &MRI = MF->getRegInfo();
1187
25
      MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1188
25
    }
1189
889
1190
889
    if (RI.spillSGPRToVGPR())
1191
839
      FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1192
889
    MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
1193
889
      .addFrameIndex(FrameIndex) // addr
1194
889
      .addMemOperand(MMO)
1195
889
      .addReg(MFI->getScratchRSrcReg(), RegState::Implicit)
1196
889
      .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1197
889
1198
889
    if (ST.hasScalarStores()) {
1199
478
      // m0 is used for offset to scalar stores if used to spill.
1200
478
      Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1201
478
    }
1202
889
1203
889
    return;
1204
889
  }
1205
1.30k
1206
1.30k
  unsigned Opcode = RI.hasAGPRs(RC) ? 
getAGPRSpillRestoreOpcode(SpillSize)14
1207
1.30k
                                    : 
getVGPRSpillRestoreOpcode(SpillSize)1.28k
;
1208
1.30k
  auto MIB = BuildMI(MBB, MI, DL, get(Opcode), DestReg);
1209
1.30k
  if (RI.hasAGPRs(RC)) {
1210
14
    MachineRegisterInfo &MRI = MF->getRegInfo();
1211
14
    unsigned Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1212
14
    MIB.addReg(Tmp, RegState::Define);
1213
14
  }
1214
1.30k
  MIB.addFrameIndex(FrameIndex)        // vaddr
1215
1.30k
     .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1216
1.30k
     .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1217
1.30k
     .addImm(0)                           // offset
1218
1.30k
     .addMemOperand(MMO);
1219
1.30k
}
1220
1221
/// \param @Offset Offset in bytes of the FrameIndex being spilled
1222
unsigned SIInstrInfo::calculateLDSSpillAddress(
1223
    MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1224
0
    unsigned FrameOffset, unsigned Size) const {
1225
0
  MachineFunction *MF = MBB.getParent();
1226
0
  SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
1227
0
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1228
0
  const DebugLoc &DL = MBB.findDebugLoc(MI);
1229
0
  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1230
0
  unsigned WavefrontSize = ST.getWavefrontSize();
1231
0
1232
0
  unsigned TIDReg = MFI->getTIDReg();
1233
0
  if (!MFI->hasCalculatedTID()) {
1234
0
    MachineBasicBlock &Entry = MBB.getParent()->front();
1235
0
    MachineBasicBlock::iterator Insert = Entry.front();
1236
0
    const DebugLoc &DL = Insert->getDebugLoc();
1237
0
1238
0
    TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1239
0
                                   *MF);
1240
0
    if (TIDReg == AMDGPU::NoRegister)
1241
0
      return TIDReg;
1242
0
1243
0
    if (!AMDGPU::isShader(MF->getFunction().getCallingConv()) &&
1244
0
        WorkGroupSize > WavefrontSize) {
1245
0
      unsigned TIDIGXReg
1246
0
        = MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
1247
0
      unsigned TIDIGYReg
1248
0
        = MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
1249
0
      unsigned TIDIGZReg
1250
0
        = MFI->getPreloadedReg(AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
1251
0
      unsigned InputPtrReg =
1252
0
          MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1253
0
      for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1254
0
        if (!Entry.isLiveIn(Reg))
1255
0
          Entry.addLiveIn(Reg);
1256
0
      }
1257
0
1258
0
      RS->enterBasicBlock(Entry);
1259
0
      // FIXME: Can we scavenge an SReg_64 and access the subregs?
1260
0
      unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1261
0
      unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1262
0
      BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1263
0
              .addReg(InputPtrReg)
1264
0
              .addImm(SI::KernelInputOffsets::NGROUPS_Z);
1265
0
      BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1266
0
              .addReg(InputPtrReg)
1267
0
              .addImm(SI::KernelInputOffsets::NGROUPS_Y);
1268
0
1269
0
      // NGROUPS.X * NGROUPS.Y
1270
0
      BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1271
0
              .addReg(STmp1)
1272
0
              .addReg(STmp0);
1273
0
      // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1274
0
      BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1275
0
              .addReg(STmp1)
1276
0
              .addReg(TIDIGXReg);
1277
0
      // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1278
0
      BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1279
0
              .addReg(STmp0)
1280
0
              .addReg(TIDIGYReg)
1281
0
              .addReg(TIDReg);
1282
0
      // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1283
0
      getAddNoCarry(Entry, Insert, DL, TIDReg)
1284
0
        .addReg(TIDReg)
1285
0
        .addReg(TIDIGZReg)
1286
0
        .addImm(0); // clamp bit
1287
0
    } else {
1288
0
      // Get the wave id
1289
0
      BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1290
0
              TIDReg)
1291
0
              .addImm(-1)
1292
0
              .addImm(0);
1293
0
1294
0
      BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1295
0
              TIDReg)
1296
0
              .addImm(-1)
1297
0
              .addReg(TIDReg);
1298
0
    }
1299
0
1300
0
    BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1301
0
            TIDReg)
1302
0
            .addImm(2)
1303
0
            .addReg(TIDReg);
1304
0
    MFI->setTIDReg(TIDReg);
1305
0
  }
1306
0
1307
0
  // Add FrameIndex to LDS offset
1308
0
  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1309
0
  getAddNoCarry(MBB, MI, DL, TmpReg)
1310
0
    .addImm(LDSOffset)
1311
0
    .addReg(TIDReg)
1312
0
    .addImm(0); // clamp bit
1313
0
1314
0
  return TmpReg;
1315
0
}
1316
1317
void SIInstrInfo::insertWaitStates(MachineBasicBlock &MBB,
1318
                                   MachineBasicBlock::iterator MI,
1319
3.87k
                                   int Count) const {
1320
3.87k
  DebugLoc DL = MBB.findDebugLoc(MI);
1321
7.75k
  while (Count > 0) {
1322
3.87k
    int Arg;
1323
3.87k
    if (Count >= 8)
1324
0
      Arg = 7;
1325
3.87k
    else
1326
3.87k
      Arg = Count - 1;
1327
3.87k
    Count -= 8;
1328
3.87k
    BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1329
3.87k
            .addImm(Arg);
1330
3.87k
  }
1331
3.87k
}
1332
1333
void SIInstrInfo::insertNoop(MachineBasicBlock &MBB,
1334
3.87k
                             MachineBasicBlock::iterator MI) const {
1335
3.87k
  insertWaitStates(MBB, MI, 1);
1336
3.87k
}
1337
1338
0
void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
1339
0
  auto MF = MBB.getParent();
1340
0
  SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1341
0
1342
0
  assert(Info->isEntryFunction());
1343
0
1344
0
  if (MBB.succ_empty()) {
1345
0
    bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1346
0
    if (HasNoTerminator) {
1347
0
      if (Info->returnsVoid()) {
1348
0
        BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1349
0
      } else {
1350
0
        BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1351
0
      }
1352
0
    }
1353
0
  }
1354
0
}
1355
1356
1.42M
unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
1357
1.42M
  switch (MI.getOpcode()) {
1358
1.42M
  
default: return 11.40M
; // FIXME: Do wait states equal cycles?
1359
1.42M
1360
1.42M
  case AMDGPU::S_NOP:
1361
15.4k
    return MI.getOperand(0).getImm() + 1;
1362
1.42M
  }
1363
1.42M
}
1364
1365
338k
bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
1366
338k
  MachineBasicBlock &MBB = *MI.getParent();
1367
338k
  DebugLoc DL = MBB.findDebugLoc(MI);
1368
338k
  switch (MI.getOpcode()) {
1369
338k
  
default: return TargetInstrInfo::expandPostRAPseudo(MI)334k
;
1370
338k
  case AMDGPU::S_MOV_B64_term:
1371
19
    // This is only a terminator to get the correct spill code placement during
1372
19
    // register allocation.
1373
19
    MI.setDesc(get(AMDGPU::S_MOV_B64));
1374
19
    break;
1375
338k
1376
338k
  case AMDGPU::S_MOV_B32_term:
1377
0
    // This is only a terminator to get the correct spill code placement during
1378
0
    // register allocation.
1379
0
    MI.setDesc(get(AMDGPU::S_MOV_B32));
1380
0
    break;
1381
338k
1382
338k
  case AMDGPU::S_XOR_B64_term:
1383
17
    // This is only a terminator to get the correct spill code placement during
1384
17
    // register allocation.
1385
17
    MI.setDesc(get(AMDGPU::S_XOR_B64));
1386
17
    break;
1387
338k
1388
338k
  case AMDGPU::S_XOR_B32_term:
1389
0
    // This is only a terminator to get the correct spill code placement during
1390
0
    // register allocation.
1391
0
    MI.setDesc(get(AMDGPU::S_XOR_B32));
1392
0
    break;
1393
338k
1394
338k
  case AMDGPU::S_OR_B32_term:
1395
0
    // This is only a terminator to get the correct spill code placement during
1396
0
    // register allocation.
1397
0
    MI.setDesc(get(AMDGPU::S_OR_B32));
1398
0
    break;
1399
338k
1400
338k
  case AMDGPU::S_ANDN2_B64_term:
1401
0
    // This is only a terminator to get the correct spill code placement during
1402
0
    // register allocation.
1403
0
    MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1404
0
    break;
1405
338k
1406
338k
  case AMDGPU::S_ANDN2_B32_term:
1407
0
    // This is only a terminator to get the correct spill code placement during
1408
0
    // register allocation.
1409
0
    MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1410
0
    break;
1411
338k
1412
338k
  case AMDGPU::V_MOV_B64_PSEUDO: {
1413
377
    unsigned Dst = MI.getOperand(0).getReg();
1414
377
    unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1415
377
    unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1416
377
1417
377
    const MachineOperand &SrcOp = MI.getOperand(1);
1418
377
    // FIXME: Will this work for 64-bit floating point immediates?
1419
377
    assert(!SrcOp.isFPImm());
1420
377
    if (SrcOp.isImm()) {
1421
362
      APInt Imm(64, SrcOp.getImm());
1422
362
      BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1423
362
        .addImm(Imm.getLoBits(32).getZExtValue())
1424
362
        .addReg(Dst, RegState::Implicit | RegState::Define);
1425
362
      BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1426
362
        .addImm(Imm.getHiBits(32).getZExtValue())
1427
362
        .addReg(Dst, RegState::Implicit | RegState::Define);
1428
362
    } else {
1429
15
      assert(SrcOp.isReg());
1430
15
      BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1431
15
        .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1432
15
        .addReg(Dst, RegState::Implicit | RegState::Define);
1433
15
      BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1434
15
        .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1435
15
        .addReg(Dst, RegState::Implicit | RegState::Define);
1436
15
    }
1437
377
    MI.eraseFromParent();
1438
377
    break;
1439
338k
  }
1440
338k
  case AMDGPU::V_SET_INACTIVE_B32: {
1441
60
    unsigned NotOpc = ST.isWave32() ? 
AMDGPU::S_NOT_B323
:
AMDGPU::S_NOT_B6457
;
1442
60
    unsigned Exec = ST.isWave32() ? 
AMDGPU::EXEC_LO3
:
AMDGPU::EXEC57
;
1443
60
    BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1444
60
      .addReg(Exec);
1445
60
    BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1446
60
      .add(MI.getOperand(2));
1447
60
    BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1448
60
      .addReg(Exec);
1449
60
    MI.eraseFromParent();
1450
60
    break;
1451
338k
  }
1452
338k
  case AMDGPU::V_SET_INACTIVE_B64: {
1453
15
    unsigned NotOpc = ST.isWave32() ? 
AMDGPU::S_NOT_B323
:
AMDGPU::S_NOT_B6412
;
1454
15
    unsigned Exec = ST.isWave32() ? 
AMDGPU::EXEC_LO3
:
AMDGPU::EXEC12
;
1455
15
    BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1456
15
      .addReg(Exec);
1457
15
    MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1458
15
                                 MI.getOperand(0).getReg())
1459
15
      .add(MI.getOperand(2));
1460
15
    expandPostRAPseudo(*Copy);
1461
15
    BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1462
15
      .addReg(Exec);
1463
15
    MI.eraseFromParent();
1464
15
    break;
1465
338k
  }
1466
338k
  case AMDGPU::V_MOVRELD_B32_V1:
1467
38
  case AMDGPU::V_MOVRELD_B32_V2:
1468
38
  case AMDGPU::V_MOVRELD_B32_V4:
1469
38
  case AMDGPU::V_MOVRELD_B32_V8:
1470
38
  case AMDGPU::V_MOVRELD_B32_V16: {
1471
38
    const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1472
38
    unsigned VecReg = MI.getOperand(0).getReg();
1473
38
    bool IsUndef = MI.getOperand(1).isUndef();
1474
38
    unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1475
38
    assert(VecReg == MI.getOperand(1).getReg());
1476
38
1477
38
    MachineInstr *MovRel =
1478
38
        BuildMI(MBB, MI, DL, MovRelDesc)
1479
38
            .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1480
38
            .add(MI.getOperand(2))
1481
38
            .addReg(VecReg, RegState::ImplicitDefine)
1482
38
            .addReg(VecReg,
1483
38
                    RegState::Implicit | (IsUndef ? 
RegState::Undef0
: 0));
1484
38
1485
38
    const int ImpDefIdx =
1486
38
        MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1487
38
    const int ImpUseIdx = ImpDefIdx + 1;
1488
38
    MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1489
38
1490
38
    MI.eraseFromParent();
1491
38
    break;
1492
38
  }
1493
738
  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1494
738
    MachineFunction &MF = *MBB.getParent();
1495
738
    unsigned Reg = MI.getOperand(0).getReg();
1496
738
    unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1497
738
    unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1498
738
1499
738
    // Create a bundle so these instructions won't be re-ordered by the
1500
738
    // post-RA scheduler.
1501
738
    MIBundleBuilder Bundler(MBB, MI);
1502
738
    Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1503
738
1504
738
    // Add 32-bit offset from this instruction to the start of the
1505
738
    // constant data.
1506
738
    Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1507
738
                       .addReg(RegLo)
1508
738
                       .add(MI.getOperand(1)));
1509
738
1510
738
    MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1511
738
                                  .addReg(RegHi);
1512
738
    MIB.add(MI.getOperand(2));
1513
738
1514
738
    Bundler.append(MIB);
1515
738
    finalizeBundle(MBB, Bundler.begin());
1516
738
1517
738
    MI.eraseFromParent();
1518
738
    break;
1519
38
  }
1520
121
  case AMDGPU::ENTER_WWM: {
1521
121
    // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1522
121
    // WWM is entered.
1523
121
    MI.setDesc(get(ST.isWave32() ? 
AMDGPU::S_OR_SAVEEXEC_B326
1524
121
                                 : 
AMDGPU::S_OR_SAVEEXEC_B64115
));
1525
121
    break;
1526
38
  }
1527
121
  case AMDGPU::EXIT_WWM: {
1528
121
    // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1529
121
    // WWM is exited.
1530
121
    MI.setDesc(get(ST.isWave32() ? 
AMDGPU::S_MOV_B326
:
AMDGPU::S_MOV_B64115
));
1531
121
    break;
1532
38
  }
1533
2.24k
  case TargetOpcode::BUNDLE: {
1534
2.24k
    if (!MI.mayLoad() || MI.hasUnmodeledSideEffects())
1535
179
      return false;
1536
2.06k
1537
2.06k
    // If it is a load it must be a memory clause
1538
2.06k
    for (MachineBasicBlock::instr_iterator I = MI.getIterator();
1539
7.15k
         I->isBundledWithSucc(); 
++I5.09k
) {
1540
5.09k
      I->unbundleFromSucc();
1541
5.09k
      for (MachineOperand &MO : I->operands())
1542
27.4k
        if (MO.isReg())
1543
15.6k
          MO.setIsInternalRead(false);
1544
5.09k
    }
1545
2.06k
1546
2.06k
    MI.eraseFromParent();
1547
2.06k
    break;
1548
2.06k
  }
1549
3.56k
  }
1550
3.56k
  return true;
1551
3.56k
}
1552
1553
bool SIInstrInfo::swapSourceModifiers(MachineInstr &MI,
1554
                                      MachineOperand &Src0,
1555
                                      unsigned Src0OpName,
1556
                                      MachineOperand &Src1,
1557
310k
                                      unsigned Src1OpName) const {
1558
310k
  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1559
310k
  if (!Src0Mods)
1560
247k
    return false;
1561
63.3k
1562
63.3k
  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1563
63.3k
  assert(Src1Mods &&
1564
63.3k
         "All commutable instructions have both src0 and src1 modifiers");
1565
63.3k
1566
63.3k
  int Src0ModsVal = Src0Mods->getImm();
1567
63.3k
  int Src1ModsVal = Src1Mods->getImm();
1568
63.3k
1569
63.3k
  Src1Mods->setImm(Src0ModsVal);
1570
63.3k
  Src0Mods->setImm(Src1ModsVal);
1571
63.3k
  return true;
1572
63.3k
}
1573
1574
static MachineInstr *swapRegAndNonRegOperand(MachineInstr &MI,
1575
                                             MachineOperand &RegOp,
1576
52.9k
                                             MachineOperand &NonRegOp) {
1577
52.9k
  unsigned Reg = RegOp.getReg();
1578
52.9k
  unsigned SubReg = RegOp.getSubReg();
1579
52.9k
  bool IsKill = RegOp.isKill();
1580
52.9k
  bool IsDead = RegOp.isDead();
1581
52.9k
  bool IsUndef = RegOp.isUndef();
1582
52.9k
  bool IsDebug = RegOp.isDebug();
1583
52.9k
1584
52.9k
  if (NonRegOp.isImm())
1585
52.9k
    RegOp.ChangeToImmediate(NonRegOp.getImm());
1586
9
  else if (NonRegOp.isFI())
1587
0
    RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1588
9
  else
1589
9
    return nullptr;
1590
52.9k
1591
52.9k
  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1592
52.9k
  NonRegOp.setSubReg(SubReg);
1593
52.9k
1594
52.9k
  return &MI;
1595
52.9k
}
1596
1597
MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
1598
                                                  unsigned Src0Idx,
1599
369k
                                                  unsigned Src1Idx) const {
1600
369k
  assert(!NewMI && "this should never be used");
1601
369k
1602
369k
  unsigned Opc = MI.getOpcode();
1603
369k
  int CommutedOpcode = commuteOpcode(Opc);
1604
369k
  if (CommutedOpcode == -1)
1605
23.4k
    return nullptr;
1606
346k
1607
346k
  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1608
346k
           static_cast<int>(Src0Idx) &&
1609
346k
         AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1610
346k
           static_cast<int>(Src1Idx) &&
1611
346k
         "inconsistency with findCommutedOpIndices");
1612
346k
1613
346k
  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1614
346k
  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1615
346k
1616
346k
  MachineInstr *CommutedMI = nullptr;
1617
346k
  if (Src0.isReg() && 
Src1.isReg()306k
) {
1618
275k
    if (isOperandLegal(MI, Src1Idx, &Src0)) {
1619
257k
      // Be sure to copy the source modifiers to the right place.
1620
257k
      CommutedMI
1621
257k
        = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1622
257k
    }
1623
275k
1624
275k
  } else 
if (71.0k
Src0.isReg()71.0k
&&
!Src1.isReg()31.0k
) {
1625
31.0k
    // src0 should always be able to support any operand type, so no need to
1626
31.0k
    // check operand legality.
1627
31.0k
    CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1628
39.9k
  } else if (!Src0.isReg() && Src1.isReg()) {
1629
39.5k
    if (isOperandLegal(MI, Src1Idx, &Src0))
1630
21.8k
      CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1631
39.5k
  } else {
1632
424
    // FIXME: Found two non registers to commute. This does happen.
1633
424
    return nullptr;
1634
424
  }
1635
346k
1636
346k
  if (CommutedMI) {
1637
310k
    swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1638
310k
                        Src1, AMDGPU::OpName::src1_modifiers);
1639
310k
1640
310k
    CommutedMI->setDesc(get(CommutedOpcode));
1641
310k
  }
1642
346k
1643
346k
  return CommutedMI;
1644
346k
}
1645
1646
// This needs to be implemented because the source modifiers may be inserted
1647
// between the true commutable operands, and the base
1648
// TargetInstrInfo::commuteInstruction uses it.
1649
bool SIInstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx0,
1650
433k
                                        unsigned &SrcOpIdx1) const {
1651
433k
  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1652
433k
}
1653
1654
bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1655
434k
                                        unsigned &SrcOpIdx1) const {
1656
434k
  if (!Desc.isCommutable())
1657
67.9k
    return false;
1658
366k
1659
366k
  unsigned Opc = Desc.getOpcode();
1660
366k
  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1661
366k
  if (Src0Idx == -1)
1662
0
    return false;
1663
366k
1664
366k
  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1665
366k
  if (Src1Idx == -1)
1666
96
    return false;
1667
366k
1668
366k
  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1669
366k
}
1670
1671
bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1672
1.96k
                                        int64_t BrOffset) const {
1673
1.96k
  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1674
1.96k
  // block is unanalyzable.
1675
1.96k
  assert(BranchOp != AMDGPU::S_SETPC_B64);
1676
1.96k
1677
1.96k
  // Convert to dwords.
1678
1.96k
  BrOffset /= 4;
1679
1.96k
1680
1.96k
  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1681
1.96k
  // from the next instruction.
1682
1.96k
  BrOffset -= 1;
1683
1.96k
1684
1.96k
  return isIntN(BranchOffsetBits, BrOffset);
1685
1.96k
}
1686
1687
MachineBasicBlock *SIInstrInfo::getBranchDestBlock(
1688
2.00k
  const MachineInstr &MI) const {
1689
2.00k
  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1690
0
    // This would be a difficult analysis to perform, but can always be legal so
1691
0
    // there's no need to analyze it.
1692
0
    return nullptr;
1693
0
  }
1694
2.00k
1695
2.00k
  return MI.getOperand(0).getMBB();
1696
2.00k
}
1697
1698
unsigned SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1699
                                           MachineBasicBlock &DestBB,
1700
                                           const DebugLoc &DL,
1701
                                           int64_t BrOffset,
1702
38
                                           RegScavenger *RS) const {
1703
38
  assert(RS && "RegScavenger required for long branching");
1704
38
  assert(MBB.empty() &&
1705
38
         "new block should be inserted for expanding unconditional branch");
1706
38
  assert(MBB.pred_size() == 1);
1707
38
1708
38
  MachineFunction *MF = MBB.getParent();
1709
38
  MachineRegisterInfo &MRI = MF->getRegInfo();
1710
38
1711
38
  // FIXME: Virtual register workaround for RegScavenger not working with empty
1712
38
  // blocks.
1713
38
  unsigned PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1714
38
1715
38
  auto I = MBB.end();
1716
38
1717
38
  // We need to compute the offset relative to the instruction immediately after
1718
38
  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1719
38
  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1720
38
1721
38
  // TODO: Handle > 32-bit block address.
1722
38
  if (BrOffset >= 0) {
1723
30
    BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1724
30
      .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1725
30
      .addReg(PCReg, 0, AMDGPU::sub0)
1726
30
      .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
1727
30
    BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1728
30
      .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1729
30
      .addReg(PCReg, 0, AMDGPU::sub1)
1730
30
      .addImm(0);
1731
30
  } else {
1732
8
    // Backwards branch.
1733
8
    BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1734
8
      .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1735
8
      .addReg(PCReg, 0, AMDGPU::sub0)
1736
8
      .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
1737
8
    BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1738
8
      .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1739
8
      .addReg(PCReg, 0, AMDGPU::sub1)
1740
8
      .addImm(0);
1741
8
  }
1742
38
1743
38
  // Insert the indirect branch after the other terminator.
1744
38
  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1745
38
    .addReg(PCReg);
1746
38
1747
38
  // FIXME: If spilling is necessary, this will fail because this scavenger has
1748
38
  // no emergency stack slots. It is non-trivial to spill in this situation,
1749
38
  // because the restore code needs to be specially placed after the
1750
38
  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1751
38
  // block.
1752
38
  //
1753
38
  // If a spill is needed for the pc register pair, we need to insert a spill
1754
38
  // restore block right before the destination block, and insert a short branch
1755
38
  // into the old destination block's fallthrough predecessor.
1756
38
  // e.g.:
1757
38
  //
1758
38
  // s_cbranch_scc0 skip_long_branch:
1759
38
  //
1760
38
  // long_branch_bb:
1761
38
  //   spill s[8:9]
1762
38
  //   s_getpc_b64 s[8:9]
1763
38
  //   s_add_u32 s8, s8, restore_bb
1764
38
  //   s_addc_u32 s9, s9, 0
1765
38
  //   s_setpc_b64 s[8:9]
1766
38
  //
1767
38
  // skip_long_branch:
1768
38
  //   foo;
1769
38
  //
1770
38
  // .....
1771
38
  //
1772
38
  // dest_bb_fallthrough_predecessor:
1773
38
  // bar;
1774
38
  // s_branch dest_bb
1775
38
  //
1776
38
  // restore_bb:
1777
38
  //  restore s[8:9]
1778
38
  //  fallthrough dest_bb
1779
38
  ///
1780
38
  // dest_bb:
1781
38
  //   buzz;
1782
38
1783
38
  RS->enterBasicBlockEnd(MBB);
1784
38
  unsigned Scav = RS->scavengeRegisterBackwards(
1785
38
    AMDGPU::SReg_64RegClass,
1786
38
    MachineBasicBlock::iterator(GetPC), false, 0);
1787
38
  MRI.replaceRegWith(PCReg, Scav);
1788
38
  MRI.clearVirtRegs();
1789
38
  RS->setRegUsed(Scav);
1790
38
1791
38
  return 4 + 8 + 4 + 4;
1792
38
}
1793
1794
1.98k
unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1795
1.98k
  switch (Cond) {
1796
1.98k
  case SIInstrInfo::SCC_TRUE:
1797
472
    return AMDGPU::S_CBRANCH_SCC1;
1798
1.98k
  case SIInstrInfo::SCC_FALSE:
1799
468
    return AMDGPU::S_CBRANCH_SCC0;
1800
1.98k
  case SIInstrInfo::VCCNZ:
1801
332
    return AMDGPU::S_CBRANCH_VCCNZ;
1802
1.98k
  case SIInstrInfo::VCCZ:
1803
315
    return AMDGPU::S_CBRANCH_VCCZ;
1804
1.98k
  case SIInstrInfo::EXECNZ:
1805
215
    return AMDGPU::S_CBRANCH_EXECNZ;
1806
1.98k
  case SIInstrInfo::EXECZ:
1807
178
    return AMDGPU::S_CBRANCH_EXECZ;
1808
1.98k
  default:
1809
0
    llvm_unreachable("invalid branch predicate");
1810
1.98k
  }
1811
1.98k
}
1812
1813
1.31M
SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1814
1.31M
  switch (Opcode) {
1815
1.31M
  case AMDGPU::S_CBRANCH_SCC0:
1816
2.91k
    return SCC_FALSE;
1817
1.31M
  case AMDGPU::S_CBRANCH_SCC1:
1818
27.2k
    return SCC_TRUE;
1819
1.31M
  case AMDGPU::S_CBRANCH_VCCNZ:
1820
12.9k
    return VCCNZ;
1821
1.31M
  case AMDGPU::S_CBRANCH_VCCZ:
1822
2.08k
    return VCCZ;
1823
1.31M
  case AMDGPU::S_CBRANCH_EXECNZ:
1824
9.28k
    return EXECNZ;
1825
1.31M
  case AMDGPU::S_CBRANCH_EXECZ:
1826
2.23k
    return EXECZ;
1827
1.31M
  default:
1828
1.25M
    return INVALID_BR;
1829
1.31M
  }
1830
1.31M
}
1831
1832
bool SIInstrInfo::analyzeBranchImpl(MachineBasicBlock &MBB,
1833
                                    MachineBasicBlock::iterator I,
1834
                                    MachineBasicBlock *&TBB,
1835
                                    MachineBasicBlock *&FBB,
1836
                                    SmallVectorImpl<MachineOperand> &Cond,
1837
1.36M
                                    bool AllowModify) const {
1838
1.36M
  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1839
54.2k
    // Unconditional Branch
1840
54.2k
    TBB = I->getOperand(0).getMBB();
1841
54.2k
    return false;
1842
54.2k
  }
1843
1.31M
1844
1.31M
  MachineBasicBlock *CondBB = nullptr;
1845
1.31M
1846
1.31M
  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1847
0
    CondBB = I->getOperand(1).getMBB();
1848
0
    Cond.push_back(I->getOperand(0));
1849
1.31M
  } else {
1850
1.31M
    BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1851
1.31M
    if (Pred == INVALID_BR)
1852
1.25M
      return true;
1853
56.7k
1854
56.7k
    CondBB = I->getOperand(0).getMBB();
1855
56.7k
    Cond.push_back(MachineOperand::CreateImm(Pred));
1856
56.7k
    Cond.push_back(I->getOperand(1)); // Save the branch register.
1857
56.7k
  }
1858
1.31M
  ++I;
1859
56.7k
1860
56.7k
  if (I == MBB.end()) {
1861
31.4k
    // Conditional branch followed by fall-through.
1862
31.4k
    TBB = CondBB;
1863
31.4k
    return false;
1864
31.4k
  }
1865
25.3k
1866
25.3k
  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1867
25.2k
    TBB = CondBB;
1868
25.2k
    FBB = I->getOperand(0).getMBB();
1869
25.2k
    return false;
1870
25.2k
  }
1871
109
1872
109
  return true;
1873
109
}
1874
1875
bool SIInstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
1876
                                MachineBasicBlock *&FBB,
1877
                                SmallVectorImpl<MachineOperand> &Cond,
1878
1.47M
                                bool AllowModify) const {
1879
1.47M
  MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1880
1.47M
  auto E = MBB.end();
1881
1.47M
  if (I == E)
1882
89.4k
    return false;
1883
1.38M
1884
1.38M
  // Skip over the instructions that are artificially terminators for special
1885
1.38M
  // exec management.
1886
1.39M
  
while (1.38M
I != E &&
!I->isBranch()1.39M
&&
!I->isReturn()1.31M
&&
1887
1.39M
         
I->getOpcode() != AMDGPU::SI_MASK_BRANCH60.1k
) {
1888
29.7k
    switch (I->getOpcode()) {
1889
29.7k
    case AMDGPU::SI_MASK_BRANCH:
1890
11.2k
    case AMDGPU::S_MOV_B64_term:
1891
11.2k
    case AMDGPU::S_XOR_B64_term:
1892
11.2k
    case AMDGPU::S_ANDN2_B64_term:
1893
11.2k
    case AMDGPU::S_MOV_B32_term:
1894
11.2k
    case AMDGPU::S_XOR_B32_term:
1895
11.2k
    case AMDGPU::S_OR_B32_term:
1896
11.2k
    case AMDGPU::S_ANDN2_B32_term:
1897
11.2k
      break;
1898
18.4k
    case AMDGPU::SI_IF:
1899
18.4k
    case AMDGPU::SI_ELSE:
1900
18.4k
    case AMDGPU::SI_KILL_I1_TERMINATOR:
1901
18.4k
    case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1902
18.4k
      // FIXME: It's messy that these need to be considered here at all.
1903
18.4k
      return true;
1904
18.4k
    default:
1905
0
      llvm_unreachable("unexpected non-branch terminator inst");
1906
11.2k
    }
1907
11.2k
1908
11.2k
    ++I;
1909
11.2k
  }
1910
1.38M
1911
1.38M
  
if (1.37M
I == E1.37M
)
1912
48
    return false;
1913
1.37M
1914
1.37M
  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1915
1.33M
    return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1916
30.4k
1917
30.4k
  ++I;
1918
30.4k
1919
30.4k
  // TODO: Should be able to treat as fallthrough?
1920
30.4k
  if (I == MBB.end())
1921
387
    return true;
1922
30.0k
1923
30.0k
  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1924
0
    return true;
1925
30.0k
1926
30.0k
  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1927
30.0k
1928
30.0k
  // Specifically handle the case where the conditional branch is to the same
1929
30.0k
  // destination as the mask branch. e.g.
1930
30.0k
  //
1931
30.0k
  // si_mask_branch BB8
1932
30.0k
  // s_cbranch_execz BB8
1933
30.0k
  // s_cbranch BB9
1934
30.0k
  //
1935
30.0k
  // This is required to understand divergent loops which may need the branches
1936
30.0k
  // to be relaxed.
1937
30.0k
  if (TBB != MaskBrDest || Cond.empty())
1938
28.7k
    return true;
1939
1.32k
1940
1.32k
  auto Pred = Cond[0].getImm();
1941
1.32k
  return (Pred != EXECZ && 
Pred != EXECNZ16
);
1942
1.32k
}
1943
1944
unsigned SIInstrInfo::removeBranch(MachineBasicBlock &MBB,
1945
3.18k
                                   int *BytesRemoved) const {
1946
3.18k
  MachineBasicBlock::iterator I = MBB.getFirstTerminator();
1947
3.18k
1948
3.18k
  unsigned Count = 0;
1949
3.18k
  unsigned RemovedSize = 0;
1950
7.18k
  while (I != MBB.end()) {
1951
4.00k
    MachineBasicBlock::iterator Next = std::next(I);
1952
4.00k
    if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1953
8
      I = Next;
1954
8
      continue;
1955
8
    }
1956
3.99k
1957
3.99k
    RemovedSize += getInstSizeInBytes(*I);
1958
3.99k
    I->eraseFromParent();
1959
3.99k
    ++Count;
1960
3.99k
    I = Next;
1961
3.99k
  }
1962
3.18k
1963
3.18k
  if (BytesRemoved)
1964
34
    *BytesRemoved = RemovedSize;
1965
3.18k
1966
3.18k
  return Count;
1967
3.18k
}
1968
1969
// Copy the flags onto the implicit condition register operand.
1970
static void preserveCondRegFlags(MachineOperand &CondReg,
1971
1.92k
                                 const MachineOperand &OrigCond) {
1972
1.92k
  CondReg.setIsUndef(OrigCond.isUndef());
1973
1.92k
  CondReg.setIsKill(OrigCond.isKill());
1974
1.92k
}
1975
1976
unsigned SIInstrInfo::insertBranch(MachineBasicBlock &MBB,
1977
                                   MachineBasicBlock *TBB,
1978
                                   MachineBasicBlock *FBB,
1979
                                   ArrayRef<MachineOperand> Cond,
1980
                                   const DebugLoc &DL,
1981
2.97k
                                   int *BytesAdded) const {
1982
2.97k
  if (!FBB && 
Cond.empty()2.88k
) {
1983
991
    BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1984
991
      .addMBB(TBB);
1985
991
    if (BytesAdded)
1986
0
      *BytesAdded = 4;
1987
991
    return 1;
1988
991
  }
1989
1.98k
1990
1.98k
  if(Cond.size() == 1 && 
Cond[0].isReg()0
) {
1991
0
     BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
1992
0
       .add(Cond[0])
1993
0
       .addMBB(TBB);
1994
0
     return 1;
1995
0
  }
1996
1.98k
1997
1.98k
  assert(TBB && Cond[0].isImm());
1998
1.98k
1999
1.98k
  unsigned Opcode
2000
1.98k
    = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2001
1.98k
2002
1.98k
  if (!FBB) {
2003
1.89k
    Cond[1].isUndef();
2004
1.89k
    MachineInstr *CondBr =
2005
1.89k
      BuildMI(&MBB, DL, get(Opcode))
2006
1.89k
      .addMBB(TBB);
2007
1.89k
2008
1.89k
    // Copy the flags onto the implicit condition register operand.
2009
1.89k
    preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2010
1.89k
2011
1.89k
    if (BytesAdded)
2012
0
      *BytesAdded = 4;
2013
1.89k
    return 1;
2014
1.89k
  }
2015
89
2016
89
  assert(TBB && FBB);
2017
89
2018
89
  MachineInstr *CondBr =
2019
89
    BuildMI(&MBB, DL, get(Opcode))
2020
89
    .addMBB(TBB);
2021
89
  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2022
89
    .addMBB(FBB);
2023
89
2024
89
  MachineOperand &CondReg = CondBr->getOperand(1);
2025
89
  CondReg.setIsUndef(Cond[1].isUndef());
2026
89
  CondReg.setIsKill(Cond[1].isKill());
2027
89
2028
89
  if (BytesAdded)
2029
34
      *BytesAdded = 8;
2030
89
2031
89
  return 2;
2032
89
}
2033
2034
bool SIInstrInfo::reverseBranchCondition(
2035
1.62k
  SmallVectorImpl<MachineOperand> &Cond) const {
2036
1.62k
  if (Cond.size() != 2) {
2037
0
    return true;
2038
0
  }
2039
1.62k
2040
1.62k
  if (Cond[0].isImm()) {
2041
1.62k
    Cond[0].setImm(-Cond[0].getImm());
2042
1.62k
    return false;
2043
1.62k
  }
2044
0
2045
0
  return true;
2046
0
}
2047
2048
bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
2049
                                  ArrayRef<MachineOperand> Cond,
2050
                                  unsigned TrueReg, unsigned FalseReg,
2051
                                  int &CondCycles,
2052
34
                                  int &TrueCycles, int &FalseCycles) const {
2053
34
  switch (Cond[0].getImm()) {
2054
34
  case VCCNZ:
2055
23
  case VCCZ: {
2056
23
    const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2057
23
    const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2058
23
    assert(MRI.getRegClass(FalseReg) == RC);
2059
23
2060
23
    int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2061
23
    CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2062
23
2063
23
    // Limit to equal cost for branch vs. N v_cndmask_b32s.
2064
23
    return RI.hasVGPRs(RC) && 
NumInsts <= 620
;
2065
23
  }
2066
23
  case SCC_TRUE:
2067
11
  case SCC_FALSE: {
2068
11
    // FIXME: We could insert for VGPRs if we could replace the original compare
2069
11
    // with a vector one.
2070
11
    const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2071
11
    const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2072
11
    assert(MRI.getRegClass(FalseReg) == RC);
2073
11
2074
11
    int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2075
11
2076
11
    // Multiples of 8 can do s_cselect_b64
2077
11
    if (NumInsts % 2 == 0)
2078
3
      NumInsts /= 2;
2079
11
2080
11
    CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2081
11
    return RI.isSGPRClass(RC);
2082
11
  }
2083
11
  default:
2084
0
    return false;
2085
34
  }
2086
34
}
2087
2088
void SIInstrInfo::insertSelect(MachineBasicBlock &MBB,
2089
                               MachineBasicBlock::iterator I, const DebugLoc &DL,
2090
                               unsigned DstReg, ArrayRef<MachineOperand> Cond,
2091
20
                               unsigned TrueReg, unsigned FalseReg) const {
2092
20
  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2093
20
  if (Pred == VCCZ || Pred == SCC_FALSE) {
2094
0
    Pred = static_cast<BranchPredicate>(-Pred);
2095
0
    std::swap(TrueReg, FalseReg);
2096
0
  }
2097
20
2098
20
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2099
20
  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2100
20
  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2101
20
2102
20
  if (DstSize == 32) {
2103
9
    unsigned SelOp = Pred == SCC_TRUE ?
2104
6
      
AMDGPU::S_CSELECT_B323
: AMDGPU::V_CNDMASK_B32_e32;
2105
9
2106
9
    // Instruction's operands are backwards from what is expected.
2107
9
    MachineInstr *Select =
2108
9
      BuildMI(MBB, I, DL, get(SelOp), DstReg)
2109
9
      .addReg(FalseReg)
2110
9
      .addReg(TrueReg);
2111
9
2112
9
    preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2113
9
    return;
2114
9
  }
2115
11
2116
11
  if (DstSize == 64 && 
Pred == SCC_TRUE5
) {
2117
1
    MachineInstr *Select =
2118
1
      BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2119
1
      .addReg(FalseReg)
2120
1
      .addReg(TrueReg);
2121
1
2122
1
    preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2123
1
    return;
2124
1
  }
2125
10
2126
10
  static const int16_t Sub0_15[] = {
2127
10
    AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2128
10
    AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2129
10
    AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2130
10
    AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2131
10
  };
2132
10
2133
10
  static const int16_t Sub0_15_64[] = {
2134
10
    AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2135
10
    AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2136
10
    AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2137
10
    AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2138
10
  };
2139
10
2140
10
  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2141
10
  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2142
10
  const int16_t *SubIndices = Sub0_15;
2143
10
  int NElts = DstSize / 32;
2144
10
2145
10
  // 64-bit select is only available for SALU.
2146
10
  // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2147
10
  if (Pred == SCC_TRUE) {
2148
2
    if (NElts % 2) {
2149
1
      SelOp = AMDGPU::S_CSELECT_B32;
2150
1
      EltRC = &AMDGPU::SGPR_32RegClass;
2151
1
    } else {
2152
1
      SelOp = AMDGPU::S_CSELECT_B64;
2153
1
      EltRC = &AMDGPU::SGPR_64RegClass;
2154
1
      SubIndices = Sub0_15_64;
2155
1
      NElts /= 2;
2156
1
    }
2157
2
  }
2158
10
2159
10
  MachineInstrBuilder MIB = BuildMI(
2160
10
    MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2161
10
2162
10
  I = MIB->getIterator();
2163
10
2164
10
  SmallVector<unsigned, 8> Regs;
2165
37
  for (int Idx = 0; Idx != NElts; 
++Idx27
) {
2166
27
    unsigned DstElt = MRI.createVirtualRegister(EltRC);
2167
27
    Regs.push_back(DstElt);
2168
27
2169
27
    unsigned SubIdx = SubIndices[Idx];
2170
27
2171
27
    MachineInstr *Select =
2172
27
      BuildMI(MBB, I, DL, get(SelOp), DstElt)
2173
27
      .addReg(FalseReg, 0, SubIdx)
2174
27
      .addReg(TrueReg, 0, SubIdx);
2175
27
    preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2176
27
    fixImplicitOperands(*Select);
2177
27
2178
27
    MIB.addReg(DstElt)
2179
27
       .addImm(SubIdx);
2180
27
  }
2181
10
}
2182
2183
1.18M
bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) const {
2184
1.18M
  switch (MI.getOpcode()) {
2185
1.18M
  case AMDGPU::V_MOV_B32_e32:
2186
34.7k
  case AMDGPU::V_MOV_B32_e64:
2187
34.7k
  case AMDGPU::V_MOV_B64_PSEUDO: {
2188
34.7k
    // If there are additional implicit register operands, this may be used for
2189
34.7k
    // register indexing so the source register operand isn't simply copied.
2190
34.7k
    unsigned NumOps = MI.getDesc().getNumOperands() +
2191
34.7k
      MI.getDesc().getNumImplicitUses();
2192
34.7k
2193
34.7k
    return MI.getNumOperands() == NumOps;
2194
34.7k
  }
2195
501k
  case AMDGPU::S_MOV_B32:
2196
501k
  case AMDGPU::S_MOV_B64:
2197
501k
  case AMDGPU::COPY:
2198
501k
  case AMDGPU::V_ACCVGPR_WRITE_B32:
2199
501k
  case AMDGPU::V_ACCVGPR_READ_B32:
2200
501k
    return true;
2201
644k
  default:
2202
644k
    return false;
2203
1.18M
  }
2204
1.18M
}
2205
2206
unsigned SIInstrInfo::getAddressSpaceForPseudoSourceKind(
2207
131k
    unsigned Kind) const {
2208
131k
  switch(Kind) {
2209
131k
  case PseudoSourceValue::Stack:
2210
34.3k
  case PseudoSourceValue::FixedStack:
2211
34.3k
    return AMDGPUAS::PRIVATE_ADDRESS;
2212
95.2k
  case PseudoSourceValue::ConstantPool:
2213
95.2k
  case PseudoSourceValue::GOT:
2214
95.2k
  case PseudoSourceValue::JumpTable:
2215
95.2k
  case PseudoSourceValue::GlobalValueCallEntry:
2216
95.2k
  case PseudoSourceValue::ExternalSymbolCallEntry:
2217
95.2k
  case PseudoSourceValue::TargetCustom:
2218
95.2k
    return AMDGPUAS::CONSTANT_ADDRESS;
2219
1.37k
  }
2220
1.37k
  return AMDGPUAS::FLAT_ADDRESS;
2221
1.37k
}
2222
2223
46
static void removeModOperands(MachineInstr &MI) {
2224
46
  unsigned Opc = MI.getOpcode();
2225
46
  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2226
46
                                              AMDGPU::OpName::src0_modifiers);
2227
46
  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2228
46
                                              AMDGPU::OpName::src1_modifiers);
2229
46
  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2230
46
                                              AMDGPU::OpName::src2_modifiers);
2231
46
2232
46
  MI.RemoveOperand(Src2ModIdx);
2233
46
  MI.RemoveOperand(Src1ModIdx);
2234
46
  MI.RemoveOperand(Src0ModIdx);
2235
46
}
2236
2237
bool SIInstrInfo::FoldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
2238
68.5k
                                unsigned Reg, MachineRegisterInfo *MRI) const {
2239
68.5k
  if (!MRI->hasOneNonDBGUse(Reg))
2240
40.2k
    return false;
2241
28.2k
2242
28.2k
  switch (DefMI.getOpcode()) {
2243
28.2k
  default:
2244
226
    return false;
2245
28.2k
  case AMDGPU::S_MOV_B64:
2246
673
    // TODO: We could fold 64-bit immediates, but this get compilicated
2247
673
    // when there are sub-registers.
2248
673
    return false;
2249
28.2k
2250
28.2k
  case AMDGPU::V_MOV_B32_e32:
2251
27.3k
  case AMDGPU::S_MOV_B32:
2252
27.3k
  case AMDGPU::V_ACCVGPR_WRITE_B32:
2253
27.3k
    break;
2254
27.3k
  }
2255
27.3k
2256
27.3k
  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2257
27.3k
  assert(ImmOp);
2258
27.3k
  // FIXME: We could handle FrameIndex values here.
2259
27.3k
  if (!ImmOp->isImm())
2260
602
    return false;
2261
26.7k
2262
26.7k
  unsigned Opc = UseMI.getOpcode();
2263
26.7k
  if (Opc == AMDGPU::COPY) {
2264
7.74k
    bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2265
7.74k
    unsigned NewOpc = isVGPRCopy ? 
AMDGPU::V_MOV_B32_e327.67k
:
AMDGPU::S_MOV_B3274
;
2266
7.74k
    if (RI.isAGPR(*MRI, UseMI.getOperand(0).getReg())) {
2267
50
      if (!isInlineConstant(*ImmOp, AMDGPU::OPERAND_REG_INLINE_AC_INT32))
2268
0
        return false;
2269
50
      NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32;
2270
50
    }
2271
7.74k
    UseMI.setDesc(get(NewOpc));
2272
7.74k
    UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2273
7.74k
    UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2274
7.74k
    return true;
2275
19.0k
  }
2276
19.0k
2277
19.0k
  if (Opc == AMDGPU::V_MAD_F32 || 
Opc == AMDGPU::V_MAC_F32_e6419.0k
||
2278
19.0k
      
Opc == AMDGPU::V_MAD_F1618.8k
||
Opc == AMDGPU::V_MAC_F16_e6418.7k
||
2279
19.0k
      
Opc == AMDGPU::V_FMA_F3218.7k
||
Opc == AMDGPU::V_FMAC_F32_e6418.6k
||
2280
19.0k
      
Opc == AMDGPU::V_FMA_F1618.6k
||
Opc == AMDGPU::V_FMAC_F16_e6418.6k
) {
2281
423
    // Don't fold if we are using source or output modifiers. The new VOP2
2282
423
    // instructions don't have them.
2283
423
    if (hasAnyModifiersSet(UseMI))
2284
103
      return false;
2285
320
2286
320
    // If this is a free constant, there's no reason to do this.
2287
320
    // TODO: We could fold this here instead of letting SIFoldOperands do it
2288
320
    // later.
2289
320
    MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2290
320
2291
320
    // Any src operand can be used for the legality check.
2292
320
    if (isInlineConstant(UseMI, *Src0, *ImmOp))
2293
171
      return false;
2294
149
2295
149
    bool IsF32 = Opc == AMDGPU::V_MAD_F32 || 
Opc == AMDGPU::V_MAC_F32_e64145
||
2296
149
                 
Opc == AMDGPU::V_FMA_F3234
||
Opc == AMDGPU::V_FMAC_F32_e6427
;
2297
149
    bool IsFMA = Opc == AMDGPU::V_FMA_F32 || 
Opc == AMDGPU::V_FMAC_F32_e64142
||
2298
149
                 
Opc == AMDGPU::V_FMA_F16135
||
Opc == AMDGPU::V_FMAC_F16_e64130
;
2299
149
    MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2300
149
    MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2301
149
2302
149
    // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2303
149
    // We should only expect these to be on src0 due to canonicalizations.
2304
149
    if (Src0->isReg() && Src0->getReg() == Reg) {
2305
7
      if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2306
0
        return false;
2307
7
2308
7
      if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2309
0
        return false;
2310
7
2311
7
      unsigned NewOpc =
2312
7
        IsFMA ? 
(IsF32 0
?
AMDGPU::V_FMAMK_F320
:
AMDGPU::V_FMAMK_F160
)
2313
7
              : (IsF32 ? 
AMDGPU::V_MADMK_F324
:
AMDGPU::V_MADMK_F163
);
2314
7
      if (pseudoToMCOpcode(NewOpc) == -1)
2315
0
        return false;
2316
7
2317
7
      // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2318
7
2319
7
      const int64_t Imm = ImmOp->getImm();
2320
7
2321
7
      // FIXME: This would be a lot easier if we could return a new instruction
2322
7
      // instead of having to modify in place.
2323
7
2324
7
      // Remove these first since they are at the end.
2325
7
      UseMI.RemoveOperand(
2326
7
          AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2327
7
      UseMI.RemoveOperand(
2328
7
          AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2329
7
2330
7
      unsigned Src1Reg = Src1->getReg();
2331
7
      unsigned Src1SubReg = Src1->getSubReg();
2332
7
      Src0->setReg(Src1Reg);
2333
7
      Src0->setSubReg(Src1SubReg);
2334
7
      Src0->setIsKill(Src1->isKill());
2335
7
2336
7
      if (Opc == AMDGPU::V_MAC_F32_e64 ||
2337
7
          Opc == AMDGPU::V_MAC_F16_e64 ||
2338
7
          Opc == AMDGPU::V_FMAC_F32_e64 ||
2339
7
          Opc == AMDGPU::V_FMAC_F16_e64)
2340
0
        UseMI.untieRegOperand(
2341
0
            AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2342
7
2343
7
      Src1->ChangeToImmediate(Imm);
2344
7
2345
7
      removeModOperands(UseMI);
2346
7
      UseMI.setDesc(get(NewOpc));
2347
7
2348
7
      bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2349
7
      if (DeleteDef)
2350
0
        DefMI.eraseFromParent();
2351
7
2352
7
      return true;
2353
7
    }
2354
142
2355
142
    // Added part is the constant: Use v_madak_{f16, f32}.
2356
142
    if (Src2->isReg() && Src2->getReg() == Reg) {
2357
62
      // Not allowed to use constant bus for another operand.
2358
62
      // We can however allow an inline immediate as src0.
2359
62
      bool Src0Inlined = false;
2360
62
      if (Src0->isReg()) {
2361
62
        // Try to inline constant if possible.
2362
62
        // If the Def moves immediate and the use is single
2363
62
        // We are saving VGPR here.
2364
62
        MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2365
62
        if (Def && 
Def->isMoveImmediate()61
&&
2366
62
          
isInlineConstant(Def->getOperand(1))3
&&
2367
62
          
MRI->hasOneUse(Src0->getReg())3
) {
2368
3
          Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2369
3
          Src0Inlined = true;
2370
59
        } else if ((RI.isPhysicalRegister(Src0->getReg()) &&
2371
59
            
(2
ST.getConstantBusLimit(Opc) <= 12
&&
2372
2
             RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2373
59
            
(58
RI.isVirtualRegister(Src0->getReg())58
&&
2374
58
            
(57
ST.getConstantBusLimit(Opc) <= 157
&&
2375
57
             
RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))45
)))
2376
7
          return false;
2377
55
          // VGPR is okay as Src0 - fallthrough
2378
55
      }
2379
55
2380
55
      if (Src1->isReg() && !Src0Inlined ) {
2381
52
        // We have one slot for inlinable constant so far - try to fill it
2382
52
        MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2383
52
        if (Def && 
Def->isMoveImmediate()51
&&
2384
52
            
isInlineConstant(Def->getOperand(1))20
&&
2385
52
            
MRI->hasOneUse(Src1->getReg())18
&&
2386
52
            
commuteInstruction(UseMI)16
) {
2387
16
            Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2388
36
        } else if ((RI.isPhysicalRegister(Src1->getReg()) &&
2389
36
            
RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))1
) ||
2390
36
            
(35
RI.isVirtualRegister(Src1->getReg())35
&&
2391
35
            RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2392
10
          return false;
2393
45
          // VGPR is okay as Src1 - fallthrough
2394
45
      }
2395
45
2396
45
      unsigned NewOpc =
2397
45
        IsFMA ? 
(IsF32 10
?
AMDGPU::V_FMAAK_F329
:
AMDGPU::V_FMAAK_F161
)
2398
45
              : 
(IsF32 35
?
AMDGPU::V_MADAK_F3231
:
AMDGPU::V_MADAK_F164
);
2399
45
      if (pseudoToMCOpcode(NewOpc) == -1)
2400
6
        return false;
2401
39
2402
39
      const int64_t Imm = ImmOp->getImm();
2403
39
2404
39
      // FIXME: This would be a lot easier if we could return a new instruction
2405
39
      // instead of having to modify in place.
2406
39
2407
39
      // Remove these first since they are at the end.
2408
39
      UseMI.RemoveOperand(
2409
39
          AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2410
39
      UseMI.RemoveOperand(
2411
39
          AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2412
39
2413
39
      if (Opc == AMDGPU::V_MAC_F32_e64 ||
2414
39
          
Opc == AMDGPU::V_MAC_F16_e648
||
2415
39
          
Opc == AMDGPU::V_FMAC_F32_e647
||
2416
39
          
Opc == AMDGPU::V_FMAC_F16_e643
)
2417
36
        UseMI.untieRegOperand(
2418
36
            AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2419
39
2420
39
      // ChangingToImmediate adds Src2 back to the instruction.
2421
39
      Src2->ChangeToImmediate(Imm);
2422
39
2423
39
      // These come before src2.
2424
39
      removeModOperands(UseMI);
2425
39
      UseMI.setDesc(get(NewOpc));
2426
39
      // It might happen that UseMI was commuted
2427
39
      // and we now have SGPR as SRC1. If so 2 inlined
2428
39
      // constant and SGPR are illegal.
2429
39
      legalizeOperands(UseMI);
2430
39
2431
39
      bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2432
39
      if (DeleteDef)
2433
0
        DefMI.eraseFromParent();
2434
39
2435
39
      return true;
2436
39
    }
2437
142
  }
2438
18.6k
2439
18.6k
  return false;
2440
18.6k
}
2441
2442
static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2443
28.0k
                                int WidthB, int OffsetB) {
2444
28.0k
  int LowOffset = OffsetA < OffsetB ? 
OffsetA6.70k
:
OffsetB21.3k
;
2445
28.0k
  int HighOffset = OffsetA < OffsetB ? 
OffsetB6.70k
:
OffsetA21.3k
;
2446
28.0k
  int LowWidth = (LowOffset == OffsetA) ? 
WidthA12.6k
:
WidthB15.4k
;
2447
28.0k
  return LowOffset + LowWidth <= HighOffset;
2448
28.0k
}
2449
2450
bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2451
1.00M
                                               const MachineInstr &MIb) const {
2452
1.00M
  const MachineOperand *BaseOp0, *BaseOp1;
2453
1.00M
  int64_t Offset0, Offset1;
2454
1.00M
2455
1.00M
  if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2456
1.00M
      
getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)98.7k
) {
2457
94.0k
    if (!BaseOp0->isIdenticalTo(*BaseOp1))
2458
51.3k
      return false;
2459
42.6k
2460
42.6k
    if (!MIa.hasOneMemOperand() || 
!MIb.hasOneMemOperand()29.3k
) {
2461
14.6k
      // FIXME: Handle ds_read2 / ds_write2.
2462
14.6k
      return false;
2463
14.6k
    }
2464
28.0k
    unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2465
28.0k
    unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2466
28.0k
    if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2467
22.0k
      return true;
2468
22.0k
    }
2469
915k
  }
2470
915k
2471
915k
  return false;
2472
915k
}
2473
2474
bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
2475
                                                  const MachineInstr &MIb,
2476
1.08M
                                                  AliasAnalysis *AA) const {
2477
1.08M
  assert((MIa.mayLoad() || MIa.mayStore()) &&
2478
1.08M
         "MIa must load from or modify a memory location");
2479
1.08M
  assert((MIb.mayLoad() || MIb.mayStore()) &&
2480
1.08M
         "MIb must load from or modify a memory location");
2481
1.08M
2482
1.08M
  if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects())
2483
0
    return false;
2484
1.08M
2485
1.08M
  // XXX - Can we relax this between address spaces?
2486
1.08M
  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2487
324
    return false;
2488
1.08M
2489
1.08M
  // TODO: Should we check the address space from the MachineMemOperand? That
2490
1.08M
  // would allow us to distinguish objects we know don't alias based on the
2491
1.08M
  // underlying address space, even if it was lowered to a different one,
2492
1.08M
  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2493
1.08M
  // buffer.
2494
1.08M
  if (isDS(MIa)) {
2495
121k
    if (isDS(MIb))
2496
66.6k
      return checkInstOffsetsDoNotOverlap(MIa, MIb);
2497
54.3k
2498
54.3k
    return !isFLAT(MIb) || 
isSegmentSpecificFLAT(MIb)41.2k
;
2499
54.3k
  }
2500
967k
2501
967k
  if (isMUBUF(MIa) || 
isMTBUF(MIa)56.6k
) {
2502
911k
    if (isMUBUF(MIb) || 
isMTBUF(MIb)13.6k
)
2503
897k
      return checkInstOffsetsDoNotOverlap(MIa, MIb);
2504
13.5k
2505
13.5k
    return !isFLAT(MIb) && 
!isSMRD(MIb)1.98k
;
2506
13.5k
  }
2507
56.4k
2508
56.4k
  if (isSMRD(MIa)) {
2509
5.26k
    if (isSMRD(MIb))
2510
0
      return checkInstOffsetsDoNotOverlap(MIa, MIb);
2511
5.26k
2512
5.26k
    return !isFLAT(MIb) && 
!isMUBUF(MIa)2.11k
&&
!isMTBUF(MIa)2.11k
;
2513
5.26k
  }
2514
51.2k
2515
51.2k
  if (isFLAT(MIa)) {
2516
50.9k
    if (isFLAT(MIb))
2517
39.0k
      return checkInstOffsetsDoNotOverlap(MIa, MIb);
2518
11.9k
2519
11.9k
    return false;
2520
11.9k
  }
2521
251
2522
251
  return false;
2523
251
}
2524
2525
819
static int64_t getFoldableImm(const MachineOperand* MO) {
2526
819
  if (!MO->isReg())
2527
2
    return false;
2528
817
  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2529
817
  const MachineRegisterInfo &MRI = MF->getRegInfo();
2530
817
  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2531
817
  if (Def && 
Def->getOpcode() == AMDGPU::V_MOV_B32_e32806
&&
2532
817
      
Def->getOperand(1).isImm()27
)
2533
27
    return Def->getOperand(1).getImm();
2534
790
  return AMDGPU::NoRegister;
2535
790
}
2536
2537
MachineInstr *SIInstrInfo::convertToThreeAddress(MachineFunction::iterator &MBB,
2538
                                                 MachineInstr &MI,
2539
325
                                                 LiveVariables *LV) const {
2540
325
  unsigned Opc = MI.getOpcode();
2541
325
  bool IsF16 = false;
2542
325
  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || 
Opc == AMDGPU::V_FMAC_F32_e64311
||
2543
325
               
Opc == AMDGPU::V_FMAC_F16_e32308
||
Opc == AMDGPU::V_FMAC_F16_e64303
;
2544
325
2545
325
  switch (Opc) {
2546
325
  default:
2547
1
    return nullptr;
2548
325
  case AMDGPU::V_MAC_F16_e64:
2549
1
  case AMDGPU::V_FMAC_F16_e64:
2550
1
    IsF16 = true;
2551
1
    LLVM_FALLTHROUGH;
2552
16
  case AMDGPU::V_MAC_F32_e64:
2553
16
  case AMDGPU::V_FMAC_F32_e64:
2554
16
    break;
2555
16
  case AMDGPU::V_MAC_F16_e32:
2556
11
  case AMDGPU::V_FMAC_F16_e32:
2557
11
    IsF16 = true;
2558
11
    LLVM_FALLTHROUGH;
2559
308
  case AMDGPU::V_MAC_F32_e32:
2560
308
  case AMDGPU::V_FMAC_F32_e32: {
2561
308
    int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2562
308
                                             AMDGPU::OpName::src0);
2563
308
    const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2564
308
    if (!Src0->isReg() && 
!Src0->isImm()9
)
2565
2
      return nullptr;
2566
306
2567
306
    if (Src0->isImm() && 
!isInlineConstant(MI, Src0Idx, *Src0)7
)
2568
3
      return nullptr;
2569
303
2570
303
    break;
2571
303
  }
2572
319
  }
2573
319
2574
319
  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2575
319
  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2576
319
  const MachineOperand *Src0Mods =
2577
319
    getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2578
319
  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2579
319
  const MachineOperand *Src1Mods =
2580
319
    getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2581
319
  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2582
319
  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2583
319
  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2584
319
2585
319
  if (!Src0Mods && 
!Src1Mods303
&&
!Clamp303
&&
!Omod303
&&
2586
319
      // If we have an SGPR input, we will violate the constant bus restriction.
2587
319
      
(303
ST.getConstantBusLimit(Opc) > 1303
||
2588
303
       
!Src0->isReg()291
||
2589
303
       
!RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg())288
)) {
2590
286
    if (auto Imm = getFoldableImm(Src2)) {
2591
16
      unsigned NewOpc =
2592
16
         IsFMA ? 
(IsF16 3
?
AMDGPU::V_FMAAK_F161
:
AMDGPU::V_FMAAK_F322
)
2593
16
               : 
(IsF16 13
?
AMDGPU::V_MADAK_F162
:
AMDGPU::V_MADAK_F3211
);
2594
16
      if (pseudoToMCOpcode(NewOpc) != -1)
2595
16
        return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2596
16
                 .add(*Dst)
2597
16
                 .add(*Src0)
2598
16
                 .add(*Src1)
2599
16
                 .addImm(Imm);
2600
270
    }
2601
270
    unsigned NewOpc =
2602
270
      IsFMA ? 
(IsF16 14
?
AMDGPU::V_FMAMK_F164
:
AMDGPU::V_FMAMK_F3210
)
2603
270
            : 
(IsF16 256
?
AMDGPU::V_MADMK_F164
:
AMDGPU::V_MADMK_F32252
);
2604
270
    if (auto Imm = getFoldableImm(Src1)) {
2605
7
      if (pseudoToMCOpcode(NewOpc) != -1)
2606
7
        return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2607
7
                 .add(*Dst)
2608
7
                 .add(*Src0)
2609
7
                 .addImm(Imm)
2610
7
                 .add(*Src2);
2611
263
    }
2612
263
    if (auto Imm = getFoldableImm(Src0)) {
2613
4
      if (pseudoToMCOpcode(NewOpc) != -1 &&
2614
4
          isOperandLegal(MI, AMDGPU::getNamedOperandIdx(NewOpc,
2615
4
                           AMDGPU::OpName::src0), Src1))
2616
4
        return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2617
4
                 .add(*Dst)
2618
4
                 .add(*Src1)
2619
4
                 .addImm(Imm)
2620
4
                 .add(*Src2);
2621
292
    }
2622
263
  }
2623
292
2624
292
  unsigned NewOpc = IsFMA ? 
(IsF16 12
?
AMDGPU::V_FMA_F163
:
AMDGPU::V_FMA_F329
)
2625
292
                          : 
(IsF16 280
?
AMDGPU::V_MAD_F162
:
AMDGPU::V_MAD_F32278
);
2626
292
  if (pseudoToMCOpcode(NewOpc) == -1)
2627
3
    return nullptr;
2628
289
2629
289
  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2630
289
      .add(*Dst)
2631
289
      .addImm(Src0Mods ? 
Src0Mods->getImm()15
:
0274
)
2632
289
      .add(*Src0)
2633
289
      .addImm(Src1Mods ? 
Src1Mods->getImm()15
:
0274
)
2634
289
      .add(*Src1)
2635
289
      .addImm(0) // Src mods
2636
289
      .add(*Src2)
2637
289
      .addImm(Clamp ? 
Clamp->getImm()15
:
0274
)
2638
289
      .addImm(Omod ? 
Omod->getImm()15
:
0274
);
2639
289
}
2640
2641
// It's not generally safe to move VALU instructions across these since it will
2642
// start using the register as a base index rather than directly.
2643
// XXX - Why isn't hasSideEffects sufficient for these?
2644
650k
static bool changesVGPRIndexingMode(const MachineInstr &MI) {
2645
650k
  switch (MI.getOpcode()) {
2646
650k
  case AMDGPU::S_SET_GPR_IDX_ON:
2647
202
  case AMDGPU::S_SET_GPR_IDX_MODE:
2648
202
  case AMDGPU::S_SET_GPR_IDX_OFF:
2649
202
    return true;
2650
650k
  default:
2651
650k
    return false;
2652
650k
  }
2653
650k
}
2654
2655
bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
2656
                                       const MachineBasicBlock *MBB,
2657
709k
                                       const MachineFunction &MF) const {
2658
709k
  // XXX - Do we want the SP check in the base implementation?
2659
709k
2660
709k
  // Target-independent instructions do not have an implicit-use of EXEC, even
2661
709k
  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2662
709k
  // boundaries prevents incorrect movements of such instructions.
2663
709k
  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2664
709k
         
MI.modifiesRegister(AMDGPU::EXEC, &RI)656k
||
2665
709k
         
MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32651k
||
2666
709k
         
MI.getOpcode() == AMDGPU::S_SETREG_B32651k
||
2667
709k
         
changesVGPRIndexingMode(MI)650k
;
2668
709k
}
2669
2670
558k
bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
2671
558k
  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
2672
558k
         
Opcode == AMDGPU::DS_GWS_INIT558k
||
2673
558k
         
Opcode == AMDGPU::DS_GWS_SEMA_V558k
||
2674
558k
         
Opcode == AMDGPU::DS_GWS_SEMA_BR558k
||
2675
558k
         
Opcode == AMDGPU::DS_GWS_SEMA_P558k
||
2676
558k
         
Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL558k
||
2677
558k
         
Opcode == AMDGPU::DS_GWS_BARRIER558k
;
2678
558k
}
2679
2680
2.05k
bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
2681
2.05k
  unsigned Opcode = MI.getOpcode();
2682
2.05k
2683
2.05k
  if (MI.mayStore() && 
isSMRD(MI)369
)
2684
0
    return true; // scalar store or atomic
2685
2.05k
2686
2.05k
  // This will terminate the function when other lanes may need to continue.
2687
2.05k
  if (MI.isReturn())
2688
14
    return true;
2689
2.03k
2690
2.03k
  // These instructions cause shader I/O that may cause hardware lockups
2691
2.03k
  // when executed with an empty EXEC mask.
2692
2.03k
  //
2693
2.03k
  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2694
2.03k
  //       EXEC = 0, but checking for that case here seems not worth it
2695
2.03k
  //       given the typical code patterns.
2696
2.03k
  if (Opcode == AMDGPU::S_SENDMSG || 
Opcode == AMDGPU::S_SENDMSGHALT2.03k
||
2697
2.03k
      
Opcode == AMDGPU::EXP2.03k
||
Opcode == AMDGPU::EXP_DONE2.03k
||
2698
2.03k
      
Opcode == AMDGPU::DS_ORDERED_COUNT2.02k
||
Opcode == AMDGPU::S_TRAP2.01k
||
2699
2.03k
      
Opcode == AMDGPU::DS_GWS_INIT2.01k
||
Opcode == AMDGPU::DS_GWS_BARRIER2.01k
)
2700
20
    return true;
2701
2.01k
2702
2.01k
  if (MI.isCall() || 
MI.isInlineAsm()2.01k
)
2703
13
    return true; // conservative assumption
2704
2.00k
2705
2.00k
  // These are like SALU instructions in terms of effects, so it's questionable
2706
2.00k
  // whether we should return true for those.
2707
2.00k
  //
2708
2.00k
  // However, executing them with EXEC = 0 causes them to operate on undefined
2709
2.00k
  // data, which we avoid by returning true here.
2710
2.00k
  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || 
Opcode == AMDGPU::V_READLANE_B321.99k
)
2711
10
    return true;
2712
1.99k
2713
1.99k
  return false;
2714
1.99k
}
2715
2716
bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
2717
288
                              const MachineInstr &MI) const {
2718
288
  if (MI.isMetaInstruction())
2719
6
    return false;
2720
282
2721
282
  // This won't read exec if this is an SGPR->SGPR copy.
2722
282
  if (MI.isCopyLike()) {
2723
63
    if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
2724
32
      return true;
2725
31
2726
31
    // Make sure this isn't copying exec as a normal operand
2727
31
    return MI.readsRegister(AMDGPU::EXEC, &RI);
2728
31
  }
2729
219
2730
219
  // Make a conservative assumption about the callee.
2731
219
  if (MI.isCall())
2732
1
    return true;
2733
218
2734
218
  // Be conservative with any unhandled generic opcodes.
2735
218
  if (!isTargetSpecificOpcode(MI.getOpcode()))
2736
0
    return true;
2737
218
2738
218
  return !isSALU(MI) || 
MI.readsRegister(AMDGPU::EXEC, &RI)172
;
2739
218
}
2740
2741
9.20k
bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2742
9.20k
  switch (Imm.getBitWidth()) {
2743
9.20k
  case 1: // This likely will be a condition code mask.
2744
1
    return true;
2745
9.20k
2746
9.20k
  case 32:
2747
1.64k
    return AMDGPU::isInlinableLiteral32(Imm.getSExtValue(),
2748
1.64k
                                        ST.hasInv2PiInlineImm());
2749
9.20k
  case 64:
2750
7.32k
    return AMDGPU::isInlinableLiteral64(Imm.getSExtValue(),
2751
7.32k
                                        ST.hasInv2PiInlineImm());
2752
9.20k
  case 16:
2753
222
    return ST.has16BitInsts() &&
2754
222
           AMDGPU::isInlinableLiteral16(Imm.getSExtValue(),
2755
222
                                        ST.hasInv2PiInlineImm());
2756
9.20k
  default:
2757
0
    llvm_unreachable("invalid bitwidth");
2758
9.20k
  }
2759
9.20k
}
2760
2761
bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
2762
5.26M
                                   uint8_t OperandType) const {
2763
5.26M
  if (!MO.isImm() ||
2764
5.26M
      
OperandType < AMDGPU::OPERAND_SRC_FIRST5.24M
||
2765
5.26M
      
OperandType > AMDGPU::OPERAND_SRC_LAST5.18M
)
2766
82.5k
    return false;
2767
5.18M
2768
5.18M
  // MachineOperand provides no way to tell the true operand size, since it only
2769
5.18M
  // records a 64-bit value. We need to know the size to determine if a 32-bit
2770
5.18M
  // floating point immediate bit pattern is legal for an integer immediate. It
2771
5.18M
  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2772
5.18M
2773
5.18M
  int64_t Imm = MO.getImm();
2774
5.18M
  switch (OperandType) {
2775
5.18M
  case AMDGPU::OPERAND_REG_IMM_INT32:
2776
4.96M
  case AMDGPU::OPERAND_REG_IMM_FP32:
2777
4.96M
  case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2778
4.96M
  case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2779
4.96M
  case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2780
4.96M
  case AMDGPU::OPERAND_REG_INLINE_AC_FP32: {
2781
4.96M
    int32_t Trunc = static_cast<int32_t>(Imm);
2782
4.96M
    return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());
2783
4.96M
  }
2784
4.96M
  case AMDGPU::OPERAND_REG_IMM_INT64:
2785
36.8k
  case AMDGPU::OPERAND_REG_IMM_FP64:
2786
36.8k
  case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2787
36.8k
  case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2788
36.8k
    return AMDGPU::isInlinableLiteral64(MO.getImm(),
2789
36.8k
                                        ST.hasInv2PiInlineImm());
2790
174k
  case AMDGPU::OPERAND_REG_IMM_INT16:
2791
174k
  case AMDGPU::OPERAND_REG_IMM_FP16:
2792
174k
  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2793
174k
  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2794
174k
  case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
2795
174k
  case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
2796
174k
    if (isInt<16>(Imm) || 
isUInt<16>(Imm)3.19k
) {
2797
173k
      // A few special case instructions have 16-bit operands on subtargets
2798
173k
      // where 16-bit instructions are not legal.
2799
173k
      // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2800
173k
      // constants in these cases
2801
173k
      int16_t Trunc = static_cast<int16_t>(Imm);
2802
173k
      return ST.has16BitInsts() &&
2803
173k
             
AMDGPU::isInlinableLiteral16(Trunc, ST.hasInv2PiInlineImm())173k
;
2804
173k
    }
2805
123
2806
123
    return false;
2807
123
  }
2808
11.8k
  case AMDGPU::OPERAND_REG_IMM_V2INT16:
2809
11.8k
  case AMDGPU::OPERAND_REG_IMM_V2FP16:
2810
11.8k
  case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2811
11.8k
  case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2812
11.8k
  case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
2813
11.8k
  case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
2814
11.8k
    uint32_t Trunc = static_cast<uint32_t>(Imm);
2815
11.8k
    return AMDGPU::isInlinableLiteralV216(Trunc, ST.hasInv2PiInlineImm());
2816
11.8k
  }
2817
11.8k
  default:
2818
0
    llvm_unreachable("invalid bitwidth");
2819
5.18M
  }
2820
5.18M
}
2821
2822
bool SIInstrInfo::isLiteralConstantLike(const MachineOperand &MO,
2823
903k
                                        const MCOperandInfo &OpInfo) const {
2824
903k
  switch (MO.getType()) {
2825
903k
  case MachineOperand::MO_Register:
2826
647k
    return false;
2827
903k
  case MachineOperand::MO_Immediate:
2828
249k
    return !isInlineConstant(MO, OpInfo);
2829
903k
  case MachineOperand::MO_FrameIndex:
2830
6.32k
  case MachineOperand::MO_MachineBasicBlock:
2831
6.32k
  case MachineOperand::MO_ExternalSymbol:
2832
6.32k
  case MachineOperand::MO_GlobalAddress:
2833
6.32k
  case MachineOperand::MO_MCSymbol:
2834
6.32k
    return true;
2835
6.32k
  default:
2836
0
    llvm_unreachable("unexpected operand type");
2837
903k
  }
2838
903k
}
2839
2840
static bool compareMachineOp(const MachineOperand &Op0,
2841
22.4k
                             const MachineOperand &Op1) {
2842
22.4k
  if (Op0.getType() != Op1.getType())
2843
0
    return false;
2844
22.4k
2845
22.4k
  switch (Op0.getType()) {
2846
22.4k
  case MachineOperand::MO_Register:
2847
22.4k
    return Op0.getReg() == Op1.getReg();
2848
22.4k
  case MachineOperand::MO_Immediate:
2849
0
    return Op0.getImm() == Op1.getImm();
2850
22.4k
  default:
2851
0
    llvm_unreachable("Didn't expect to be comparing these operand types");
2852
22.4k
  }
2853
22.4k
}
2854
2855
bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
2856
105k
                                    const MachineOperand &MO) const {
2857
105k
  const MCInstrDesc &InstDesc = MI.getDesc();
2858
105k
  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
2859
105k
2860
105k
  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
2861
105k
2862
105k
  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2863
0
    return true;
2864
105k
2865
105k
  if (OpInfo.RegClass < 0)
2866
0
    return false;
2867
105k
2868
105k
  if (MO.isImm() && 
isInlineConstant(MO, OpInfo)104k
)
2869
78.2k
    return RI.opCanUseInlineConstant(OpInfo.OperandType);
2870
27.1k
2871
27.1k
  if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
2872
21.7k
    return false;
2873
5.39k
2874
5.39k
  if (!isVOP3(MI) || 
!AMDGPU::isSISrcOperand(InstDesc, OpNo)148
)
2875
5.24k
    return true;
2876
148
2877
148
  const MachineFunction *MF = MI.getParent()->getParent();
2878
148
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2879
148
  return ST.hasVOP3Literal();
2880
148
}
2881
2882
1.00M
bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2883
1.00M
  int Op32 = AMDGPU::getVOPe32(Opcode);
2884
1.00M
  if (Op32 == -1)
2885
790k
    return false;
2886
216k
2887
216k
  return pseudoToMCOpcode(Op32) != -1;
2888
216k
}
2889
2890
0
bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2891
0
  // The src0_modifier operand is present on all instructions
2892
0
  // that have modifiers.
2893
0
2894
0
  return AMDGPU::getNamedOperandIdx(Opcode,
2895
0
                                    AMDGPU::OpName::src0_modifiers) != -1;
2896
0
}
2897
2898
bool SIInstrInfo::hasModifiersSet(const MachineInstr &MI,
2899
250k
                                  unsigned OpName) const {
2900
250k
  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2901
250k
  return Mods && 
Mods->getImm()95.6k
;
2902
250k
}
2903
2904
423
bool SIInstrInfo::hasAnyModifiersSet(const MachineInstr &MI) const {
2905
423
  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2906
423
         
hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)360
||
2907
423
         
hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers)353
||
2908
423
         
hasModifiersSet(MI, AMDGPU::OpName::clamp)324
||
2909
423
         
hasModifiersSet(MI, AMDGPU::OpName::omod)322
;
2910
423
}
2911
2912
bool SIInstrInfo::canShrink(const MachineInstr &MI,
2913
104k
                            const MachineRegisterInfo &MRI) const {
2914
104k
  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2915
104k
  // Can't shrink instruction with three operands.
2916
104k
  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2917
104k
  // a special case for it.  It can only be shrunk if the third operand
2918
104k
  // is vcc, and src0_modifiers and src1_modifiers are not set.
2919
104k
  // We should handle this the same way we handle vopc, by addding
2920
104k
  // a register allocation hint pre-regalloc and then do the shrinking
2921
104k
  // post-regalloc.
2922
104k
  if (Src2) {
2923
30.8k
    switch (MI.getOpcode()) {
2924
30.8k
      
default: return false0
;
2925
30.8k
2926
30.8k
      case AMDGPU::V_ADDC_U32_e64:
2927
17.9k
      case AMDGPU::V_SUBB_U32_e64:
2928
17.9k
      case AMDGPU::V_SUBBREV_U32_e64: {
2929
17.9k
        const MachineOperand *Src1
2930
17.9k
          = getNamedOperand(MI, AMDGPU::OpName::src1);
2931
17.9k
        if (!Src1->isReg() || 
!RI.isVGPR(MRI, Src1->getReg())13.0k
)
2932
5.39k
          return false;
2933
12.5k
        // Additional verification is needed for sdst/src2.
2934
12.5k
        return true;
2935
12.5k
      }
2936
12.5k
      case AMDGPU::V_MAC_F32_e64:
2937
1.75k
      case AMDGPU::V_MAC_F16_e64:
2938
1.75k
      case AMDGPU::V_FMAC_F32_e64:
2939
1.75k
      case AMDGPU::V_FMAC_F16_e64:
2940
1.75k
        if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2941
1.75k
            hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2942
0
          return false;
2943
1.75k
        break;
2944
1.75k
2945
11.1k
      case AMDGPU::V_CNDMASK_B32_e64:
2946
11.1k
        break;
2947
86.6k
    }
2948
86.6k
  }
2949
86.6k
2950
86.6k
  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2951
86.6k
  if (Src1 && 
(80.4k
!Src1->isReg()80.4k
||
!RI.isVGPR(MRI, Src1->getReg())67.1k
||
2952
80.4k
               
hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)55.5k
))
2953
26.3k
    return false;
2954
60.3k
2955
60.3k
  // We don't need to check src0, all input types are legal, so just make sure
2956
60.3k
  // src0 isn't using any modifiers.
2957
60.3k
  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2958
1.00k
    return false;
2959
59.3k
2960
59.3k
  // Can it be shrunk to a valid 32 bit opcode?
2961
59.3k
  if (!hasVALU32BitEncoding(MI.getOpcode()))
2962
0
    return false;
2963
59.3k
2964
59.3k
  // Check output modifiers
2965
59.3k
  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2966
59.3k
         
!hasModifiersSet(MI, AMDGPU::OpName::clamp)59.2k
;
2967
59.3k
}
2968
2969
// Set VCC operand with all flags from \p Orig, except for setting it as
2970
// implicit.
2971
static void copyFlagsToImplicitVCC(MachineInstr &MI,
2972
8.92k
                                   const MachineOperand &Orig) {
2973
8.92k
2974
14.8k
  for (MachineOperand &Use : MI.implicit_operands()) {
2975
14.8k
    if (Use.isUse() && 
Use.getReg() == AMDGPU::VCC8.92k
) {
2976
8.92k
      Use.setIsUndef(Orig.isUndef());
2977
8.92k
      Use.setIsKill(Orig.isKill());
2978
8.92k
      return;
2979
8.92k
    }
2980
14.8k
  }
2981
8.92k
}
2982
2983
MachineInstr *SIInstrInfo::buildShrunkInst(MachineInstr &MI,
2984
47.9k
                                           unsigned Op32) const {
2985
47.9k
  MachineBasicBlock *MBB = MI.getParent();;
2986
47.9k
  MachineInstrBuilder Inst32 =
2987
47.9k
    BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
2988
47.9k
2989
47.9k
  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
2990
47.9k
  // For VOPC instructions, this is replaced by an implicit def of vcc.
2991
47.9k
  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
2992
47.9k
  if (Op32DstIdx != -1) {
2993
45.0k
    // dst
2994
45.0k
    Inst32.add(MI.getOperand(0));
2995
45.0k
  } else {
2996
2.81k
    assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
2997
2.81k
            (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
2998
2.81k
           "Unexpected case");
2999
2.81k
  }
3000
47.9k
3001
47.9k
  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3002
47.9k
3003
47.9k
  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3004
47.9k
  if (Src1)
3005
42.1k
    Inst32.add(*Src1);
3006
47.9k
3007
47.9k
  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3008
47.9k
3009
47.9k
  if (Src2) {
3010
10.1k
    int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3011
10.1k
    if (Op32Src2Idx != -1) {
3012
1.26k
      Inst32.add(*Src2);
3013
8.92k
    } else {
3014
8.92k
      // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3015
8.92k
      // replaced with an implicit read of vcc. This was already added
3016
8.92k
      // during the initial BuildMI, so find it to preserve the flags.
3017
8.92k
      copyFlagsToImplicitVCC(*Inst32, *Src2);
3018
8.92k
    }
3019
10.1k
  }
3020
47.9k
3021
47.9k
  return Inst32;
3022
47.9k
}
3023
3024
bool SIInstrInfo::usesConstantBus(const MachineRegisterInfo &MRI,
3025
                                  const MachineOperand &MO,
3026
11.1M
                                  const MCOperandInfo &OpInfo) const {
3027
11.1M
  // Literal constants use the constant bus.
3028
11.1M
  //if (isLiteralConstantLike(MO, OpInfo))
3029
11.1M
  // return true;
3030
11.1M
  if (MO.isImm())
3031
2.86M
    return !isInlineConstant(MO, OpInfo);
3032
8.29M
3033
8.29M
  if (!MO.isReg())
3034
24.1k
    return true; // Misc other operands like FrameIndex
3035
8.26M
3036
8.26M
  if (!MO.isUse())
3037
214k
    return false;
3038
8.05M
3039
8.05M
  if (TargetRegisterInfo::isVirtualRegister(MO.getReg()))
3040
4.10M
    return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3041
3.94M
3042
3.94M
  // Null is free
3043
3.94M
  if (MO.getReg() == AMDGPU::SGPR_NULL)
3044
0
    return false;
3045
3.94M
3046
3.94M
  // SGPRs use the constant bus
3047
3.94M
  if (MO.isImplicit()) {
3048
106k
    return MO.getReg() == AMDGPU::M0 ||
3049
106k
           
MO.getReg() == AMDGPU::VCC106k
||
3050
106k
           
MO.getReg() == AMDGPU::VCC_LO101k
;
3051
3.83M
  } else {
3052
3.83M
    return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3053
3.83M
           
AMDGPU::SReg_64RegClass.contains(MO.getReg())2.80M
;
3054
3.83M
  }
3055
3.94M
}
3056
3057
5.46M
static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
3058
6.44M
  for (const MachineOperand &MO : MI.implicit_operands()) {
3059
6.44M
    // We only care about reads.
3060
6.44M
    if (MO.isDef())
3061
330k
      continue;
3062
6.11M
3063
6.11M
    switch (MO.getReg()) {
3064
6.11M
    case AMDGPU::VCC:
3065
81.4k
    case AMDGPU::VCC_LO:
3066
81.4k
    case AMDGPU::VCC_HI:
3067
81.4k
    case AMDGPU::M0:
3068
81.4k
    case AMDGPU::FLAT_SCR:
3069
81.4k
      return MO.getReg();
3070
81.4k
3071
6.03M
    default:
3072
6.03M
      break;
3073
6.11M
    }
3074
6.11M
  }
3075
5.46M
3076
5.46M
  
return AMDGPU::NoRegister5.38M
;
3077
5.46M
}
3078
3079
14.8M
static bool shouldReadExec(const MachineInstr &MI) {
3080
14.8M
  if (SIInstrInfo::isVALU(MI)) {
3081
5.45M
    switch (MI.getOpcode()) {
3082
5.45M
    case AMDGPU::V_READLANE_B32:
3083
53.0k
    case AMDGPU::V_READLANE_B32_gfx6_gfx7:
3084
53.0k
    case AMDGPU::V_READLANE_B32_gfx10:
3085
53.0k
    case AMDGPU::V_READLANE_B32_vi:
3086
53.0k
    case AMDGPU::V_WRITELANE_B32:
3087
53.0k
    case AMDGPU::V_WRITELANE_B32_gfx6_gfx7:
3088
53.0k
    case AMDGPU::V_WRITELANE_B32_gfx10:
3089
53.0k
    case AMDGPU::V_WRITELANE_B32_vi:
3090
53.0k
      return false;
3091
5.40M
    }
3092
5.40M
3093
5.40M
    return true;
3094
5.40M
  }
3095
9.43M
3096
9.43M
  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3097
9.43M
      SIInstrInfo::isSALU(MI) ||
3098
9.43M
      
SIInstrInfo::isSMRD(MI)4.52M
)
3099
6.51M
    return false;
3100
2.92M
3101
2.92M
  return true;
3102
2.92M
}
3103
3104
static bool isSubRegOf(const SIRegisterInfo &TRI,
3105
                       const MachineOperand &SuperVec,
3106
1.66k
                       const MachineOperand &SubReg) {
3107
1.66k
  if (TargetRegisterInfo::isPhysicalRegister(SubReg.getReg()))
3108
1.04k
    return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3109
627
3110
627
  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3111
627
         SubReg.getReg() == SuperVec.getReg();
3112
627
}
3113
3114
bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
3115
22.1M
                                    StringRef &ErrInfo) const {
3116
22.1M
  uint16_t Opcode = MI.getOpcode();
3117
22.1M
  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3118
7.23M
    return true;
3119
14.8M
3120
14.8M
  const MachineFunction *MF = MI.getParent()->getParent();
3121
14.8M
  const MachineRegisterInfo &MRI = MF->getRegInfo();
3122
14.8M
3123
14.8M
  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3124
14.8M
  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3125
14.8M
  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3126
14.8M
3127
14.8M
  // Make sure the number of operands is correct.
3128
14.8M
  const MCInstrDesc &Desc = get(Opcode);
3129
14.8M
  if (!Desc.isVariadic() &&
3130
14.8M
      
Desc.getNumOperands() != MI.getNumExplicitOperands()14.7M
) {
3131
0
    ErrInfo = "Instruction has wrong number of operands.";
3132
0
    return false;
3133
0
  }
3134
14.8M
3135
14.8M
  if (MI.isInlineAsm()) {
3136
0
    // Verify register classes for inlineasm constraints.
3137
0
    for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
3138
0
         I != E; ++I) {
3139
0
      const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
3140
0
      if (!RC)
3141
0
        continue;
3142
0
3143
0
      const MachineOperand &Op = MI.getOperand(I);
3144
0
      if (!Op.isReg())
3145
0
        continue;
3146
0
3147
0
      unsigned Reg = Op.getReg();
3148
0
      if (!TargetRegisterInfo::isVirtualRegister(Reg) && !RC->contains(Reg)) {
3149
0
        ErrInfo = "inlineasm operand has incorrect register class.";
3150
0
        return false;
3151
0
      }
3152
0
    }
3153
0
3154
0
    return true;
3155
14.8M
  }
3156
14.8M
3157
14.8M
  // Make sure the register classes are correct.
3158
73.0M
  
for (int i = 0, e = Desc.getNumOperands(); 14.8M
i != e;
++i58.1M
) {
3159
58.1M
    if (MI.getOperand(i).isFPImm()) {
3160
0
      ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3161
0
                "all fp values to integers.";
3162
0
      return false;
3163
0
    }
3164
58.1M
3165
58.1M
    int RegClass = Desc.OpInfo[i].RegClass;
3166
58.1M
3167
58.1M
    switch (Desc.OpInfo[i].OperandType) {
3168
58.1M
    case MCOI::OPERAND_REGISTER:
3169
21.1M
      if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
3170
0
        ErrInfo = "Illegal immediate value for operand.";
3171
0
        return false;
3172
0
      }
3173
21.1M
      break;
3174
21.1M
    case AMDGPU::OPERAND_REG_IMM_INT32:
3175
10.8M
    case AMDGPU::OPERAND_REG_IMM_FP32:
3176
10.8M
      break;
3177
10.8M
    case AMDGPU::OPERAND_REG_INLINE_C_INT32:
3178
1.80M
    case AMDGPU::OPERAND_REG_INLINE_C_FP32:
3179
1.80M
    case AMDGPU::OPERAND_REG_INLINE_C_INT64:
3180
1.80M
    case AMDGPU::OPERAND_REG_INLINE_C_FP64:
3181
1.80M
    case AMDGPU::OPERAND_REG_INLINE_C_INT16:
3182
1.80M
    case AMDGPU::OPERAND_REG_INLINE_C_FP16:
3183
1.80M
    case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
3184
1.80M
    case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
3185
1.80M
    case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
3186
1.80M
    case AMDGPU::OPERAND_REG_INLINE_AC_FP16: {
3187
1.80M
      const MachineOperand &MO = MI.getOperand(i);
3188
1.80M
      if (!MO.isReg() && 
(1.16M
!MO.isImm()1.16M
||
!isInlineConstant(MI, i)1.16M
)) {
3189
0
        ErrInfo = "Illegal immediate value for operand.";
3190
0
        return false;
3191
0
      }
3192
1.80M
      break;
3193
1.80M
    }
3194
20.1M
    case MCOI::OPERAND_IMMEDIATE:
3195
20.1M
    case AMDGPU::OPERAND_KIMM32:
3196
20.1M
      // Check if this operand is an immediate.
3197
20.1M
      // FrameIndex operands will be replaced by immediates, so they are
3198
20.1M
      // allowed.
3199
20.1M
      if (!MI.getOperand(i).isImm() && 
!MI.getOperand(i).isFI()10.0k
) {
3200
0
        ErrInfo = "Expected immediate, but got non-immediate";
3201
0
        return false;
3202
0
      }
3203
20.1M
      LLVM_FALLTHROUGH;
3204
24.4M
    default:
3205
24.4M
      continue;
3206
33.7M
    }
3207
33.7M
3208
33.7M
    if (!MI.getOperand(i).isReg())
3209
6.05M
      continue;
3210
27.6M
3211
27.6M
    if (RegClass != -1) {
3212
27.6M
      unsigned Reg = MI.getOperand(i).getReg();
3213
27.6M
      if (Reg == AMDGPU::NoRegister ||
3214
27.6M
          
TargetRegisterInfo::isVirtualRegister(Reg)27.6M
)
3215
13.0M
        continue;
3216
14.6M
3217
14.6M
      const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3218
14.6M
      if (!RC->contains(Reg)) {
3219
0
        ErrInfo = "Operand has incorrect register class.";
3220
0
        return false;
3221
0
      }
3222
14.6M
    }
3223
27.6M
  }
3224
14.8M
3225
14.8M
  // Verify SDWA
3226
14.8M
  if (isSDWA(MI)) {
3227
78.6k
    if (!ST.hasSDWA()) {
3228
0
      ErrInfo = "SDWA is not supported on this target";
3229
0
      return false;
3230
0
    }
3231
78.6k
3232
78.6k
    int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3233
78.6k
3234
78.6k
    const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3235
78.6k
3236
314k
    for (int OpIdx: OpIndicies) {
3237
314k
      if (OpIdx == -1)
3238
87.2k
        continue;
3239
227k
      const MachineOperand &MO = MI.getOperand(OpIdx);
3240
227k
3241
227k
      if (!ST.hasSDWAScalar()) {
3242
160k
        // Only VGPRS on VI
3243
160k
        if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3244
0
          ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3245
0
          return false;
3246
0
        }
3247
67.3k
      } else {
3248
67.3k
        // No immediates on GFX9
3249
67.3k
        if (!MO.isReg()) {
3250
0
          ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
3251
0
          return false;
3252
0
        }
3253
67.3k
      }
3254
227k
    }
3255
78.6k
3256
78.6k
    if (!ST.hasSDWAOmod()) {
3257
55.9k
      // No omod allowed on VI
3258
55.9k
      const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3259
55.9k
      if (OMod != nullptr &&
3260
55.9k
        
(16.4k
!OMod->isImm()16.4k
||
OMod->getImm() != 016.4k
)) {
3261
0
        ErrInfo = "OMod not allowed in SDWA instructions on VI";
3262
0
        return false;
3263
0
      }
3264
78.6k
    }
3265
78.6k
3266
78.6k
    uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3267
78.6k
    if (isVOPC(BasicOpcode)) {
3268
277
      if (!ST.hasSDWASdst() && 
DstIdx != -113
) {
3269
0
        // Only vcc allowed as dst on VI for VOPC
3270
0
        const MachineOperand &Dst = MI.getOperand(DstIdx);
3271
0
        if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3272
0
          ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3273
0
          return false;
3274
0
        }
3275
277
      } else if (!ST.hasSDWAOutModsVOPC()) {
3276
264
        // No clamp allowed on GFX9 for VOPC
3277
264
        const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3278
264
        if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3279
0
          ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3280
0
          return false;
3281
0
        }
3282
264
3283
264
        // No omod allowed on GFX9 for VOPC
3284
264
        const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3285
264
        if (OMod && 
(0
!OMod->isImm()0
||
OMod->getImm() != 00
)) {
3286
0
          ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3287
0
          return false;
3288
0
        }
3289
78.6k
      }
3290
277
    }
3291
78.6k
3292
78.6k
    const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3293
78.6k
    if (DstUnused && 
DstUnused->isImm()78.3k
&&
3294
78.6k
        
DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE78.3k
) {
3295
260
      const MachineOperand &Dst = MI.getOperand(DstIdx);
3296
260
      if (!Dst.isReg() || !Dst.isTied()) {
3297
0
        ErrInfo = "Dst register should have tied register";
3298
0
        return false;
3299
0
      }
3300
260
3301
260
      const MachineOperand &TiedMO =
3302
260
          MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3303
260
      if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3304
0
        ErrInfo =
3305
0
            "Dst register should be tied to implicit use of preserved register";
3306
0
        return false;
3307
260
      } else if (TargetRegisterInfo::isPhysicalRegister(TiedMO.getReg()) &&
3308
260
                 
Dst.getReg() != TiedMO.getReg()150
) {
3309
0
        ErrInfo = "Dst register should use same physical register as preserved";
3310
0
        return false;
3311
0
      }
3312
14.8M
    }
3313
78.6k
  }
3314
14.8M
3315
14.8M
  // Verify MIMG
3316
14.8M
  if (isMIMG(MI.getOpcode()) && 
!MI.mayStore()65.2k
) {
3317
52.0k
    // Ensure that the return type used is large enough for all the options
3318
52.0k
    // being used TFE/LWE require an extra result register.
3319
52.0k
    const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3320
52.0k
    if (DMask) {
3321
52.0k
      uint64_t DMaskImm = DMask->getImm();
3322
52.0k
      uint32_t RegCount =
3323
52.0k
          isGather4(MI.getOpcode()) ? 
45.35k
:
countPopulation(DMaskImm)46.6k
;
3324
52.0k
      const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3325
52.0k
      const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3326
52.0k
      const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3327
52.0k
3328
52.0k
      // Adjust for packed 16 bit values
3329
52.0k
      if (D16 && 
D16->getImm()49.0k
&&
!ST.hasUnpackedD16VMem()3.12k
)
3330
2.49k
        RegCount >>= 1;
3331
52.0k
3332
52.0k
      // Adjust if using LWE or TFE
3333
52.0k
      if ((LWE && LWE->getImm()) || 
(50.3k
TFE50.3k
&&
TFE->getImm()50.3k
))
3334
7.80k
        RegCount += 1;
3335
52.0k
3336
52.0k
      const uint32_t DstIdx =
3337
52.0k
          AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3338
52.0k
      const MachineOperand &Dst = MI.getOperand(DstIdx);
3339
52.0k
      if (Dst.isReg()) {
3340
52.0k
        const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3341
52.0k
        uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3342
52.0k
        if (RegCount > DstSize) {
3343
0
          ErrInfo = "MIMG instruction returns too many registers for dst "
3344
0
                    "register class";
3345
0
          return false;
3346
0
        }
3347
14.8M
      }
3348
52.0k
    }
3349
52.0k
  }
3350
14.8M
3351
14.8M
  // Verify VOP*. Ignore multiple sgpr operands on writelane.
3352
14.8M
  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3353
14.8M
      && 
(14.8M
isVOP1(MI)14.8M
||
isVOP2(MI)13.4M
||
isVOP3(MI)12.0M
||
isVOPC(MI)9.61M
||
isSDWA(MI)9.59M
)) {
3354
5.37M
    // Only look at the true operands. Only a real operand can use the constant
3355
5.37M
    // bus, and we don't want to check pseudo-operands like the source modifier
3356
5.37M
    // flags.
3357
5.37M
    const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3358
5.37M
3359
5.37M
    unsigned ConstantBusCount = 0;
3360
5.37M
    unsigned LiteralCount = 0;
3361
5.37M
3362
5.37M
    if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3363
2.25k
      ++ConstantBusCount;
3364
5.37M
3365
5.37M
    SmallVector<unsigned, 2> SGPRsUsed;
3366
5.37M
    unsigned SGPRUsed = findImplicitSGPRRead(MI);
3367
5.37M
    if (SGPRUsed != AMDGPU::NoRegister) {
3368
80.9k
      ++ConstantBusCount;
3369
80.9k
      SGPRsUsed.push_back(SGPRUsed);
3370
80.9k
    }
3371
5.37M
3372
14.5M
    for (int OpIdx : OpIndices) {
3373
14.5M
      if (OpIdx == -1)
3374
4.27M
        break;
3375
10.2M
      const MachineOperand &MO = MI.getOperand(OpIdx);
3376
10.2M
      if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3377
2.47M
        if (MO.isReg()) {
3378
2.20M
          SGPRUsed = MO.getReg();
3379
2.20M
          if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
3380
71.3k
                return !RI.regsOverlap(SGPRUsed, SGPR);
3381
2.17M
              })) {
3382
2.17M
            ++ConstantBusCount;
3383
2.17M
            SGPRsUsed.push_back(SGPRUsed);
3384
2.17M
          }
3385
2.20M
        } else {
3386
273k
          ++ConstantBusCount;
3387
273k
          ++LiteralCount;
3388
273k
        }
3389
2.47M
      }
3390
10.2M
    }
3391
5.37M
    const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3392
5.37M
    // v_writelane_b32 is an exception from constant bus restriction:
3393
5.37M
    // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3394
5.37M
    if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
3395
5.37M
        
Opcode != AMDGPU::V_WRITELANE_B320
) {
3396
0
      ErrInfo = "VOP* instruction violates constant bus restriction";
3397
0
      return false;
3398
0
    }
3399
5.37M
3400
5.37M
    if (isVOP3(MI) && 
LiteralCount2.44M
) {
3401
3.95k
      if (LiteralCount && !ST.hasVOP3Literal()) {
3402
0
        ErrInfo = "VOP3 instruction uses literal";
3403
0
        return false;
3404
0
      }
3405
3.95k
      if (LiteralCount > 1) {
3406
0
        ErrInfo = "VOP3 instruction uses more than one literal";
3407
0
        return false;
3408
0
      }
3409
14.8M
    }
3410
5.37M
  }
3411
14.8M
3412
14.8M
  // Verify misc. restrictions on specific instructions.
3413
14.8M
  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3414
14.8M
      
Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F6414.8M
) {
3415
15.6k
    const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3416
15.6k
    const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3417
15.6k
    const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3418
15.6k
    if (Src0.isReg() && 
Src1.isReg()15.3k
&&
Src2.isReg()15.3k
) {
3419
15.0k
      if (!compareMachineOp(Src0, Src1) &&
3420
15.0k
          
!compareMachineOp(Src0, Src2)7.37k
) {
3421
0
        ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3422
0
        return false;
3423
0
      }
3424
14.8M
    }
3425
15.6k
  }
3426
14.8M
3427
14.8M
  if (isSOP2(MI) || 
isSOPC(MI)13.7M
) {
3428
1.14M
    const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3429
1.14M
    const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3430
1.14M
    unsigned Immediates = 0;
3431
1.14M
3432
1.14M
    if (!Src0.isReg() &&
3433
1.14M
        
!isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType)9.92k
)
3434
1.74k
      Immediates++;
3435
1.14M
    if (!Src1.isReg() &&
3436
1.14M
        
!isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType)709k
)
3437
271k
      Immediates++;
3438
1.14M
3439
1.14M
    if (Immediates > 1) {
3440
2
      ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
3441
2
      return false;
3442
2
    }
3443
14.8M
  }
3444
14.8M
3445
14.8M
  if (isSOPK(MI)) {
3446
33.7k
    auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
3447
33.7k
    if (Desc.isBranch()) {
3448
60
      if (!Op->isMBB()) {
3449
0
        ErrInfo = "invalid branch target for SOPK instruction";
3450
0
        return false;
3451
0
      }
3452
33.6k
    } else {
3453
33.6k
      uint64_t Imm = Op->getImm();
3454
33.6k
      if (sopkIsZext(MI)) {
3455
888
        if (!isUInt<16>(Imm)) {
3456
0
          ErrInfo = "invalid immediate for SOPK instruction";
3457
0
          return false;
3458
0
        }
3459
32.7k
      } else {
3460
32.7k
        if (!isInt<16>(Imm)) {
3461
0
          ErrInfo = "invalid immediate for SOPK instruction";
3462
0
          return false;
3463
0
        }
3464
14.8M
      }
3465
33.6k
    }
3466
33.7k
  }
3467
14.8M
3468
14.8M
  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3469
14.8M
      
Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e6414.8M
||
3470
14.8M
      
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e3214.8M
||
3471
14.8M
      
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e6414.8M
) {
3472
1.66k
    const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3473
1.66k
                       
Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e641.23k
;
3474
1.66k
3475
1.66k
    const unsigned StaticNumOps = Desc.getNumOperands() +
3476
1.66k
      Desc.getNumImplicitUses();
3477
1.66k
    const unsigned NumImplicitOps = IsDst ? 
2435
:
11.23k
;
3478
1.66k
3479
1.66k
    // Allow additional implicit operands. This allows a fixup done by the post
3480
1.66k
    // RA scheduler where the main implicit operand is killed and implicit-defs
3481
1.66k
    // are added for sub-registers that remain live after this instruction.
3482
1.66k
    if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3483
0
      ErrInfo = "missing implicit register operands";
3484
0
      return false;
3485
0
    }
3486
1.66k
3487
1.66k
    const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3488
1.66k
    if (IsDst) {
3489
435
      if (!Dst->isUse()) {
3490
0
        ErrInfo = "v_movreld_b32 vdst should be a use operand";
3491
0
        return false;
3492
0
      }
3493
435
3494
435
      unsigned UseOpIdx;
3495
435
      if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3496
435
          UseOpIdx != StaticNumOps + 1) {
3497
0
        ErrInfo = "movrel implicit operands should be tied";
3498
0
        return false;
3499
0
      }
3500
1.66k
    }
3501
1.66k
3502
1.66k
    const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3503
1.66k
    const MachineOperand &ImpUse
3504
1.66k
      = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3505
1.66k
    if (!ImpUse.isReg() || !ImpUse.isUse() ||
3506
1.66k
        !isSubRegOf(RI, ImpUse, IsDst ? 
*Dst435
:
Src01.23k
)) {
3507
0
      ErrInfo = "src0 should be subreg of implicit vector use";
3508
0
      return false;
3509
0
    }
3510
14.8M
  }
3511
14.8M
3512
14.8M
  // Make sure we aren't losing exec uses in the td files. This mostly requires
3513
14.8M
  // being careful when using let Uses to try to add other use registers.
3514
14.8M
  if (shouldReadExec(MI)) {
3515
8.33M
    if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3516
0
      ErrInfo = "VALU instruction does not implicitly read exec mask";
3517
0
      return false;
3518
0
    }
3519
14.8M
  }
3520
14.8M
3521
14.8M
  if (isSMRD(MI)) {
3522
1.59M
    if (MI.mayStore()) {
3523
1.30k
      // The register offset form of scalar stores may only use m0 as the
3524
1.30k
      // soffset register.
3525
1.30k
      const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3526
1.30k
      if (Soff && 
Soff->getReg() != AMDGPU::M0265
) {
3527
0
        ErrInfo = "scalar stores must use m0 as offset register";
3528
0
        return false;
3529
0
      }
3530
14.8M
    }
3531
1.59M
  }
3532
14.8M
3533
14.8M
  if (isFLAT(MI) && 
!MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()858k
) {
3534
481k
    const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3535
481k
    if (Offset->getImm() != 0) {
3536
0
      ErrInfo = "subtarget does not support offsets in flat instructions";
3537
0
      return false;
3538
0
    }
3539
14.8M
  }
3540
14.8M
3541
14.8M
  if (isMIMG(MI)) {
3542
65.2k
    const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
3543
65.2k
    if (DimOp) {
3544
12.4k
      int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
3545
12.4k
                                                 AMDGPU::OpName::vaddr0);
3546
12.4k
      int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
3547
12.4k
      const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
3548
12.4k
      const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3549
12.4k
          AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
3550
12.4k
      const AMDGPU::MIMGDimInfo *Dim =
3551
12.4k
          AMDGPU::getMIMGDimInfoByEncoding(DimOp->getImm());
3552
12.4k
3553
12.4k
      if (!Dim) {
3554
0
        ErrInfo = "dim is out of range";
3555
0
        return false;
3556
0
      }
3557
12.4k
3558
12.4k
      bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
3559
12.4k
      unsigned AddrWords = BaseOpcode->NumExtraArgs +
3560
12.4k
                           (BaseOpcode->Gradients ? 
Dim->NumGradients1.25k
:
011.1k
) +
3561
12.4k
                           (BaseOpcode->Coordinates ? 
Dim->NumCoords12.0k
:
0416
) +
3562
12.4k
                           (BaseOpcode->LodOrClampOrMip ? 
13.32k
:
09.12k
);
3563
12.4k
3564
12.4k
      unsigned VAddrWords;
3565
12.4k
      if (IsNSA) {
3566
5.07k
        VAddrWords = SRsrcIdx - VAddr0Idx;
3567
7.37k
      } else {
3568
7.37k
        const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
3569
7.37k
        VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
3570
7.37k
        if (AddrWords > 8)
3571
66
          AddrWords = 16;
3572
7.30k
        else if (AddrWords > 4)
3573
320
          AddrWords = 8;
3574
6.98k
        else if (AddrWords == 3 && 
VAddrWords == 4666
) {
3575
260
          // CodeGen uses the V4 variant of instructions for three addresses,
3576
260
          // because the selection DAG does not support non-power-of-two types.
3577
260
          AddrWords = 4;
3578
260
        }
3579
7.37k
      }
3580
12.4k
3581
12.4k
      if (VAddrWords != AddrWords) {
3582
0
        ErrInfo = "bad vaddr size";
3583
0
        return false;
3584
0
      }
3585
14.8M
    }
3586
65.2k
  }
3587
14.8M
3588
14.8M
  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3589
14.8M
  if (DppCt) {
3590
14.4k
    using namespace AMDGPU::DPP;
3591
14.4k
3592
14.4k
    unsigned DC = DppCt->getImm();
3593
14.4k
    if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3594
14.4k
        DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3595
14.4k
        (DC >= DppCtrl::DPP_UNUSED4_FIRST && 
DC <= DppCtrl::DPP_UNUSED4_LAST5.87k
) ||
3596
14.4k
        (DC >= DppCtrl::DPP_UNUSED5_FIRST && 
DC <= DppCtrl::DPP_UNUSED5_LAST5.87k
) ||
3597
14.4k
        (DC >= DppCtrl::DPP_UNUSED6_FIRST && 
DC <= DppCtrl::DPP_UNUSED6_LAST4.00k
) ||
3598
14.4k
        (DC >= DppCtrl::DPP_UNUSED7_FIRST && 
DC <= DppCtrl::DPP_UNUSED7_LAST4.00k
) ||
3599
14.4k
        (DC >= DppCtrl::DPP_UNUSED8_FIRST && 
DC <= DppCtrl::DPP_UNUSED8_LAST0
)) {
3600
0
      ErrInfo = "Invalid dpp_ctrl value";
3601
0
      return false;
3602
0
    }
3603
14.4k
    if (DC >= DppCtrl::WAVE_SHL1 && 
DC <= DppCtrl::WAVE_ROR15.87k
&&
3604
14.4k
        
ST.getGeneration() >= AMDGPUSubtarget::GFX101.86k
) {
3605
0
      ErrInfo = "Invalid dpp_ctrl value: "
3606
0
                "wavefront shifts are not supported on GFX10+";
3607
0
      return false;
3608
0
    }
3609
14.4k
    if (DC >= DppCtrl::BCAST15 && 
DC <= DppCtrl::BCAST314.00k
&&
3610
14.4k
        
ST.getGeneration() >= AMDGPUSubtarget::GFX104.00k
) {
3611
0
      ErrInfo = "Invalid dpp_ctrl value: "
3612
0
                "broadcats are not supported on GFX10+";
3613
0
      return false;
3614
0
    }
3615
14.4k
    if (DC >= DppCtrl::ROW_SHARE_FIRST && 
DC <= DppCtrl::ROW_XMASK_LAST0
&&
3616
14.4k
        
ST.getGeneration() < AMDGPUSubtarget::GFX100
) {
3617
0
      ErrInfo = "Invalid dpp_ctrl value: "
3618
0
                "row_share and row_xmask are not supported before GFX10";
3619
0
      return false;
3620
0
    }
3621
14.8M
  }
3622
14.8M
3623
14.8M
  return true;
3624
14.8M
}
3625
3626
111k
unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3627
111k
  switch (MI.getOpcode()) {
3628
111k
  
default: return AMDGPU::INSTRUCTION_LIST_END9.08k
;
3629
111k
  
case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE21.0k
;
3630
111k
  
case AMDGPU::COPY: return AMDGPU::COPY49.4k
;
3631
111k
  
case AMDGPU::PHI: return AMDGPU::PHI665
;
3632
111k
  
case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG8
;
3633
111k
  
case AMDGPU::WQM: return AMDGPU::WQM9
;
3634
111k
  
case AMDGPU::WWM: return AMDGPU::WWM61
;
3635
111k
  case AMDGPU::S_MOV_B32: {
3636
29
    const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3637
29
    return MI.getOperand(1).isReg() ||
3638
29
           
RI.isAGPR(MRI, MI.getOperand(0).getReg())0
?
3639
29
           AMDGPU::COPY : 
AMDGPU::V_MOV_B32_e320
;
3640
111k
  }
3641
111k
  case AMDGPU::S_ADD_I32:
3642
2.14k
    return ST.hasAddNoCarry() ? 
AMDGPU::V_ADD_U32_e64435
:
AMDGPU::V_ADD_I32_e321.71k
;
3643
111k
  case AMDGPU::S_ADDC_U32:
3644
290
    return AMDGPU::V_ADDC_U32_e32;
3645
111k
  case AMDGPU::S_SUB_I32:
3646
844
    return ST.hasAddNoCarry() ? 
AMDGPU::V_SUB_U32_e6485
:
AMDGPU::V_SUB_I32_e32759
;
3647
111k
    // FIXME: These are not consistently handled, and selected when the carry is
3648
111k
    // used.
3649
111k
  case AMDGPU::S_ADD_U32:
3650
145
    return AMDGPU::V_ADD_I32_e32;
3651
111k
  case AMDGPU::S_SUB_U32:
3652
0
    return AMDGPU::V_SUB_I32_e32;
3653
111k
  
case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e320
;
3654
111k
  
case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U321.40k
;
3655
111k
  
case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U320
;
3656
111k
  
case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I320
;
3657
111k
  
case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e643.65k
;
3658
111k
  
case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e642.22k
;
3659
111k
  
case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64511
;
3660
111k
  case AMDGPU::S_XNOR_B32:
3661
31
    return ST.hasDLInsts() ? 
AMDGPU::V_XNOR_B32_e6411
:
AMDGPU::INSTRUCTION_LIST_END20
;
3662
111k
  
case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e6424
;
3663
111k
  
case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e640
;
3664
111k
  
case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e6429
;
3665
111k
  
case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e643
;
3666
111k
  
case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e322.88k
;
3667
111k
  
case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64211
;
3668
111k
  
case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e325.57k
;
3669
111k
  
case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64844
;
3670
111k
  
case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e324.29k
;
3671
111k
  
case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64219
;
3672
111k
  
case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32586
;
3673
111k
  
case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32978
;
3674
111k
  
case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U321.31k
;
3675
111k
  
case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I321.97k
;
3676
111k
  
case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e640
;
3677
111k
  
case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e3215
;
3678
111k
  
case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32129
;
3679
111k
  
case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e3211
;
3680
111k
  
case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e320
;
3681
111k
  
case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e320
;
3682
111k
  
case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e3220
;
3683
111k
  
case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e324
;
3684
111k
  
case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e325
;
3685
111k
  
case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e320
;
3686
111k
  
case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e3223
;
3687
111k
  
case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e3234
;
3688
111k
  
case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e322
;
3689
111k
  
case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e323
;
3690
111k
  
case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e323
;
3691
111k
  
case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e320
;
3692
111k
  
case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e321
;
3693
111k
  
case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e321
;
3694
111k
  
case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64128
;
3695
111k
  
case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e3254
;
3696
111k
  
case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32158
;
3697
111k
  
case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e642
;
3698
111k
  
case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ0
;
3699
111k
  
case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ96
;
3700
0
  }
3701
0
  llvm_unreachable(
3702
0
      "Unexpected scalar opcode without corresponding vector one!");
3703
0
}
3704
3705
const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
3706
2.49M
                                                      unsigned OpNo) const {
3707
2.49M
  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3708
2.49M
  const MCInstrDesc &Desc = get(MI.getOpcode());
3709
2.49M
  if (MI.isVariadic() || 
OpNo >= Desc.getNumOperands()2.32M
||
3710
2.49M
      
Desc.OpInfo[OpNo].RegClass == -11.95M
) {
3711
1.10M
    unsigned Reg = MI.getOperand(OpNo).getReg();
3712
1.10M
3713
1.10M
    if (TargetRegisterInfo::isVirtualRegister(Reg))
3714
643k
      return MRI.getRegClass(Reg);
3715
463k
    return RI.getPhysRegClass(Reg);
3716
463k
  }
3717
1.38M
3718
1.38M
  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3719
1.38M
  return RI.getRegClass(RCID);
3720
1.38M
}
3721
3722
24.8k
void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3723
24.8k
  MachineBasicBlock::iterator I = MI;
3724
24.8k
  MachineBasicBlock *MBB = MI.getParent();
3725
24.8k
  MachineOperand &MO = MI.getOperand(OpIdx);
3726
24.8k
  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
3727
24.8k
  const SIRegisterInfo *TRI =
3728
24.8k
      static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3729
24.8k
  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3730
24.8k
  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3731
24.8k
  unsigned Size = TRI->getRegSizeInBits(*RC);
3732
24.8k
  unsigned Opcode = (Size == 64) ? 
AMDGPU::V_MOV_B64_PSEUDO1.05k
:
AMDGPU::V_MOV_B32_e3223.7k
;
3733
24.8k
  if (MO.isReg())
3734
24.8k
    Opcode = AMDGPU::COPY;
3735
0
  else if (RI.isSGPRClass(RC))
3736
0
    Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
3737
24.8k
3738
24.8k
  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3739
24.8k
  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3740
1.05k
    VRC = &AMDGPU::VReg_64RegClass;
3741
23.7k
  else
3742
23.7k
    VRC = &AMDGPU::VGPR_32RegClass;
3743
24.8k
3744
24.8k
  unsigned Reg = MRI.createVirtualRegister(VRC);
3745
24.8k
  DebugLoc DL = MBB->findDebugLoc(I);
3746
24.8k
  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3747
24.8k
  MO.ChangeToRegister(Reg, false);
3748
24.8k
}
3749
3750
unsigned SIInstrInfo::buildExtractSubReg(MachineBasicBlock::iterator MI,
3751
                                         MachineRegisterInfo &MRI,
3752
                                         MachineOperand &SuperReg,
3753
                                         const TargetRegisterClass *SuperRC,
3754
                                         unsigned SubIdx,
3755
                                         const TargetRegisterClass *SubRC)
3756
37.7k
                                         const {
3757
37.7k
  MachineBasicBlock *MBB = MI->getParent();
3758
37.7k
  DebugLoc DL = MI->getDebugLoc();
3759
37.7k
  unsigned SubReg = MRI.createVirtualRegister(SubRC);
3760
37.7k
3761
37.7k
  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3762
37.7k
    BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3763
37.7k
      .addReg(SuperReg.getReg(), 0, SubIdx);
3764
37.7k
    return SubReg;
3765
37.7k
  }
3766
0
3767
0
  // Just in case the super register is itself a sub-register, copy it to a new
3768
0
  // value so we don't need to worry about merging its subreg index with the
3769
0
  // SubIdx passed to this function. The register coalescer should be able to
3770
0
  // eliminate this extra copy.
3771
0
  unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC);
3772
0
3773
0
  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3774
0
    .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3775
0
3776
0
  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3777
0
    .addReg(NewSuperReg, 0, SubIdx);
3778
0
3779
0
  return SubReg;
3780
0
}
3781
3782
MachineOperand SIInstrInfo::buildExtractSubRegOrImm(
3783
  MachineBasicBlock::iterator MII,
3784
  MachineRegisterInfo &MRI,
3785
  MachineOperand &Op,
3786
  const TargetRegisterClass *SuperRC,
3787
  unsigned SubIdx,
3788
37.7k
  const TargetRegisterClass *SubRC) const {
3789
37.7k
  if (Op.isImm()) {
3790
0
    if (SubIdx == AMDGPU::sub0)
3791
0
      return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3792
0
    if (SubIdx == AMDGPU::sub1)
3793
0
      return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3794
0
3795
0
    llvm_unreachable("Unhandled register index for immediate");
3796
0
  }
3797
37.7k
3798
37.7k
  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3799
37.7k
                                       SubIdx, SubRC);
3800
37.7k
  return MachineOperand::CreateReg(SubReg, false);
3801
37.7k
}
3802
3803
// Change the order of operands from (0, 1, 2) to (0, 2, 1)
3804
8.30k
void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3805
8.30k
  assert(Inst.getNumExplicitOperands() == 3);
3806
8.30k
  MachineOperand Op1 = Inst.getOperand(1);
3807
8.30k
  Inst.RemoveOperand(1);
3808
8.30k
  Inst.addOperand(Op1);
3809
8.30k
}
3810
3811
bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI,
3812
                                    const MCOperandInfo &OpInfo,
3813
469k
                                    const MachineOperand &MO) const {
3814
469k
  if (!MO.isReg())
3815
554
    return false;
3816
468k
3817
468k
  unsigned Reg = MO.getReg();
3818
468k
  const TargetRegisterClass *RC =
3819
468k
    TargetRegisterInfo::isVirtualRegister(Reg) ?
3820
454k
    MRI.getRegClass(Reg) :
3821
468k
    
RI.getPhysRegClass(Reg)14.2k
;
3822
468k
3823
468k
  const SIRegisterInfo *TRI =
3824
468k
      static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3825
468k
  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3826
468k
3827
468k
  // In order to be legal, the common sub-class must be equal to the
3828
468k
  // class of the current operand.  For example:
3829
468k
  //
3830
468k
  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3831
468k
  //              ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3832
468k
  //
3833
468k
  // s_sendmsg 0, s0 ; Operand defined as m0reg
3834
468k
  //                 ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3835
468k
3836
468k
  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3837
468k
}
3838
3839
bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
3840
                                     const MCOperandInfo &OpInfo,
3841
0
                                     const MachineOperand &MO) const {
3842
0
  if (MO.isReg())
3843
0
    return isLegalRegOperand(MRI, OpInfo, MO);
3844
0
3845
0
  // Handle non-register types that are treated like immediates.
3846
0
  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3847
0
  return true;
3848
0
}
3849
3850
bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3851
622k
                                 const MachineOperand *MO) const {
3852
622k
  const MachineFunction &MF = *MI.getParent()->getParent();
3853
622k
  const MachineRegisterInfo &MRI = MF.getRegInfo();
3854
622k
  const MCInstrDesc &InstDesc = MI.getDesc();
3855
622k
  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3856
622k
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3857
622k
  const TargetRegisterClass *DefinedRC =
3858
622k
      OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : 
nullptr0
;
3859
622k
  if (!MO)
3860
0
    MO = &MI.getOperand(OpIdx);
3861
622k
3862
622k
  int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
3863
622k
  int VOP3LiteralLimit = ST.hasVOP3Literal() ? 
133.5k
:
0588k
;
3864
622k
  if (isVALU(MI) && 
usesConstantBus(MRI, *MO, OpInfo)427k
) {
3865
167k
    if (isVOP3(MI) && 
isLiteralConstantLike(*MO, OpInfo)125k
&&
!VOP3LiteralLimit--8.89k
)
3866
8.53k
      return false;
3867
159k
3868
159k
    SmallDenseSet<RegSubRegPair> SGPRsUsed;
3869
159k
    if (MO->isReg())
3870
138k
      SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
3871
159k
3872
884k
    for (unsigned i = 0, e = MI.getNumOperands(); i != e; 
++i725k
) {
3873
782k
      if (i == OpIdx)
3874
142k
        continue;
3875
639k
      const MachineOperand &Op = MI.getOperand(i);
3876
639k
      if (Op.isReg()) {
3877
494k
        RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
3878
494k
        if (!SGPRsUsed.count(SGPR) &&
3879
494k
            
usesConstantBus(MRI, Op, InstDesc.OpInfo[i])427k
) {
3880
59.4k
          if (--ConstantBusLimit <= 0)
3881
56.8k
            return false;
3882
2.52k
          SGPRsUsed.insert(SGPR);
3883
2.52k
        }
3884
494k
      } else 
if (145k
InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32145k
) {
3885
44
        if (--ConstantBusLimit <= 0)
3886
34
          return false;
3887
145k
      } else if (isVOP3(MI) && 
AMDGPU::isSISrcOperand(InstDesc, i)126k
&&
3888
145k
                 
isLiteralConstantLike(Op, InstDesc.OpInfo[i])8.65k
) {
3889
185
        if (!VOP3LiteralLimit--)
3890
173
          return false;
3891
12
        if (--ConstantBusLimit <= 0)
3892
0
          return false;
3893
12
      }
3894
639k
    }
3895
159k
  }
3896
622k
3897
622k
  
if (556k
MO->isReg()556k
) {
3898
451k
    assert(DefinedRC);
3899
451k
    return isLegalRegOperand(MRI, OpInfo, *MO);
3900
451k
  }
3901
105k
3902
105k
  // Handle non-register types that are treated like immediates.
3903
105k
  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
3904
105k
3905
105k
  if (!DefinedRC) {
3906
0
    // This operand expects an immediate.
3907
0
    return true;
3908
0
  }
3909
105k
3910
105k
  return isImmOperandLegal(MI, OpIdx, *MO);
3911
105k
}
3912
3913
void SIInstrInfo::legalizeOperandsVOP2(MachineRegisterInfo &MRI,
3914
10.7k
                                       MachineInstr &MI) const {
3915
10.7k
  unsigned Opc = MI.getOpcode();
3916
10.7k
  const MCInstrDesc &InstrDesc = get(Opc);
3917
10.7k
3918
10.7k
  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3919
10.7k
  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3920
10.7k
3921
10.7k
  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3922
10.7k
  MachineOperand &Src1 = MI.getOperand(Src1Idx);
3923
10.7k
3924
10.7k
  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3925
10.7k
  // we need to only have one constant bus use before GFX10.
3926
10.7k
  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3927
10.7k
  if (HasImplicitSGPR && 
ST.getConstantBusLimit(Opc) <= 1290
&&
3928
10.7k
      
Src0.isReg()260
&&
(260
RI.isSGPRReg(MRI, Src0.getReg())260
||
3929
260
       
isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])75
))
3930
185
    legalizeOpWithMove(MI, Src0Idx);
3931
10.7k
3932
10.7k
  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3933
10.7k
  // both the value to write (src0) and lane select (src1).  Fix up non-SGPR
3934
10.7k
  // src0/src1 with V_READFIRSTLANE.
3935
10.7k
  if (Opc == AMDGPU::V_WRITELANE_B32) {
3936
2
    const DebugLoc &DL = MI.getDebugLoc();
3937
2
    if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3938
0
      unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3939
0
      BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3940
0
          .add(Src0);
3941
0
      Src0.ChangeToRegister(Reg, false);
3942
0
    }
3943
2
    if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3944
2
      unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3945
2
      const DebugLoc &DL = MI.getDebugLoc();
3946
2
      BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3947
2
          .add(Src1);
3948
2
      Src1.ChangeToRegister(Reg, false);
3949
2
    }
3950
2
    return;
3951
2
  }
3952
10.7k
3953
10.7k
  // No VOP2 instructions support AGPRs.
3954
10.7k
  if (Src0.isReg() && 
RI.isAGPR(MRI, Src0.getReg())10.6k
)
3955
0
    legalizeOpWithMove(MI, Src0Idx);
3956
10.7k
3957
10.7k
  if (Src1.isReg() && 
RI.isAGPR(MRI, Src1.getReg())10.1k
)
3958
0
    legalizeOpWithMove(MI, Src1Idx);
3959
10.7k
3960
10.7k
  // VOP2 src0 instructions support all operand types, so we don't need to check
3961
10.7k
  // their legality. If src1 is already legal, we don't need to do anything.
3962
10.7k
  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3963
3.64k
    return;
3964
7.09k
3965
7.09k
  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3966
7.09k
  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3967
7.09k
  // select is uniform.
3968
7.09k
  if (Opc == AMDGPU::V_READLANE_B32 && 
Src1.isReg()1
&&
3969
7.09k
      
RI.isVGPR(MRI, Src1.getReg())1
) {
3970
1
    unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3971
1
    const DebugLoc &DL = MI.getDebugLoc();
3972
1
    BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3973
1
        .add(Src1);
3974
1
    Src1.ChangeToRegister(Reg, false);
3975
1
    return;
3976
1
  }
3977
7.09k
3978
7.09k
  // We do not use commuteInstruction here because it is too aggressive and will
3979
7.09k
  // commute if it is possible. We only want to commute here if it improves
3980
7.09k
  // legality. This can be called a fairly large number of times so don't waste
3981
7.09k
  // compile time pointlessly swapping and checking legality again.
3982
7.09k
  if (HasImplicitSGPR || 
!MI.isCommutable()6.86k
) {
3983
229
    legalizeOpWithMove(MI, Src1Idx);
3984
229
    return;
3985
229
  }
3986
6.86k
3987
6.86k
  // If src0 can be used as src1, commuting will make the operands legal.
3988
6.86k
  // Otherwise we have to give up and insert a move.
3989
6.86k
  //
3990
6.86k
  // TODO: Other immediate-like operand kinds could be commuted if there was a
3991
6.86k
  // MachineOperand::ChangeTo* for them.
3992
6.86k
  if ((!Src1.isImm() && 
!Src1.isReg()6.31k
) ||
3993
6.86k
      !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
3994
0
    legalizeOpWithMove(MI, Src1Idx);
3995
0
    return;
3996
0
  }
3997
6.86k
3998
6.86k
  int CommutedOpc = commuteOpcode(MI);
3999
6.86k
  if (CommutedOpc == -1) {
4000
0
    legalizeOpWithMove(MI, Src1Idx);
4001
0
    return;
4002
0
  }
4003
6.86k
4004
6.86k
  MI.setDesc(get(CommutedOpc));
4005
6.86k
4006
6.86k
  unsigned Src0Reg = Src0.getReg();
4007
6.86k
  unsigned Src0SubReg = Src0.getSubReg();
4008
6.86k
  bool Src0Kill = Src0.isKill();
4009
6.86k
4010
6.86k
  if (Src1.isImm())
4011
554
    Src0.ChangeToImmediate(Src1.getImm());
4012
6.31k
  else if (Src1.isReg()) {
4013
6.31k
    Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
4014
6.31k
    Src0.setSubReg(Src1.getSubReg());
4015
6.31k
  } else
4016
6.31k
    
llvm_unreachable0
("Should only have register or immediate operands");
4017
6.86k
4018
6.86k
  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
4019
6.86k
  Src1.setSubReg(Src0SubReg);
4020
6.86k
  fixImplicitOperands(MI);
4021
6.86k
}
4022
4023
// Legalize VOP3 operands. All operand types are supported for any operand
4024
// but only one literal constant and only starting from GFX10.
4025
void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
4026
77.3k
                                       MachineInstr &MI) const {
4027
77.3k
  unsigned Opc = MI.getOpcode();
4028
77.3k
4029
77.3k
  int VOP3Idx[3] = {
4030
77.3k
    AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
4031
77.3k
    AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
4032
77.3k
    AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
4033
77.3k
  };
4034
77.3k
4035
77.3k
  if (Opc == AMDGPU::V_PERMLANE16_B32 ||
4036
77.3k
      
Opc == AMDGPU::V_PERMLANEX16_B3277.3k
) {
4037
38
    // src1 and src2 must be scalar
4038
38
    MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
4039
38
    MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
4040
38
    const DebugLoc &DL = MI.getDebugLoc();
4041
38
    if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
4042
4
      unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4043
4
      BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4044
4
        .add(Src1);
4045
4
      Src1.ChangeToRegister(Reg, false);
4046
4
    }
4047
38
    if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
4048
4
      unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4049
4
      BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4050
4
        .add(Src2);
4051
4
      Src2.ChangeToRegister(Reg, false);
4052
4
    }
4053
38
  }
4054
77.3k
4055
77.3k
  // Find the one SGPR operand we are allowed to use.
4056
77.3k
  int ConstantBusLimit = ST.getConstantBusLimit(Opc);
4057
77.3k
  int LiteralLimit = ST.hasVOP3Literal() ? 
14.48k
:
072.8k
;
4058
77.3k
  SmallDenseSet<unsigned> SGPRsUsed;
4059
77.3k
  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
4060
77.3k
  if (SGPRReg != AMDGPU::NoRegister) {
4061
13.3k
    SGPRsUsed.insert(SGPRReg);
4062
13.3k
    --ConstantBusLimit;
4063
13.3k
  }
4064
77.3k
4065
249k
  for (unsigned i = 0; i < 3; 
++i172k
) {
4066
225k
    int Idx = VOP3Idx[i];
4067
225k
    if (Idx == -1)
4068
53.5k
      break;
4069
172k
    MachineOperand &MO = MI.getOperand(Idx);
4070
172k
4071
172k
    if (!MO.isReg()) {
4072
13.1k
      if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
4073
13.1k
        continue;
4074
0
4075
0
      if (LiteralLimit > 0 && ConstantBusLimit > 0) {
4076
0
        --LiteralLimit;
4077
0
        --ConstantBusLimit;
4078
0
        continue;
4079
0
      }
4080
0
4081
0
      --LiteralLimit;
4082
0
      --ConstantBusLimit;
4083
0
      legalizeOpWithMove(MI, Idx);
4084
0
      continue;
4085
0
    }
4086
159k
4087
159k
    if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) &&
4088
159k
        
!isOperandLegal(MI, Idx, &MO)79
) {
4089
0
      legalizeOpWithMove(MI, Idx);
4090
0
      continue;
4091
0
    }
4092
159k
4093
159k
    if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
4094
74.8k
      continue; // VGPRs are legal
4095
84.2k
4096
84.2k
    // We can use one SGPR in each VOP3 instruction prior to GFX10
4097
84.2k
    // and two starting from GFX10.
4098
84.2k
    if (SGPRsUsed.count(MO.getReg()))
4099
13.9k
      continue;
4100
70.2k
    if (ConstantBusLimit > 0) {
4101
45.8k
      SGPRsUsed.insert(MO.getReg());
4102
45.8k
      --ConstantBusLimit;
4103
45.8k
      continue;
4104
45.8k
    }
4105
24.4k
4106
24.4k
    // If we make it this far, then the operand is not legal and we must
4107
24.4k
    // legalize it.
4108
24.4k
    legalizeOpWithMove(MI, Idx);
4109
24.4k
  }
4110
77.3k
}
4111
4112
unsigned SIInstrInfo::readlaneVGPRToSGPR(unsigned SrcReg, MachineInstr &UseMI,
4113
80
                                         MachineRegisterInfo &MRI) const {
4114
80
  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
4115
80
  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
4116
80
  unsigned DstReg = MRI.createVirtualRegister(SRC);
4117
80
  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
4118
80
4119
80
  if (RI.hasAGPRs(VRC)) {
4120
0
    VRC = RI.getEquivalentVGPRClass(VRC);
4121
0
    unsigned NewSrcReg = MRI.createVirtualRegister(VRC);
4122
0
    BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4123
0
            get(TargetOpcode::COPY), NewSrcReg)
4124
0
        .addReg(SrcReg);
4125
0
    SrcReg = NewSrcReg;
4126
0
  }
4127
80
4128
80
  if (SubRegs == 1) {
4129
25
    BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4130
25
            get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
4131
25
        .addReg(SrcReg);
4132
25
    return DstReg;
4133
25
  }
4134
55
4135
55
  SmallVector<unsigned, 8> SRegs;
4136
203
  for (unsigned i = 0; i < SubRegs; 
++i148
) {
4137
148
    unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4138
148
    BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4139
148
            get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
4140
148
        .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
4141
148
    SRegs.push_back(SGPR);
4142
148
  }
4143
55
4144
55
  MachineInstrBuilder MIB =
4145
55
      BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4146
55
              get(AMDGPU::REG_SEQUENCE), DstReg);
4147
203
  for (unsigned i = 0; i < SubRegs; 
++i148
) {
4148
148
    MIB.addReg(SRegs[i]);
4149
148
    MIB.addImm(RI.getSubRegFromChannel(i));
4150
148
  }
4151
55
  return DstReg;
4152
55
}
4153
4154
void SIInstrInfo::legalizeOperandsSMRD(MachineRegisterInfo &MRI,
4155
54
                                       MachineInstr &MI) const {
4156
54
4157
54
  // If the pointer is store in VGPRs, then we need to move them to
4158
54
  // SGPRs using v_readfirstlane.  This is safe because we only select
4159
54
  // loads with uniform pointers to SMRD instruction so we know the
4160
54
  // pointer value is uniform.
4161
54
  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
4162
54
  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
4163
42
    unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
4164
42
    SBase->setReg(SGPR);
4165
42
  }
4166
54
  MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
4167
54
  if (SOff && 
!RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))13
) {
4168
12
    unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
4169
12
    SOff->setReg(SGPR);
4170
12
  }
4171
54
}
4172
4173
void SIInstrInfo::legalizeGenericOperand(MachineBasicBlock &InsertMBB,
4174
                                         MachineBasicBlock::iterator I,
4175
                                         const TargetRegisterClass *DstRC,
4176
                                         MachineOperand &Op,
4177
                                         MachineRegisterInfo &MRI,
4178
26.3k
                                         const DebugLoc &DL) const {
4179
26.3k
  unsigned OpReg = Op.getReg();
4180
26.3k
  unsigned OpSubReg = Op.getSubReg();
4181
26.3k
4182
26.3k
  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
4183
26.3k
      RI.getRegClassForReg(MRI, OpReg), OpSubReg);
4184
26.3k
4185
26.3k
  // Check if operand is already the correct register class.
4186
26.3k
  if (DstRC == OpRC)
4187
496
    return;
4188
25.8k
4189
25.8k
  unsigned DstReg = MRI.createVirtualRegister(DstRC);
4190
25.8k
  MachineInstr *Copy =
4191
25.8k
      BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
4192
25.8k
4193
25.8k
  Op.setReg(DstReg);
4194
25.8k
  Op.setSubReg(0);
4195
25.8k
4196
25.8k
  MachineInstr *Def = MRI.getVRegDef(OpReg);
4197
25.8k
  if (!Def)
4198
0
    return;
4199
25.8k
4200
25.8k
  // Try to eliminate the copy if it is copying an immediate value.
4201
25.8k
  if (Def->isMoveImmediate())
4202
7.16k
    FoldImmediate(*Copy, *Def, OpReg, &MRI);
4203
25.8k
}
4204
4205
// Emit the actual waterfall loop, executing the wrapped instruction for each
4206
// unique value of \p Rsrc across all lanes. In the best case we execute 1
4207
// iteration, in the worst case we execute 64 (once per lane).
4208
static void
4209
emitLoadSRsrcFromVGPRLoop(const SIInstrInfo &TII, MachineRegisterInfo &MRI,
4210
                          MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
4211
35
                          const DebugLoc &DL, MachineOperand &Rsrc) {
4212
35
  MachineFunction &MF = *OrigBB.getParent();
4213
35
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4214
35
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4215
35
  unsigned Exec = ST.isWave32() ? 
AMDGPU::EXEC_LO9
:
AMDGPU::EXEC26
;
4216
35
  unsigned SaveExecOpc =
4217
35
      ST.isWave32() ? 
AMDGPU::S_AND_SAVEEXEC_B329
:
AMDGPU::S_AND_SAVEEXEC_B6426
;
4218
35
  unsigned XorTermOpc =
4219
35
      ST.isWave32() ? 
AMDGPU::S_XOR_B32_term9
:
AMDGPU::S_XOR_B64_term26
;
4220
35
  unsigned AndOpc =
4221
35
      ST.isWave32() ? 
AMDGPU::S_AND_B329
:
AMDGPU::S_AND_B6426
;
4222
35
  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4223
35
4224
35
  MachineBasicBlock::iterator I = LoopBB.begin();
4225
35
4226
35
  unsigned VRsrc = Rsrc.getReg();
4227
35
  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
4228
35
4229
35
  unsigned SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4230
35
  unsigned CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4231
35
  unsigned CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4232
35
  unsigned AndCond = MRI.createVirtualRegister(BoolXExecRC);
4233
35
  unsigned SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4234
35
  unsigned SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4235
35
  unsigned SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4236
35
  unsigned SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4237
35
  unsigned SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4238
35
4239
35
  // Beginning of the loop, read the next Rsrc variant.
4240
35
  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
4241
35
      .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
4242
35
  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
4243
35
      .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
4244
35
  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
4245
35
      .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
4246
35
  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
4247
35
      .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
4248
35
4249
35
  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
4250
35
      .addReg(SRsrcSub0)
4251
35
      .addImm(AMDGPU::sub0)
4252
35
      .addReg(SRsrcSub1)
4253
35
      .addImm(AMDGPU::sub1)
4254
35
      .addReg(SRsrcSub2)
4255
35
      .addImm(AMDGPU::sub2)
4256
35
      .addReg(SRsrcSub3)
4257
35
      .addImm(AMDGPU::sub3);
4258
35
4259
35
  // Update Rsrc operand to use the SGPR Rsrc.
4260
35
  Rsrc.setReg(SRsrc);
4261
35
  Rsrc.setIsKill(true);
4262
35
4263
35
  // Identify all lanes with identical Rsrc operands in their VGPRs.
4264
35
  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
4265
35
      .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
4266
35
      .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
4267
35
  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
4268
35
      .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
4269
35
      .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
4270
35
  BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndCond)
4271
35
      .addReg(CondReg0)
4272
35
      .addReg(CondReg1);
4273
35
4274
35
  MRI.setSimpleHint(SaveExec, AndCond);
4275
35
4276
35
  // Update EXEC to matching lanes, saving original to SaveExec.
4277
35
  BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
4278
35
      .addReg(AndCond, RegState::Kill);
4279
35
4280
35
  // The original instruction is here; we insert the terminators after it.
4281
35
  I = LoopBB.end();
4282
35
4283
35
  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4284
35
  BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
4285
35
      .addReg(Exec)
4286
35
      .addReg(SaveExec);
4287
35
  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
4288
35
}
4289
4290
// Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4291
// with SGPRs by iterating over all unique values across all lanes.
4292
static void loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI,
4293
35
                              MachineOperand &Rsrc, MachineDominatorTree *MDT) {
4294
35
  MachineBasicBlock &MBB = *MI.getParent();
4295
35
  MachineFunction &MF = *MBB.getParent();
4296
35
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4297
35
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4298
35
  MachineRegisterInfo &MRI = MF.getRegInfo();
4299
35
  MachineBasicBlock::iterator I(&MI);
4300
35
  const DebugLoc &DL = MI.getDebugLoc();
4301
35
  unsigned Exec = ST.isWave32() ? 
AMDGPU::EXEC_LO9
:
AMDGPU::EXEC26
;
4302
35
  unsigned MovExecOpc = ST.isWave32() ? 
AMDGPU::S_MOV_B329
:
AMDGPU::S_MOV_B6426
;
4303
35
  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4304
35
4305
35
  unsigned SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4306
35
4307
35
  // Save the EXEC mask
4308
35
  BuildMI(MBB, I, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
4309
35
4310
35
  // Killed uses in the instruction we are waterfalling around will be
4311
35
  // incorrect due to the added control-flow.
4312
312
  for (auto &MO : MI.uses()) {
4313
312
    if (MO.isReg() && 
MO.isUse()122
) {
4314
122
      MRI.clearKillFlags(MO.getReg());
4315
122
    }
4316
312
  }
4317
35
4318
35
  // To insert the loop we need to split the block. Move everything after this
4319
35
  // point to a new block, and insert a new empty block between the two.
4320
35
  MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
4321
35
  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
4322
35
  MachineFunction::iterator MBBI(MBB);
4323
35
  ++MBBI;
4324
35
4325
35
  MF.insert(MBBI, LoopBB);
4326
35
  MF.insert(MBBI, RemainderBB);
4327
35
4328
35
  LoopBB->addSuccessor(LoopBB);
4329
35
  LoopBB->addSuccessor(RemainderBB);
4330
35
4331
35
  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4332
35
  MachineBasicBlock::iterator J = I++;
4333
35
  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
4334
35
  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
4335
35
  LoopBB->splice(LoopBB->begin(), &MBB, J);
4336
35
4337
35
  MBB.addSuccessor(LoopBB);
4338
35
4339
35
  // Update dominators. We know that MBB immediately dominates LoopBB, that
4340
35
  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4341
35
  // dominates all of the successors transferred to it from MBB that MBB used
4342
35
  // to dominate.
4343
35
  if (MDT) {
4344
35
    MDT->addNewBlock(LoopBB, &MBB);
4345
35
    MDT->addNewBlock(RemainderBB, LoopBB);
4346
35
    for (auto &Succ : RemainderBB->successors()) {
4347
16
      if (MDT->dominates(&MBB, Succ)) {
4348
12
        MDT->changeImmediateDominator(Succ, RemainderBB);
4349
12
      }
4350
16
    }
4351
35
  }
4352
35
4353
35
  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
4354
35
4355
35
  // Restore the EXEC mask
4356
35
  MachineBasicBlock::iterator First = RemainderBB->begin();
4357
35
  BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
4358
35
}
4359
4360
// Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4361
static std::tuple<unsigned, unsigned>
4362
8
extractRsrcPtr(const SIInstrInfo &TII, MachineInstr &MI, MachineOperand &Rsrc) {
4363
8
  MachineBasicBlock &MBB = *MI.getParent();
4364
8
  MachineFunction &MF = *MBB.getParent();
4365
8
  MachineRegisterInfo &MRI = MF.getRegInfo();
4366
8
4367
8
  // Extract the ptr from the resource descriptor.
4368
8
  unsigned RsrcPtr =
4369
8
      TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
4370
8
                             AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
4371
8
4372
8
  // Create an empty resource descriptor
4373
8
  unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4374
8
  unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4375
8
  unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4376
8
  unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4377
8
  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
4378
8
4379
8
  // Zero64 = 0
4380
8
  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
4381
8
      .addImm(0);
4382
8
4383
8
  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4384
8
  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
4385
8
      .addImm(RsrcDataFormat & 0xFFFFFFFF);
4386
8
4387
8
  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4388
8
  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
4389
8
      .addImm(RsrcDataFormat >> 32);
4390
8
4391
8
  // NewSRsrc = {Zero64, SRsrcFormat}
4392
8
  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
4393
8
      .addReg(Zero64)
4394
8
      .addImm(AMDGPU::sub0_sub1)
4395
8
      .addReg(SRsrcFormatLo)
4396
8
      .addImm(AMDGPU::sub2)
4397
8
      .addReg(SRsrcFormatHi)
4398
8
      .addImm(AMDGPU::sub3);
4399
8
4400
8
  return std::make_tuple(RsrcPtr, NewSRsrc);
4401
8
}
4402
4403
void SIInstrInfo::legalizeOperands(MachineInstr &MI,
4404
96.2k
                                   MachineDominatorTree *MDT) const {
4405
96.2k
  MachineFunction &MF = *MI.getParent()->getParent();
4406
96.2k
  MachineRegisterInfo &MRI = MF.getRegInfo();
4407
96.2k
4408
96.2k
  // Legalize VOP2
4409
96.2k
  if (isVOP2(MI) || 
isVOPC(MI)85.6k
) {
4410
10.7k
    legalizeOperandsVOP2(MRI, MI);
4411
10.7k
    return;
4412
10.7k
  }
4413
85.5k
4414
85.5k
  // Legalize VOP3
4415
85.5k
  if (isVOP3(MI)) {
4416
34.8k
    legalizeOperandsVOP3(MRI, MI);
4417
34.8k
    return;
4418
34.8k
  }
4419
50.6k
4420
50.6k
  // Legalize SMRD
4421
50.6k
  if (isSMRD(MI)) {
4422
54
    legalizeOperandsSMRD(MRI, MI);
4423
54
    return;
4424
54
  }
4425
50.5k
4426
50.5k
  // Legalize REG_SEQUENCE and PHI
4427
50.5k
  // The register class of the operands much be the same type as the register
4428
50.5k
  // class of the output.
4429
50.5k
  if (MI.getOpcode() == AMDGPU::PHI) {
4430
665
    const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
4431
1.94k
    for (unsigned i = 1, e = MI.getNumOperands(); i != e; 
i += 21.27k
) {
4432
1.27k
      if (!MI.getOperand(i).isReg() ||
4433
1.27k
          !TargetRegisterInfo::isVirtualRegister(MI.getOperand(i).getReg()))
4434
0
        continue;
4435
1.27k
      const TargetRegisterClass *OpRC =
4436
1.27k
          MRI.getRegClass(MI.getOperand(i).getReg());
4437
1.27k
      if (RI.hasVectorRegisters(OpRC)) {
4438
485
        VRC = OpRC;
4439
791
      } else {
4440
791
        SRC = OpRC;
4441
791
      }
4442
1.27k
    }
4443
665
4444
665
    // If any of the operands are VGPR registers, then they all most be
4445
665
    // otherwise we will create illegal VGPR->SGPR copies when legalizing
4446
665
    // them.
4447
665
    if (VRC || 
!RI.isSGPRClass(getOpRegClass(MI, 0))194
) {
4448
665
      if (!VRC) {
4449
194
        assert(SRC);
4450
194
        VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) ? 
RI.getEquivalentAGPRClass(SRC)0
4451
194
                                                : RI.getEquivalentVGPRClass(SRC);
4452
194
      }
4453
665
      RC = VRC;
4454
665
    } else {
4455
0
      RC = SRC;
4456
0
    }
4457
665
4458
665
    // Update all the operands so they have the same type.
4459
1.94k
    for (unsigned I = 1, E = MI.getNumOperands(); I != E; 
I += 21.27k
) {
4460
1.27k
      MachineOperand &Op = MI.getOperand(I);
4461
1.27k
      if (!Op.isReg() || !TargetRegisterInfo::isVirtualRegister(Op.getReg()))
4462
0
        continue;
4463
1.27k
4464
1.27k
      // MI is a PHI instruction.
4465
1.27k
      MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
4466
1.27k
      MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
4467
1.27k
4468
1.27k
      // Avoid creating no-op copies with the same src and dst reg class.  These
4469
1.27k
      // confuse some of the machine passes.
4470
1.27k
      legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
4471
1.27k
    }
4472
665
  }
4473
50.5k
4474
50.5k
  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4475
50.5k
  // VGPR dest type and SGPR sources, insert copies so all operands are
4476
50.5k
  // VGPRs. This seems to help operand folding / the register coalescer.
4477
50.5k
  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
4478
21.0k
    MachineBasicBlock *MBB = MI.getParent();
4479
21.0k
    const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
4480
21.0k
    if (RI.hasVGPRs(DstRC)) {
4481
21.0k
      // Update all the operands so they are VGPR register classes. These may
4482
21.0k
      // not be the same register class because REG_SEQUENCE supports mixing
4483
21.0k
      // subregister index types e.g. sub0_sub1 + sub2 + sub3
4484
76.7k
      for (unsigned I = 1, E = MI.getNumOperands(); I != E; 
I += 255.6k
) {
4485
55.6k
        MachineOperand &Op = MI.getOperand(I);
4486
55.6k
        if (!Op.isReg() || !TargetRegisterInfo::isVirtualRegister(Op.getReg()))
4487
0
          continue;
4488
55.6k
4489
55.6k
        const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
4490
55.6k
        const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
4491
55.6k
        if (VRC == OpRC)
4492
30.5k
          continue;
4493
25.0k
4494
25.0k
        legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
4495
25.0k
        Op.setIsKill();
4496
25.0k
      }
4497
21.0k
    }
4498
21.0k
4499
21.0k
    return;
4500
21.0k
  }
4501
29.4k
4502
29.4k
  // Legalize INSERT_SUBREG
4503
29.4k
  // src0 must have the same register class as dst
4504
29.4k
  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
4505
8
    unsigned Dst = MI.getOperand(0).getReg();
4506
8
    unsigned Src0 = MI.getOperand(1).getReg();
4507
8
    const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
4508
8
    const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
4509
8
    if (DstRC != Src0RC) {
4510
6
      MachineBasicBlock *MBB = MI.getParent();
4511
6
      MachineOperand &Op = MI.getOperand(1);
4512
6
      legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
4513
6
    }
4514
8
    return;
4515
8
  }
4516
29.4k
4517
29.4k
  // Legalize SI_INIT_M0
4518
29.4k
  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
4519
13
    MachineOperand &Src = MI.getOperand(0);
4520
13
    if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
4521
13
      Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
4522
13
    return;
4523
13
  }
4524
29.4k
4525
29.4k
  // Legalize MIMG and MUBUF/MTBUF for shaders.
4526
29.4k
  //
4527
29.4k
  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4528
29.4k
  // scratch memory access. In both cases, the legalization never involves
4529
29.4k
  // conversion to the addr64 form.
4530
29.4k
  if (isMIMG(MI) ||
4531
29.4k
      
(29.4k
AMDGPU::isShader(MF.getFunction().getCallingConv())29.4k
&&
4532
29.4k
       
(697
isMUBUF(MI)697
||
isMTBUF(MI)688
))) {
4533
13
    MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
4534
13
    if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
4535
11
      unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
4536
11
      SRsrc->setReg(SGPR);
4537
11
    }
4538
13
4539
13
    MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
4540
13
    if (SSamp && 
!RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))4
) {
4541
2
      unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
4542
2
      SSamp->setReg(SGPR);
4543
2
    }
4544
13
    return;
4545
13
  }
4546
29.4k
4547
29.4k
  // Legalize MUBUF* instructions.
4548
29.4k
  int RsrcIdx =
4549
29.4k
      AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
4550
29.4k
  if (RsrcIdx != -1) {
4551
43
    // We have an MUBUF instruction
4552
43
    MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
4553
43
    unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
4554
43
    if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
4555
43
                             RI.getRegClass(RsrcRC))) {
4556
0
      // The operands are legal.
4557
0
      // FIXME: We may need to legalize operands besided srsrc.
4558
0
      return;
4559
0
    }
4560
43
4561
43
    // Legalize a VGPR Rsrc.
4562
43
    //
4563
43
    // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4564
43
    // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4565
43
    // a zero-value SRsrc.
4566
43
    //
4567
43
    // If the instruction is _OFFSET (both idxen and offen disabled), and we
4568
43
    // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4569
43
    // above.
4570
43
    //
4571
43
    // Otherwise we are on non-ADDR64 hardware, and/or we have
4572
43
    // idxen/offen/bothen and we fall back to a waterfall loop.
4573
43
4574
43
    MachineBasicBlock &MBB = *MI.getParent();
4575
43
4576
43
    MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4577
43
    if (VAddr && 
AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -138
) {
4578
6
      // This is already an ADDR64 instruction so we need to add the pointer
4579
6
      // extracted from the resource descriptor to the current value of VAddr.
4580
6
      unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4581
6
      unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4582
6
      unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4583
6
4584
6
      const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4585
6
      unsigned CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4586
6
      unsigned CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4587
6
4588
6
      unsigned RsrcPtr, NewSRsrc;
4589
6
      std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4590
6
4591
6
      // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4592
6
      const DebugLoc &DL = MI.getDebugLoc();
4593
6
      BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e64), NewVAddrLo)
4594
6
        .addDef(CondReg0)
4595
6
        .addReg(RsrcPtr, 0, AMDGPU::sub0)
4596
6
        .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
4597
6
        .addImm(0);
4598
6
4599
6
      // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4600
6
      BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
4601
6
        .addDef(CondReg1, RegState::Dead)
4602
6
        .addReg(RsrcPtr, 0, AMDGPU::sub1)
4603
6
        .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
4604
6
        .addReg(CondReg0, RegState::Kill)
4605
6
        .addImm(0);
4606
6
4607
6
      // NewVaddr = {NewVaddrHi, NewVaddrLo}
4608
6
      BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4609
6
          .addReg(NewVAddrLo)
4610
6
          .addImm(AMDGPU::sub0)
4611
6
          .addReg(NewVAddrHi)
4612
6
          .addImm(AMDGPU::sub1);
4613
6
4614
6
      VAddr->setReg(NewVAddr);
4615
6
      Rsrc->setReg(NewSRsrc);
4616
37
    } else if (!VAddr && 
ST.hasAddr64()5
) {
4617
2
      // This instructions is the _OFFSET variant, so we need to convert it to
4618
2
      // ADDR64.
4619
2
      assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4620
2
             < AMDGPUSubtarget::VOLCANIC_ISLANDS &&
4621
2
             "FIXME: Need to emit flat atomics here");
4622
2
4623
2
      unsigned RsrcPtr, NewSRsrc;
4624
2
      std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4625
2
4626
2
      unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4627
2
      MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4628
2
      MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4629
2
      MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4630
2
      unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4631
2
4632
2
      // Atomics rith return have have an additional tied operand and are
4633
2
      // missing some of the special bits.
4634
2
      MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4635
2
      MachineInstr *Addr64;
4636
2
4637
2
      if (!VDataIn) {
4638
2
        // Regular buffer load / store.
4639
2
        MachineInstrBuilder MIB =
4640
2
            BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4641
2
                .add(*VData)
4642
2
                .addReg(NewVAddr)
4643
2
                .addReg(NewSRsrc)
4644
2
                .add(*SOffset)
4645
2
                .add(*Offset);
4646
2
4647
2
        // Atomics do not have this operand.
4648
2
        if (const MachineOperand *GLC =
4649
2
                getNamedOperand(MI, AMDGPU::OpName::glc)) {
4650
2
          MIB.addImm(GLC->getImm());
4651
2
        }
4652
2
        if (const MachineOperand *DLC =
4653
2
                getNamedOperand(MI, AMDGPU::OpName::dlc)) {
4654
2
          MIB.addImm(DLC->getImm());
4655
2
        }
4656
2
4657
2
        MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4658
2
4659
2
        if (const MachineOperand *TFE =
4660
2
                getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4661
2
          MIB.addImm(TFE->getImm());
4662
2
        }
4663
2
4664
2
        MIB.cloneMemRefs(MI);
4665
2
        Addr64 = MIB;
4666
2
      } else {
4667
0
        // Atomics with return.
4668
0
        Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4669
0
                     .add(*VData)
4670
0
                     .add(*VDataIn)
4671
0
                     .addReg(NewVAddr)
4672
0
                     .addReg(NewSRsrc)
4673
0
                     .add(*SOffset)
4674
0
                     .add(*Offset)
4675
0
                     .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4676
0
                     .cloneMemRefs(MI);
4677
0
      }
4678
2
4679
2
      MI.removeFromParent();
4680
2
4681
2
      // NewVaddr = {NewVaddrHi, NewVaddrLo}
4682
2
      BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4683
2
              NewVAddr)
4684
2
          .addReg(RsrcPtr, 0, AMDGPU::sub0)
4685
2
          .addImm(AMDGPU::sub0)
4686
2
          .addReg(RsrcPtr, 0, AMDGPU::sub1)
4687
2
          .addImm(AMDGPU::sub1);
4688
35
    } else {
4689
35
      // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4690
35
      // to SGPRs.
4691
35
      loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4692
35
    }
4693
43
  }
4694
29.4k
}
4695
4696
void SIInstrInfo::moveToVALU(MachineInstr &TopInst,
4697
39.7k
                             MachineDominatorTree *MDT) const {
4698
39.7k
  SetVectorType Worklist;
4699
39.7k
  Worklist.insert(&TopInst);
4700
39.7k
4701
151k
  while (!Worklist.empty()) {
4702
111k
    MachineInstr &Inst = *Worklist.pop_back_val();
4703
111k
    MachineBasicBlock *MBB = Inst.getParent();
4704
111k
    MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
4705
111k
4706
111k
    unsigned Opcode = Inst.getOpcode();
4707
111k
    unsigned NewOpcode = getVALUOp(Inst);
4708
111k
4709
111k
    // Handle some special cases
4710
111k
    switch (Opcode) {
4711
111k
    default:
4712
85.2k
      break;
4713
111k
    case AMDGPU::S_ADD_U64_PSEUDO:
4714
6.28k
    case AMDGPU::S_SUB_U64_PSEUDO:
4715
6.28k
      splitScalar64BitAddSub(Worklist, Inst, MDT);
4716
6.28k
      Inst.eraseFromParent();
4717
6.28k
      continue;
4718
6.28k
    case AMDGPU::S_ADD_I32:
4719
2.99k
    case AMDGPU::S_SUB_I32:
4720
2.99k
      // FIXME: The u32 versions currently selected use the carry.
4721
2.99k
      if (moveScalarAddSub(Worklist, Inst, MDT))
4722
520
        continue;
4723
2.47k
4724
2.47k
      // Default handling
4725
2.47k
      break;
4726
2.47k
    case AMDGPU::S_AND_B64:
4727
40
      splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4728
40
      Inst.eraseFromParent();
4729
40
      continue;
4730
2.47k
4731
2.47k
    case AMDGPU::S_OR_B64:
4732
119
      splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4733
119
      Inst.eraseFromParent();
4734
119
      continue;
4735
2.47k
4736
2.47k
    case AMDGPU::S_XOR_B64:
4737
120
      splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4738
120
      Inst.eraseFromParent();
4739
120
      continue;
4740
2.47k
4741
2.47k
    case AMDGPU::S_NAND_B64:
4742
4
      splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4743
4
      Inst.eraseFromParent();
4744
4
      continue;
4745
2.47k
4746
2.47k
    case AMDGPU::S_NOR_B64:
4747
4
      splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4748
4
      Inst.eraseFromParent();
4749
4
      continue;
4750
2.47k
4751
2.47k
    case AMDGPU::S_XNOR_B64:
4752
15
      if (ST.hasDLInsts())
4753
3
        splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4754
12
      else
4755
12
        splitScalar64BitXnor(Worklist, Inst, MDT);
4756
15
      Inst.eraseFromParent();
4757
15
      continue;
4758
2.47k
4759
2.47k
    case AMDGPU::S_ANDN2_B64:
4760
22
      splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4761
22
      Inst.eraseFromParent();
4762
22
      continue;
4763
2.47k
4764
2.47k
    case AMDGPU::S_ORN2_B64:
4765
2
      splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4766
2
      Inst.eraseFromParent();
4767
2
      continue;
4768
2.47k
4769
2.47k
    case AMDGPU::S_NOT_B64:
4770
11
      splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4771
11
      Inst.eraseFromParent();
4772
11
      continue;
4773
2.47k
4774
2.47k
    case AMDGPU::S_BCNT1_I32_B64:
4775
26
      splitScalar64BitBCNT(Worklist, Inst);
4776
26
      Inst.eraseFromParent();
4777
26
      continue;
4778
2.47k
4779
2.47k
    case AMDGPU::S_BFE_I64:
4780
1.81k
      splitScalar64BitBFE(Worklist, Inst);
4781
1.81k
      Inst.eraseFromParent();
4782
1.81k
      continue;
4783
2.47k
4784
5.57k
    case AMDGPU::S_LSHL_B32:
4785
5.57k
      if (ST.hasOnlyRevVALUShifts()) {
4786
3.14k
        NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4787
3.14k
        swapOperands(Inst);
4788
3.14k
      }
4789
5.57k
      break;
4790
2.88k
    case AMDGPU::S_ASHR_I32:
4791
2.88k
      if (ST.hasOnlyRevVALUShifts()) {
4792
1.77k
        NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4793
1.77k
        swapOperands(Inst);
4794
1.77k
      }
4795
2.88k
      break;
4796
4.29k
    case AMDGPU::S_LSHR_B32:
4797
4.29k
      if (ST.hasOnlyRevVALUShifts()) {
4798
2.74k
        NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4799
2.74k
        swapOperands(Inst);
4800
2.74k
      }
4801
4.29k
      break;
4802
2.47k
    case AMDGPU::S_LSHL_B64:
4803
844
      if (ST.hasOnlyRevVALUShifts()) {
4804
433
        NewOpcode = AMDGPU::V_LSHLREV_B64;
4805
433
        swapOperands(Inst);
4806
433
      }
4807
844
      break;
4808
2.47k
    case AMDGPU::S_ASHR_I64:
4809
211
      if (ST.hasOnlyRevVALUShifts()) {
4810
74
        NewOpcode = AMDGPU::V_ASHRREV_I64;
4811
74
        swapOperands(Inst);
4812
74
      }
4813
211
      break;
4814
2.47k
    case AMDGPU::S_LSHR_B64:
4815
219
      if (ST.hasOnlyRevVALUShifts()) {
4816
130
        NewOpcode = AMDGPU::V_LSHRREV_B64;
4817
130
        swapOperands(Inst);
4818
130
      }
4819
219
      break;
4820
2.47k
4821
2.47k
    case AMDGPU::S_ABS_I32:
4822
24
      lowerScalarAbs(Worklist, Inst);
4823
24
      Inst.eraseFromParent();
4824
24
      continue;
4825
2.47k
4826
2.47k
    case AMDGPU::S_CBRANCH_SCC0:
4827
96
    case AMDGPU::S_CBRANCH_SCC1:
4828
96
      // Clear unused bits of vcc
4829
96
      if (ST.isWave32())
4830
4
        BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
4831
4
                AMDGPU::VCC_LO)
4832
4
            .addReg(AMDGPU::EXEC_LO)
4833
4
            .addReg(AMDGPU::VCC_LO);
4834
92
      else
4835
92
        BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4836
92
                AMDGPU::VCC)
4837
92
            .addReg(AMDGPU::EXEC)
4838
92
            .addReg(AMDGPU::VCC);
4839
96
      break;
4840
96
4841
96
    case AMDGPU::S_BFE_U64:
4842
0
    case AMDGPU::S_BFM_B64:
4843
0
      llvm_unreachable("Moving this op to VALU not implemented");
4844
0
4845
397
    case AMDGPU::S_PACK_LL_B32_B16:
4846
397
    case AMDGPU::S_PACK_LH_B32_B16:
4847
397
    case AMDGPU::S_PACK_HH_B32_B16:
4848
397
      movePackToVALU(Worklist, MRI, Inst);
4849
397
      Inst.eraseFromParent();
4850
397
      continue;
4851
397
4852
397
    case AMDGPU::S_XNOR_B32:
4853
31
      lowerScalarXnor(Worklist, Inst);
4854
31
      Inst.eraseFromParent();
4855
31
      continue;
4856
397
4857
397
    case AMDGPU::S_NAND_B32:
4858
12
      splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4859
12
      Inst.eraseFromParent();
4860
12
      continue;
4861
397
4862
397
    case AMDGPU::S_NOR_B32:
4863
12
      splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4864
12
      Inst.eraseFromParent();
4865
12
      continue;
4866
397
4867
397
    case AMDGPU::S_ANDN2_B32:
4868
46
      splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4869
46
      Inst.eraseFromParent();
4870
46
      continue;
4871
397
4872
397
    case AMDGPU::S_ORN2_B32:
4873
4
      splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
4874
4
      Inst.eraseFromParent();
4875
4
      continue;
4876
101k
    }
4877
101k
4878
101k
    if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4879
140
      // We cannot move this instruction to the VALU, so we should try to
4880
140
      // legalize its operands instead.
4881
140
      legalizeOperands(Inst, MDT);
4882
140
      continue;
4883
140
    }
4884
101k
4885
101k
    // Use the new VALU Opcode.
4886
101k
    const MCInstrDesc &NewDesc = get(NewOpcode);
4887
101k
    Inst.setDesc(NewDesc);
4888
101k
4889
101k
    // Remove any references to SCC. Vector instructions can't read from it, and
4890
101k
    // We're just about to add the implicit use / defs of VCC, and we don't want
4891
101k
    // both.
4892
350k
    for (unsigned i = Inst.getNumOperands() - 1; i > 0; 
--i249k
) {
4893
249k
      MachineOperand &Op = Inst.getOperand(i);
4894
249k
      if (Op.isReg() && 
Op.getReg() == AMDGPU::SCC187k
) {
4895
27.4k
        // Only propagate through live-def of SCC.
4896
27.4k
        if (Op.isDef() && 
!Op.isDead()27.0k
)
4897
1.14k
          addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
4898
27.4k
        Inst.RemoveOperand(i);
4899
27.4k
      }
4900
249k
    }
4901
101k
4902
101k
    if (Opcode == AMDGPU::S_SEXT_I32_I8 || 
Opcode == AMDGPU::S_SEXT_I32_I16101k
) {
4903
1.56k
      // We are converting these to a BFE, so we need to add the missing
4904
1.56k
      // operands for the size and offset.
4905
1.56k
      unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 
8586
:
16978
;
4906
1.56k
      Inst.addOperand(MachineOperand::CreateImm(0));
4907
1.56k
      Inst.addOperand(MachineOperand::CreateImm(Size));
4908
1.56k
4909
100k
    } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4910
128
      // The VALU version adds the second operand to the result, so insert an
4911
128
      // extra 0 operand.
4912
128
      Inst.addOperand(MachineOperand::CreateImm(0));
4913
128
    }
4914
101k
4915
101k
    Inst.addImplicitDefUseOperands(*Inst.getParent()->getParent());
4916
101k
    fixImplicitOperands(Inst);
4917
101k
4918
101k
    if (Opcode == AMDGPU::S_BFE_I32 || 
Opcode == AMDGPU::S_BFE_U3299.6k
) {
4919
3.28k
      const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4920
3.28k
      // If we need to move this to VGPRs, we need to unpack the second operand
4921
3.28k
      // back into the 2 separate ones for bit offset and width.
4922
3.28k
      assert(OffsetWidthOp.isImm() &&
4923
3.28k
             "Scalar BFE is only implemented for constant width and offset");
4924
3.28k
      uint32_t Imm = OffsetWidthOp.getImm();
4925
3.28k
4926
3.28k
      uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4927
3.28k
      uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4928
3.28k
      Inst.RemoveOperand(2);                     // Remove old immediate.
4929
3.28k
      Inst.addOperand(MachineOperand::CreateImm(Offset));
4930
3.28k
      Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4931
3.28k
    }
4932
101k
4933
101k
    bool HasDst = Inst.getOperand(0).isReg() && 
Inst.getOperand(0).isDef()101k
;
4934
101k
    unsigned NewDstReg = AMDGPU::NoRegister;
4935
101k
    if (HasDst) {
4936
101k
      unsigned DstReg = Inst.getOperand(0).getReg();
4937
101k
      if (TargetRegisterInfo::isPhysicalRegister(DstReg))
4938
61
        continue;
4939
101k
4940
101k
      // Update the destination register class.
4941
101k
      const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4942
101k
      if (!NewDstRC)
4943
0
        continue;
4944
101k
4945
101k
      if (Inst.isCopy() &&
4946
101k
          
TargetRegisterInfo::isVirtualRegister(Inst.getOperand(1).getReg())49.4k
&&
4947
101k
          
NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())49.0k
) {
4948
21.2k
        // Instead of creating a copy where src and dst are the same register
4949
21.2k
        // class, we just replace all uses of dst with src.  These kinds of
4950
21.2k
        // copies interfere with the heuristics MachineSink uses to decide
4951
21.2k
        // whether or not to split a critical edge.  Since the pass assumes
4952
21.2k
        // that copies will end up as machine instructions and not be
4953
21.2k
        // eliminated.
4954
21.2k
        addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4955
21.2k
        MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4956
21.2k
        MRI.clearKillFlags(Inst.getOperand(1).getReg());
4957
21.2k
        Inst.getOperand(0).setReg(DstReg);
4958
21.2k
4959
21.2k
        // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4960
21.2k
        // these are deleted later, but at -O0 it would leave a suspicious
4961
21.2k
        // looking illegal copy of an undef register.
4962
42.5k
        for (unsigned I = Inst.getNumOperands() - 1; I != 0; 
--I21.2k
)
4963
21.2k
          Inst.RemoveOperand(I);
4964
21.2k
        Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4965
21.2k
        continue;
4966
21.2k
      }
4967
80.1k
4968
80.1k
      NewDstReg = MRI.createVirtualRegister(NewDstRC);
4969
80.1k
      MRI.replaceRegWith(DstReg, NewDstReg);
4970
80.1k
    }
4971
101k
4972
101k
    // Legalize the operands
4973
101k
    legalizeOperands(Inst, MDT);
4974
80.3k
4975
80.3k
    if (HasDst)
4976
80.1k
     addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
4977
80.3k
  }
4978
39.7k
}
4979
4980
// Add/sub require special handling to deal with carry outs.
4981
bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
4982
2.99k
                                   MachineDominatorTree *MDT) const {
4983
2.99k
  if (ST.hasAddNoCarry()) {
4984
520
    // Assume there is no user of scc since we don't select this in that case.
4985
520
    // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
4986
520
    // is used.
4987
520
4988
520
    MachineBasicBlock &MBB = *Inst.getParent();
4989
520
    MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
4990
520
4991
520
    unsigned OldDstReg = Inst.getOperand(0).getReg();
4992
520
    unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4993
520
4994
520
    unsigned Opc = Inst.getOpcode();
4995
520
    assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
4996
520
4997
520
    unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
4998
435
      AMDGPU::V_ADD_U32_e64 : 
AMDGPU::V_SUB_U32_e6485
;
4999
520
5000
520
    assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
5001
520
    Inst.RemoveOperand(3);
5002
520
5003
520
    Inst.setDesc(get(NewOpc));
5004
520
    Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
5005
520
    Inst.addImplicitDefUseOperands(*MBB.getParent());
5006
520
    MRI.replaceRegWith(OldDstReg, ResultReg);
5007