Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// Custom DAG lowering for SI
11
//
12
//===----------------------------------------------------------------------===//
13
14
#if defined(_MSC_VER) || defined(__MINGW32__)
15
// Provide M_PI.
16
#define _USE_MATH_DEFINES
17
#endif
18
19
#include "SIISelLowering.h"
20
#include "AMDGPU.h"
21
#include "AMDGPUSubtarget.h"
22
#include "AMDGPUTargetMachine.h"
23
#include "SIDefines.h"
24
#include "SIInstrInfo.h"
25
#include "SIMachineFunctionInfo.h"
26
#include "SIRegisterInfo.h"
27
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
28
#include "Utils/AMDGPUBaseInfo.h"
29
#include "llvm/ADT/APFloat.h"
30
#include "llvm/ADT/APInt.h"
31
#include "llvm/ADT/ArrayRef.h"
32
#include "llvm/ADT/BitVector.h"
33
#include "llvm/ADT/SmallVector.h"
34
#include "llvm/ADT/Statistic.h"
35
#include "llvm/ADT/StringRef.h"
36
#include "llvm/ADT/StringSwitch.h"
37
#include "llvm/ADT/Twine.h"
38
#include "llvm/CodeGen/Analysis.h"
39
#include "llvm/CodeGen/CallingConvLower.h"
40
#include "llvm/CodeGen/DAGCombine.h"
41
#include "llvm/CodeGen/ISDOpcodes.h"
42
#include "llvm/CodeGen/MachineBasicBlock.h"
43
#include "llvm/CodeGen/MachineFrameInfo.h"
44
#include "llvm/CodeGen/MachineFunction.h"
45
#include "llvm/CodeGen/MachineInstr.h"
46
#include "llvm/CodeGen/MachineInstrBuilder.h"
47
#include "llvm/CodeGen/MachineMemOperand.h"
48
#include "llvm/CodeGen/MachineModuleInfo.h"
49
#include "llvm/CodeGen/MachineOperand.h"
50
#include "llvm/CodeGen/MachineRegisterInfo.h"
51
#include "llvm/CodeGen/SelectionDAG.h"
52
#include "llvm/CodeGen/SelectionDAGNodes.h"
53
#include "llvm/CodeGen/TargetCallingConv.h"
54
#include "llvm/CodeGen/TargetRegisterInfo.h"
55
#include "llvm/CodeGen/ValueTypes.h"
56
#include "llvm/IR/Constants.h"
57
#include "llvm/IR/DataLayout.h"
58
#include "llvm/IR/DebugLoc.h"
59
#include "llvm/IR/DerivedTypes.h"
60
#include "llvm/IR/DiagnosticInfo.h"
61
#include "llvm/IR/Function.h"
62
#include "llvm/IR/GlobalValue.h"
63
#include "llvm/IR/InstrTypes.h"
64
#include "llvm/IR/Instruction.h"
65
#include "llvm/IR/Instructions.h"
66
#include "llvm/IR/IntrinsicInst.h"
67
#include "llvm/IR/Type.h"
68
#include "llvm/Support/Casting.h"
69
#include "llvm/Support/CodeGen.h"
70
#include "llvm/Support/CommandLine.h"
71
#include "llvm/Support/Compiler.h"
72
#include "llvm/Support/ErrorHandling.h"
73
#include "llvm/Support/KnownBits.h"
74
#include "llvm/Support/MachineValueType.h"
75
#include "llvm/Support/MathExtras.h"
76
#include "llvm/Target/TargetOptions.h"
77
#include <cassert>
78
#include <cmath>
79
#include <cstdint>
80
#include <iterator>
81
#include <tuple>
82
#include <utility>
83
#include <vector>
84
85
using namespace llvm;
86
87
#define DEBUG_TYPE "si-lower"
88
89
STATISTIC(NumTailCalls, "Number of tail calls");
90
91
static cl::opt<bool> EnableVGPRIndexMode(
92
  "amdgpu-vgpr-index-mode",
93
  cl::desc("Use GPR indexing mode instead of movrel for vector indexing"),
94
  cl::init(false));
95
96
static cl::opt<bool> DisableLoopAlignment(
97
  "amdgpu-disable-loop-alignment",
98
  cl::desc("Do not align and prefetch loops"),
99
  cl::init(false));
100
101
2.91k
static unsigned findFirstFreeSGPR(CCState &CCInfo) {
102
2.91k
  unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
103
19.9k
  for (unsigned Reg = 0; Reg < NumSGPRs; 
++Reg17.0k
) {
104
19.9k
    if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
105
2.91k
      return AMDGPU::SGPR0 + Reg;
106
2.91k
    }
107
19.9k
  }
108
2.91k
  
llvm_unreachable0
("Cannot allocate sgpr");
109
2.91k
}
110
111
SITargetLowering::SITargetLowering(const TargetMachine &TM,
112
                                   const GCNSubtarget &STI)
113
    : AMDGPUTargetLowering(TM, STI),
114
3.64k
      Subtarget(&STI) {
115
3.64k
  addRegisterClass(MVT::i1, &AMDGPU::VReg_1RegClass);
116
3.64k
  addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass);
117
3.64k
118
3.64k
  addRegisterClass(MVT::i32, &AMDGPU::SReg_32_XM0RegClass);
119
3.64k
  addRegisterClass(MVT::f32, &AMDGPU::VGPR_32RegClass);
120
3.64k
121
3.64k
  addRegisterClass(MVT::f64, &AMDGPU::VReg_64RegClass);
122
3.64k
  addRegisterClass(MVT::v2i32, &AMDGPU::SReg_64RegClass);
123
3.64k
  addRegisterClass(MVT::v2f32, &AMDGPU::VReg_64RegClass);
124
3.64k
125
3.64k
  addRegisterClass(MVT::v3i32, &AMDGPU::SGPR_96RegClass);
126
3.64k
  addRegisterClass(MVT::v3f32, &AMDGPU::VReg_96RegClass);
127
3.64k
128
3.64k
  addRegisterClass(MVT::v2i64, &AMDGPU::SReg_128RegClass);
129
3.64k
  addRegisterClass(MVT::v2f64, &AMDGPU::SReg_128RegClass);
130
3.64k
131
3.64k
  addRegisterClass(MVT::v4i32, &AMDGPU::SReg_128RegClass);
132
3.64k
  addRegisterClass(MVT::v4f32, &AMDGPU::VReg_128RegClass);
133
3.64k
134
3.64k
  addRegisterClass(MVT::v5i32, &AMDGPU::SGPR_160RegClass);
135
3.64k
  addRegisterClass(MVT::v5f32, &AMDGPU::VReg_160RegClass);
136
3.64k
137
3.64k
  addRegisterClass(MVT::v8i32, &AMDGPU::SReg_256RegClass);
138
3.64k
  addRegisterClass(MVT::v8f32, &AMDGPU::VReg_256RegClass);
139
3.64k
140
3.64k
  addRegisterClass(MVT::v16i32, &AMDGPU::SReg_512RegClass);
141
3.64k
  addRegisterClass(MVT::v16f32, &AMDGPU::VReg_512RegClass);
142
3.64k
143
3.64k
  if (Subtarget->has16BitInsts()) {
144
2.01k
    addRegisterClass(MVT::i16, &AMDGPU::SReg_32_XM0RegClass);
145
2.01k
    addRegisterClass(MVT::f16, &AMDGPU::SReg_32_XM0RegClass);
146
2.01k
147
2.01k
    // Unless there are also VOP3P operations, not operations are really legal.
148
2.01k
    addRegisterClass(MVT::v2i16, &AMDGPU::SReg_32_XM0RegClass);
149
2.01k
    addRegisterClass(MVT::v2f16, &AMDGPU::SReg_32_XM0RegClass);
150
2.01k
    addRegisterClass(MVT::v4i16, &AMDGPU::SReg_64RegClass);
151
2.01k
    addRegisterClass(MVT::v4f16, &AMDGPU::SReg_64RegClass);
152
2.01k
  }
153
3.64k
154
3.64k
  if (Subtarget->hasMAIInsts()) {
155
21
    addRegisterClass(MVT::v32i32, &AMDGPU::VReg_1024RegClass);
156
21
    addRegisterClass(MVT::v32f32, &AMDGPU::VReg_1024RegClass);
157
21
  }
158
3.64k
159
3.64k
  computeRegisterProperties(Subtarget->getRegisterInfo());
160
3.64k
161
3.64k
  // We need to custom lower vector stores from local memory
162
3.64k
  setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
163
3.64k
  setOperationAction(ISD::LOAD, MVT::v3i32, Custom);
164
3.64k
  setOperationAction(ISD::LOAD, MVT::v4i32, Custom);
165
3.64k
  setOperationAction(ISD::LOAD, MVT::v5i32, Custom);
166
3.64k
  setOperationAction(ISD::LOAD, MVT::v8i32, Custom);
167
3.64k
  setOperationAction(ISD::LOAD, MVT::v16i32, Custom);
168
3.64k
  setOperationAction(ISD::LOAD, MVT::i1, Custom);
169
3.64k
  setOperationAction(ISD::LOAD, MVT::v32i32, Custom);
170
3.64k
171
3.64k
  setOperationAction(ISD::STORE, MVT::v2i32, Custom);
172
3.64k
  setOperationAction(ISD::STORE, MVT::v3i32, Custom);
173
3.64k
  setOperationAction(ISD::STORE, MVT::v4i32, Custom);
174
3.64k
  setOperationAction(ISD::STORE, MVT::v5i32, Custom);
175
3.64k
  setOperationAction(ISD::STORE, MVT::v8i32, Custom);
176
3.64k
  setOperationAction(ISD::STORE, MVT::v16i32, Custom);
177
3.64k
  setOperationAction(ISD::STORE, MVT::i1, Custom);
178
3.64k
  setOperationAction(ISD::STORE, MVT::v32i32, Custom);
179
3.64k
180
3.64k
  setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
181
3.64k
  setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand);
182
3.64k
  setTruncStoreAction(MVT::v8i32, MVT::v8i16, Expand);
183
3.64k
  setTruncStoreAction(MVT::v16i32, MVT::v16i16, Expand);
184
3.64k
  setTruncStoreAction(MVT::v32i32, MVT::v32i16, Expand);
185
3.64k
  setTruncStoreAction(MVT::v2i32, MVT::v2i8, Expand);
186
3.64k
  setTruncStoreAction(MVT::v4i32, MVT::v4i8, Expand);
187
3.64k
  setTruncStoreAction(MVT::v8i32, MVT::v8i8, Expand);
188
3.64k
  setTruncStoreAction(MVT::v16i32, MVT::v16i8, Expand);
189
3.64k
  setTruncStoreAction(MVT::v32i32, MVT::v32i8, Expand);
190
3.64k
191
3.64k
  setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
192
3.64k
  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
193
3.64k
194
3.64k
  setOperationAction(ISD::SELECT, MVT::i1, Promote);
195
3.64k
  setOperationAction(ISD::SELECT, MVT::i64, Custom);
196
3.64k
  setOperationAction(ISD::SELECT, MVT::f64, Promote);
197
3.64k
  AddPromotedToType(ISD::SELECT, MVT::f64, MVT::i64);
198
3.64k
199
3.64k
  setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
200
3.64k
  setOperationAction(ISD::SELECT_CC, MVT::i32, Expand);
201
3.64k
  setOperationAction(ISD::SELECT_CC, MVT::i64, Expand);
202
3.64k
  setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
203
3.64k
  setOperationAction(ISD::SELECT_CC, MVT::i1, Expand);
204
3.64k
205
3.64k
  setOperationAction(ISD::SETCC, MVT::i1, Promote);
206
3.64k
  setOperationAction(ISD::SETCC, MVT::v2i1, Expand);
207
3.64k
  setOperationAction(ISD::SETCC, MVT::v4i1, Expand);
208
3.64k
  AddPromotedToType(ISD::SETCC, MVT::i1, MVT::i32);
209
3.64k
210
3.64k
  setOperationAction(ISD::TRUNCATE, MVT::v2i32, Expand);
211
3.64k
  setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand);
212
3.64k
213
3.64k
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i1, Custom);
214
3.64k
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i1, Custom);
215
3.64k
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Custom);
216
3.64k
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Custom);
217
3.64k
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Custom);
218
3.64k
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Custom);
219
3.64k
  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::Other, Custom);
220
3.64k
221
3.64k
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
222
3.64k
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f32, Custom);
223
3.64k
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v4f32, Custom);
224
3.64k
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
225
3.64k
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::f16, Custom);
226
3.64k
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2i16, Custom);
227
3.64k
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::v2f16, Custom);
228
3.64k
229
3.64k
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2f16, Custom);
230
3.64k
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4f16, Custom);
231
3.64k
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v8f16, Custom);
232
3.64k
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
233
3.64k
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
234
3.64k
  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
235
3.64k
236
3.64k
  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
237
3.64k
  setOperationAction(ISD::INTRINSIC_VOID, MVT::v2i16, Custom);
238
3.64k
  setOperationAction(ISD::INTRINSIC_VOID, MVT::v2f16, Custom);
239
3.64k
  setOperationAction(ISD::INTRINSIC_VOID, MVT::v4f16, Custom);
240
3.64k
  setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom);
241
3.64k
  setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom);
242
3.64k
243
3.64k
  setOperationAction(ISD::BRCOND, MVT::Other, Custom);
244
3.64k
  setOperationAction(ISD::BR_CC, MVT::i1, Expand);
245
3.64k
  setOperationAction(ISD::BR_CC, MVT::i32, Expand);
246
3.64k
  setOperationAction(ISD::BR_CC, MVT::i64, Expand);
247
3.64k
  setOperationAction(ISD::BR_CC, MVT::f32, Expand);
248
3.64k
  setOperationAction(ISD::BR_CC, MVT::f64, Expand);
249
3.64k
250
3.64k
  setOperationAction(ISD::UADDO, MVT::i32, Legal);
251
3.64k
  setOperationAction(ISD::USUBO, MVT::i32, Legal);
252
3.64k
253
3.64k
  setOperationAction(ISD::ADDCARRY, MVT::i32, Legal);
254
3.64k
  setOperationAction(ISD::SUBCARRY, MVT::i32, Legal);
255
3.64k
256
3.64k
  setOperationAction(ISD::SHL_PARTS, MVT::i64, Expand);
257
3.64k
  setOperationAction(ISD::SRA_PARTS, MVT::i64, Expand);
258
3.64k
  setOperationAction(ISD::SRL_PARTS, MVT::i64, Expand);
259
3.64k
260
#if 0
261
  setOperationAction(ISD::ADDCARRY, MVT::i64, Legal);
262
  setOperationAction(ISD::SUBCARRY, MVT::i64, Legal);
263
#endif
264
265
3.64k
  // We only support LOAD/STORE and vector manipulation ops for vectors
266
3.64k
  // with > 4 elements.
267
3.64k
  for (MVT VT : { MVT::v8i32, MVT::v8f32, MVT::v16i32, MVT::v16f32,
268
3.64k
                  MVT::v2i64, MVT::v2f64, MVT::v4i16, MVT::v4f16,
269
36.4k
                  MVT::v32i32, MVT::v32f32 }) {
270
10.4M
    for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; 
++Op10.4M
) {
271
10.4M
      switch (Op) {
272
10.4M
      case ISD::LOAD:
273
327k
      case ISD::STORE:
274
327k
      case ISD::BUILD_VECTOR:
275
327k
      case ISD::BITCAST:
276
327k
      case ISD::EXTRACT_VECTOR_ELT:
277
327k
      case ISD::INSERT_VECTOR_ELT:
278
327k
      case ISD::INSERT_SUBVECTOR:
279
327k
      case ISD::EXTRACT_SUBVECTOR:
280
327k
      case ISD::SCALAR_TO_VECTOR:
281
327k
        break;
282
327k
      case ISD::CONCAT_VECTORS:
283
36.4k
        setOperationAction(Op, VT, Custom);
284
36.4k
        break;
285
10.0M
      default:
286
10.0M
        setOperationAction(Op, VT, Expand);
287
10.0M
        break;
288
10.4M
      }
289
10.4M
    }
290
36.4k
  }
291
3.64k
292
3.64k
  setOperationAction(ISD::FP_EXTEND, MVT::v4f32, Expand);
293
3.64k
294
3.64k
  // TODO: For dynamic 64-bit vector inserts/extracts, should emit a pseudo that
295
3.64k
  // is expanded to avoid having two separate loops in case the index is a VGPR.
296
3.64k
297
3.64k
  // Most operations are naturally 32-bit vector operations. We only support
298
3.64k
  // load and store of i64 vectors, so promote v2i64 vector operations to v4i32.
299
7.28k
  for (MVT Vec64 : { MVT::v2i64, MVT::v2f64 }) {
300
7.28k
    setOperationAction(ISD::BUILD_VECTOR, Vec64, Promote);
301
7.28k
    AddPromotedToType(ISD::BUILD_VECTOR, Vec64, MVT::v4i32);
302
7.28k
303
7.28k
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, Vec64, Promote);
304
7.28k
    AddPromotedToType(ISD::EXTRACT_VECTOR_ELT, Vec64, MVT::v4i32);
305
7.28k
306
7.28k
    setOperationAction(ISD::INSERT_VECTOR_ELT, Vec64, Promote);
307
7.28k
    AddPromotedToType(ISD::INSERT_VECTOR_ELT, Vec64, MVT::v4i32);
308
7.28k
309
7.28k
    setOperationAction(ISD::SCALAR_TO_VECTOR, Vec64, Promote);
310
7.28k
    AddPromotedToType(ISD::SCALAR_TO_VECTOR, Vec64, MVT::v4i32);
311
7.28k
  }
312
3.64k
313
3.64k
  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i32, Expand);
314
3.64k
  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8f32, Expand);
315
3.64k
  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i32, Expand);
316
3.64k
  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16f32, Expand);
317
3.64k
318
3.64k
  setOperationAction(ISD::BUILD_VECTOR, MVT::v4f16, Custom);
319
3.64k
  setOperationAction(ISD::BUILD_VECTOR, MVT::v4i16, Custom);
320
3.64k
321
3.64k
  // Avoid stack access for these.
322
3.64k
  // TODO: Generalize to more vector types.
323
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i16, Custom);
324
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f16, Custom);
325
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom);
326
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f16, Custom);
327
3.64k
328
3.64k
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom);
329
3.64k
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom);
330
3.64k
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i8, Custom);
331
3.64k
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i8, Custom);
332
3.64k
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i8, Custom);
333
3.64k
334
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i8, Custom);
335
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i8, Custom);
336
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i8, Custom);
337
3.64k
338
3.64k
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i16, Custom);
339
3.64k
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f16, Custom);
340
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i16, Custom);
341
3.64k
  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f16, Custom);
342
3.64k
343
3.64k
  // Deal with vec3 vector operations when widened to vec4.
344
3.64k
  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v3i32, Custom);
345
3.64k
  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v3f32, Custom);
346
3.64k
  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i32, Custom);
347
3.64k
  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4f32, Custom);
348
3.64k
349
3.64k
  // Deal with vec5 vector operations when widened to vec8.
350
3.64k
  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v5i32, Custom);
351
3.64k
  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v5f32, Custom);
352
3.64k
  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i32, Custom);
353
3.64k
  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8f32, Custom);
354
3.64k
355
3.64k
  // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling,
356
3.64k
  // and output demarshalling
357
3.64k
  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom);
358
3.64k
  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom);
359
3.64k
360
3.64k
  // We can't return success/failure, only the old value,
361
3.64k
  // let LLVM add the comparison
362
3.64k
  setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i32, Expand);
363
3.64k
  setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i64, Expand);
364
3.64k
365
3.64k
  if (Subtarget->hasFlatAddressSpace()) {
366
2.56k
    setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
367
2.56k
    setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
368
2.56k
  }
369
3.64k
370
3.64k
  setOperationAction(ISD::BSWAP, MVT::i32, Legal);
371
3.64k
  setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
372
3.64k
373
3.64k
  // On SI this is s_memtime and s_memrealtime on VI.
374
3.64k
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
375
3.64k
  setOperationAction(ISD::TRAP, MVT::Other, Custom);
376
3.64k
  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Custom);
377
3.64k
378
3.64k
  if (Subtarget->has16BitInsts()) {
379
2.01k
    setOperationAction(ISD::FLOG, MVT::f16, Custom);
380
2.01k
    setOperationAction(ISD::FEXP, MVT::f16, Custom);
381
2.01k
    setOperationAction(ISD::FLOG10, MVT::f16, Custom);
382
2.01k
  }
383
3.64k
384
3.64k
  // v_mad_f32 does not support denormals according to some sources.
385
3.64k
  if (!Subtarget->hasFP32Denormals())
386
3.56k
    setOperationAction(ISD::FMAD, MVT::f32, Legal);
387
3.64k
388
3.64k
  if (!Subtarget->hasBFI()) {
389
0
    // fcopysign can be done in a single instruction with BFI.
390
0
    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
391
0
    setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
392
0
  }
393
3.64k
394
3.64k
  if (!Subtarget->hasBCNT(32))
395
0
    setOperationAction(ISD::CTPOP, MVT::i32, Expand);
396
3.64k
397
3.64k
  if (!Subtarget->hasBCNT(64))
398
0
    setOperationAction(ISD::CTPOP, MVT::i64, Expand);
399
3.64k
400
3.64k
  if (Subtarget->hasFFBH())
401
3.64k
    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
402
3.64k
403
3.64k
  if (Subtarget->hasFFBL())
404
3.64k
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
405
3.64k
406
3.64k
  // We only really have 32-bit BFE instructions (and 16-bit on VI).
407
3.64k
  //
408
3.64k
  // On SI+ there are 64-bit BFEs, but they are scalar only and there isn't any
409
3.64k
  // effort to match them now. We want this to be false for i64 cases when the
410
3.64k
  // extraction isn't restricted to the upper or lower half. Ideally we would
411
3.64k
  // have some pass reduce 64-bit extracts to 32-bit if possible. Extracts that
412
3.64k
  // span the midpoint are probably relatively rare, so don't worry about them
413
3.64k
  // for now.
414
3.64k
  if (Subtarget->hasBFE())
415
3.64k
    setHasExtractBitsInsn(true);
416
3.64k
417
3.64k
  setOperationAction(ISD::FMINNUM, MVT::f32, Custom);
418
3.64k
  setOperationAction(ISD::FMAXNUM, MVT::f32, Custom);
419
3.64k
  setOperationAction(ISD::FMINNUM, MVT::f64, Custom);
420
3.64k
  setOperationAction(ISD::FMAXNUM, MVT::f64, Custom);
421
3.64k
422
3.64k
423
3.64k
  // These are really only legal for ieee_mode functions. We should be avoiding
424
3.64k
  // them for functions that don't have ieee_mode enabled, so just say they are
425
3.64k
  // legal.
426
3.64k
  setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal);
427
3.64k
  setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal);
428
3.64k
  setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal);
429
3.64k
  setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal);
430
3.64k
431
3.64k
432
3.64k
  if (Subtarget->haveRoundOpsF64()) {
433
2.33k
    setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
434
2.33k
    setOperationAction(ISD::FCEIL, MVT::f64, Legal);
435
2.33k
    setOperationAction(ISD::FRINT, MVT::f64, Legal);
436
2.33k
  } else {
437
1.31k
    setOperationAction(ISD::FCEIL, MVT::f64, Custom);
438
1.31k
    setOperationAction(ISD::FTRUNC, MVT::f64, Custom);
439
1.31k
    setOperationAction(ISD::FRINT, MVT::f64, Custom);
440
1.31k
    setOperationAction(ISD::FFLOOR, MVT::f64, Custom);
441
1.31k
  }
442
3.64k
443
3.64k
  setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
444
3.64k
445
3.64k
  setOperationAction(ISD::FSIN, MVT::f32, Custom);
446
3.64k
  setOperationAction(ISD::FCOS, MVT::f32, Custom);
447
3.64k
  setOperationAction(ISD::FDIV, MVT::f32, Custom);
448
3.64k
  setOperationAction(ISD::FDIV, MVT::f64, Custom);
449
3.64k
450
3.64k
  if (Subtarget->has16BitInsts()) {
451
2.01k
    setOperationAction(ISD::Constant, MVT::i16, Legal);
452
2.01k
453
2.01k
    setOperationAction(ISD::SMIN, MVT::i16, Legal);
454
2.01k
    setOperationAction(ISD::SMAX, MVT::i16, Legal);
455
2.01k
456
2.01k
    setOperationAction(ISD::UMIN, MVT::i16, Legal);
457
2.01k
    setOperationAction(ISD::UMAX, MVT::i16, Legal);
458
2.01k
459
2.01k
    setOperationAction(ISD::SIGN_EXTEND, MVT::i16, Promote);
460
2.01k
    AddPromotedToType(ISD::SIGN_EXTEND, MVT::i16, MVT::i32);
461
2.01k
462
2.01k
    setOperationAction(ISD::ROTR, MVT::i16, Promote);
463
2.01k
    setOperationAction(ISD::ROTL, MVT::i16, Promote);
464
2.01k
465
2.01k
    setOperationAction(ISD::SDIV, MVT::i16, Promote);
466
2.01k
    setOperationAction(ISD::UDIV, MVT::i16, Promote);
467
2.01k
    setOperationAction(ISD::SREM, MVT::i16, Promote);
468
2.01k
    setOperationAction(ISD::UREM, MVT::i16, Promote);
469
2.01k
470
2.01k
    setOperationAction(ISD::BSWAP, MVT::i16, Promote);
471
2.01k
    setOperationAction(ISD::BITREVERSE, MVT::i16, Promote);
472
2.01k
473
2.01k
    setOperationAction(ISD::CTTZ, MVT::i16, Promote);
474
2.01k
    setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16, Promote);
475
2.01k
    setOperationAction(ISD::CTLZ, MVT::i16, Promote);
476
2.01k
    setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16, Promote);
477
2.01k
    setOperationAction(ISD::CTPOP, MVT::i16, Promote);
478
2.01k
479
2.01k
    setOperationAction(ISD::SELECT_CC, MVT::i16, Expand);
480
2.01k
481
2.01k
    setOperationAction(ISD::BR_CC, MVT::i16, Expand);
482
2.01k
483
2.01k
    setOperationAction(ISD::LOAD, MVT::i16, Custom);
484
2.01k
485
2.01k
    setTruncStoreAction(MVT::i64, MVT::i16, Expand);
486
2.01k
487
2.01k
    setOperationAction(ISD::FP16_TO_FP, MVT::i16, Promote);
488
2.01k
    AddPromotedToType(ISD::FP16_TO_FP, MVT::i16, MVT::i32);
489
2.01k
    setOperationAction(ISD::FP_TO_FP16, MVT::i16, Promote);
490
2.01k
    AddPromotedToType(ISD::FP_TO_FP16, MVT::i16, MVT::i32);
491
2.01k
492
2.01k
    setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
493
2.01k
    setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
494
2.01k
    setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
495
2.01k
    setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
496
2.01k
497
2.01k
    // F16 - Constant Actions.
498
2.01k
    setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
499
2.01k
500
2.01k
    // F16 - Load/Store Actions.
501
2.01k
    setOperationAction(ISD::LOAD, MVT::f16, Promote);
502
2.01k
    AddPromotedToType(ISD::LOAD, MVT::f16, MVT::i16);
503
2.01k
    setOperationAction(ISD::STORE, MVT::f16, Promote);
504
2.01k
    AddPromotedToType(ISD::STORE, MVT::f16, MVT::i16);
505
2.01k
506
2.01k
    // F16 - VOP1 Actions.
507
2.01k
    setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
508
2.01k
    setOperationAction(ISD::FCOS, MVT::f16, Promote);
509
2.01k
    setOperationAction(ISD::FSIN, MVT::f16, Promote);
510
2.01k
    setOperationAction(ISD::FP_TO_SINT, MVT::f16, Promote);
511
2.01k
    setOperationAction(ISD::FP_TO_UINT, MVT::f16, Promote);
512
2.01k
    setOperationAction(ISD::SINT_TO_FP, MVT::f16, Promote);
513
2.01k
    setOperationAction(ISD::UINT_TO_FP, MVT::f16, Promote);
514
2.01k
    setOperationAction(ISD::FROUND, MVT::f16, Custom);
515
2.01k
516
2.01k
    // F16 - VOP2 Actions.
517
2.01k
    setOperationAction(ISD::BR_CC, MVT::f16, Expand);
518
2.01k
    setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
519
2.01k
520
2.01k
    setOperationAction(ISD::FDIV, MVT::f16, Custom);
521
2.01k
522
2.01k
    // F16 - VOP3 Actions.
523
2.01k
    setOperationAction(ISD::FMA, MVT::f16, Legal);
524
2.01k
    if (!Subtarget->hasFP16Denormals() && 
STI.hasMadF16()34
)
525
29
      setOperationAction(ISD::FMAD, MVT::f16, Legal);
526
2.01k
527
8.06k
    for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16}) {
528
2.31M
      for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; 
++Op2.30M
) {
529
2.30M
        switch (Op) {
530
2.30M
        case ISD::LOAD:
531
72.5k
        case ISD::STORE:
532
72.5k
        case ISD::BUILD_VECTOR:
533
72.5k
        case ISD::BITCAST:
534
72.5k
        case ISD::EXTRACT_VECTOR_ELT:
535
72.5k
        case ISD::INSERT_VECTOR_ELT:
536
72.5k
        case ISD::INSERT_SUBVECTOR:
537
72.5k
        case ISD::EXTRACT_SUBVECTOR:
538
72.5k
        case ISD::SCALAR_TO_VECTOR:
539
72.5k
          break;
540
72.5k
        case ISD::CONCAT_VECTORS:
541
8.06k
          setOperationAction(Op, VT, Custom);
542
8.06k
          break;
543
2.22M
        default:
544
2.22M
          setOperationAction(Op, VT, Expand);
545
2.22M
          break;
546
2.30M
        }
547
2.30M
      }
548
8.06k
    }
549
2.01k
550
2.01k
    // XXX - Do these do anything? Vector constants turn into build_vector.
551
2.01k
    setOperationAction(ISD::Constant, MVT::v2i16, Legal);
552
2.01k
    setOperationAction(ISD::ConstantFP, MVT::v2f16, Legal);
553
2.01k
554
2.01k
    setOperationAction(ISD::UNDEF, MVT::v2i16, Legal);
555
2.01k
    setOperationAction(ISD::UNDEF, MVT::v2f16, Legal);
556
2.01k
557
2.01k
    setOperationAction(ISD::STORE, MVT::v2i16, Promote);
558
2.01k
    AddPromotedToType(ISD::STORE, MVT::v2i16, MVT::i32);
559
2.01k
    setOperationAction(ISD::STORE, MVT::v2f16, Promote);
560
2.01k
    AddPromotedToType(ISD::STORE, MVT::v2f16, MVT::i32);
561
2.01k
562
2.01k
    setOperationAction(ISD::LOAD, MVT::v2i16, Promote);
563
2.01k
    AddPromotedToType(ISD::LOAD, MVT::v2i16, MVT::i32);
564
2.01k
    setOperationAction(ISD::LOAD, MVT::v2f16, Promote);
565
2.01k
    AddPromotedToType(ISD::LOAD, MVT::v2f16, MVT::i32);
566
2.01k
567
2.01k
    setOperationAction(ISD::AND, MVT::v2i16, Promote);
568
2.01k
    AddPromotedToType(ISD::AND, MVT::v2i16, MVT::i32);
569
2.01k
    setOperationAction(ISD::OR, MVT::v2i16, Promote);
570
2.01k
    AddPromotedToType(ISD::OR, MVT::v2i16, MVT::i32);
571
2.01k
    setOperationAction(ISD::XOR, MVT::v2i16, Promote);
572
2.01k
    AddPromotedToType(ISD::XOR, MVT::v2i16, MVT::i32);
573
2.01k
574
2.01k
    setOperationAction(ISD::LOAD, MVT::v4i16, Promote);
575
2.01k
    AddPromotedToType(ISD::LOAD, MVT::v4i16, MVT::v2i32);
576
2.01k
    setOperationAction(ISD::LOAD, MVT::v4f16, Promote);
577
2.01k
    AddPromotedToType(ISD::LOAD, MVT::v4f16, MVT::v2i32);
578
2.01k
579
2.01k
    setOperationAction(ISD::STORE, MVT::v4i16, Promote);
580
2.01k
    AddPromotedToType(ISD::STORE, MVT::v4i16, MVT::v2i32);
581
2.01k
    setOperationAction(ISD::STORE, MVT::v4f16, Promote);
582
2.01k
    AddPromotedToType(ISD::STORE, MVT::v4f16, MVT::v2i32);
583
2.01k
584
2.01k
    setOperationAction(ISD::ANY_EXTEND, MVT::v2i32, Expand);
585
2.01k
    setOperationAction(ISD::ZERO_EXTEND, MVT::v2i32, Expand);
586
2.01k
    setOperationAction(ISD::SIGN_EXTEND, MVT::v2i32, Expand);
587
2.01k
    setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Expand);
588
2.01k
589
2.01k
    setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Expand);
590
2.01k
    setOperationAction(ISD::ZERO_EXTEND, MVT::v4i32, Expand);
591
2.01k
    setOperationAction(ISD::SIGN_EXTEND, MVT::v4i32, Expand);
592
2.01k
593
2.01k
    if (!Subtarget->hasVOP3PInsts()) {
594
1.24k
      setOperationAction(ISD::BUILD_VECTOR, MVT::v2i16, Custom);
595
1.24k
      setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom);
596
1.24k
    }
597
2.01k
598
2.01k
    setOperationAction(ISD::FNEG, MVT::v2f16, Legal);
599
2.01k
    // This isn't really legal, but this avoids the legalizer unrolling it (and
600
2.01k
    // allows matching fneg (fabs x) patterns)
601
2.01k
    setOperationAction(ISD::FABS, MVT::v2f16, Legal);
602
2.01k
603
2.01k
    setOperationAction(ISD::FMAXNUM, MVT::f16, Custom);
604
2.01k
    setOperationAction(ISD::FMINNUM, MVT::f16, Custom);
605
2.01k
    setOperationAction(ISD::FMAXNUM_IEEE, MVT::f16, Legal);
606
2.01k
    setOperationAction(ISD::FMINNUM_IEEE, MVT::f16, Legal);
607
2.01k
608
2.01k
    setOperationAction(ISD::FMINNUM_IEEE, MVT::v4f16, Custom);
609
2.01k
    setOperationAction(ISD::FMAXNUM_IEEE, MVT::v4f16, Custom);
610
2.01k
611
2.01k
    setOperationAction(ISD::FMINNUM, MVT::v4f16, Expand);
612
2.01k
    setOperationAction(ISD::FMAXNUM, MVT::v4f16, Expand);
613
2.01k
  }
614
3.64k
615
3.64k
  if (Subtarget->hasVOP3PInsts()) {
616
775
    setOperationAction(ISD::ADD, MVT::v2i16, Legal);
617
775
    setOperationAction(ISD::SUB, MVT::v2i16, Legal);
618
775
    setOperationAction(ISD::MUL, MVT::v2i16, Legal);
619
775
    setOperationAction(ISD::SHL, MVT::v2i16, Legal);
620
775
    setOperationAction(ISD::SRL, MVT::v2i16, Legal);
621
775
    setOperationAction(ISD::SRA, MVT::v2i16, Legal);
622
775
    setOperationAction(ISD::SMIN, MVT::v2i16, Legal);
623
775
    setOperationAction(ISD::UMIN, MVT::v2i16, Legal);
624
775
    setOperationAction(ISD::SMAX, MVT::v2i16, Legal);
625
775
    setOperationAction(ISD::UMAX, MVT::v2i16, Legal);
626
775
627
775
    setOperationAction(ISD::FADD, MVT::v2f16, Legal);
628
775
    setOperationAction(ISD::FMUL, MVT::v2f16, Legal);
629
775
    setOperationAction(ISD::FMA, MVT::v2f16, Legal);
630
775
631
775
    setOperationAction(ISD::FMINNUM_IEEE, MVT::v2f16, Legal);
632
775
    setOperationAction(ISD::FMAXNUM_IEEE, MVT::v2f16, Legal);
633
775
634
775
    setOperationAction(ISD::FCANONICALIZE, MVT::v2f16, Legal);
635
775
636
775
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i16, Custom);
637
775
    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f16, Custom);
638
775
639
775
    setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f16, Custom);
640
775
    setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i16, Custom);
641
775
642
775
    setOperationAction(ISD::SHL, MVT::v4i16, Custom);
643
775
    setOperationAction(ISD::SRA, MVT::v4i16, Custom);
644
775
    setOperationAction(ISD::SRL, MVT::v4i16, Custom);
645
775
    setOperationAction(ISD::ADD, MVT::v4i16, Custom);
646
775
    setOperationAction(ISD::SUB, MVT::v4i16, Custom);
647
775
    setOperationAction(ISD::MUL, MVT::v4i16, Custom);
648
775
649
775
    setOperationAction(ISD::SMIN, MVT::v4i16, Custom);
650
775
    setOperationAction(ISD::SMAX, MVT::v4i16, Custom);
651
775
    setOperationAction(ISD::UMIN, MVT::v4i16, Custom);
652
775
    setOperationAction(ISD::UMAX, MVT::v4i16, Custom);
653
775
654
775
    setOperationAction(ISD::FADD, MVT::v4f16, Custom);
655
775
    setOperationAction(ISD::FMUL, MVT::v4f16, Custom);
656
775
657
775
    setOperationAction(ISD::FMAXNUM, MVT::v2f16, Custom);
658
775
    setOperationAction(ISD::FMINNUM, MVT::v2f16, Custom);
659
775
660
775
    setOperationAction(ISD::FMINNUM, MVT::v4f16, Custom);
661
775
    setOperationAction(ISD::FMAXNUM, MVT::v4f16, Custom);
662
775
    setOperationAction(ISD::FCANONICALIZE, MVT::v4f16, Custom);
663
775
664
775
    setOperationAction(ISD::FEXP, MVT::v2f16, Custom);
665
775
    setOperationAction(ISD::SELECT, MVT::v4i16, Custom);
666
775
    setOperationAction(ISD::SELECT, MVT::v4f16, Custom);
667
775
  }
668
3.64k
669
3.64k
  setOperationAction(ISD::FNEG, MVT::v4f16, Custom);
670
3.64k
  setOperationAction(ISD::FABS, MVT::v4f16, Custom);
671
3.64k
672
3.64k
  if (Subtarget->has16BitInsts()) {
673
2.01k
    setOperationAction(ISD::SELECT, MVT::v2i16, Promote);
674
2.01k
    AddPromotedToType(ISD::SELECT, MVT::v2i16, MVT::i32);
675
2.01k
    setOperationAction(ISD::SELECT, MVT::v2f16, Promote);
676
2.01k
    AddPromotedToType(ISD::SELECT, MVT::v2f16, MVT::i32);
677
2.01k
  } else {
678
1.62k
    // Legalization hack.
679
1.62k
    setOperationAction(ISD::SELECT, MVT::v2i16, Custom);
680
1.62k
    setOperationAction(ISD::SELECT, MVT::v2f16, Custom);
681
1.62k
682
1.62k
    setOperationAction(ISD::FNEG, MVT::v2f16, Custom);
683
1.62k
    setOperationAction(ISD::FABS, MVT::v2f16, Custom);
684
1.62k
  }
685
3.64k
686
18.2k
  for (MVT VT : { MVT::v4i16, MVT::v4f16, MVT::v2i8, MVT::v4i8, MVT::v8i8 }) {
687
18.2k
    setOperationAction(ISD::SELECT, VT, Custom);
688
18.2k
  }
689
3.64k
690
3.64k
  setTargetDAGCombine(ISD::ADD);
691
3.64k
  setTargetDAGCombine(ISD::ADDCARRY);
692
3.64k
  setTargetDAGCombine(ISD::SUB);
693
3.64k
  setTargetDAGCombine(ISD::SUBCARRY);
694
3.64k
  setTargetDAGCombine(ISD::FADD);
695
3.64k
  setTargetDAGCombine(ISD::FSUB);
696
3.64k
  setTargetDAGCombine(ISD::FMINNUM);
697
3.64k
  setTargetDAGCombine(ISD::FMAXNUM);
698
3.64k
  setTargetDAGCombine(ISD::FMINNUM_IEEE);
699
3.64k
  setTargetDAGCombine(ISD::FMAXNUM_IEEE);
700
3.64k
  setTargetDAGCombine(ISD::FMA);
701
3.64k
  setTargetDAGCombine(ISD::SMIN);
702
3.64k
  setTargetDAGCombine(ISD::SMAX);
703
3.64k
  setTargetDAGCombine(ISD::UMIN);
704
3.64k
  setTargetDAGCombine(ISD::UMAX);
705
3.64k
  setTargetDAGCombine(ISD::SETCC);
706
3.64k
  setTargetDAGCombine(ISD::AND);
707
3.64k
  setTargetDAGCombine(ISD::OR);
708
3.64k
  setTargetDAGCombine(ISD::XOR);
709
3.64k
  setTargetDAGCombine(ISD::SINT_TO_FP);
710
3.64k
  setTargetDAGCombine(ISD::UINT_TO_FP);
711
3.64k
  setTargetDAGCombine(ISD::FCANONICALIZE);
712
3.64k
  setTargetDAGCombine(ISD::SCALAR_TO_VECTOR);
713
3.64k
  setTargetDAGCombine(ISD::ZERO_EXTEND);
714
3.64k
  setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
715
3.64k
  setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
716
3.64k
  setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
717
3.64k
718
3.64k
  // All memory operations. Some folding on the pointer operand is done to help
719
3.64k
  // matching the constant offsets in the addressing modes.
720
3.64k
  setTargetDAGCombine(ISD::LOAD);
721
3.64k
  setTargetDAGCombine(ISD::STORE);
722
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD);
723
3.64k
  setTargetDAGCombine(ISD::ATOMIC_STORE);
724
3.64k
  setTargetDAGCombine(ISD::ATOMIC_CMP_SWAP);
725
3.64k
  setTargetDAGCombine(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
726
3.64k
  setTargetDAGCombine(ISD::ATOMIC_SWAP);
727
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_ADD);
728
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_SUB);
729
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_AND);
730
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_OR);
731
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_XOR);
732
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_NAND);
733
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_MIN);
734
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_MAX);
735
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_UMIN);
736
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_UMAX);
737
3.64k
  setTargetDAGCombine(ISD::ATOMIC_LOAD_FADD);
738
3.64k
739
3.64k
  setSchedulingPreference(Sched::RegPressure);
740
3.64k
}
741
742
1.51M
const GCNSubtarget *SITargetLowering::getSubtarget() const {
743
1.51M
  return Subtarget;
744
1.51M
}
745
746
//===----------------------------------------------------------------------===//
747
// TargetLowering queries
748
//===----------------------------------------------------------------------===//
749
750
// v_mad_mix* support a conversion from f16 to f32.
751
//
752
// There is only one special case when denormals are enabled we don't currently,
753
// where this is OK to use.
754
bool SITargetLowering::isFPExtFoldable(unsigned Opcode,
755
24
                                           EVT DestVT, EVT SrcVT) const {
756
24
  return ((Opcode == ISD::FMAD && 
Subtarget->hasMadMixInsts()22
) ||
757
24
          
(13
Opcode == ISD::FMA13
&&
Subtarget->hasFmaMixInsts()2
)) &&
758
24
         
DestVT.getScalarType() == MVT::f3211
&&
!Subtarget->hasFP32Denormals()11
&&
759
24
         
SrcVT.getScalarType() == MVT::f1611
;
760
24
}
761
762
32
bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const {
763
32
  // SI has some legal vector types, but no legal vector operations. Say no
764
32
  // shuffles are legal in order to prefer scalarizing some vector operations.
765
32
  return false;
766
32
}
767
768
MVT SITargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
769
                                                    CallingConv::ID CC,
770
184k
                                                    EVT VT) const {
771
184k
  if (CC == CallingConv::AMDGPU_KERNEL)
772
142k
    return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
773
41.5k
774
41.5k
  if (VT.isVector()) {
775
12.8k
    EVT ScalarVT = VT.getScalarType();
776
12.8k
    unsigned Size = ScalarVT.getSizeInBits();
777
12.8k
    if (Size == 32)
778
10.5k
      return ScalarVT.getSimpleVT();
779
2.27k
780
2.27k
    if (Size > 32)
781
206
      return MVT::i32;
782
2.06k
783
2.06k
    if (Size == 16 && 
Subtarget->has16BitInsts()2.03k
)
784
1.62k
      return VT.isInteger() ? 
MVT::v2i16573
:
MVT::v2f161.05k
;
785
28.7k
  } else if (VT.getSizeInBits() > 32)
786
2.95k
    return MVT::i32;
787
26.2k
788
26.2k
  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
789
26.2k
}
790
791
unsigned SITargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
792
                                                         CallingConv::ID CC,
793
184k
                                                         EVT VT) const {
794
184k
  if (CC == CallingConv::AMDGPU_KERNEL)
795
142k
    return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
796
41.6k
797
41.6k
  if (VT.isVector()) {
798
12.8k
    unsigned NumElts = VT.getVectorNumElements();
799
12.8k
    EVT ScalarVT = VT.getScalarType();
800
12.8k
    unsigned Size = ScalarVT.getSizeInBits();
801
12.8k
802
12.8k
    if (Size == 32)
803
10.5k
      return NumElts;
804
2.27k
805
2.27k
    if (Size > 32)
806
206
      return NumElts * ((Size + 31) / 32);
807
2.06k
808
2.06k
    if (Size == 16 && 
Subtarget->has16BitInsts()2.03k
)
809
1.62k
      return (NumElts + 1) / 2;
810
28.8k
  } else if (VT.getSizeInBits() > 32)
811
2.95k
    return (VT.getSizeInBits() + 31) / 32;
812
26.2k
813
26.2k
  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
814
26.2k
}
815
816
unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
817
  LLVMContext &Context, CallingConv::ID CC,
818
  EVT VT, EVT &IntermediateVT,
819
5.17k
  unsigned &NumIntermediates, MVT &RegisterVT) const {
820
5.17k
  if (CC != CallingConv::AMDGPU_KERNEL && 
VT.isVector()5.17k
) {
821
5.17k
    unsigned NumElts = VT.getVectorNumElements();
822
5.17k
    EVT ScalarVT = VT.getScalarType();
823
5.17k
    unsigned Size = ScalarVT.getSizeInBits();
824
5.17k
    if (Size == 32) {
825
4.54k
      RegisterVT = ScalarVT.getSimpleVT();
826
4.54k
      IntermediateVT = RegisterVT;
827
4.54k
      NumIntermediates = NumElts;
828
4.54k
      return NumIntermediates;
829
4.54k
    }
830
631
831
631
    if (Size > 32) {
832
107
      RegisterVT = MVT::i32;
833
107
      IntermediateVT = RegisterVT;
834
107
      NumIntermediates = NumElts * ((Size + 31) / 32);
835
107
      return NumIntermediates;
836
107
    }
837
524
838
524
    // FIXME: We should fix the ABI to be the same on targets without 16-bit
839
524
    // support, but unless we can properly handle 3-vectors, it will be still be
840
524
    // inconsistent.
841
524
    if (Size == 16 && 
Subtarget->has16BitInsts()506
) {
842
308
      RegisterVT = VT.isInteger() ? 
MVT::v2i1667
:
MVT::v2f16241
;
843
308
      IntermediateVT = RegisterVT;
844
308
      NumIntermediates = (NumElts + 1) / 2;
845
308
      return NumIntermediates;
846
308
    }
847
217
  }
848
217
849
217
  return TargetLowering::getVectorTypeBreakdownForCallingConv(
850
217
    Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
851
217
}
852
853
150
static MVT memVTFromAggregate(Type *Ty) {
854
150
  // Only limited forms of aggregate type currently expected.
855
150
  assert(Ty->isStructTy() && "Expected struct type");
856
150
857
150
858
150
  Type *ElementType = nullptr;
859
150
  unsigned NumElts;
860
150
  if (Ty->getContainedType(0)->isVectorTy()) {
861
133
    VectorType *VecComponent = cast<VectorType>(Ty->getContainedType(0));
862
133
    ElementType = VecComponent->getElementType();
863
133
    NumElts = VecComponent->getNumElements();
864
133
  } else {
865
17
    ElementType = Ty->getContainedType(0);
866
17
    NumElts = 1;
867
17
  }
868
150
869
150
  assert((Ty->getContainedType(1) && Ty->getContainedType(1)->isIntegerTy(32)) && "Expected int32 type");
870
150
871
150
  // Calculate the size of the memVT type from the aggregate
872
150
  unsigned Pow2Elts = 0;
873
150
  unsigned ElementSize;
874
150
  switch (ElementType->getTypeID()) {
875
150
    default:
876
0
      llvm_unreachable("Unknown type!");
877
150
    case Type::IntegerTyID:
878
0
      ElementSize = cast<IntegerType>(ElementType)->getBitWidth();
879
0
      break;
880
150
    case Type::HalfTyID:
881
12
      ElementSize = 16;
882
12
      break;
883
150
    case Type::FloatTyID:
884
138
      ElementSize = 32;
885
138
      break;
886
150
  }
887
150
  unsigned AdditionalElts = ElementSize == 16 ? 
212
:
1138
;
888
150
  Pow2Elts = 1 << Log2_32_Ceil(NumElts + AdditionalElts);
889
150
890
150
  return MVT::getVectorVT(MVT::getVT(ElementType, false),
891
150
                          Pow2Elts);
892
150
}
893
894
bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
895
                                          const CallInst &CI,
896
                                          MachineFunction &MF,
897
32.7k
                                          unsigned IntrID) const {
898
32.7k
  if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
899
3.38k
          AMDGPU::lookupRsrcIntrinsic(IntrID)) {
900
3.38k
    AttributeList Attr = Intrinsic::getAttributes(CI.getContext(),
901
3.38k
                                                  (Intrinsic::ID)IntrID);
902
3.38k
    if (Attr.hasFnAttribute(Attribute::ReadNone))
903
695
      return false;
904
2.69k
905
2.69k
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
906
2.69k
907
2.69k
    if (RsrcIntr->IsImage) {
908
1.21k
      Info.ptrVal = MFI->getImagePSV(
909
1.21k
        *MF.getSubtarget<GCNSubtarget>().getInstrInfo(),
910
1.21k
        CI.getArgOperand(RsrcIntr->RsrcArg));
911
1.21k
      Info.align = 0;
912
1.48k
    } else {
913
1.48k
      Info.ptrVal = MFI->getBufferPSV(
914
1.48k
        *MF.getSubtarget<GCNSubtarget>().getInstrInfo(),
915
1.48k
        CI.getArgOperand(RsrcIntr->RsrcArg));
916
1.48k
    }
917
2.69k
918
2.69k
    Info.flags = MachineMemOperand::MODereferenceable;
919
2.69k
    if (Attr.hasFnAttribute(Attribute::ReadOnly)) {
920
1.66k
      Info.opc = ISD::INTRINSIC_W_CHAIN;
921
1.66k
      Info.memVT = MVT::getVT(CI.getType(), true);
922
1.66k
      if (Info.memVT == MVT::Other) {
923
150
        // Some intrinsics return an aggregate type - special case to work out
924
150
        // the correct memVT
925
150
        Info.memVT = memVTFromAggregate(CI.getType());
926
150
      }
927
1.66k
      Info.flags |= MachineMemOperand::MOLoad;
928
1.66k
    } else 
if (1.03k
Attr.hasFnAttribute(Attribute::WriteOnly)1.03k
) {
929
722
      Info.opc = ISD::INTRINSIC_VOID;
930
722
      Info.memVT = MVT::getVT(CI.getArgOperand(0)->getType());
931
722
      Info.flags |= MachineMemOperand::MOStore;
932
722
    } else {
933
309
      // Atomic
934
309
      Info.opc = ISD::INTRINSIC_W_CHAIN;
935
309
      Info.memVT = MVT::getVT(CI.getType());
936
309
      Info.flags = MachineMemOperand::MOLoad |
937
309
                   MachineMemOperand::MOStore |
938
309
                   MachineMemOperand::MODereferenceable;
939
309
940
309
      // XXX - Should this be volatile without known ordering?
941
309
      Info.flags |= MachineMemOperand::MOVolatile;
942
309
    }
943
2.69k
    return true;
944
2.69k
  }
945
29.4k
946
29.4k
  switch (IntrID) {
947
29.4k
  case Intrinsic::amdgcn_atomic_inc:
948
314
  case Intrinsic::amdgcn_atomic_dec:
949
314
  case Intrinsic::amdgcn_ds_ordered_add:
950
314
  case Intrinsic::amdgcn_ds_ordered_swap:
951
314
  case Intrinsic::amdgcn_ds_fadd:
952
314
  case Intrinsic::amdgcn_ds_fmin:
953
314
  case Intrinsic::amdgcn_ds_fmax: {
954
314
    Info.opc = ISD::INTRINSIC_W_CHAIN;
955
314
    Info.memVT = MVT::getVT(CI.getType());
956
314
    Info.ptrVal = CI.getOperand(0);
957
314
    Info.align = 0;
958
314
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
959
314
960
314
    const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
961
314
    if (!Vol->isZero())
962
0
      Info.flags |= MachineMemOperand::MOVolatile;
963
314
964
314
    return true;
965
314
  }
966
314
  case Intrinsic::amdgcn_buffer_atomic_fadd: {
967
4
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
968
4
969
4
    Info.opc = ISD::INTRINSIC_VOID;
970
4
    Info.memVT = MVT::getVT(CI.getOperand(0)->getType());
971
4
    Info.ptrVal = MFI->getBufferPSV(
972
4
      *MF.getSubtarget<GCNSubtarget>().getInstrInfo(),
973
4
      CI.getArgOperand(1));
974
4
    Info.align = 0;
975
4
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
976
4
977
4
    const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(4));
978
4
    if (!Vol || !Vol->isZero())
979
2
      Info.flags |= MachineMemOperand::MOVolatile;
980
4
981
4
    return true;
982
314
  }
983
314
  case Intrinsic::amdgcn_global_atomic_fadd: {
984
4
    Info.opc = ISD::INTRINSIC_VOID;
985
4
    Info.memVT = MVT::getVT(CI.getOperand(0)->getType()
986
4
                            ->getPointerElementType());
987
4
    Info.ptrVal = CI.getOperand(0);
988
4
    Info.align = 0;
989
4
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
990
4
991
4
    return true;
992
314
  }
993
314
  case Intrinsic::amdgcn_ds_append:
994
72
  case Intrinsic::amdgcn_ds_consume: {
995
72
    Info.opc = ISD::INTRINSIC_W_CHAIN;
996
72
    Info.memVT = MVT::getVT(CI.getType());
997
72
    Info.ptrVal = CI.getOperand(0);
998
72
    Info.align = 0;
999
72
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1000
72
1001
72
    const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1));
1002
72
    if (!Vol->isZero())
1003
0
      Info.flags |= MachineMemOperand::MOVolatile;
1004
72
1005
72
    return true;
1006
72
  }
1007
180
  case Intrinsic::amdgcn_ds_gws_init:
1008
180
  case Intrinsic::amdgcn_ds_gws_barrier:
1009
180
  case Intrinsic::amdgcn_ds_gws_sema_v:
1010
180
  case Intrinsic::amdgcn_ds_gws_sema_br:
1011
180
  case Intrinsic::amdgcn_ds_gws_sema_p:
1012
180
  case Intrinsic::amdgcn_ds_gws_sema_release_all: {
1013
180
    Info.opc = ISD::INTRINSIC_VOID;
1014
180
1015
180
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1016
180
    Info.ptrVal =
1017
180
        MFI->getGWSPSV(*MF.getSubtarget<GCNSubtarget>().getInstrInfo());
1018
180
1019
180
    // This is an abstract access, but we need to specify a type and size.
1020
180
    Info.memVT = MVT::i32;
1021
180
    Info.size = 4;
1022
180
    Info.align = 4;
1023
180
1024
180
    Info.flags = MachineMemOperand::MOStore;
1025
180
    if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
1026
99
      Info.flags = MachineMemOperand::MOLoad;
1027
180
    return true;
1028
180
  }
1029
28.8k
  default:
1030
28.8k
    return false;
1031
29.4k
  }
1032
29.4k
}
1033
1034
bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
1035
                                            SmallVectorImpl<Value*> &Ops,
1036
39.3k
                                            Type *&AccessTy) const {
1037
39.3k
  switch (II->getIntrinsicID()) {
1038
39.3k
  case Intrinsic::amdgcn_atomic_inc:
1039
338
  case Intrinsic::amdgcn_atomic_dec:
1040
338
  case Intrinsic::amdgcn_ds_ordered_add:
1041
338
  case Intrinsic::amdgcn_ds_ordered_swap:
1042
338
  case Intrinsic::amdgcn_ds_fadd:
1043
338
  case Intrinsic::amdgcn_ds_fmin:
1044
338
  case Intrinsic::amdgcn_ds_fmax: {
1045
338
    Value *Ptr = II->getArgOperand(0);
1046
338
    AccessTy = II->getType();
1047
338
    Ops.push_back(Ptr);
1048
338
    return true;
1049
338
  }
1050
38.9k
  default:
1051
38.9k
    return false;
1052
39.3k
  }
1053
39.3k
}
1054
1055
83.5k
bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM) const {
1056
83.5k
  if (!Subtarget->hasFlatInstOffsets()) {
1057
74.9k
    // Flat instructions do not have offsets, and only have the register
1058
74.9k
    // address.
1059
74.9k
    return AM.BaseOffs == 0 && 
AM.Scale == 044.7k
;
1060
74.9k
  }
1061
8.61k
1062
8.61k
  // GFX9 added a 13-bit signed offset. When using regular flat instructions,
1063
8.61k
  // the sign bit is ignored and is treated as a 12-bit unsigned offset.
1064
8.61k
1065
8.61k
  // GFX10 shrinked signed offset to 12 bits. When using regular flat
1066
8.61k
  // instructions, the sign bit is also ignored and is treated as 11-bit
1067
8.61k
  // unsigned offset.
1068
8.61k
1069
8.61k
  if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10)
1070
5.10k
    return isUInt<11>(AM.BaseOffs) && AM.Scale == 0;
1071
3.51k
1072
3.51k
  // Just r + i
1073
3.51k
  return isUInt<12>(AM.BaseOffs) && 
AM.Scale == 03.44k
;
1074
3.51k
}
1075
1076
172k
bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const {
1077
172k
  if (Subtarget->hasFlatGlobalInsts())
1078
43.2k
    return isInt<13>(AM.BaseOffs) && 
AM.Scale == 042.2k
;
1079
129k
1080
129k
  if (!Subtarget->hasAddr64() || 
Subtarget->useFlatForGlobal()71.0k
) {
1081
63.5k
      // Assume the we will use FLAT for all global memory accesses
1082
63.5k
      // on VI.
1083
63.5k
      // FIXME: This assumption is currently wrong.  On VI we still use
1084
63.5k
      // MUBUF instructions for the r + i addressing mode.  As currently
1085
63.5k
      // implemented, the MUBUF instructions only work on buffer < 4GB.
1086
63.5k
      // It may be possible to support > 4GB buffers with MUBUF instructions,
1087
63.5k
      // by setting the stride value in the resource descriptor which would
1088
63.5k
      // increase the size limit to (stride * 4GB).  However, this is risky,
1089
63.5k
      // because it has never been validated.
1090
63.5k
    return isLegalFlatAddressingMode(AM);
1091
63.5k
  }
1092
65.4k
1093
65.4k
  return isLegalMUBUFAddressingMode(AM);
1094
65.4k
}
1095
1096
93.5k
bool SITargetLowering::isLegalMUBUFAddressingMode(const AddrMode &AM) const {
1097
93.5k
  // MUBUF / MTBUF instructions have a 12-bit unsigned byte offset, and
1098
93.5k
  // additionally can do r + r + i with addr64. 32-bit has more addressing
1099
93.5k
  // mode options. Depending on the resource constant, it can also do
1100
93.5k
  // (i64 r0) + (i32 r1) * (i14 i).
1101
93.5k
  //
1102
93.5k
  // Private arrays end up using a scratch buffer most of the time, so also
1103
93.5k
  // assume those use MUBUF instructions. Scratch loads / stores are currently
1104
93.5k
  // implemented as mubuf instructions with offen bit set, so slightly
1105
93.5k
  // different than the normal addr64.
1106
93.5k
  if (!isUInt<12>(AM.BaseOffs))
1107
1.00k
    return false;
1108
92.5k
1109
92.5k
  // FIXME: Since we can split immediate into soffset and immediate offset,
1110
92.5k
  // would it make sense to allow any immediate?
1111
92.5k
1112
92.5k
  switch (AM.Scale) {
1113
92.5k
  case 0: // r + i or just i, depending on HasBaseReg.
1114
76.9k
    return true;
1115
92.5k
  case 1:
1116
1.20k
    return true; // We have r + r or r + i.
1117
92.5k
  case 2:
1118
808
    if (AM.HasBaseReg) {
1119
808
      // Reject 2 * r + r.
1120
808
      return false;
1121
808
    }
1122
0
1123
0
    // Allow 2 * r as r + r
1124
0
    // Or  2 * r + i is allowed as r + r + i.
1125
0
    return true;
1126
13.5k
  default: // Don't allow n * r
1127
13.5k
    return false;
1128
92.5k
  }
1129
92.5k
}
1130
1131
bool SITargetLowering::isLegalAddressingMode(const DataLayout &DL,
1132
                                             const AddrMode &AM, Type *Ty,
1133
350k
                                             unsigned AS, Instruction *I) const {
1134
350k
  // No global is ever allowed as a base.
1135
350k
  if (AM.BaseGV)
1136
3.18k
    return false;
1137
347k
1138
347k
  if (AS == AMDGPUAS::GLOBAL_ADDRESS)
1139
134k
    return isLegalGlobalAddressingMode(AM);
1140
212k
1141
212k
  if (AS == AMDGPUAS::CONSTANT_ADDRESS ||
1142
212k
      
AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT85.2k
||
1143
212k
      
AS == AMDGPUAS::BUFFER_FAT_POINTER84.8k
) {
1144
127k
    // If the offset isn't a multiple of 4, it probably isn't going to be
1145
127k
    // correctly aligned.
1146
127k
    // FIXME: Can we get the real alignment here?
1147
127k
    if (AM.BaseOffs % 4 != 0)
1148
422
      return isLegalMUBUFAddressingMode(AM);
1149
127k
1150
127k
    // There are no SMRD extloads, so if we have to do a small type access we
1151
127k
    // will use a MUBUF load.
1152
127k
    // FIXME?: We also need to do this if unaligned, but we don't know the
1153
127k
    // alignment here.
1154
127k
    if (Ty->isSized() && DL.getTypeStoreSize(Ty) < 4)
1155
37.4k
      return isLegalGlobalAddressingMode(AM);
1156
90.0k
1157
90.0k
    if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS) {
1158
20.8k
      // SMRD instructions have an 8-bit, dword offset on SI.
1159
20.8k
      if (!isUInt<8>(AM.BaseOffs / 4))
1160
81
        return false;
1161
69.2k
    } else if (Subtarget->getGeneration() == AMDGPUSubtarget::SEA_ISLANDS) {
1162
11.8k
      // On CI+, this can also be a 32-bit literal constant offset. If it fits
1163
11.8k
      // in 8-bits, it can use a smaller encoding.
1164
11.8k
      if (!isUInt<32>(AM.BaseOffs / 4))
1165
13
        return false;
1166
57.3k
    } else if (Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
1167
57.3k
      // On VI, these use the SMEM format and the offset is 20-bit in bytes.
1168
57.3k
      if (!isUInt<20>(AM.BaseOffs))
1169
62
        return false;
1170
57.3k
    } else
1171
57.3k
      llvm_unreachable("unhandled generation");
1172
90.0k
1173
90.0k
    
if (89.9k
AM.Scale == 089.9k
) // r + i or just i, depending on HasBaseReg.
1174
89.4k
      return true;
1175
483
1176
483
    if (AM.Scale == 1 && 
AM.HasBaseReg54
)
1177
54
      return true;
1178
429
1179
429
    return false;
1180
429
1181
84.8k
  } else if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
1182
27.6k
    return isLegalMUBUFAddressingMode(AM);
1183
57.1k
  } else if (AS == AMDGPUAS::LOCAL_ADDRESS ||
1184
57.1k
             
AS == AMDGPUAS::REGION_ADDRESS20.1k
) {
1185
37.1k
    // Basic, single offset DS instructions allow a 16-bit unsigned immediate
1186
37.1k
    // field.
1187
37.1k
    // XXX - If doing a 4-byte aligned 8-byte type access, we effectively have
1188
37.1k
    // an 8-bit dword offset but we don't know the alignment here.
1189
37.1k
    if (!isUInt<16>(AM.BaseOffs))
1190
1.83k
      return false;
1191
35.3k
1192
35.3k
    if (AM.Scale == 0) // r + i or just i, depending on HasBaseReg.
1193
31.2k
      return true;
1194
4.10k
1195
4.10k
    if (AM.Scale == 1 && 
AM.HasBaseReg1.36k
)
1196
1.36k
      return true;
1197
2.74k
1198
2.74k
    return false;
1199
19.9k
  } else if (AS == AMDGPUAS::FLAT_ADDRESS ||
1200
19.9k
             
AS == AMDGPUAS::UNKNOWN_ADDRESS_SPACE8.77k
) {
1201
19.9k
    // For an unknown address space, this usually means that this is for some
1202
19.9k
    // reason being used for pure arithmetic, and not based on some addressing
1203
19.9k
    // computation. We don't have instructions that compute pointers with any
1204
19.9k
    // addressing modes, so treat them as having no offset like flat
1205
19.9k
    // instructions.
1206
19.9k
    return isLegalFlatAddressingMode(AM);
1207
19.9k
  } else {
1208
0
    llvm_unreachable("unhandled address space");
1209
0
  }
1210
212k
}
1211
1212
bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
1213
32.6k
                                        const SelectionDAG &DAG) const {
1214
32.6k
  if (AS == AMDGPUAS::GLOBAL_ADDRESS || 
AS == AMDGPUAS::FLAT_ADDRESS12.4k
) {
1215
20.1k
    return (MemVT.getSizeInBits() <= 4 * 32);
1216
20.1k
  } else 
if (12.4k
AS == AMDGPUAS::PRIVATE_ADDRESS12.4k
) {
1217
3.84k
    unsigned MaxPrivateBits = 8 * getSubtarget()->getMaxPrivateElementSize();
1218
3.84k
    return (MemVT.getSizeInBits() <= MaxPrivateBits);
1219
8.61k
  } else if (AS == AMDGPUAS::LOCAL_ADDRESS || 
AS == AMDGPUAS::REGION_ADDRESS0
) {
1220
8.61k
    return (MemVT.getSizeInBits() <= 2 * 32);
1221
8.61k
  }
1222
0
  return true;
1223
0
}
1224
1225
bool SITargetLowering::allowsMisalignedMemoryAccesses(
1226
    EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
1227
178k
    bool *IsFast) const {
1228
178k
  if (IsFast)
1229
124k
    *IsFast = false;
1230
178k
1231
178k
  // TODO: I think v3i32 should allow unaligned accesses on CI with DS_READ_B96,
1232
178k
  // which isn't a simple VT.
1233
178k
  // Until MVT is extended to handle this, simply check for the size and
1234
178k
  // rely on the condition below: allow accesses if the size is a multiple of 4.
1235
178k
  if (VT == MVT::Other || (VT != MVT::Other && VT.getSizeInBits() > 1024 &&
1236
178k
                           
VT.getStoreSize() > 160
)) {
1237
0
    return false;
1238
0
  }
1239
178k
1240
178k
  if (AddrSpace == AMDGPUAS::LOCAL_ADDRESS ||
1241
178k
      
AddrSpace == AMDGPUAS::REGION_ADDRESS152k
) {
1242
26.1k
    // ds_read/write_b64 require 8-byte alignment, but we can do a 4 byte
1243
26.1k
    // aligned, 8 byte access in a single operation using ds_read2/write2_b32
1244
26.1k
    // with adjacent offsets.
1245
26.1k
    bool AlignedBy4 = (Align % 4 == 0);
1246
26.1k
    if (IsFast)
1247
23.5k
      *IsFast = AlignedBy4;
1248
26.1k
1249
26.1k
    return AlignedBy4;
1250
26.1k
  }
1251
152k
1252
152k
  // FIXME: We have to be conservative here and assume that flat operations
1253
152k
  // will access scratch.  If we had access to the IR function, then we
1254
152k
  // could determine if any private memory was used in the function.
1255
152k
  if (!Subtarget->hasUnalignedScratchAccess() &&
1256
152k
      
(152k
AddrSpace == AMDGPUAS::PRIVATE_ADDRESS152k
||
1257
152k
       
AddrSpace == AMDGPUAS::FLAT_ADDRESS151k
)) {
1258
1.07k
    bool AlignedBy4 = Align >= 4;
1259
1.07k
    if (IsFast)
1260
843
      *IsFast = AlignedBy4;
1261
1.07k
1262
1.07k
    return AlignedBy4;
1263
1.07k
  }
1264
151k
1265
151k
  if (Subtarget->hasUnalignedBufferAccess()) {
1266
12.1k
    // If we have an uniform constant load, it still requires using a slow
1267
12.1k
    // buffer instruction if unaligned.
1268
12.1k
    if (IsFast) {
1269
8.25k
      *IsFast = (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
1270
8.25k
                 
AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT7.44k
) ?
1271
7.44k
        
(Align % 4 == 0)812
: true;
1272
8.25k
    }
1273
12.1k
1274
12.1k
    return true;
1275
12.1k
  }
1276
139k
1277
139k
  // Smaller than dword value must be aligned.
1278
139k
  if (VT.bitsLT(MVT::i32))
1279
4.53k
    return false;
1280
134k
1281
134k
  // 8.1.6 - For Dword or larger reads or writes, the two LSBs of the
1282
134k
  // byte-address are ignored, thus forcing Dword alignment.
1283
134k
  // This applies to private, global, and constant memory.
1284
134k
  if (IsFast)
1285
87.0k
    *IsFast = true;
1286
134k
1287
134k
  return VT.bitsGT(MVT::i32) && 
Align % 4 == 0128k
;
1288
134k
}
1289
1290
EVT SITargetLowering::getOptimalMemOpType(
1291
    uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset,
1292
    bool ZeroMemset, bool MemcpyStrSrc,
1293
124
    const AttributeList &FuncAttributes) const {
1294
124
  // FIXME: Should account for address space here.
1295
124
1296
124
  // The default fallback uses the private pointer size as a guess for a type to
1297
124
  // use. Make sure we switch these to 64-bit accesses.
1298
124
1299
124
  if (Size >= 16 && 
DstAlign >= 4104
) // XXX: Should only do for global
1300
94
    return MVT::v4i32;
1301
30
1302
30
  if (Size >= 8 && 
DstAlign >= 422
)
1303
12
    return MVT::v2i32;
1304
18
1305
18
  // Use the default.
1306
18
  return MVT::Other;
1307
18
}
1308
1309
1.52k
static bool isFlatGlobalAddrSpace(unsigned AS) {
1310
1.52k
  return AS == AMDGPUAS::GLOBAL_ADDRESS ||
1311
1.52k
         
AS == AMDGPUAS::FLAT_ADDRESS1.30k
||
1312
1.52k
         
AS == AMDGPUAS::CONSTANT_ADDRESS414
||
1313
1.52k
         
AS > AMDGPUAS::MAX_AMDGPU_ADDRESS377
;
1314
1.52k
}
1315
1316
bool SITargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
1317
432
                                           unsigned DestAS) const {
1318
432
  return isFlatGlobalAddrSpace(SrcAS) && 
isFlatGlobalAddrSpace(DestAS)252
;
1319
432
}
1320
1321
6.49k
bool SITargetLowering::isMemOpHasNoClobberedMemOperand(const SDNode *N) const {
1322
6.49k
  const MemSDNode *MemNode = cast<MemSDNode>(N);
1323
6.49k
  const Value *Ptr = MemNode->getMemOperand()->getValue();
1324
6.49k
  const Instruction *I = dyn_cast_or_null<Instruction>(Ptr);
1325
6.49k
  return I && 
I->getMetadata("amdgpu.noclobber")6.20k
;
1326
6.49k
}
1327
1328
bool SITargetLowering::isFreeAddrSpaceCast(unsigned SrcAS,
1329
313
                                           unsigned DestAS) const {
1330
313
  // Flat -> private/local is a simple truncate.
1331
313
  // Flat -> global is no-op
1332
313
  if (SrcAS == AMDGPUAS::FLAT_ADDRESS)
1333
133
    return true;
1334
180
1335
180
  return isNoopAddrSpaceCast(SrcAS, DestAS);
1336
180
}
1337
1338
0
bool SITargetLowering::isMemOpUniform(const SDNode *N) const {
1339
0
  const MemSDNode *MemNode = cast<MemSDNode>(N);
1340
0
1341
0
  return AMDGPUInstrInfo::isUniformMMO(MemNode->getMemOperand());
1342
0
}
1343
1344
TargetLoweringBase::LegalizeTypeAction
1345
345k
SITargetLowering::getPreferredVectorAction(MVT VT) const {
1346
345k
  if (VT.getVectorNumElements() != 1 && 
VT.getScalarType().bitsLE(MVT::i16)290k
)
1347
155k
    return TypeSplitVector;
1348
189k
1349
189k
  return TargetLoweringBase::getPreferredVectorAction(VT);
1350
189k
}
1351
1352
bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
1353
32
                                                         Type *Ty) const {
1354
32
  // FIXME: Could be smarter if called for vector constants.
1355
32
  return true;
1356
32
}
1357
1358
349k
bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
1359
349k
  if (Subtarget->has16BitInsts() && 
VT == MVT::i16221k
) {
1360
38.7k
    switch (Op) {
1361
38.7k
    case ISD::LOAD:
1362
5.62k
    case ISD::STORE:
1363
5.62k
1364
5.62k
    // These operations are done with 32-bit instructions anyway.
1365
5.62k
    case ISD::AND:
1366
5.62k
    case ISD::OR:
1367
5.62k
    case ISD::XOR:
1368
5.62k
    case ISD::SELECT:
1369
5.62k
      // TODO: Extensions?
1370
5.62k
      return true;
1371
33.0k
    default:
1372
33.0k
      return false;
1373
311k
    }
1374
311k
  }
1375
311k
1376
311k
  // SimplifySetCC uses this function to determine whether or not it should
1377
311k
  // create setcc with i1 operands.  We don't have instructions for i1 setcc.
1378
311k
  if (VT == MVT::i1 && 
Op == ISD::SETCC1.71k
)
1379
25
    return false;
1380
311k
1381
311k
  return TargetLowering::isTypeDesirableForOp(Op, VT);
1382
311k
}
1383
1384
SDValue SITargetLowering::lowerKernArgParameterPtr(SelectionDAG &DAG,
1385
                                                   const SDLoc &SL,
1386
                                                   SDValue Chain,
1387
49.5k
                                                   uint64_t Offset) const {
1388
49.5k
  const DataLayout &DL = DAG.getDataLayout();
1389
49.5k
  MachineFunction &MF = DAG.getMachineFunction();
1390
49.5k
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
1391
49.5k
1392
49.5k
  const ArgDescriptor *InputPtrReg;
1393
49.5k
  const TargetRegisterClass *RC;
1394
49.5k
1395
49.5k
  std::tie(InputPtrReg, RC)
1396
49.5k
    = Info->getPreloadedValue(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
1397
49.5k
1398
49.5k
  MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
1399
49.5k
  MVT PtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS);
1400
49.5k
  SDValue BasePtr = DAG.getCopyFromReg(Chain, SL,
1401
49.5k
    MRI.getLiveInVirtReg(InputPtrReg->getRegister()), PtrVT);
1402
49.5k
1403
49.5k
  return DAG.getObjectPtrOffset(SL, BasePtr, Offset);
1404
49.5k
}
1405
1406
SDValue SITargetLowering::getImplicitArgPtr(SelectionDAG &DAG,
1407
42
                                            const SDLoc &SL) const {
1408
42
  uint64_t Offset = getImplicitParameterOffset(DAG.getMachineFunction(),
1409
42
                                               FIRST_IMPLICIT);
1410
42
  return lowerKernArgParameterPtr(DAG, SL, DAG.getEntryNode(), Offset);
1411
42
}
1412
1413
SDValue SITargetLowering::convertArgType(SelectionDAG &DAG, EVT VT, EVT MemVT,
1414
                                         const SDLoc &SL, SDValue Val,
1415
                                         bool Signed,
1416
49.5k
                                         const ISD::InputArg *Arg) const {
1417
49.5k
  // First, if it is a widened vector, narrow it.
1418
49.5k
  if (VT.isVector() &&
1419
49.5k
      
VT.getVectorNumElements() != MemVT.getVectorNumElements()1.91k
) {
1420
63
    EVT NarrowedVT =
1421
63
        EVT::getVectorVT(*DAG.getContext(), MemVT.getVectorElementType(),
1422
63
                         VT.getVectorNumElements());
1423
63
    Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, NarrowedVT, Val,
1424
63
                      DAG.getConstant(0, SL, MVT::i32));
1425
63
  }
1426
49.5k
1427
49.5k
  // Then convert the vector elements or scalar value.
1428
49.5k
  if (Arg && 
(49.4k
Arg->Flags.isSExt()49.4k
||
Arg->Flags.isZExt()49.4k
) &&
1429
49.5k
      
VT.bitsLT(MemVT)112
) {
1430
0
    unsigned Opc = Arg->Flags.isZExt() ? ISD::AssertZext : ISD::AssertSext;
1431
0
    Val = DAG.getNode(Opc, SL, MemVT, Val, DAG.getValueType(VT));
1432
0
  }
1433
49.5k
1434
49.5k
  if (MemVT.isFloatingPoint())
1435
2.81k
    Val = getFPExtOrFPTrunc(DAG, Val, SL, VT);
1436
46.7k
  else if (Signed)
1437
16
    Val = DAG.getSExtOrTrunc(Val, SL, VT);
1438
46.6k
  else
1439
46.6k
    Val = DAG.getZExtOrTrunc(Val, SL, VT);
1440
49.5k
1441
49.5k
  return Val;
1442
49.5k
}
1443
1444
SDValue SITargetLowering::lowerKernargMemParameter(
1445
  SelectionDAG &DAG, EVT VT, EVT MemVT,
1446
  const SDLoc &SL, SDValue Chain,
1447
  uint64_t Offset, unsigned Align, bool Signed,
1448
49.5k
  const ISD::InputArg *Arg) const {
1449
49.5k
  Type *Ty = MemVT.getTypeForEVT(*DAG.getContext());
1450
49.5k
  PointerType *PtrTy = PointerType::get(Ty, AMDGPUAS::CONSTANT_ADDRESS);
1451
49.5k
  MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
1452
49.5k
1453
49.5k
  // Try to avoid using an extload by loading earlier than the argument address,
1454
49.5k
  // and extracting the relevant bits. The load should hopefully be merged with
1455
49.5k
  // the previous argument.
1456
49.5k
  if (MemVT.getStoreSize() < 4 && 
Align < 43.22k
) {
1457
1.86k
    // TODO: Handle align < 4 and size >= 4 (can happen with packed structs).
1458
1.86k
    int64_t AlignDownOffset = alignDown(Offset, 4);
1459
1.86k
    int64_t OffsetDiff = Offset - AlignDownOffset;
1460
1.86k
1461
1.86k
    EVT IntVT = MemVT.changeTypeToInteger();
1462
1.86k
1463
1.86k
    // TODO: If we passed in the base kernel offset we could have a better
1464
1.86k
    // alignment than 4, but we don't really need it.
1465
1.86k
    SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, AlignDownOffset);
1466
1.86k
    SDValue Load = DAG.getLoad(MVT::i32, SL, Chain, Ptr, PtrInfo, 4,
1467
1.86k
                               MachineMemOperand::MODereferenceable |
1468
1.86k
                               MachineMemOperand::MOInvariant);
1469
1.86k
1470
1.86k
    SDValue ShiftAmt = DAG.getConstant(OffsetDiff * 8, SL, MVT::i32);
1471
1.86k
    SDValue Extract = DAG.getNode(ISD::SRL, SL, MVT::i32, Load, ShiftAmt);
1472
1.86k
1473
1.86k
    SDValue ArgVal = DAG.getNode(ISD::TRUNCATE, SL, IntVT, Extract);
1474
1.86k
    ArgVal = DAG.getNode(ISD::BITCAST, SL, MemVT, ArgVal);
1475
1.86k
    ArgVal = convertArgType(DAG, VT, MemVT, SL, ArgVal, Signed, Arg);
1476
1.86k
1477
1.86k
1478
1.86k
    return DAG.getMergeValues({ ArgVal, Load.getValue(1) }, SL);
1479
1.86k
  }
1480
47.6k
1481
47.6k
  SDValue Ptr = lowerKernArgParameterPtr(DAG, SL, Chain, Offset);
1482
47.6k
  SDValue Load = DAG.getLoad(MemVT, SL, Chain, Ptr, PtrInfo, Align,
1483
47.6k
                             MachineMemOperand::MODereferenceable |
1484
47.6k
                             MachineMemOperand::MOInvariant);
1485
47.6k
1486
47.6k
  SDValue Val = convertArgType(DAG, VT, MemVT, SL, Load, Signed, Arg);
1487
47.6k
  return DAG.getMergeValues({ Val, Load.getValue(1) }, SL);
1488
47.6k
}
1489
1490
SDValue SITargetLowering::lowerStackParameter(SelectionDAG &DAG, CCValAssign &VA,
1491
                                              const SDLoc &SL, SDValue Chain,
1492
433
                                              const ISD::InputArg &Arg) const {
1493
433
  MachineFunction &MF = DAG.getMachineFunction();
1494
433
  MachineFrameInfo &MFI = MF.getFrameInfo();
1495
433
1496
433
  if (Arg.Flags.isByVal()) {
1497
88
    unsigned Size = Arg.Flags.getByValSize();
1498
88
    int FrameIdx = MFI.CreateFixedObject(Size, VA.getLocMemOffset(), false);
1499
88
    return DAG.getFrameIndex(FrameIdx, MVT::i32);
1500
88
  }
1501
345
1502
345
  unsigned ArgOffset = VA.getLocMemOffset();
1503
345
  unsigned ArgSize = VA.getValVT().getStoreSize();
1504
345
1505
345
  int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, true);
1506
345
1507
345
  // Create load nodes to retrieve arguments from the stack.
1508
345
  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1509
345
  SDValue ArgValue;
1510
345
1511
345
  // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
1512
345
  ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
1513
345
  MVT MemVT = VA.getValVT();
1514
345
1515
345
  switch (VA.getLocInfo()) {
1516
345
  default:
1517
342
    break;
1518
345
  case CCValAssign::BCvt:
1519
0
    MemVT = VA.getLocVT();
1520
0
    break;
1521
345
  case CCValAssign::SExt:
1522
0
    ExtType = ISD::SEXTLOAD;
1523
0
    break;
1524
345
  case CCValAssign::ZExt:
1525
0
    ExtType = ISD::ZEXTLOAD;
1526
0
    break;
1527
345
  case CCValAssign::AExt:
1528
3
    ExtType = ISD::EXTLOAD;
1529
3
    break;
1530
345
  }
1531
345
1532
345
  ArgValue = DAG.getExtLoad(
1533
345
    ExtType, SL, VA.getLocVT(), Chain, FIN,
1534
345
    MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
1535
345
    MemVT);
1536
345
  return ArgValue;
1537
345
}
1538
1539
SDValue SITargetLowering::getPreloadedValue(SelectionDAG &DAG,
1540
  const SIMachineFunctionInfo &MFI,
1541
  EVT VT,
1542
16.6k
  AMDGPUFunctionArgInfo::PreloadedValue PVID) const {
1543
16.6k
  const ArgDescriptor *Reg;
1544
16.6k
  const TargetRegisterClass *RC;
1545
16.6k
1546
16.6k
  std::tie(Reg, RC) = MFI.getPreloadedValue(PVID);
1547
16.6k
  return CreateLiveInRegister(DAG, RC, Reg->getRegister(), VT);
1548
16.6k
}
1549
1550
static void processShaderInputArgs(SmallVectorImpl<ISD::InputArg> &Splits,
1551
                                   CallingConv::ID CallConv,
1552
                                   ArrayRef<ISD::InputArg> Ins,
1553
                                   BitVector &Skipped,
1554
                                   FunctionType *FType,
1555
2.97k
                                   SIMachineFunctionInfo *Info) {
1556
27.2k
  for (unsigned I = 0, E = Ins.size(), PSInputNum = 0; I != E; 
++I24.2k
) {
1557
24.2k
    const ISD::InputArg *Arg = &Ins[I];
1558
24.2k
1559
24.2k
    assert((!Arg->VT.isVector() || Arg->VT.getScalarSizeInBits() == 16) &&
1560
24.2k
           "vector type argument should have been split");
1561
24.2k
1562
24.2k
    // First check if it's a PS input addr.
1563
24.2k
    if (CallConv == CallingConv::AMDGPU_PS &&
1564
24.2k
        
!Arg->Flags.isInReg()22.3k
&&
PSInputNum <= 156.77k
) {
1565
6.76k
      bool SkipArg = !Arg->Used && 
!Info->isPSInputAllocated(PSInputNum)1.84k
;
1566
6.76k
1567
6.76k
      // Inconveniently only the first part of the split is marked as isSplit,
1568
6.76k
      // so skip to the end. We only want to increment PSInputNum once for the
1569
6.76k
      // entire split argument.
1570
6.76k
      if (Arg->Flags.isSplit()) {
1571
3.26k
        while (!Arg->Flags.isSplitEnd()) {
1572
2.01k
          assert(!Arg->VT.isVector() &&
1573
2.01k
                 "unexpected vector split in ps argument type");
1574
2.01k
          if (!SkipArg)
1575
1.19k
            Splits.push_back(*Arg);
1576
2.01k
          Arg = &Ins[++I];
1577
2.01k
        }
1578
1.24k
      }
1579
6.76k
1580
6.76k
      if (SkipArg) {
1581
1.72k
        // We can safely skip PS inputs.
1582
1.72k
        Skipped.set(Arg->getOrigArgIndex());
1583
1.72k
        ++PSInputNum;
1584
1.72k
        continue;
1585
1.72k
      }
1586
5.03k
1587
5.03k
      Info->markPSInputAllocated(PSInputNum);
1588
5.03k
      if (Arg->Used)
1589
4.92k
        Info->markPSInputEnabled(PSInputNum);
1590
5.03k
1591
5.03k
      ++PSInputNum;
1592
5.03k
    }
1593
24.2k
1594
24.2k
    Splits.push_back(*Arg);
1595
22.5k
  }
1596
2.97k
}
1597
1598
// Allocate special inputs passed in VGPRs.
1599
void SITargetLowering::allocateSpecialEntryInputVGPRs(CCState &CCInfo,
1600
                                                      MachineFunction &MF,
1601
                                                      const SIRegisterInfo &TRI,
1602
23.3k
                                                      SIMachineFunctionInfo &Info) const {
1603
23.3k
  const LLT S32 = LLT::scalar(32);
1604
23.3k
  MachineRegisterInfo &MRI = MF.getRegInfo();
1605
23.3k
1606
23.3k
  if (Info.hasWorkItemIDX()) {
1607
20.3k
    Register Reg = AMDGPU::VGPR0;
1608
20.3k
    MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1609
20.3k
1610
20.3k
    CCInfo.AllocateReg(Reg);
1611
20.3k
    Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg));
1612
20.3k
  }
1613
23.3k
1614
23.3k
  if (Info.hasWorkItemIDY()) {
1615
172
    Register Reg = AMDGPU::VGPR1;
1616
172
    MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1617
172
1618
172
    CCInfo.AllocateReg(Reg);
1619
172
    Info.setWorkItemIDY(ArgDescriptor::createRegister(Reg));
1620
172
  }
1621
23.3k
1622
23.3k
  if (Info.hasWorkItemIDZ()) {
1623
84
    Register Reg = AMDGPU::VGPR2;
1624
84
    MRI.setType(MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass), S32);
1625
84
1626
84
    CCInfo.AllocateReg(Reg);
1627
84
    Info.setWorkItemIDZ(ArgDescriptor::createRegister(Reg));
1628
84
  }
1629
23.3k
}
1630
1631
// Try to allocate a VGPR at the end of the argument list, or if no argument
1632
// VGPRs are left allocating a stack slot.
1633
// If \p Mask is is given it indicates bitfield position in the register.
1634
// If \p Arg is given use it with new ]p Mask instead of allocating new.
1635
static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
1636
43
                                         ArgDescriptor Arg = ArgDescriptor()) {
1637
43
  if (Arg.isSet())
1638
11
    return ArgDescriptor::createArg(Arg, Mask);
1639
32
1640
32
  ArrayRef<MCPhysReg> ArgVGPRs
1641
32
    = makeArrayRef(AMDGPU::VGPR_32RegClass.begin(), 32);
1642
32
  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgVGPRs);
1643
32
  if (RegIdx == ArgVGPRs.size()) {
1644
4
    // Spill to stack required.
1645
4
    int64_t Offset = CCInfo.AllocateStack(4, 4);
1646
4
1647
4
    return ArgDescriptor::createStack(Offset, Mask);
1648
4
  }
1649
28
1650
28
  unsigned Reg = ArgVGPRs[RegIdx];
1651
28
  Reg = CCInfo.AllocateReg(Reg);
1652
28
  assert(Reg != AMDGPU::NoRegister);
1653
28
1654
28
  MachineFunction &MF = CCInfo.getMachineFunction();
1655
28
  MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
1656
28
  return ArgDescriptor::createRegister(Reg, Mask);
1657
28
}
1658
1659
static ArgDescriptor allocateSGPR32InputImpl(CCState &CCInfo,
1660
                                             const TargetRegisterClass *RC,
1661
121
                                             unsigned NumArgRegs) {
1662
121
  ArrayRef<MCPhysReg> ArgSGPRs = makeArrayRef(RC->begin(), 32);
1663
121
  unsigned RegIdx = CCInfo.getFirstUnallocated(ArgSGPRs);
1664
121
  if (RegIdx == ArgSGPRs.size())
1665
0
    report_fatal_error("ran out of SGPRs for arguments");
1666
121
1667
121
  unsigned Reg = ArgSGPRs[RegIdx];
1668
121
  Reg = CCInfo.AllocateReg(Reg);
1669
121
  assert(Reg != AMDGPU::NoRegister);
1670
121
1671
121
  MachineFunction &MF = CCInfo.getMachineFunction();
1672
121
  MF.addLiveIn(Reg, RC);
1673
121
  return ArgDescriptor::createRegister(Reg);
1674
121
}
1675
1676
62
static ArgDescriptor allocateSGPR32Input(CCState &CCInfo) {
1677
62
  return allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_32RegClass, 32);
1678
62
}
1679
1680
59
static ArgDescriptor allocateSGPR64Input(CCState &CCInfo) {
1681
59
  return allocateSGPR32InputImpl(CCInfo, &AMDGPU::SGPR_64RegClass, 16);
1682
59
}
1683
1684
void SITargetLowering::allocateSpecialInputVGPRs(CCState &CCInfo,
1685
                                                 MachineFunction &MF,
1686
                                                 const SIRegisterInfo &TRI,
1687
2.32k
                                                 SIMachineFunctionInfo &Info) const {
1688
2.32k
  const unsigned Mask = 0x3ff;
1689
2.32k
  ArgDescriptor Arg;
1690
2.32k
1691
2.32k
  if (Info.hasWorkItemIDX()) {
1692
25
    Arg = allocateVGPR32Input(CCInfo, Mask);
1693
25
    Info.setWorkItemIDX(Arg);
1694
25
  }
1695
2.32k
1696
2.32k
  if (Info.hasWorkItemIDY()) {
1697
10
    Arg = allocateVGPR32Input(CCInfo, Mask << 10, Arg);
1698
10
    Info.setWorkItemIDY(Arg);
1699
10
  }
1700
2.32k
1701
2.32k
  if (Info.hasWorkItemIDZ())
1702
8
    Info.setWorkItemIDZ(allocateVGPR32Input(CCInfo, Mask << 20, Arg));
1703
2.32k
}
1704
1705
void SITargetLowering::allocateSpecialInputSGPRs(
1706
  CCState &CCInfo,
1707
  MachineFunction &MF,
1708
  const SIRegisterInfo &TRI,
1709
2.32k
  SIMachineFunctionInfo &Info) const {
1710
2.32k
  auto &ArgInfo = Info.getArgInfo();
1711
2.32k
1712
2.32k
  // TODO: Unify handling with private memory pointers.
1713
2.32k
1714
2.32k
  if (Info.hasDispatchPtr())
1715
10
    ArgInfo.DispatchPtr = allocateSGPR64Input(CCInfo);
1716
2.32k
1717
2.32k
  if (Info.hasQueuePtr())
1718
13
    ArgInfo.QueuePtr = allocateSGPR64Input(CCInfo);
1719
2.32k
1720
2.32k
  if (Info.hasKernargSegmentPtr())
1721
14
    ArgInfo.KernargSegmentPtr = allocateSGPR64Input(CCInfo);
1722
2.32k
1723
2.32k
  if (Info.hasDispatchID())
1724
10
    ArgInfo.DispatchID = allocateSGPR64Input(CCInfo);
1725
2.32k
1726
2.32k
  // flat_scratch_init is not applicable for non-kernel functions.
1727
2.32k
1728
2.32k
  if (Info.hasWorkGroupIDX())
1729
22
    ArgInfo.WorkGroupIDX = allocateSGPR32Input(CCInfo);
1730
2.32k
1731
2.32k
  if (Info.hasWorkGroupIDY())
1732
20
    ArgInfo.WorkGroupIDY = allocateSGPR32Input(CCInfo);
1733
2.32k
1734
2.32k
  if (Info.hasWorkGroupIDZ())
1735
20
    ArgInfo.WorkGroupIDZ = allocateSGPR32Input(CCInfo);
1736
2.32k
1737
2.32k
  if (Info.hasImplicitArgPtr())
1738
12
    ArgInfo.ImplicitArgPtr = allocateSGPR64Input(CCInfo);
1739
2.32k
}
1740
1741
// Allocate special inputs passed in user SGPRs.
1742
void SITargetLowering::allocateHSAUserSGPRs(CCState &CCInfo,
1743
                                            MachineFunction &MF,
1744
                                            const SIRegisterInfo &TRI,
1745
22.9k
                                            SIMachineFunctionInfo &Info) const {
1746
22.9k
  if (Info.hasImplicitBufferPtr()) {
1747
164
    unsigned ImplicitBufferPtrReg = Info.addImplicitBufferPtr(TRI);
1748
164
    MF.addLiveIn(ImplicitBufferPtrReg, &AMDGPU::SGPR_64RegClass);
1749
164
    CCInfo.AllocateReg(ImplicitBufferPtrReg);
1750
164
  }
1751
22.9k
1752
22.9k
  // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
1753
22.9k
  if (Info.hasPrivateSegmentBuffer()) {
1754
4.29k
    unsigned PrivateSegmentBufferReg = Info.addPrivateSegmentBuffer(TRI);
1755
4.29k
    MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SGPR_128RegClass);
1756
4.29k
    CCInfo.AllocateReg(PrivateSegmentBufferReg);
1757
4.29k
  }
1758
22.9k
1759
22.9k
  if (Info.hasDispatchPtr()) {
1760
39
    unsigned DispatchPtrReg = Info.addDispatchPtr(TRI);
1761
39
    MF.addLiveIn(DispatchPtrReg, &AMDGPU::SGPR_64RegClass);
1762
39
    CCInfo.AllocateReg(DispatchPtrReg);
1763
39
  }
1764
22.9k
1765
22.9k
  if (Info.hasQueuePtr()) {
1766
59
    unsigned QueuePtrReg = Info.addQueuePtr(TRI);
1767
59
    MF.addLiveIn(QueuePtrReg, &AMDGPU::SGPR_64RegClass);
1768
59
    CCInfo.AllocateReg(QueuePtrReg);
1769
59
  }
1770
22.9k
1771
22.9k
  if (Info.hasKernargSegmentPtr()) {
1772
18.3k
    MachineRegisterInfo &MRI = MF.getRegInfo();
1773
18.3k
    Register InputPtrReg = Info.addKernargSegmentPtr(TRI);
1774
18.3k
    CCInfo.AllocateReg(InputPtrReg);
1775
18.3k
1776
18.3k
    Register VReg = MF.addLiveIn(InputPtrReg, &AMDGPU::SGPR_64RegClass);
1777
18.3k
    MRI.setType(VReg, LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
1778
18.3k
  }
1779
22.9k
1780
22.9k
  if (Info.hasDispatchID()) {
1781
5
    unsigned DispatchIDReg = Info.addDispatchID(TRI);
1782
5
    MF.addLiveIn(DispatchIDReg, &AMDGPU::SGPR_64RegClass);
1783
5
    CCInfo.AllocateReg(DispatchIDReg);
1784
5
  }
1785
22.9k
1786
22.9k
  if (Info.hasFlatScratchInit()) {
1787
431
    unsigned FlatScratchInitReg = Info.addFlatScratchInit(TRI);
1788
431
    MF.addLiveIn(FlatScratchInitReg, &AMDGPU::SGPR_64RegClass);
1789
431
    CCInfo.AllocateReg(FlatScratchInitReg);
1790
431
  }
1791
22.9k
1792
22.9k
  // TODO: Add GridWorkGroupCount user SGPRs when used. For now with HSA we read
1793
22.9k
  // these from the dispatch pointer.
1794
22.9k
}
1795
1796
// Allocate special input registers that are initialized per-wave.
1797
void SITargetLowering::allocateSystemSGPRs(CCState &CCInfo,
1798
                                           MachineFunction &MF,
1799
                                           SIMachineFunctionInfo &Info,
1800
                                           CallingConv::ID CallConv,
1801
23.3k
                                           bool IsShader) const {
1802
23.3k
  if (Info.hasWorkGroupIDX()) {
1803
20.3k
    unsigned Reg = Info.addWorkGroupIDX();
1804
20.3k
    MF.addLiveIn(Reg, &AMDGPU::SReg_32_XM0RegClass);
1805
20.3k
    CCInfo.AllocateReg(Reg);
1806
20.3k
  }
1807
23.3k
1808
23.3k
  if (Info.hasWorkGroupIDY()) {
1809
30
    unsigned Reg = Info.addWorkGroupIDY();
1810
30
    MF.addLiveIn(Reg, &AMDGPU::SReg_32_XM0RegClass);
1811
30
    CCInfo.AllocateReg(Reg);
1812
30
  }
1813
23.3k
1814
23.3k
  if (Info.hasWorkGroupIDZ()) {
1815
28
    unsigned Reg = Info.addWorkGroupIDZ();
1816
28
    MF.addLiveIn(Reg, &AMDGPU::SReg_32_XM0RegClass);
1817
28
    CCInfo.AllocateReg(Reg);
1818
28
  }
1819
23.3k
1820
23.3k
  if (Info.hasWorkGroupInfo()) {
1821
0
    unsigned Reg = Info.addWorkGroupInfo();
1822
0
    MF.addLiveIn(Reg, &AMDGPU::SReg_32_XM0RegClass);
1823
0
    CCInfo.AllocateReg(Reg);
1824
0
  }
1825
23.3k
1826
23.3k
  if (Info.hasPrivateSegmentWaveByteOffset()) {
1827
23.3k
    // Scratch wave offset passed in system SGPR.
1828
23.3k
    unsigned PrivateSegmentWaveByteOffsetReg;
1829
23.3k
1830
23.3k
    if (IsShader) {
1831
2.98k
      PrivateSegmentWaveByteOffsetReg =
1832
2.98k
        Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
1833
2.98k
1834
2.98k
      // This is true if the scratch wave byte offset doesn't have a fixed
1835
2.98k
      // location.
1836
2.98k
      if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
1837
2.91k
        PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
1838
2.91k
        Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
1839
2.91k
      }
1840
2.98k
    } else
1841
20.3k
      PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
1842
23.3k
1843
23.3k
    MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
1844
23.3k
    CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
1845
23.3k
  }
1846
23.3k
}
1847
1848
static void reservePrivateMemoryRegs(const TargetMachine &TM,
1849
                                     MachineFunction &MF,
1850
                                     const SIRegisterInfo &TRI,
1851
46.4k
                                     SIMachineFunctionInfo &Info) {
1852
46.4k
  // Now that we've figured out where the scratch register inputs are, see if
1853
46.4k
  // should reserve the arguments and use them directly.
1854
46.4k
  MachineFrameInfo &MFI = MF.getFrameInfo();
1855
46.4k
  bool HasStackObjects = MFI.hasStackObjects();
1856
46.4k
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1857
46.4k
1858
46.4k
  // Record that we know we have non-spill stack objects so we don't need to
1859
46.4k
  // check all stack objects later.
1860
46.4k
  if (HasStackObjects)
1861
960
    Info.setHasNonSpillStackObjects(true);
1862
46.4k
1863
46.4k
  // Everything live out of a block is spilled with fast regalloc, so it's
1864
46.4k
  // almost certain that spilling will be required.
1865
46.4k
  if (TM.getOptLevel() == CodeGenOpt::None)
1866
401
    HasStackObjects = true;
1867
46.4k
1868
46.4k
  // For now assume stack access is needed in any callee functions, so we need
1869
46.4k
  // the scratch registers to pass in.
1870
46.4k
  bool RequiresStackAccess = HasStackObjects || 
MFI.hasCalls()45.1k
;
1871
46.4k
1872
46.4k
  if (RequiresStackAccess && 
ST.isAmdHsaOrMesa(MF.getFunction())2.24k
) {
1873
1.05k
    // If we have stack objects, we unquestionably need the private buffer
1874
1.05k
    // resource. For the Code Object V2 ABI, this will be the first 4 user
1875
1.05k
    // SGPR inputs. We can reserve those and use them directly.
1876
1.05k
1877
1.05k
    unsigned PrivateSegmentBufferReg =
1878
1.05k
        Info.getPreloadedReg(AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
1879
1.05k
    Info.setScratchRSrcReg(PrivateSegmentBufferReg);
1880
45.3k
  } else {
1881
45.3k
    unsigned ReservedBufferReg = TRI.reservedPrivateSegmentBufferReg(MF);
1882
45.3k
    // We tentatively reserve the last registers (skipping the last registers
1883
45.3k
    // which may contain VCC, FLAT_SCR, and XNACK). After register allocation,
1884
45.3k
    // we'll replace these with the ones immediately after those which were
1885
45.3k
    // really allocated. In the prologue copies will be inserted from the
1886
45.3k
    // argument to these reserved registers.
1887
45.3k
1888
45.3k
    // Without HSA, relocations are used for the scratch pointer and the
1889
45.3k
    // buffer resource setup is always inserted in the prologue. Scratch wave
1890
45.3k
    // offset is still in an input SGPR.
1891
45.3k
    Info.setScratchRSrcReg(ReservedBufferReg);
1892
45.3k
  }
1893
46.4k
1894
46.4k
  // hasFP should be accurate for kernels even before the frame is finalized.
1895
46.4k
  if (ST.getFrameLowering()->hasFP(MF)) {
1896
1.00k
    MachineRegisterInfo &MRI = MF.getRegInfo();
1897
1.00k
1898
1.00k
    // Try to use s32 as the SP, but move it if it would interfere with input
1899
1.00k
    // arguments. This won't work with calls though.
1900
1.00k
    //
1901
1.00k
    // FIXME: Move SP to avoid any possible inputs, or find a way to spill input
1902
1.00k
    // registers.
1903
1.00k
    if (!MRI.isLiveIn(AMDGPU::SGPR32)) {
1904
1.00k
      Info.setStackPtrOffsetReg(AMDGPU::SGPR32);
1905
1.00k
    } else {
1906
0
      assert(AMDGPU::isShader(MF.getFunction().getCallingConv()));
1907
0
1908
0
      if (MFI.hasCalls())
1909
0
        report_fatal_error("call in graphics shader with too many input SGPRs");
1910
0
1911
0
      for (unsigned Reg : AMDGPU::SGPR_32RegClass) {
1912
0
        if (!MRI.isLiveIn(Reg)) {
1913
0
          Info.setStackPtrOffsetReg(Reg);
1914
0
          break;
1915
0
        }
1916
0
      }
1917
0
1918
0
      if (Info.getStackPtrOffsetReg() == AMDGPU::SP_REG)
1919
0
        report_fatal_error("failed to find register for SP");
1920
1.00k
    }
1921
1.00k
1922
1.00k
    if (MFI.hasCalls()) {
1923
994
      Info.setScratchWaveOffsetReg(AMDGPU::SGPR33);
1924
994
      Info.setFrameOffsetReg(AMDGPU::SGPR33);
1925
994
    } else {
1926
6
      unsigned ReservedOffsetReg =
1927
6
        TRI.reservedPrivateSegmentWaveByteOffsetReg(MF);
1928
6
      Info.setScratchWaveOffsetReg(ReservedOffsetReg);
1929
6
      Info.setFrameOffsetReg(ReservedOffsetReg);
1930
6
    }
1931
45.4k
  } else if (RequiresStackAccess) {
1932
1.24k
    assert(!MFI.hasCalls());
1933
1.24k
    // We know there are accesses and they will be done relative to SP, so just
1934
1.24k
    // pin it to the input.
1935
1.24k
    //
1936
1.24k
    // FIXME: Should not do this if inline asm is reading/writing these
1937
1.24k
    // registers.
1938
1.24k
    unsigned PreloadedSP = Info.getPreloadedReg(
1939
1.24k
        AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET);
1940
1.24k
1941
1.24k
    Info.setStackPtrOffsetReg(PreloadedSP);
1942
1.24k
    Info.setScratchWaveOffsetReg(PreloadedSP);
1943
1.24k
    Info.setFrameOffsetReg(PreloadedSP);
1944
44.2k
  } else {
1945
44.2k
    assert(!MFI.hasCalls());
1946
44.2k
1947
44.2k
    // There may not be stack access at all. There may still be spills, or
1948
44.2k
    // access of a constant pointer (in which cases an extra copy will be
1949
44.2k
    // emitted in the prolog).
1950
44.2k
    unsigned ReservedOffsetReg
1951
44.2k
      = TRI.reservedPrivateSegmentWaveByteOffsetReg(MF);
1952
44.2k
    Info.setStackPtrOffsetReg(ReservedOffsetReg);
1953
44.2k
    Info.setScratchWaveOffsetReg(ReservedOffsetReg);
1954
44.2k
    Info.setFrameOffsetReg(ReservedOffsetReg);
1955
44.2k
  }
1956
46.4k
}
1957
1958
24.9k
bool SITargetLowering::supportSplitCSR(MachineFunction *MF) const {
1959
24.9k
  const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1960
24.9k
  return !Info->isEntryFunction();
1961
24.9k
}
1962
1963
2.23k
void SITargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
1964
2.23k
1965
2.23k
}
1966
1967
void SITargetLowering::insertCopiesSplitCSR(
1968
  MachineBasicBlock *Entry,
1969
2.23k
  const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
1970
2.23k
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
1971
2.23k
1972
2.23k
  const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
1973
2.23k
  if (!IStart)
1974
2.23k
    return;
1975
0
1976
0
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1977
0
  MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
1978
0
  MachineBasicBlock::iterator MBBI = Entry->begin();
1979
0
  for (const MCPhysReg *I = IStart; *I; ++I) {
1980
0
    const TargetRegisterClass *RC = nullptr;
1981
0
    if (AMDGPU::SReg_64RegClass.contains(*I))
1982
0
      RC = &AMDGPU::SGPR_64RegClass;
1983
0
    else if (AMDGPU::SReg_32RegClass.contains(*I))
1984
0
      RC = &AMDGPU::SGPR_32RegClass;
1985
0
    else
1986
0
      llvm_unreachable("Unexpected register class in CSRsViaCopy!");
1987
0
1988
0
    unsigned NewVR = MRI->createVirtualRegister(RC);
1989
0
    // Create copy from CSR to a virtual register.
1990
0
    Entry->addLiveIn(*I);
1991
0
    BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
1992
0
      .addReg(*I);
1993
0
1994
0
    // Insert the copy-back instructions right before the terminator.
1995
0
    for (auto *Exit : Exits)
1996
0
      BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
1997
0
              TII->get(TargetOpcode::COPY), *I)
1998
0
        .addReg(NewVR);
1999
0
  }
2000
0
}
2001
2002
SDValue SITargetLowering::LowerFormalArguments(
2003
    SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
2004
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2005
25.1k
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2006
25.1k
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
2007
25.1k
2008
25.1k
  MachineFunction &MF = DAG.getMachineFunction();
2009
25.1k
  const Function &Fn = MF.getFunction();
2010
25.1k
  FunctionType *FType = MF.getFunction().getFunctionType();
2011
25.1k
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2012
25.1k
2013
25.1k
  if (Subtarget->isAmdHsaOS() && 
AMDGPU::isShader(CallConv)4.40k
) {
2014
3
    DiagnosticInfoUnsupported NoGraphicsHSA(
2015
3
        Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc());
2016
3
    DAG.getContext()->diagnose(NoGraphicsHSA);
2017
3
    return DAG.getEntryNode();
2018
3
  }
2019
25.1k
2020
25.1k
  SmallVector<ISD::InputArg, 16> Splits;
2021
25.1k
  SmallVector<CCValAssign, 16> ArgLocs;
2022
25.1k
  BitVector Skipped(Ins.size());
2023
25.1k
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2024
25.1k
                 *DAG.getContext());
2025
25.1k
2026
25.1k
  bool IsShader = AMDGPU::isShader(CallConv);
2027
25.1k
  bool IsKernel = AMDGPU::isKernel(CallConv);
2028
25.1k
  bool IsEntryFunc = AMDGPU::isEntryFunctionCC(CallConv);
2029
25.1k
2030
25.1k
  if (IsShader) {
2031
2.97k
    processShaderInputArgs(Splits, CallConv, Ins, Skipped, FType, Info);
2032
2.97k
2033
2.97k
    // At least one interpolation mode must be enabled or else the GPU will
2034
2.97k
    // hang.
2035
2.97k
    //
2036
2.97k
    // Check PSInputAddr instead of PSInputEnable. The idea is that if the user
2037
2.97k
    // set PSInputAddr, the user wants to enable some bits after the compilation
2038
2.97k
    // based on run-time states. Since we can't know what the final PSInputEna
2039
2.97k
    // will look like, so we shouldn't do anything here and the user should take
2040
2.97k
    // responsibility for the correct programming.
2041
2.97k
    //
2042
2.97k
    // Otherwise, the following restrictions apply:
2043
2.97k
    // - At least one of PERSP_* (0xF) or LINEAR_* (0x70) must be enabled.
2044
2.97k
    // - If POS_W_FLOAT (11) is enabled, at least one of PERSP_* must be
2045
2.97k
    //   enabled too.
2046
2.97k
    if (CallConv == CallingConv::AMDGPU_PS) {
2047
2.53k
      if ((Info->getPSInputAddr() & 0x7F) == 0 ||
2048
2.53k
           
(2.02k
(Info->getPSInputAddr() & 0xF) == 02.02k
&&
2049
2.02k
            
Info->isPSInputAllocated(11)7
)) {
2050
504
        CCInfo.AllocateReg(AMDGPU::VGPR0);
2051
504
        CCInfo.AllocateReg(AMDGPU::VGPR1);
2052
504
        Info->markPSInputAllocated(0);
2053
504
        Info->markPSInputEnabled(0);
2054
504
      }
2055
2.53k
      if (Subtarget->isAmdPalOS()) {
2056
33
        // For isAmdPalOS, the user does not enable some bits after compilation
2057
33
        // based on run-time states; the register values being generated here are
2058
33
        // the final ones set in hardware. Therefore we need to apply the
2059
33
        // workaround to PSInputAddr and PSInputEnable together.  (The case where
2060
33
        // a bit is set in PSInputAddr but not PSInputEnable is where the
2061
33
        // frontend set up an input arg for a particular interpolation mode, but
2062
33
        // nothing uses that input arg. Really we should have an earlier pass
2063
33
        // that removes such an arg.)
2064
33
        unsigned PsInputBits = Info->getPSInputAddr() & Info->getPSInputEnable();
2065
33
        if ((PsInputBits & 0x7F) == 0 ||
2066
33
            
(27
(PsInputBits & 0xF) == 027
&&
2067
27
             
(PsInputBits >> 11 & 1)0
))
2068
6
          Info->markPSInputEnabled(
2069
6
              countTrailingZeros(Info->getPSInputAddr(), ZB_Undefined));
2070
33
      }
2071
2.53k
    }
2072
2.97k
2073
2.97k
    assert(!Info->hasDispatchPtr() &&
2074
2.97k
           !Info->hasKernargSegmentPtr() && !Info->hasFlatScratchInit() &&
2075
2.97k
           !Info->hasWorkGroupIDX() && !Info->hasWorkGroupIDY() &&
2076
2.97k
           !Info->hasWorkGroupIDZ() && !Info->hasWorkGroupInfo() &&
2077
2.97k
           !Info->hasWorkItemIDX() && !Info->hasWorkItemIDY() &&
2078
2.97k
           !Info->hasWorkItemIDZ());
2079
22.2k
  } else if (IsKernel) {
2080
19.9k
    assert(Info->hasWorkGroupIDX() && Info->hasWorkItemIDX());
2081
19.9k
  } else {
2082
2.25k
    Splits.append(Ins.begin(), Ins.end());
2083
2.25k
  }
2084
25.1k
2085
25.1k
  if (IsEntryFunc) {
2086
22.9k
    allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
2087
22.9k
    allocateHSAUserSGPRs(CCInfo, MF, *TRI, *Info);
2088
22.9k
  }
2089
25.1k
2090
25.1k
  if (IsKernel) {
2091
19.9k
    analyzeFormalArgumentsCompute(CCInfo, Ins);
2092
19.9k
  } else {
2093
5.23k
    CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
2094
5.23k
    CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
2095
5.23k
  }
2096
25.1k
2097
25.1k
  SmallVector<SDValue, 16> Chains;
2098
25.1k
2099
25.1k
  // FIXME: This is the minimum kernel argument alignment. We should improve
2100
25.1k
  // this to the maximum alignment of the arguments.
2101
25.1k
  //
2102
25.1k
  // FIXME: Alignment of explicit arguments totally broken with non-0 explicit
2103
25.1k
  // kern arg offset.
2104
25.1k
  const unsigned KernelArgBaseAlign = 16;
2105
25.1k
2106
108k
   for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; 
++i83.5k
) {
2107
83.5k
    const ISD::InputArg &Arg = Ins[i];
2108
83.5k
    if (Arg.isOrigArg() && 
Skipped[Arg.getOrigArgIndex()]83.5k
) {
2109
2.55k
      InVals.push_back(DAG.getUNDEF(Arg.VT));
2110
2.55k
      continue;
2111
2.55k
    }
2112
80.9k
2113
80.9k
    CCValAssign &VA = ArgLocs[ArgIdx++];
2114
80.9k
    MVT VT = VA.getLocVT();
2115
80.9k
2116
80.9k
    if (IsEntryFunc && 
VA.isMemLoc()73.1k
) {
2117
49.4k
      VT = Ins[i].VT;
2118
49.4k
      EVT MemVT = VA.getLocVT();
2119
49.4k
2120
49.4k
      const uint64_t Offset = VA.getLocMemOffset();
2121
49.4k
      unsigned Align = MinAlign(KernelArgBaseAlign, Offset);
2122
49.4k
2123
49.4k
      SDValue Arg = lowerKernargMemParameter(
2124
49.4k
        DAG, VT, MemVT, DL, Chain, Offset, Align, Ins[i].Flags.isSExt(), &Ins[i]);
2125
49.4k
      Chains.push_back(Arg.getValue(1));
2126
49.4k
2127
49.4k
      auto *ParamTy =
2128
49.4k
        dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
2129
49.4k
      if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
2130
49.4k
          
ParamTy13.5k
&&
(8.23k
ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS8.23k
||
2131
8.23k
                      
ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS7.57k
)) {
2132
675
        // On SI local pointers are just offsets into LDS, so they are always
2133
675
        // less than 16-bits.  On CI and newer they could potentially be
2134
675
        // real pointers, so we can't guarantee their size.
2135
675
        Arg = DAG.getNode(ISD::AssertZext, DL, Arg.getValueType(), Arg,
2136
675
                          DAG.getValueType(MVT::i16));
2137
675
      }
2138
49.4k
2139
49.4k
      InVals.push_back(Arg);
2140
49.4k
      continue;
2141
49.4k
    } else 
if (31.5k
!IsEntryFunc31.5k
&&
VA.isMemLoc()7.87k
) {
2142
433
      SDValue Val = lowerStackParameter(DAG, VA, DL, Chain, Arg);
2143
433
      InVals.push_back(Val);
2144
433
      if (!Arg.Flags.isByVal())
2145
345
        Chains.push_back(Val.getValue(1));
2146
433
      continue;
2147
433
    }
2148
31.1k
2149
31.1k
    assert(VA.isRegLoc() && "Parameter must be in a register!");
2150
31.1k
2151
31.1k
    unsigned Reg = VA.getLocReg();
2152
31.1k
    const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg, VT);
2153
31.1k
    EVT ValVT = VA.getValVT();
2154
31.1k
2155
31.1k
    Reg = MF.addLiveIn(Reg, RC);
2156
31.1k
    SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
2157
31.1k
2158
31.1k
    if (Arg.Flags.isSRet()) {
2159
15
      // The return object should be reasonably addressable.
2160
15
2161
15
      // FIXME: This helps when the return is a real sret. If it is a
2162
15
      // automatically inserted sret (i.e. CanLowerReturn returns false), an
2163
15
      // extra copy is inserted in SelectionDAGBuilder which obscures this.
2164
15
      unsigned NumBits
2165
15
        = 32 - getSubtarget()->getKnownHighZeroBitsForFrameIndex();
2166
15
      Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
2167
15
        DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(), NumBits)));
2168
15
    }
2169
31.1k
2170
31.1k
    // If this is an 8 or 16-bit value, it is really passed promoted
2171
31.1k
    // to 32 bits. Insert an assert[sz]ext to capture this, then
2172
31.1k
    // truncate to the right size.
2173
31.1k
    switch (VA.getLocInfo()) {
2174
31.1k
    case CCValAssign::Full:
2175
31.1k
      break;
2176
31.1k
    case CCValAssign::BCvt:
2177
0
      Val = DAG.getNode(ISD::BITCAST, DL, ValVT, Val);
2178
0
      break;
2179
31.1k
    case CCValAssign::SExt:
2180
8
      Val = DAG.getNode(ISD::AssertSext, DL, VT, Val,
2181
8
                        DAG.getValueType(ValVT));
2182
8
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2183
8
      break;
2184
31.1k
    case CCValAssign::ZExt:
2185
12
      Val = DAG.getNode(ISD::AssertZext, DL, VT, Val,
2186
12
                        DAG.getValueType(ValVT));
2187
12
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2188
12
      break;
2189
31.1k
    case CCValAssign::AExt:
2190
15
      Val = DAG.getNode(ISD::TRUNCATE, DL, ValVT, Val);
2191
15
      break;
2192
31.1k
    default:
2193
0
      llvm_unreachable("Unknown loc info!");
2194
31.1k
    }
2195
31.1k
2196
31.1k
    InVals.push_back(Val);
2197
31.1k
  }
2198
25.1k
2199
25.1k
  if (!IsEntryFunc) {
2200
2.25k
    // Special inputs come after user arguments.
2201
2.25k
    allocateSpecialInputVGPRs(CCInfo, MF, *TRI, *Info);
2202
2.25k
  }
2203
25.1k
2204
25.1k
  // Start adding system SGPRs.
2205
25.1k
  if (IsEntryFunc) {
2206
22.9k
    allocateSystemSGPRs(CCInfo, MF, *Info, CallConv, IsShader);
2207
22.9k
  } else {
2208
2.25k
    CCInfo.AllocateReg(Info->getScratchRSrcReg());
2209
2.25k
    CCInfo.AllocateReg(Info->getScratchWaveOffsetReg());
2210
2.25k
    CCInfo.AllocateReg(Info->getFrameOffsetReg());
2211
2.25k
    allocateSpecialInputSGPRs(CCInfo, MF, *TRI, *Info);
2212
2.25k
  }
2213
25.1k
2214
25.1k
  auto &ArgUsageInfo =
2215
25.1k
    DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>();
2216
25.1k
  ArgUsageInfo.setFuncArgInfo(Fn, Info->getArgInfo());
2217
25.1k
2218
25.1k
  unsigned StackArgSize = CCInfo.getNextStackOffset();
2219
25.1k
  Info->setBytesInStackArgArea(StackArgSize);
2220
25.1k
2221
25.1k
  return Chains.empty() ? 
Chain6.82k
:
2222
25.1k
    
DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains)18.3k
;
2223
25.1k
}
2224
2225
// TODO: If return values can't fit in registers, we should return as many as
2226
// possible in registers before passing on stack.
2227
bool SITargetLowering::CanLowerReturn(
2228
  CallingConv::ID CallConv,
2229
  MachineFunction &MF, bool IsVarArg,
2230
  const SmallVectorImpl<ISD::OutputArg> &Outs,
2231
25.8k
  LLVMContext &Context) const {
2232
25.8k
  // Replacing returns with sret/stack usage doesn't make sense for shaders.
2233
25.8k
  // FIXME: Also sort of a workaround for custom vector splitting in LowerReturn
2234
25.8k
  // for shaders. Vector types should be explicitly handled by CC.
2235
25.8k
  if (AMDGPU::isEntryFunctionCC(CallConv))
2236
22.9k
    return true;
2237
2.95k
2238
2.95k
  SmallVector<CCValAssign, 16> RVLocs;
2239
2.95k
  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
2240
2.95k
  return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, IsVarArg));
2241
2.95k
}
2242
2243
SDValue
2244
SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2245
                              bool isVarArg,
2246
                              const SmallVectorImpl<ISD::OutputArg> &Outs,
2247
                              const SmallVectorImpl<SDValue> &OutVals,
2248
25.1k
                              const SDLoc &DL, SelectionDAG &DAG) const {
2249
25.1k
  MachineFunction &MF = DAG.getMachineFunction();
2250
25.1k
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2251
25.1k
2252
25.1k
  if (AMDGPU::isKernel(CallConv)) {
2253
19.9k
    return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs,
2254
19.9k
                                             OutVals, DL, DAG);
2255
19.9k
  }
2256
5.17k
2257
5.17k
  bool IsShader = AMDGPU::isShader(CallConv);
2258
5.17k
2259
5.17k
  Info->setIfReturnsVoid(Outs.empty());
2260
5.17k
  bool IsWaveEnd = Info->returnsVoid() && 
IsShader2.04k
;
2261
5.17k
2262
5.17k
  // CCValAssign - represent the assignment of the return value to a location.
2263
5.17k
  SmallVector<CCValAssign, 48> RVLocs;
2264
5.17k
  SmallVector<ISD::OutputArg, 48> Splits;
2265
5.17k
2266
5.17k
  // CCState - Info about the registers and stack slots.
2267
5.17k
  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2268
5.17k
                 *DAG.getContext());
2269
5.17k
2270
5.17k
  // Analyze outgoing return values.
2271
5.17k
  CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
2272
5.17k
2273
5.17k
  SDValue Flag;
2274
5.17k
  SmallVector<SDValue, 48> RetOps;
2275
5.17k
  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2276
5.17k
2277
5.17k
  // Add return address for callable functions.
2278
5.17k
  if (!Info->isEntryFunction()) {
2279
2.20k
    const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
2280
2.20k
    SDValue ReturnAddrReg = CreateLiveInRegister(
2281
2.20k
      DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64);
2282
2.20k
2283
2.20k
    SDValue ReturnAddrVirtualReg = DAG.getRegister(
2284
2.20k
        MF.getRegInfo().createVirtualRegister(&AMDGPU::CCR_SGPR_64RegClass),
2285
2.20k
        MVT::i64);
2286
2.20k
    Chain =
2287
2.20k
        DAG.getCopyToReg(Chain, DL, ReturnAddrVirtualReg, ReturnAddrReg, Flag);
2288
2.20k
    Flag = Chain.getValue(1);
2289
2.20k
    RetOps.push_back(ReturnAddrVirtualReg);
2290
2.20k
  }
2291
5.17k
2292
5.17k
  // Copy the result values into the output registers.
2293
13.4k
  for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
2294
8.23k
       ++I, ++RealRVLocIdx) {
2295
8.23k
    CCValAssign &VA = RVLocs[I];
2296
8.23k
    assert(VA.isRegLoc() && "Can only return in registers!");
2297
8.23k
    // TODO: Partially return in registers if return values don't fit.
2298
8.23k
    SDValue Arg = OutVals[RealRVLocIdx];
2299
8.23k
2300
8.23k
    // Copied from other backends.
2301
8.23k
    switch (VA.getLocInfo()) {
2302
8.23k
    case CCValAssign::Full:
2303
8.22k
      break;
2304
8.23k
    case CCValAssign::BCvt:
2305
0
      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
2306
0
      break;
2307
8.23k
    case CCValAssign::SExt:
2308
0
      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
2309
0
      break;
2310
8.23k
    case CCValAssign::ZExt:
2311
0
      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
2312
0
      break;
2313
8.23k
    case CCValAssign::AExt:
2314
6
      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
2315
6
      break;
2316
8.23k
    default:
2317
0
      llvm_unreachable("Unknown loc info!");
2318
8.23k
    }
2319
8.23k
2320
8.23k
    Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Flag);
2321
8.23k
    Flag = Chain.getValue(1);
2322
8.23k
    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2323
8.23k
  }
2324
5.17k
2325
5.17k
  // FIXME: Does sret work properly?
2326
5.17k
  if (!Info->isEntryFunction()) {
2327
2.20k
    const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
2328
2.20k
    const MCPhysReg *I =
2329
2.20k
      TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
2330
2.20k
    if (I) {
2331
0
      for (; *I; ++I) {
2332
0
        if (AMDGPU::SReg_64RegClass.contains(*I))
2333
0
          RetOps.push_back(DAG.getRegister(*I, MVT::i64));
2334
0
        else if (AMDGPU::SReg_32RegClass.contains(*I))
2335
0
          RetOps.push_back(DAG.getRegister(*I, MVT::i32));
2336
0
        else
2337
0
          llvm_unreachable("Unexpected register class in CSRsViaCopy!");
2338
0
      }
2339
0
    }
2340
2.20k
  }
2341
5.17k
2342
5.17k
  // Update chain and glue.
2343
5.17k
  RetOps[0] = Chain;
2344
5.17k
  if (Flag.getNode())
2345
4.22k
    RetOps.push_back(Flag);
2346
5.17k
2347
5.17k
  unsigned Opc = AMDGPUISD::ENDPGM;
2348
5.17k
  if (!IsWaveEnd)
2349
4.22k
    Opc = IsShader ? 
AMDGPUISD::RETURN_TO_EPILOG2.02k
:
AMDGPUISD::RET_FLAG2.20k
;
2350
5.17k
  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
2351
5.17k
}
2352
2353
SDValue SITargetLowering::LowerCallResult(
2354
    SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
2355
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2356
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool IsThisReturn,
2357
646
    SDValue ThisVal) const {
2358
646
  CCAssignFn *RetCC = CCAssignFnForReturn(CallConv, IsVarArg);
2359
646
2360
646
  // Assign locations to each value returned by this call.
2361
646
  SmallVector<CCValAssign, 16> RVLocs;
2362
646
  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
2363
646
                 *DAG.getContext());
2364
646
  CCInfo.AnalyzeCallResult(Ins, RetCC);
2365
646
2366
646
  // Copy all of the result registers out of their specified physreg.
2367
1.07k
  for (unsigned i = 0; i != RVLocs.size(); 
++i426
) {
2368
426
    CCValAssign VA = RVLocs[i];
2369
426
    SDValue Val;
2370
426
2371
426
    if (VA.isRegLoc()) {
2372
426
      Val = DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
2373
426
      Chain = Val.getValue(1);
2374
426
      InFlag = Val.getValue(2);
2375
426
    } else 
if (0
VA.isMemLoc()0
) {
2376
0
      report_fatal_error("TODO: return values in memory");
2377
0
    } else
2378
0
      llvm_unreachable("unknown argument location type");
2379
426
2380
426
    switch (VA.getLocInfo()) {
2381
426
    case CCValAssign::Full:
2382
409
      break;
2383
426
    case CCValAssign::BCvt:
2384
0
      Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
2385
0
      break;
2386
426
    case CCValAssign::ZExt:
2387
7
      Val = DAG.getNode(ISD::AssertZext, DL, VA.getLocVT(), Val,
2388
7
                        DAG.getValueType(VA.getValVT()));
2389
7
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2390
7
      break;
2391
426
    case CCValAssign::SExt:
2392
7
      Val = DAG.getNode(ISD::AssertSext, DL, VA.getLocVT(), Val,
2393
7
                        DAG.getValueType(VA.getValVT()));
2394
7
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2395
7
      break;
2396
426
    case CCValAssign::AExt:
2397
3
      Val = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Val);
2398
3
      break;
2399
426
    default:
2400
0
      llvm_unreachable("Unknown loc info!");
2401
426
    }
2402
426
2403
426
    InVals.push_back(Val);
2404
426
  }
2405
646
2406
646
  return Chain;
2407
646
}
2408
2409
// Add code to pass special inputs required depending on used features separate
2410
// from the explicit user arguments present in the IR.
2411
void SITargetLowering::passSpecialInputs(
2412
    CallLoweringInfo &CLI,
2413
    CCState &CCInfo,
2414
    const SIMachineFunctionInfo &Info,
2415
    SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass,
2416
    SmallVectorImpl<SDValue> &MemOpChains,
2417
694
    SDValue Chain) const {
2418
694
  // If we don't have a call site, this was a call inserted by
2419
694
  // legalization. These can never use special inputs.
2420
694
  if (!CLI.CS)
2421
0
    return;
2422
694
2423
694
  const Function *CalleeFunc = CLI.CS.getCalledFunction();
2424
694
  assert(CalleeFunc);
2425
694
2426
694
  SelectionDAG &DAG = CLI.DAG;
2427
694
  const SDLoc &DL = CLI.DL;
2428
694
2429
694
  const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
2430
694
2431
694
  auto &ArgUsageInfo =
2432
694
    DAG.getPass()->getAnalysis<AMDGPUArgumentUsageInfo>();
2433
694
  const AMDGPUFunctionArgInfo &CalleeArgInfo
2434
694
    = ArgUsageInfo.lookupFuncArgInfo(*CalleeFunc);
2435
694
2436
694
  const AMDGPUFunctionArgInfo &CallerArgInfo = Info.getArgInfo();
2437
694
2438
694
  // TODO: Unify with private memory register handling. This is complicated by
2439
694
  // the fact that at least in kernels, the input argument is not necessarily
2440
694
  // in the same location as the input.
2441
694
  AMDGPUFunctionArgInfo::PreloadedValue InputRegs[] = {
2442
694
    AMDGPUFunctionArgInfo::DISPATCH_PTR,
2443
694
    AMDGPUFunctionArgInfo::QUEUE_PTR,
2444
694
    AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR,
2445
694
    AMDGPUFunctionArgInfo::DISPATCH_ID,
2446
694
    AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
2447
694
    AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
2448
694
    AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
2449
694
    AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR
2450
694
  };
2451
694
2452
5.55k
  for (auto InputID : InputRegs) {
2453
5.55k
    const ArgDescriptor *OutgoingArg;
2454
5.55k
    const TargetRegisterClass *ArgRC;
2455
5.55k
2456
5.55k
    std::tie(OutgoingArg, ArgRC) = CalleeArgInfo.getPreloadedValue(InputID);
2457
5.55k
    if (!OutgoingArg)
2458
5.44k
      continue;
2459
103
2460
103
    const ArgDescriptor *IncomingArg;
2461
103
    const TargetRegisterClass *IncomingArgRC;
2462
103
    std::tie(IncomingArg, IncomingArgRC)
2463
103
      = CallerArgInfo.getPreloadedValue(InputID);
2464
103
    assert(IncomingArgRC == ArgRC);
2465
103
2466
103
    // All special arguments are ints for now.
2467
103
    EVT ArgVT = TRI->getSpillSize(*ArgRC) == 8 ? 
MVT::i6442
:
MVT::i3261
;
2468
103
    SDValue InputReg;
2469
103
2470
103
    if (IncomingArg) {
2471
93
      InputReg = loadInputValue(DAG, ArgRC, ArgVT, DL, *IncomingArg);
2472
93
    } else {
2473
10
      // The implicit arg ptr is special because it doesn't have a corresponding
2474
10
      // input for kernels, and is computed from the kernarg segment pointer.
2475
10
      assert(InputID == AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
2476
10
      InputReg = getImplicitArgPtr(DAG, DL);
2477
10
    }
2478
103
2479
103
    if (OutgoingArg->isRegister()) {
2480
103
      RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
2481
103
    } else {
2482
0
      unsigned SpecialArgOffset = CCInfo.AllocateStack(ArgVT.getStoreSize(), 4);
2483
0
      SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2484
0
                                              SpecialArgOffset);
2485
0
      MemOpChains.push_back(ArgStore);
2486
0
    }
2487
103
  }
2488
694
2489
694
  // Pack workitem IDs into a single register or pass it as is if already
2490
694
  // packed.
2491
694
  const ArgDescriptor *OutgoingArg;
2492
694
  const TargetRegisterClass *ArgRC;
2493
694
2494
694
  std::tie(OutgoingArg, ArgRC) =
2495
694
    CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X);
2496
694
  if (!OutgoingArg)
2497
679
    std::tie(OutgoingArg, ArgRC) =
2498
679
      CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
2499
694
  if (!OutgoingArg)
2500
675
    std::tie(OutgoingArg, ArgRC) =
2501
675
      CalleeArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
2502
694
  if (!OutgoingArg)
2503
672
    return;
2504
22
2505
22
  const ArgDescriptor *IncomingArgX
2506
22
    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_X).first;
2507
22
  const ArgDescriptor *IncomingArgY
2508
22
    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Y).first;
2509
22
  const ArgDescriptor *IncomingArgZ
2510
22
    = CallerArgInfo.getPreloadedValue(AMDGPUFunctionArgInfo::WORKITEM_ID_Z).first;
2511
22
2512
22
  SDValue InputReg;
2513
22
  SDLoc SL;
2514
22
2515
22
  // If incoming ids are not packed we need to pack them.
2516
22
  if (IncomingArgX && 
!IncomingArgX->isMasked()20
&&
CalleeArgInfo.WorkItemIDX16
)
2517
11
    InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgX);
2518
22
2519
22
  if (IncomingArgY && 
!IncomingArgY->isMasked()11
&&
CalleeArgInfo.WorkItemIDY10
) {
2520
7
    SDValue Y = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgY);
2521
7
    Y = DAG.getNode(ISD::SHL, SL, MVT::i32, Y,
2522
7
                    DAG.getShiftAmountConstant(10, MVT::i32, SL));
2523
7
    InputReg = InputReg.getNode() ?
2524
4
                 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Y) : 
Y3
;
2525
7
  }
2526
22
2527
22
  if (IncomingArgZ && 
!IncomingArgZ->isMasked()8
&&
CalleeArgInfo.WorkItemIDZ7
) {
2528
7
    SDValue Z = loadInputValue(DAG, ArgRC, MVT::i32, DL, *IncomingArgZ);
2529
7
    Z = DAG.getNode(ISD::SHL, SL, MVT::i32, Z,
2530
7
                    DAG.getShiftAmountConstant(20, MVT::i32, SL));
2531
7
    InputReg = InputReg.getNode() ?
2532
5
                 DAG.getNode(ISD::OR, SL, MVT::i32, InputReg, Z) : 
Z2
;
2533
7
  }
2534
22
2535
22
  if (!InputReg.getNode()) {
2536
6
    // Workitem ids are already packed, any of present incoming arguments
2537
6
    // will carry all required fields.
2538
6
    ArgDescriptor IncomingArg = ArgDescriptor::createArg(
2539
6
      IncomingArgX ? 
*IncomingArgX4
:
2540
6
      
IncomingArgY 2
?
*IncomingArgY1
:
2541
2
                     
*IncomingArgZ1
, ~0u);
2542
6
    InputReg = loadInputValue(DAG, ArgRC, MVT::i32, DL, IncomingArg);
2543
6
  }
2544
22
2545
22
  if (OutgoingArg->isRegister()) {
2546
16
    RegsToPass.emplace_back(OutgoingArg->getRegister(), InputReg);
2547
16
  } else {
2548
6
    unsigned SpecialArgOffset = CCInfo.AllocateStack(4, 4);
2549
6
    SDValue ArgStore = storeStackInputValue(DAG, DL, Chain, InputReg,
2550
6
                                            SpecialArgOffset);
2551
6
    MemOpChains.push_back(ArgStore);
2552
6
  }
2553
22
}
2554
2555
39
static bool canGuaranteeTCO(CallingConv::ID CC) {
2556
39
  return CC == CallingConv::Fast;
2557
39
}
2558
2559
/// Return true if we might ever do TCO for calls with this calling convention.
2560
62
static bool mayTailCallThisCC(CallingConv::ID CC) {
2561
62
  switch (CC) {
2562
62
  case CallingConv::C:
2563
23
    return true;
2564
62
  default:
2565
39
    return canGuaranteeTCO(CC);
2566
62
  }
2567
62
}
2568
2569
bool SITargetLowering::isEligibleForTailCallOptimization(
2570
    SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg,
2571
    const SmallVectorImpl<ISD::OutputArg> &Outs,
2572
    const SmallVectorImpl<SDValue> &OutVals,
2573
62
    const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
2574
62
  if (!mayTailCallThisCC(CalleeCC))
2575
0
    return false;
2576
62
2577
62
  MachineFunction &MF = DAG.getMachineFunction();
2578
62
  const Function &CallerF = MF.getFunction();
2579
62
  CallingConv::ID CallerCC = CallerF.getCallingConv();
2580
62
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
2581
62
  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2582
62
2583
62
  // Kernels aren't callable, and don't have a live in return address so it
2584
62
  // doesn't make sense to do a tail call with entry functions.
2585
62
  if (!CallerPreserved)
2586
8
    return false;
2587
54
2588
54
  bool CCMatch = CallerCC == CalleeCC;
2589
54
2590
54
  if (DAG.getTarget().Options.GuaranteedTailCallOpt) {
2591
0
    if (canGuaranteeTCO(CalleeCC) && CCMatch)
2592
0
      return true;
2593
0
    return false;
2594
0
  }
2595
54
2596
54
  // TODO: Can we handle var args?
2597
54
  if (IsVarArg)
2598
0
    return false;
2599
54
2600
122
  
for (const Argument &Arg : CallerF.args())54
{
2601
122
    if (Arg.hasByValAttr())
2602
3
      return false;
2603
122
  }
2604
54
2605
54
  LLVMContext &Ctx = *DAG.getContext();
2606
51
2607
51
  // Check that the call results are passed in the same way.
2608
51
  if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, Ctx, Ins,
2609
51
                                  CCAssignFnForCall(CalleeCC, IsVarArg),
2610
51
                                  CCAssignFnForCall(CallerCC, IsVarArg)))
2611
0
    return false;
2612
51
2613
51
  // The callee has to preserve all registers the caller needs to preserve.
2614
51
  if (!CCMatch) {
2615
0
    const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2616
0
    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
2617
0
      return false;
2618
51
  }
2619
51
2620
51
  // Nothing more to check if the callee is taking no arguments.
2621
51
  if (Outs.empty())
2622
8
    return true;
2623
43
2624
43
  SmallVector<CCValAssign, 16> ArgLocs;
2625
43
  CCState CCInfo(CalleeCC, IsVarArg, MF, ArgLocs, Ctx);
2626
43
2627
43
  CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, IsVarArg));
2628
43
2629
43
  const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
2630
43
  // If the stack arguments for this call do not fit into our own save area then
2631
43
  // the call cannot be made tail.
2632
43
  // TODO: Is this really necessary?
2633
43
  if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
2634
3
    return false;
2635
40
2636
40
  const MachineRegisterInfo &MRI = MF.getRegInfo();
2637
40
  return parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals);
2638
40
}
2639
2640
50
bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
2641
50
  if (!CI->isTailCall())
2642
46
    return false;
2643
4
2644
4
  const Function *ParentFn = CI->getParent()->getParent();
2645
4
  if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv()))
2646
3
    return false;
2647
1
2648
1
  auto Attr = ParentFn->getFnAttribute("disable-tail-calls");
2649
1
  return (Attr.getValueAsString() != "true");
2650
1
}
2651
2652
// The wave scratch offset register is used as the global base pointer.
2653
SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
2654
701
                                    SmallVectorImpl<SDValue> &InVals) const {
2655
701
  SelectionDAG &DAG = CLI.DAG;
2656
701
  const SDLoc &DL = CLI.DL;
2657
701
  SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
2658
701
  SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
2659
701
  SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
2660
701
  SDValue Chain = CLI.Chain;
2661
701
  SDValue Callee = CLI.Callee;
2662
701
  bool &IsTailCall = CLI.IsTailCall;
2663
701
  CallingConv::ID CallConv = CLI.CallConv;
2664
701
  bool IsVarArg = CLI.IsVarArg;
2665
701
  bool IsSibCall = false;
2666
701
  bool IsThisReturn = false;
2667
701
  MachineFunction &MF = DAG.getMachineFunction();
2668
701
2669
701
  if (IsVarArg) {
2670
2
    return lowerUnhandledCall(CLI, InVals,
2671
2
                              "unsupported call to variadic function ");
2672
2
  }
2673
699
2674
699
  if (!CLI.CS.getInstruction())
2675
1
    report_fatal_error("unsupported libcall legalization");
2676
698
2677
698
  if (!CLI.CS.getCalledFunction()) {
2678
1
    return lowerUnhandledCall(CLI, InVals,
2679
1
                              "unsupported indirect call to function ");
2680
1
  }
2681
697
2682
697
  if (IsTailCall && 
MF.getTarget().Options.GuaranteedTailCallOpt63
) {
2683
1
    return lowerUnhandledCall(CLI, InVals,
2684
1
                              "unsupported required tail call to function ");
2685
1
  }
2686
696
2687
696
  if (AMDGPU::isShader(MF.getFunction().getCallingConv())) {
2688
1
    // Note the issue is with the CC of the calling function, not of the call
2689
1
    // itself.
2690
1
    return lowerUnhandledCall(CLI, InVals,
2691
1
                          "unsupported call from graphics shader of function ");
2692
1
  }
2693
695
2694
695
  if (IsTailCall) {
2695
62
    IsTailCall = isEligibleForTailCallOptimization(
2696
62
      Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
2697
62
    if (!IsTailCall && 
CLI.CS14
&&
CLI.CS.isMustTailCall()14
) {
2698
0
      report_fatal_error("failed to perform tail call elimination on a call "
2699
0
                         "site marked musttail");
2700
0
    }
2701
62
2702
62
    bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
2703
62
2704
62
    // A sibling call is one where we're under the usual C ABI and not planning
2705
62
    // to change that but can still do a tail call:
2706
62
    if (!TailCallOpt && IsTailCall)
2707
48
      IsSibCall = true;
2708
62
2709
62
    if (IsTailCall)
2710
48
      ++NumTailCalls;
2711
62
  }
2712
695
2713
695
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
2714
695
2715
695
  // Analyze operands of the call, assigning locations to each operand.
2716
695
  SmallVector<CCValAssign, 16> ArgLocs;
2717
695
  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2718
695
  CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, IsVarArg);
2719
695
2720
695
  CCInfo.AnalyzeCallOperands(Outs, AssignFn);
2721
695
2722
695
  // Get a count of how many bytes are to be pushed on the stack.
2723
695
  unsigned NumBytes = CCInfo.getNextStackOffset();
2724
695
2725
695
  if (IsSibCall) {
2726
48
    // Since we're not changing the ABI to make this a tail call, the memory
2727
48
    // operands are already available in the caller's incoming argument space.
2728
48
    NumBytes = 0;
2729
48
  }
2730
695
2731
695
  // FPDiff is the byte offset of the call's argument area from the callee's.
2732
695
  // Stores to callee stack arguments will be placed in FixedStackSlots offset
2733
695
  // by this amount for a tail call. In a sibling call it must be 0 because the
2734
695
  // caller will deallocate the entire stack and the callee still expects its
2735
695
  // arguments to begin at SP+0. Completely unused for non-tail calls.
2736
695
  int32_t FPDiff = 0;
2737
695
  MachineFrameInfo &MFI = MF.getFrameInfo();
2738
695
  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2739
695
2740
695
  // Adjust the stack pointer for the new arguments...
2741
695
  // These operations are automatically eliminated by the prolog/epilog pass
2742
695
  if (!IsSibCall) {
2743
646
    Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
2744
646
2745
646
    SmallVector<SDValue, 4> CopyFromChains;
2746
646
2747
646
    // In the HSA case, this should be an identity copy.
2748
646
    SDValue ScratchRSrcReg
2749
646
      = DAG.getCopyFromReg(Chain, DL, Info->getScratchRSrcReg(), MVT::v4i32);
2750
646
    RegsToPass.emplace_back(AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3, ScratchRSrcReg);
2751
646
    CopyFromChains.push_back(ScratchRSrcReg.getValue(1));
2752
646
    Chain = DAG.getTokenFactor(DL, CopyFromChains);
2753
646
  }
2754
695
2755
695
  SmallVector<SDValue, 8> MemOpChains;
2756
695
  MVT PtrVT = MVT::i32;
2757
695
2758
695
  // Walk the register/memloc assignments, inserting copies/loads.
2759
3.50k
  for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e;
2760
2.81k
       ++i, ++realArgIdx) {
2761
2.81k
    CCValAssign &VA = ArgLocs[i];
2762
2.81k
    SDValue Arg = OutVals[realArgIdx];
2763
2.81k
2764
2.81k
    // Promote the value if needed.
2765
2.81k
    switch (VA.getLocInfo()) {
2766
2.81k
    case CCValAssign::Full:
2767
2.78k
      break;
2768
2.81k
    case CCValAssign::BCvt:
2769
0
      Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
2770
0
      break;
2771
2.81k
    case CCValAssign::ZExt:
2772
10
      Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
2773
10
      break;
2774
2.81k
    case CCValAssign::SExt:
2775
10
      Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
2776
10
      break;
2777
2.81k
    case CCValAssign::AExt:
2778
4
      Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
2779
4
      break;
2780
2.81k
    case CCValAssign::FPExt:
2781
0
      Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
2782
0
      break;
2783
2.81k
    default:
2784
0
      llvm_unreachable("Unknown loc info!");
2785
2.81k
    }
2786
2.81k
2787
2.81k
    if (VA.isRegLoc()) {
2788
2.62k
      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
2789
2.62k
    } else {
2790
186
      assert(VA.isMemLoc());
2791
186
2792
186
      SDValue DstAddr;
2793
186
      MachinePointerInfo DstInfo;
2794
186
2795
186
      unsigned LocMemOffset = VA.getLocMemOffset();
2796
186
      int32_t Offset = LocMemOffset;
2797
186
2798
186
      SDValue PtrOff = DAG.getConstant(Offset, DL, PtrVT);
2799
186
      unsigned Align = 0;
2800
186
2801
186
      if (IsTailCall) {
2802
39
        ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2803
39
        unsigned OpSize = Flags.isByVal() ?
2804
32
          
Flags.getByValSize()7
: VA.getValVT().getStoreSize();
2805
39
2806
39
        // FIXME: We can have better than the minimum byval required alignment.
2807
39
        Align = Flags.isByVal() ? 
Flags.getByValAlign()7
:
2808
39
          
MinAlign(Subtarget->getStackAlignment(), Offset)32
;
2809
39
2810
39
        Offset = Offset + FPDiff;
2811
39
        int FI = MFI.CreateFixedObject(OpSize, Offset, true);
2812
39
2813
39
        DstAddr = DAG.getFrameIndex(FI, PtrVT);
2814
39
        DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
2815
39
2816
39
        // Make sure any stack arguments overlapping with where we're storing
2817
39
        // are loaded before this eventual operation. Otherwise they'll be
2818
39
        // clobbered.
2819
39
2820
39
        // FIXME: Why is this really necessary? This seems to just result in a
2821
39
        // lot of code to copy the stack and write them back to the same
2822
39
        // locations, which are supposed to be immutable?
2823
39
        Chain = addTokenForArgument(Chain, DAG, MFI, FI);
2824
147
      } else {
2825
147
        DstAddr = PtrOff;
2826
147
        DstInfo = MachinePointerInfo::getStack(MF, LocMemOffset);
2827
147
        Align = MinAlign(Subtarget->getStackAlignment(), LocMemOffset);
2828
147
      }
2829
186
2830
186
      if (Outs[i].Flags.isByVal()) {
2831
40
        SDValue SizeNode =
2832
40
            DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i32);
2833
40
        SDValue Cpy = DAG.getMemcpy(
2834
40
            Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(),
2835
40
            /*isVol = */ false, /*AlwaysInline = */ true,
2836
40
            /*isTailCall = */ false, DstInfo,
2837
40
            MachinePointerInfo(UndefValue::get(Type::getInt8PtrTy(
2838
40
                *DAG.getContext(), AMDGPUAS::PRIVATE_ADDRESS))));
2839
40
2840
40
        MemOpChains.push_back(Cpy);
2841
146
      } else {
2842
146
        SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo, Align);
2843
146
        MemOpChains.push_back(Store);
2844
146
      }
2845
186
    }
2846
2.81k
  }
2847
695
2848
695
  // Copy special input registers after user input arguments.
2849
695
  passSpecialInputs(CLI, CCInfo, *Info, RegsToPass, MemOpChains, Chain);
2850
695
2851
695
  if (!MemOpChains.empty())
2852
77
    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2853
695
2854
695
  // Build a sequence of copy-to-reg nodes chained together with token chain
2855
695
  // and flag operands which copy the outgoing args into the appropriate regs.
2856
695
  SDValue InFlag;
2857
3.39k
  for (auto &RegToPass : RegsToPass) {
2858
3.39k
    Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
2859
3.39k
                             RegToPass.second, InFlag);
2860
3.39k
    InFlag = Chain.getValue(1);
2861
3.39k
  }
2862
695
2863
695
2864
695
  SDValue PhysReturnAddrReg;
2865
695
  if (IsTailCall) {
2866
48
    // Since the return is being combined with the call, we need to pass on the
2867
48
    // return address.
2868
48
2869
48
    const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
2870
48
    SDValue ReturnAddrReg = CreateLiveInRegister(
2871
48
      DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64);
2872
48
2873
48
    PhysReturnAddrReg = DAG.getRegister(TRI->getReturnAddressReg(MF),
2874
48
                                        MVT::i64);
2875
48
    Chain = DAG.getCopyToReg(Chain, DL, PhysReturnAddrReg, ReturnAddrReg, InFlag);
2876
48
    InFlag = Chain.getValue(1);
2877
48
  }
2878
695
2879
695
  // We don't usually want to end the call-sequence here because we would tidy
2880
695
  // the frame up *after* the call, however in the ABI-changing tail-call case
2881
695
  // we've carefully laid out the parameters so that when sp is reset they'll be
2882
695
  // in the correct location.
2883
695
  if (IsTailCall && 
!IsSibCall48
) {
2884
0
    Chain = DAG.getCALLSEQ_END(Chain,
2885
0
                               DAG.getTargetConstant(NumBytes, DL, MVT::i32),
2886
0
                               DAG.getTargetConstant(0, DL, MVT::i32),
2887
0
                               InFlag, DL);
2888
0
    InFlag = Chain.getValue(1);
2889
0
  }
2890
695
2891
695
  std::vector<SDValue> Ops;
2892
695
  Ops.push_back(Chain);
2893
695
  Ops.push_back(Callee);
2894
695
  // Add a redundant copy of the callee global which will not be legalized, as
2895
695
  // we need direct access to the callee later.
2896
695
  GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Callee);
2897
695
  const GlobalValue *GV = GSD->getGlobal();
2898
695
  Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
2899
695
2900
695
  if (IsTailCall) {
2901
48
    // Each tail call may have to adjust the stack by a different amount, so
2902
48
    // this information must travel along with the operation for eventual
2903
48
    // consumption by emitEpilogue.
2904
48
    Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
2905
48
2906
48
    Ops.push_back(PhysReturnAddrReg);
2907
48
  }
2908
695
2909
695
  // Add argument registers to the end of the list so that they are known live
2910
695
  // into the call.
2911
3.39k
  for (auto &RegToPass : RegsToPass) {
2912
3.39k
    Ops.push_back(DAG.getRegister(RegToPass.first,
2913
3.39k
                                  RegToPass.second.getValueType()));
2914
3.39k
  }
2915
695
2916
695
  // Add a register mask operand representing the call-preserved registers.
2917
695
2918
695
  auto *TRI = static_cast<const SIRegisterInfo*>(Subtarget->getRegisterInfo());
2919
695
  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2920
695
  assert(Mask && "Missing call preserved mask for calling convention");
2921
695
  Ops.push_back(DAG.getRegisterMask(Mask));
2922
695
2923
695
  if (InFlag.getNode())
2924
694
    Ops.push_back(InFlag);
2925
695
2926
695
  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2927
695
2928
695
  // If we're doing a tall call, use a TC_RETURN here rather than an
2929
695
  // actual call instruction.
2930
695
  if (IsTailCall) {
2931
48
    MFI.setHasTailCall();
2932
48
    return DAG.getNode(AMDGPUISD::TC_RETURN, DL, NodeTys, Ops);
2933
48
  }
2934
647
2935
647
  // Returns a chain and a flag for retval copy to use.
2936
647
  SDValue Call = DAG.getNode(AMDGPUISD::CALL, DL, NodeTys, Ops);
2937
647
  Chain = Call.getValue(0);
2938
647
  InFlag = Call.getValue(1);
2939
647
2940
647
  uint64_t CalleePopBytes = NumBytes;
2941
647
  Chain = DAG.getCALLSEQ_END(Chain, DAG.getTargetConstant(0, DL, MVT::i32),
2942
647
                             DAG.getTargetConstant(CalleePopBytes, DL, MVT::i32),
2943
647
                             InFlag, DL);
2944
647
  if (!Ins.empty())
2945
154
    InFlag = Chain.getValue(1);
2946
647
2947
647
  // Handle result values, copying them out of physregs into vregs that we
2948
647
  // return.
2949
647
  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
2950
647
                         InVals, IsThisReturn,
2951
647
                         IsThisReturn ? 
OutVals[0]0
: SDValue());
2952
647
}
2953
2954
unsigned SITargetLowering::getRegisterByName(const char* RegName, EVT VT,
2955
27
                                             SelectionDAG &DAG) const {
2956
27
  unsigned Reg = StringSwitch<unsigned>(RegName)
2957
27
    .Case("m0", AMDGPU::M0)
2958
27
    .Case("exec", AMDGPU::EXEC)
2959
27
    .Case("exec_lo", AMDGPU::EXEC_LO)
2960
27
    .Case("exec_hi", AMDGPU::EXEC_HI)
2961
27
    .Case("flat_scratch", AMDGPU::FLAT_SCR)
2962
27
    .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2963
27
    .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2964
27
    .Default(AMDGPU::NoRegister);
2965
27
2966
27
  if (Reg == AMDGPU::NoRegister) {
2967
0
    report_fatal_error(Twine("invalid register name \""
2968
0
                             + StringRef(RegName)  + "\"."));
2969
0
2970
0
  }
2971
27
2972
27
  if (!Subtarget->hasFlatScrRegister() &&
2973
27
       
Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)3
) {
2974
1
    report_fatal_error(Twine("invalid register \""
2975
1
                             + StringRef(RegName)  + "\" for subtarget."));
2976
1
  }
2977
26
2978
26
  switch (Reg) {
2979
26
  case AMDGPU::M0:
2980
17
  case AMDGPU::EXEC_LO:
2981
17
  case AMDGPU::EXEC_HI:
2982
17
  case AMDGPU::FLAT_SCR_LO:
2983
17
  case AMDGPU::FLAT_SCR_HI:
2984
17
    if (VT.getSizeInBits() == 32)
2985
16
      return Reg;
2986
1
    break;
2987
9
  case AMDGPU::EXEC:
2988
9
  case AMDGPU::FLAT_SCR:
2989
9
    if (VT.getSizeInBits() == 64)
2990
8
      return Reg;
2991
1
    break;
2992
1
  default:
2993
0
    llvm_unreachable("missing register type checking");
2994
2
  }
2995
2
2996
2
  report_fatal_error(Twine("invalid type for register \""
2997
2
                           + StringRef(RegName) + "\"."));
2998
2
}
2999
3000
// If kill is not the last instruction, split the block so kill is always a
3001
// proper terminator.
3002
MachineBasicBlock *SITargetLowering::splitKillBlock(MachineInstr &MI,
3003
129
                                                    MachineBasicBlock *BB) const {
3004
129
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3005
129
3006
129
  MachineBasicBlock::iterator SplitPoint(&MI);
3007
129
  ++SplitPoint;
3008
129
3009
129
  if (SplitPoint == BB->end()) {
3010
4
    // Don't bother with a new block.
3011
4
    MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode()));
3012
4
    return BB;
3013
4
  }
3014
125
3015
125
  MachineFunction *MF = BB->getParent();
3016
125
  MachineBasicBlock *SplitBB
3017
125
    = MF->CreateMachineBasicBlock(BB->getBasicBlock());
3018
125
3019
125
  MF->insert(++MachineFunction::iterator(BB), SplitBB);
3020
125
  SplitBB->splice(SplitBB->begin(), BB, SplitPoint, BB->end());
3021
125
3022
125
  SplitBB->transferSuccessorsAndUpdatePHIs(BB);
3023
125
  BB->addSuccessor(SplitBB);
3024
125
3025
125
  MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode()));
3026
125
  return SplitBB;
3027
125
}
3028
3029
// Split block \p MBB at \p MI, as to insert a loop. If \p InstInLoop is true,
3030
// \p MI will be the only instruction in the loop body block. Otherwise, it will
3031
// be the first instruction in the remainder block.
3032
//
3033
/// \returns { LoopBody, Remainder }
3034
static std::pair<MachineBasicBlock *, MachineBasicBlock *>
3035
138
splitBlockForLoop(MachineInstr &MI, MachineBasicBlock &MBB, bool InstInLoop) {
3036
138
  MachineFunction *MF = MBB.getParent();
3037
138
  MachineBasicBlock::iterator I(&MI);
3038
138
3039
138
  // To insert the loop we need to split the block. Move everything after this
3040
138
  // point to a new block, and insert a new empty block between the two.
3041
138
  MachineBasicBlock *LoopBB = MF->CreateMachineBasicBlock();
3042
138
  MachineBasicBlock *RemainderBB = MF->CreateMachineBasicBlock();
3043
138
  MachineFunction::iterator MBBI(MBB);
3044
138
  ++MBBI;
3045
138
3046
138
  MF->insert(MBBI, LoopBB);
3047
138
  MF->insert(MBBI, RemainderBB);
3048
138
3049
138
  LoopBB->addSuccessor(LoopBB);
3050
138
  LoopBB->addSuccessor(RemainderBB);
3051
138
3052
138
  // Move the rest of the block into a new block.
3053
138
  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
3054
138
3055
138
  if (InstInLoop) {
3056
105
    auto Next = std::next(I);
3057
105
3058
105
    // Move instruction to loop body.
3059
105
    LoopBB->splice(LoopBB->begin(), &MBB, I, Next);
3060
105
3061
105
    // Move the rest of the block.
3062
105
    RemainderBB->splice(RemainderBB->begin(), &MBB, Next, MBB.end());
3063
105
  } else {
3064
33
    RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
3065
33
  }
3066
138
3067
138
  MBB.addSuccessor(LoopBB);
3068
138
3069
138
  return std::make_pair(LoopBB, RemainderBB);
3070
138
}
3071
3072
/// Insert \p MI into a BUNDLE with an S_WAITCNT 0 immediately following it.
3073
179
void SITargetLowering::bundleInstWithWaitcnt(MachineInstr &MI) const {
3074
179
  MachineBasicBlock *MBB = MI.getParent();
3075
179
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3076
179
  auto I = MI.getIterator();
3077
179
  auto E = std::next(I);
3078
179
3079
179
  BuildMI(*MBB, E, MI.getDebugLoc(), TII->get(AMDGPU::S_WAITCNT))
3080
179
    .addImm(0);
3081
179
3082
179
  MIBundleBuilder Bundler(*MBB, I, E);
3083
179
  finalizeBundle(*MBB, Bundler.begin());
3084
179
}
3085
3086
MachineBasicBlock *
3087
SITargetLowering::emitGWSMemViolTestLoop(MachineInstr &MI,
3088
105
                                         MachineBasicBlock *BB) const {
3089
105
  const DebugLoc &DL = MI.getDebugLoc();
3090
105
3091
105
  MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
3092
105
3093
105
  MachineBasicBlock *LoopBB;
3094
105
  MachineBasicBlock *RemainderBB;
3095
105
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3096
105
3097
105
  MachineBasicBlock::iterator Prev = std::prev(MI.getIterator());
3098
105
3099
105
  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, *BB, true);
3100
105
3101
105
  MachineBasicBlock::iterator I = LoopBB->end();
3102
105
  MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
3103
105
3104
105
  const unsigned EncodedReg = AMDGPU::Hwreg::encodeHwreg(
3105
105
    AMDGPU::Hwreg::ID_TRAPSTS, AMDGPU::Hwreg::OFFSET_MEM_VIOL, 1);
3106
105
3107
105
  // Clear TRAP_STS.MEM_VIOL
3108
105
  BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::S_SETREG_IMM32_B32))
3109
105
    .addImm(0)
3110
105
    .addImm(EncodedReg);
3111
105
3112
105
  // This is a pain, but we're not allowed to have physical register live-ins
3113
105
  // yet. Insert a pair of copies if the VGPR0 hack is necessary.
3114
105
  if (Src && 
TargetRegisterInfo::isPhysicalRegister(Src->getReg())96
) {
3115
96
    unsigned Data0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3116
96
    BuildMI(*BB, std::next(Prev), DL, TII->get(AMDGPU::COPY), Data0)
3117
96
      .add(*Src);
3118
96
3119
96
    BuildMI(*LoopBB, LoopBB->begin(), DL, TII->get(AMDGPU::COPY), Src->getReg())
3120
96
      .addReg(Data0);
3121
96
3122
96
    MRI.setSimpleHint(Data0, Src->getReg());
3123
96
  }
3124
105
3125
105
  bundleInstWithWaitcnt(MI);
3126
105
3127
105
  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3128
105
3129
105
  // Load and check TRAP_STS.MEM_VIOL
3130
105
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_GETREG_B32), Reg)
3131
105
    .addImm(EncodedReg);
3132
105
3133
105
  // FIXME: Do we need to use an isel pseudo that may clobber scc?
3134
105
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CMP_LG_U32))
3135
105
    .addReg(Reg, RegState::Kill)
3136
105
    .addImm(0);
3137
105
  BuildMI(*LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
3138
105
    .addMBB(LoopBB);
3139
105
3140
105
  return RemainderBB;
3141
105
}
3142
3143
// Do a v_movrels_b32 or v_movreld_b32 for each unique value of \p IdxReg in the
3144
// wavefront. If the value is uniform and just happens to be in a VGPR, this
3145
// will only do one iteration. In the worst case, this will loop 64 times.
3146
//
3147
// TODO: Just use v_readlane_b32 if we know the VGPR has a uniform value.
3148
static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(
3149
  const SIInstrInfo *TII,
3150
  MachineRegisterInfo &MRI,
3151
  MachineBasicBlock &OrigBB,
3152
  MachineBasicBlock &LoopBB,
3153
  const DebugLoc &DL,
3154
  const MachineOperand &IdxReg,
3155
  unsigned InitReg,
3156
  unsigned ResultReg,
3157
  unsigned PhiReg,
3158
  unsigned InitSaveExecReg,
3159
  int Offset,
3160
  bool UseGPRIdxMode,
3161
33
  bool IsIndirectSrc) {
3162
33
  MachineFunction *MF = OrigBB.getParent();
3163
33
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3164
33
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
3165
33
  MachineBasicBlock::iterator I = LoopBB.begin();
3166
33
3167
33
  const TargetRegisterClass *BoolRC = TRI->getBoolRC();
3168
33
  unsigned PhiExec = MRI.createVirtualRegister(BoolRC);
3169
33
  unsigned NewExec = MRI.createVirtualRegister(BoolRC);
3170
33
  unsigned CurrentIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3171
33
  unsigned CondReg = MRI.createVirtualRegister(BoolRC);
3172
33
3173
33
  BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg)
3174
33
    .addReg(InitReg)
3175
33
    .addMBB(&OrigBB)
3176
33
    .addReg(ResultReg)
3177
33
    .addMBB(&LoopBB);
3178
33
3179
33
  BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiExec)
3180
33
    .addReg(InitSaveExecReg)
3181
33
    .addMBB(&OrigBB)
3182
33
    .addReg(NewExec)
3183
33
    .addMBB(&LoopBB);
3184
33
3185
33
  // Read the next variant <- also loop target.
3186
33
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), CurrentIdxReg)
3187
33
    .addReg(IdxReg.getReg(), getUndefRegState(IdxReg.isUndef()));
3188
33
3189
33
  // Compare the just read M0 value to all possible Idx values.
3190
33
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::V_CMP_EQ_U32_e64), CondReg)
3191
33
    .addReg(CurrentIdxReg)
3192
33
    .addReg(IdxReg.getReg(), 0, IdxReg.getSubReg());
3193
33
3194
33
  // Update EXEC, save the original EXEC value to VCC.
3195
33
  BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? 
AMDGPU::S_AND_SAVEEXEC_B320
3196
33
                                                : AMDGPU::S_AND_SAVEEXEC_B64),
3197
33
          NewExec)
3198
33
    .addReg(CondReg, RegState::Kill);
3199
33
3200
33
  MRI.setSimpleHint(NewExec, CondReg);
3201
33
3202
33
  if (UseGPRIdxMode) {
3203
17
    unsigned IdxReg;
3204
17
    if (Offset == 0) {
3205
11
      IdxReg = CurrentIdxReg;
3206
11
    } else {
3207
6
      IdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3208
6
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), IdxReg)
3209
6
        .addReg(CurrentIdxReg, RegState::Kill)
3210
6
        .addImm(Offset);
3211
6
    }
3212
17
    unsigned IdxMode = IsIndirectSrc ?
3213
10
      
AMDGPU::VGPRIndexMode::SRC0_ENABLE7
: AMDGPU::VGPRIndexMode::DST_ENABLE;
3214
17
    MachineInstr *SetOn =
3215
17
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
3216
17
      .addReg(IdxReg, RegState::Kill)
3217
17
      .addImm(IdxMode);
3218
17
    SetOn->getOperand(3).setIsUndef();
3219
17
  } else {
3220
16
    // Move index from VCC into M0
3221
16
    if (Offset == 0) {
3222
10
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3223
10
        .addReg(CurrentIdxReg, RegState::Kill);
3224
10
    } else {
3225
6
      BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
3226
6
        .addReg(CurrentIdxReg, RegState::Kill)
3227
6
        .addImm(Offset);
3228
6
    }
3229
16
  }
3230
33
3231
33
  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
3232
33
  unsigned Exec = ST.isWave32() ? 
AMDGPU::EXEC_LO0
: AMDGPU::EXEC;
3233
33
  MachineInstr *InsertPt =
3234
33
    BuildMI(LoopBB, I, DL, TII->get(ST.isWave32() ? 
AMDGPU::S_XOR_B32_term0
3235
33
                                                  : AMDGPU::S_XOR_B64_term), Exec)
3236
33
      .addReg(Exec)
3237
33
      .addReg(NewExec);
3238
33
3239
33
  // XXX - s_xor_b64 sets scc to 1 if the result is nonzero, so can we use
3240
33
  // s_cbranch_scc0?
3241
33
3242
33
  // Loop back to V_READFIRSTLANE_B32 if there are still variants to cover.
3243
33
  BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
3244
33
    .addMBB(&LoopBB);
3245
33
3246
33
  return InsertPt->getIterator();
3247
33
}
3248
3249
// This has slightly sub-optimal regalloc when the source vector is killed by
3250
// the read. The register allocator does not understand that the kill is
3251
// per-workitem, so is kept alive for the whole loop so we end up not re-using a
3252
// subregister from it, using 1 more VGPR than necessary. This was saved when
3253
// this was expanded after register allocation.
3254
static MachineBasicBlock::iterator loadM0FromVGPR(const SIInstrInfo *TII,
3255
                                                  MachineBasicBlock &MBB,
3256
                                                  MachineInstr &MI,
3257
                                                  unsigned InitResultReg,
3258
                                                  unsigned PhiReg,
3259
                                                  int Offset,
3260
                                                  bool UseGPRIdxMode,
3261
33
                                                  bool IsIndirectSrc) {
3262
33
  MachineFunction *MF = MBB.getParent();
3263
33
  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3264
33
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
3265
33
  MachineRegisterInfo &MRI = MF->getRegInfo();
3266
33
  const DebugLoc &DL = MI.getDebugLoc();
3267
33
  MachineBasicBlock::iterator I(&MI);
3268
33
3269
33
  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
3270
33
  unsigned DstReg = MI.getOperand(0).getReg();
3271
33
  unsigned SaveExec = MRI.createVirtualRegister(BoolXExecRC);
3272
33
  unsigned TmpExec = MRI.createVirtualRegister(BoolXExecRC);
3273
33
  unsigned Exec = ST.isWave32() ? 
AMDGPU::EXEC_LO0
: AMDGPU::EXEC;
3274
33
  unsigned MovExecOpc = ST.isWave32() ? 
AMDGPU::S_MOV_B320
: AMDGPU::S_MOV_B64;
3275
33
3276
33
  BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), TmpExec);
3277
33
3278
33
  // Save the EXEC mask
3279
33
  BuildMI(MBB, I, DL, TII->get(MovExecOpc), SaveExec)
3280
33
    .addReg(Exec);
3281
33
3282
33
  MachineBasicBlock *LoopBB;
3283
33
  MachineBasicBlock *RemainderBB;
3284
33
  std::tie(LoopBB, RemainderBB) = splitBlockForLoop(MI, MBB, false);
3285
33
3286
33
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3287
33
3288
33
  auto InsPt = emitLoadM0FromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, *Idx,
3289
33
                                      InitResultReg, DstReg, PhiReg, TmpExec,
3290
33
                                      Offset, UseGPRIdxMode, IsIndirectSrc);
3291
33
3292
33
  MachineBasicBlock::iterator First = RemainderBB->begin();
3293
33
  BuildMI(*RemainderBB, First, DL, TII->get(MovExecOpc), Exec)
3294
33
    .addReg(SaveExec);
3295
33
3296
33
  return InsPt;
3297
33
}
3298
3299
// Returns subreg index, offset
3300
static std::pair<unsigned, int>
3301
computeIndirectRegAndOffset(const SIRegisterInfo &TRI,
3302
                            const TargetRegisterClass *SuperRC,
3303
                            unsigned VecReg,
3304
114
                            int Offset) {
3305
114
  int NumElts = TRI.getRegSizeInBits(*SuperRC) / 32;
3306
114
3307
114
  // Skip out of bounds offsets, or else we would end up using an undefined
3308
114
  // register.
3309
114
  if (Offset >= NumElts || Offset < 0)
3310
36
    return std::make_pair(AMDGPU::sub0, Offset);
3311
78
3312
78
  return std::make_pair(AMDGPU::sub0 + Offset, 0);
3313
78
}
3314
3315
// Return true if the index is an SGPR and was set.
3316
static bool setM0ToIndexFromSGPR(const SIInstrInfo *TII,
3317
                                 MachineRegisterInfo &MRI,
3318
                                 MachineInstr &MI,
3319
                                 int Offset,
3320
                                 bool UseGPRIdxMode,
3321
114
                                 bool IsIndirectSrc) {
3322
114
  MachineBasicBlock *MBB = MI.getParent();
3323
114
  const DebugLoc &DL = MI.getDebugLoc();
3324
114
  MachineBasicBlock::iterator I(&MI);
3325
114
3326
114
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3327
114
  const TargetRegisterClass *IdxRC = MRI.getRegClass(Idx->getReg());
3328
114
3329
114
  assert(Idx->getReg() != AMDGPU::NoRegister);
3330
114
3331
114
  if (!TII->getRegisterInfo().isSGPRClass(IdxRC))
3332
33
    return false;
3333
81
3334
81
  if (UseGPRIdxMode) {
3335
34
    unsigned IdxMode = IsIndirectSrc ?
3336
18
      
AMDGPU::VGPRIndexMode::SRC0_ENABLE16
: AMDGPU::VGPRIndexMode::DST_ENABLE;
3337
34
    if (Offset == 0) {
3338
22
      MachineInstr *SetOn =
3339
22
          BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
3340
22
              .add(*Idx)
3341
22
              .addImm(IdxMode);
3342
22
3343
22
      SetOn->getOperand(3).setIsUndef();
3344
22
    } else {
3345
12
      unsigned Tmp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3346
12
      BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), Tmp)
3347
12
          .add(*Idx)
3348
12
          .addImm(Offset);
3349
12
      MachineInstr *SetOn =
3350
12
        BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_ON))
3351
12
        .addReg(Tmp, RegState::Kill)
3352
12
        .addImm(IdxMode);
3353
12
3354
12
      SetOn->getOperand(3).setIsUndef();
3355
12
    }
3356
34
3357
34
    return true;
3358
34
  }
3359
47
3360
47
  if (Offset == 0) {
3361
35
    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3362
35
      .add(*Idx);
3363
35
  } else {
3364
12
    BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
3365
12
      .add(*Idx)
3366
12
      .addImm(Offset);
3367
12
  }
3368
47
3369
47
  return true;
3370
47
}
3371
3372
// Control flow needs to be inserted if indexing with a VGPR.
3373
static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI,
3374
                                          MachineBasicBlock &MBB,
3375
48
                                          const GCNSubtarget &ST) {
3376
48
  const SIInstrInfo *TII = ST.getInstrInfo();
3377
48
  const SIRegisterInfo &TRI = TII->getRegisterInfo();
3378
48
  MachineFunction *MF = MBB.getParent();
3379
48
  MachineRegisterInfo &MRI = MF->getRegInfo();
3380
48
3381
48
  unsigned Dst = MI.getOperand(0).getReg();
3382
48
  unsigned SrcReg = TII->getNamedOperand(MI, AMDGPU::OpName::src)->getReg();
3383
48
  int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
3384
48
3385
48
  const TargetRegisterClass *VecRC = MRI.getRegClass(SrcReg);
3386
48
3387
48
  unsigned SubReg;
3388
48
  std::tie(SubReg, Offset)
3389
48
    = computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset);
3390
48
3391
48
  bool UseGPRIdxMode = ST.useVGPRIndexMode(EnableVGPRIndexMode);
3392
48
3393
48
  if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, true)) {
3394
35
    MachineBasicBlock::iterator I(&MI);
3395
35
    const DebugLoc &DL = MI.getDebugLoc();
3396
35
3397
35
    if (UseGPRIdxMode) {
3398
16
      // TODO: Look at the uses to avoid the copy. This may require rescheduling
3399
16
      // to avoid interfering with other uses, so probably requires a new
3400
16
      // optimization pass.
3401
16
      BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
3402
16
        .addReg(SrcReg, RegState::Undef, SubReg)
3403
16
        .addReg(SrcReg, RegState::Implicit)
3404
16
        .addReg(AMDGPU::M0, RegState::Implicit);
3405
16
      BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
3406
19
    } else {
3407
19
      BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
3408
19
        .addReg(SrcReg, RegState::Undef, SubReg)
3409
19
        .addReg(SrcReg, RegState::Implicit);
3410
19
    }
3411
35
3412
35
    MI.eraseFromParent();
3413
35
3414
35
    return &MBB;
3415
35
  }
3416
13
3417
13
  const DebugLoc &DL = MI.getDebugLoc();
3418
13
  MachineBasicBlock::iterator I(&MI);
3419
13
3420
13
  unsigned PhiReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3421
13
  unsigned InitReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3422
13
3423
13
  BuildMI(MBB, I, DL, TII->get(TargetOpcode::IMPLICIT_DEF), InitReg);
3424
13
3425
13
  auto InsPt = loadM0FromVGPR(TII, MBB, MI, InitReg, PhiReg,
3426
13
                              Offset, UseGPRIdxMode, true);
3427
13
  MachineBasicBlock *LoopBB = InsPt->getParent();
3428
13
3429
13
  if (UseGPRIdxMode) {
3430
7
    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOV_B32_e32), Dst)
3431
7
      .addReg(SrcReg, RegState::Undef, SubReg)
3432
7
      .addReg(SrcReg, RegState::Implicit)
3433
7
      .addReg(AMDGPU::M0, RegState::Implicit);
3434
7
    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
3435
7
  } else {
3436
6
    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOVRELS_B32_e32), Dst)
3437
6
      .addReg(SrcReg, RegState::Undef, SubReg)
3438
6
      .addReg(SrcReg, RegState::Implicit);
3439
6
  }
3440
13
3441
13
  MI.eraseFromParent();
3442
13
3443
13
  return LoopBB;
3444
13
}
3445
3446
static unsigned getMOVRELDPseudo(const SIRegisterInfo &TRI,
3447
38
                                 const TargetRegisterClass *VecRC) {
3448
38
  switch (TRI.getRegSizeInBits(*VecRC)) {
3449
38
  case 32: // 4 bytes
3450
0
    return AMDGPU::V_MOVRELD_B32_V1;
3451
38
  case 64: // 8 bytes
3452
0
    return AMDGPU::V_MOVRELD_B32_V2;
3453
38
  case 128: // 16 bytes
3454
0
    return AMDGPU::V_MOVRELD_B32_V4;
3455
38
  case 256: // 32 bytes
3456
0
    return AMDGPU::V_MOVRELD_B32_V8;
3457
38
  case 512: // 64 bytes
3458
38
    return AMDGPU::V_MOVRELD_B32_V16;
3459
38
  default:
3460
0
    llvm_unreachable("unsupported size for MOVRELD pseudos");
3461
38
  }
3462
38
}
3463
3464
static MachineBasicBlock *emitIndirectDst(MachineInstr &MI,
3465
                                          MachineBasicBlock &MBB,
3466
66
                                          const GCNSubtarget &ST) {
3467
66
  const SIInstrInfo *TII = ST.getInstrInfo();
3468
66
  const SIRegisterInfo &TRI = TII->getRegisterInfo();
3469
66
  MachineFunction *MF = MBB.getParent();
3470
66
  MachineRegisterInfo &MRI = MF->getRegInfo();
3471
66
3472
66
  unsigned Dst = MI.getOperand(0).getReg();
3473
66
  const MachineOperand *SrcVec = TII->getNamedOperand(MI, AMDGPU::OpName::src);
3474
66
  const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
3475
66
  const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val);
3476
66
  int Offset = TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm();
3477
66
  const TargetRegisterClass *VecRC = MRI.getRegClass(SrcVec->getReg());
3478
66
3479
66
  // This can be an immediate, but will be folded later.
3480
66
  assert(Val->getReg());
3481
66
3482
66
  unsigned SubReg;
3483
66
  std::tie(SubReg, Offset) = computeIndirectRegAndOffset(TRI, VecRC,
3484
66
                                                         SrcVec->getReg(),
3485
66
                                                         Offset);
3486
66
  bool UseGPRIdxMode = ST.useVGPRIndexMode(EnableVGPRIndexMode);
3487
66
3488
66
  if (Idx->getReg() == AMDGPU::NoRegister) {
3489
0
    MachineBasicBlock::iterator I(&MI);
3490
0
    const DebugLoc &DL = MI.getDebugLoc();
3491
0
3492
0
    assert(Offset == 0);
3493
0
3494
0
    BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), Dst)
3495
0
        .add(*SrcVec)
3496
0
        .add(*Val)
3497
0
        .addImm(SubReg);
3498
0
3499
0
    MI.eraseFromParent();
3500
0
    return &MBB;
3501
0
  }
3502
66
3503
66
  if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, false)) {
3504
46
    MachineBasicBlock::iterator I(&MI);
3505
46
    const DebugLoc &DL = MI.getDebugLoc();
3506
46
3507
46
    if (UseGPRIdxMode) {
3508
18
      BuildMI(MBB, I, DL, TII->get(AMDGPU::V_MOV_B32_indirect))
3509
18
          .addReg(SrcVec->getReg(), RegState::Undef, SubReg) // vdst
3510
18
          .add(*Val)
3511
18
          .addReg(Dst, RegState::ImplicitDefine)
3512
18
          .addReg(SrcVec->getReg(), RegState::Implicit)
3513
18
          .addReg(AMDGPU::M0, RegState::Implicit);
3514
18
3515
18
      BuildMI(MBB, I, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
3516
28
    } else {
3517
28
      const MCInstrDesc &MovRelDesc = TII->get(getMOVRELDPseudo(TRI, VecRC));
3518
28
3519
28
      BuildMI(MBB, I, DL, MovRelDesc)
3520
28
          .addReg(Dst, RegState::Define)
3521
28
          .addReg(SrcVec->getReg())
3522
28
          .add(*Val)
3523
28
          .addImm(SubReg - AMDGPU::sub0);
3524
28
    }
3525
46
3526
46
    MI.eraseFromParent();
3527
46
    return &MBB;
3528
46
  }
3529
20
3530
20
  if (Val->isReg())
3531
20
    MRI.clearKillFlags(Val->getReg());
3532
20
3533
20
  const DebugLoc &DL = MI.getDebugLoc();
3534
20
3535
20
  unsigned PhiReg = MRI.createVirtualRegister(VecRC);
3536
20
3537
20
  auto InsPt = loadM0FromVGPR(TII, MBB, MI, SrcVec->getReg(), PhiReg,
3538
20
                              Offset, UseGPRIdxMode, false);
3539
20
  MachineBasicBlock *LoopBB = InsPt->getParent();
3540
20
3541
20
  if (UseGPRIdxMode) {
3542
10
    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::V_MOV_B32_indirect))
3543
10
        .addReg(PhiReg, RegState::Undef, SubReg) // vdst
3544
10
        .add(*Val)                               // src0
3545
10
        .addReg(Dst, RegState::ImplicitDefine)
3546
10
        .addReg(PhiReg, RegState::Implicit)
3547
10
        .addReg(AMDGPU::M0, RegState::Implicit);
3548
10
    BuildMI(*LoopBB, InsPt, DL, TII->get(AMDGPU::S_SET_GPR_IDX_OFF));
3549
10
  } else {
3550
10
    const MCInstrDesc &MovRelDesc = TII->get(getMOVRELDPseudo(TRI, VecRC));
3551
10
3552
10
    BuildMI(*LoopBB, InsPt, DL, MovRelDesc)
3553
10
        .addReg(Dst, RegState::Define)
3554
10
        .addReg(PhiReg)
3555
10
        .add(*Val)
3556
10
        .addImm(SubReg - AMDGPU::sub0);
3557
10
  }
3558
20
3559
20
  MI.eraseFromParent();
3560
20
3561
20
  return LoopBB;
3562
20
}
3563
3564
MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
3565
11.8k
  MachineInstr &MI, MachineBasicBlock *BB) const {
3566
11.8k
3567
11.8k
  const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3568
11.8k
  MachineFunction *MF = BB->getParent();
3569
11.8k
  SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
3570
11.8k
3571
11.8k
  if (TII->isMIMG(MI)) {
3572
1.26k
    if (MI.memoperands_empty() && 
MI.mayLoadOrStore()57
) {
3573
0
      report_fatal_error("missing mem operand from MIMG instruction");
3574
0
    }
3575
1.26k
    // Add a memoperand for mimg instructions so that they aren't assumed to
3576
1.26k
    // be ordered memory instuctions.
3577
1.26k
3578
1.26k
    return BB;
3579
1.26k
  }
3580
10.6k
3581
10.6k
  switch (MI.getOpcode()) {
3582
10.6k
  case AMDGPU::S_ADD_U64_PSEUDO:
3583
2.81k
  case AMDGPU::S_SUB_U64_PSEUDO: {
3584
2.81k
    MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
3585
2.81k
    const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3586
2.81k
    const SIRegisterInfo *TRI = ST.getRegisterInfo();
3587
2.81k
    const TargetRegisterClass *BoolRC = TRI->getBoolRC();
3588
2.81k
    const DebugLoc &DL = MI.getDebugLoc();
3589
2.81k
3590
2.81k
    MachineOperand &Dest = MI.getOperand(0);
3591
2.81k
    MachineOperand &Src0 = MI.getOperand(1);
3592
2.81k
    MachineOperand &Src1 = MI.getOperand(2);
3593
2.81k
3594
2.81k
    unsigned DestSub0 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3595
2.81k
    unsigned DestSub1 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3596
2.81k
3597
2.81k
    MachineOperand Src0Sub0 = TII->buildExtractSubRegOrImm(MI, MRI,
3598
2.81k
     Src0, BoolRC, AMDGPU::sub0,
3599
2.81k
     &AMDGPU::SReg_32_XM0RegClass);
3600
2.81k
    MachineOperand Src0Sub1 = TII->buildExtractSubRegOrImm(MI, MRI,
3601
2.81k
      Src0, BoolRC, AMDGPU::sub1,
3602
2.81k
      &AMDGPU::SReg_32_XM0RegClass);
3603
2.81k
3604
2.81k
    MachineOperand Src1Sub0 = TII->buildExtractSubRegOrImm(MI, MRI,
3605
2.81k
      Src1, BoolRC, AMDGPU::sub0,
3606
2.81k
      &AMDGPU::SReg_32_XM0RegClass);
3607
2.81k
    MachineOperand Src1Sub1 = TII->buildExtractSubRegOrImm(MI, MRI,
3608
2.81k
      Src1, BoolRC, AMDGPU::sub1,
3609
2.81k
      &AMDGPU::SReg_32_XM0RegClass);
3610
2.81k
3611
2.81k
    bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
3612
2.81k
3613
2.81k
    unsigned LoOpc = IsAdd ? 
AMDGPU::S_ADD_U322.77k
:
AMDGPU::S_SUB_U3233
;
3614
2.81k
    unsigned HiOpc = IsAdd ? 
AMDGPU::S_ADDC_U322.77k
:
AMDGPU::S_SUBB_U3233
;
3615
2.81k
    BuildMI(*BB, MI, DL, TII->get(LoOpc), DestSub0)
3616
2.81k
      .add(Src0Sub0)
3617
2.81k
      .add(Src1Sub0);
3618
2.81k
    BuildMI(*BB, MI, DL, TII->get(HiOpc), DestSub1)
3619
2.81k
      .add(Src0Sub1)
3620
2.81k
      .add(Src1Sub1);
3621
2.81k
    BuildMI(*BB, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), Dest.getReg())
3622
2.81k
      .addReg(DestSub0)
3623
2.81k
      .addImm(AMDGPU::sub0)
3624
2.81k
      .addReg(DestSub1)
3625
2.81k
      .addImm(AMDGPU::sub1);
3626
2.81k
    MI.eraseFromParent();
3627
2.81k
    return BB;
3628
2.81k
  }
3629
2.81k
  case AMDGPU::SI_INIT_M0: {
3630
2.61k
    BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(),
3631
2.61k
            TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
3632
2.61k
        .add(MI.getOperand(0));
3633
2.61k
    MI.eraseFromParent();
3634
2.61k
    return BB;
3635
2.81k
  }
3636
2.81k
  case AMDGPU::SI_INIT_EXEC:
3637
5
    // This should be before all vector instructions.
3638
5
    BuildMI(*BB, &*BB->begin(), MI.getDebugLoc(), TII->get(AMDGPU::S_MOV_B64),
3639
5
            AMDGPU::EXEC)
3640
5
        .addImm(MI.getOperand(0).getImm());
3641
5
    MI.eraseFromParent();
3642
5
    return BB;
3643
2.81k
3644
2.81k
  case AMDGPU::SI_INIT_EXEC_LO:
3645
3
    // This should be before all vector instructions.
3646
3
    BuildMI(*BB, &*BB->begin(), MI.getDebugLoc(), TII->get(AMDGPU::S_MOV_B32),
3647
3
            AMDGPU::EXEC_LO)
3648
3
        .addImm(MI.getOperand(0).getImm());
3649
3
    MI.eraseFromParent();
3650
3
    return BB;
3651
2.81k
3652
2.81k
  case AMDGPU::SI_INIT_EXEC_FROM_INPUT: {
3653
9
    // Extract the thread count from an SGPR input and set EXEC accordingly.
3654
9
    // Since BFM can't shift by 64, handle that case with CMP + CMOV.
3655
9
    //
3656
9
    // S_BFE_U32 count, input, {shift, 7}
3657
9
    // S_BFM_B64 exec, count, 0
3658
9
    // S_CMP_EQ_U32 count, 64
3659
9
    // S_CMOV_B64 exec, -1
3660
9
    MachineInstr *FirstMI = &*BB->begin();
3661
9
    MachineRegisterInfo &MRI = MF->getRegInfo();
3662
9
    unsigned InputReg = MI.getOperand(0).getReg();
3663
9
    unsigned CountReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3664
9
    bool Found = false;
3665
9
3666
9
    // Move the COPY of the input reg to the beginning, so that we can use it.
3667
28
    for (auto I = BB->begin(); I != &MI; 
I++19
) {
3668
28
      if (I->getOpcode() != TargetOpcode::COPY ||
3669
28
          
I->getOperand(0).getReg() != InputReg25
)
3670
19
        continue;
3671
9
3672
9
      if (I == FirstMI) {
3673
0
        FirstMI = &*++BB->begin();
3674
9
      } else {
3675
9
        I->removeFromParent();
3676
9
        BB->insert(FirstMI, &*I);
3677
9
      }
3678
9
      Found = true;
3679
9
      break;
3680
9
    }
3681
9
    assert(Found);
3682
9
    (void)Found;
3683
9
3684
9
    // This should be before all vector instructions.
3685
9
    unsigned Mask = (getSubtarget()->getWavefrontSize() << 1) - 1;
3686
9
    bool isWave32 = getSubtarget()->isWave32();
3687
9
    unsigned Exec = isWave32 ? 
AMDGPU::EXEC_LO3
:
AMDGPU::EXEC6
;
3688
9
    BuildMI(*BB, FirstMI, DebugLoc(), TII->get(AMDGPU::S_BFE_U32), CountReg)
3689
9
        .addReg(InputReg)
3690
9
        .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000);
3691
9
    BuildMI(*BB, FirstMI, DebugLoc(),
3692
9
            TII->get(isWave32 ? 
AMDGPU::S_BFM_B323
:
AMDGPU::S_BFM_B646
),
3693
9
            Exec)
3694
9
        .addReg(CountReg)
3695
9
        .addImm(0);
3696
9
    BuildMI(*BB, FirstMI, DebugLoc(), TII->get(AMDGPU::S_CMP_EQ_U32))
3697
9
        .addReg(CountReg, RegState::Kill)
3698
9
        .addImm(getSubtarget()->getWavefrontSize());
3699
9
    BuildMI(*BB, FirstMI, DebugLoc(),
3700
9
            TII->get(isWave32 ? 
AMDGPU::S_CMOV_B323
:
AMDGPU::S_CMOV_B646
),
3701
9
            Exec)
3702
9
        .addImm(-1);
3703
9
    MI.eraseFromParent();
3704
9
    return BB;
3705
2.81k
  }
3706
2.81k
3707
2.81k
  case AMDGPU::GET_GROUPSTATICSIZE: {
3708
58
    assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
3709
58
           getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
3710
58
    DebugLoc DL = MI.getDebugLoc();
3711
58
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_MOV_B32))
3712
58
        .add(MI.getOperand(0))
3713
58
        .addImm(MFI->getLDSSize());
3714
58
    MI.eraseFromParent();
3715
58
    return BB;
3716
2.81k
  }
3717
2.81k
  case AMDGPU::SI_INDIRECT_SRC_V1:
3718
48
  case AMDGPU::SI_INDIRECT_SRC_V2:
3719
48
  case AMDGPU::SI_INDIRECT_SRC_V4:
3720
48
  case AMDGPU::SI_INDIRECT_SRC_V8:
3721
48
  case AMDGPU::SI_INDIRECT_SRC_V16:
3722
48
    return emitIndirectSrc(MI, *BB, *getSubtarget());
3723
66
  case AMDGPU::SI_INDIRECT_DST_V1:
3724
66
  case AMDGPU::SI_INDIRECT_DST_V2:
3725
66
  case AMDGPU::SI_INDIRECT_DST_V4:
3726
66
  case AMDGPU::SI_INDIRECT_DST_V8:
3727
66
  case AMDGPU::SI_INDIRECT_DST_V16:
3728
66
    return emitIndirectDst(MI, *BB, *getSubtarget());
3729
129
  case AMDGPU::SI_KILL_F32_COND_IMM_PSEUDO:
3730
129
  case AMDGPU::SI_KILL_I1_PSEUDO:
3731
129
    return splitKillBlock(MI, BB);
3732
129
  case AMDGPU::V_CNDMASK_B64_PSEUDO: {
3733
49
    MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
3734
49
    const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3735
49
    const SIRegisterInfo *TRI = ST.getRegisterInfo();
3736
49
3737
49
    unsigned Dst = MI.getOperand(0).getReg();
3738
49
    unsigned Src0 = MI.getOperand(1).getReg();
3739
49
    unsigned Src1 = MI.getOperand(2).getReg();
3740
49
    const DebugLoc &DL = MI.getDebugLoc();
3741
49
    unsigned SrcCond = MI.getOperand(3).getReg();
3742
49
3743
49
    unsigned DstLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3744
49
    unsigned DstHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3745
49
    const auto *CondRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
3746
49
    unsigned SrcCondCopy = MRI.createVirtualRegister(CondRC);
3747
49
3748
49
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::COPY), SrcCondCopy)
3749
49
      .addReg(SrcCond);
3750
49
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstLo)
3751
49
      .addImm(0)
3752
49
      .addReg(Src0, 0, AMDGPU::sub0)
3753
49
      .addImm(0)
3754
49
      .addReg(Src1, 0, AMDGPU::sub0)
3755
49
      .addReg(SrcCondCopy);
3756
49
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_CNDMASK_B32_e64), DstHi)
3757
49
      .addImm(0)
3758
49
      .addReg(Src0, 0, AMDGPU::sub1)
3759
49
      .addImm(0)
3760
49
      .addReg(Src1, 0, AMDGPU::sub1)
3761
49
      .addReg(SrcCondCopy);
3762
49
3763
49
    BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), Dst)
3764
49
      .addReg(DstLo)
3765
49
      .addImm(AMDGPU::sub0)
3766
49
      .addReg(DstHi)
3767
49
      .addImm(AMDGPU::sub1);
3768
49
    MI.eraseFromParent();
3769
49
    return BB;
3770
129
  }
3771
129
  case AMDGPU::SI_BR_UNDEF: {
3772
88
    const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3773
88
    const DebugLoc &DL = MI.getDebugLoc();
3774
88
    MachineInstr *Br = BuildMI(*BB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC1))
3775
88
                           .add(MI.getOperand(0));
3776
88
    Br->getOperand(1).setIsUndef(true); // read undef SCC
3777
88
    MI.eraseFromParent();
3778
88
    return BB;
3779
129
  }
3780
1.29k
  case AMDGPU::ADJCALLSTACKUP:
3781
1.29k
  case AMDGPU::ADJCALLSTACKDOWN: {
3782
1.29k
    const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
3783
1.29k
    MachineInstrBuilder MIB(*MF, &MI);
3784
1.29k
3785
1.29k
    // Add an implicit use of the frame offset reg to prevent the restore copy
3786
1.29k
    // inserted after the call from being reorderd after stack operations in the
3787
1.29k
    // the caller's frame.
3788
1.29k
    MIB.addReg(Info->getStackPtrOffsetReg(), RegState::ImplicitDefine)
3789
1.29k
        .addReg(Info->getStackPtrOffsetReg(), RegState::Implicit)
3790
1.29k
        .addReg(Info->getFrameOffsetReg(), RegState::Implicit);
3791
1.29k
    return BB;
3792
1.29k
  }
3793
1.29k
  case AMDGPU::SI_CALL_ISEL: {
3794
646
    const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
3795
646
    const DebugLoc &DL = MI.getDebugLoc();
3796
646
3797
646
    unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF);
3798
646
3799
646
    MachineInstrBuilder MIB;
3800
646
    MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg);
3801
646
3802
5.72k
    for (unsigned I = 0, E = MI.getNumOperands(); I != E; 
++I5.07k
)
3803
5.07k
      MIB.add(MI.getOperand(I));
3804
646
3805
646
    MIB.cloneMemRefs(MI);
3806
646
    MI.eraseFromParent();
3807
646
    return BB;
3808
1.29k
  }
3809
2.63k
  case AMDGPU::V_ADD_I32_e32:
3810
2.63k
  case AMDGPU::V_SUB_I32_e32:
3811
2.63k
  case AMDGPU::V_SUBREV_I32_e32: {
3812
2.63k
    // TODO: Define distinct V_*_I32_Pseudo instructions instead.
3813
2.63k
    const DebugLoc &DL = MI.getDebugLoc();
3814
2.63k
    unsigned Opc = MI.getOpcode();
3815
2.63k
3816
2.63k
    bool NeedClampOperand = false;
3817
2.63k
    if (TII->pseudoToMCOpcode(Opc) == -1) {
3818
15
      Opc = AMDGPU::getVOPe64(Opc);
3819
15
      NeedClampOperand = true;
3820
15
    }
3821
2.63k
3822
2.63k
    auto I = BuildMI(*BB, MI, DL, TII->get(Opc), MI.getOperand(0).getReg());
3823
2.63k
    if (TII->isVOP3(*I)) {
3824
15
      const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3825
15
      const SIRegisterInfo *TRI = ST.getRegisterInfo();
3826
15
      I.addReg(TRI->getVCC(), RegState::Define);
3827
15
    }
3828
2.63k
    I.add(MI.getOperand(1))
3829
2.63k
     .add(MI.getOperand(2));
3830
2.63k
    if (NeedClampOperand)
3831
15
      I.addImm(0); // clamp bit for e64 encoding
3832
2.63k
3833
2.63k
    TII->legalizeOperands(*I);
3834
2.63k
3835
2.63k
    MI.eraseFromParent();
3836
2.63k
    return BB;
3837
2.63k
  }
3838
2.63k
  case AMDGPU::DS_GWS_INIT:
3839
179
  case AMDGPU::DS_GWS_SEMA_V:
3840
179
  case AMDGPU::DS_GWS_SEMA_BR:
3841
179
  case AMDGPU::DS_GWS_SEMA_P:
3842
179
  case AMDGPU::DS_GWS_SEMA_RELEASE_ALL:
3843
179
  case AMDGPU::DS_GWS_BARRIER:
3844
179
    // A s_waitcnt 0 is required to be the instruction immediately following.
3845
179
    if (getSubtarget()->hasGWSAutoReplay()) {
3846
74
      bundleInstWithWaitcnt(MI);
3847
74
      return BB;
3848
74
    }
3849
105
3850
105
    return emitGWSMemViolTestLoop(MI, BB);
3851
105
  default:
3852
0
    return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
3853
10.6k
  }
3854
10.6k
}
3855
3856
33.5k
bool SITargetLowering::hasBitPreservingFPLogic(EVT VT) const {
3857
33.5k
  return isTypeLegal(VT.getScalarType());
3858
33.5k
}
3859
3860
4.88k
bool SITargetLowering::enableAggressiveFMAFusion(EVT VT) const {
3861
4.88k
  // This currently forces unfolding various combinations of fsub into fma with
3862
4.88k
  // free fneg'd operands. As long as we have fast FMA (controlled by
3863
4.88k
  // isFMAFasterThanFMulAndFAdd), we should perform these.
3864
4.88k
3865
4.88k
  // When fma is quarter rate, for f64 where add / sub are at best half rate,
3866
4.88k
  // most of these combines appear to be cycle neutral but save on instruction
3867
4.88k
  // count / code size.
3868
4.88k
  return true;
3869
4.88k
}
3870
3871
EVT SITargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
3872
20.4k
                                         EVT VT) const {
3873
20.4k
  if (!VT.isVector()) {
3874
20.2k
    return MVT::i1;
3875
20.2k
  }
3876
142
  return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
3877
142
}
3878
3879
139k
MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const {
3880
139k
  // TODO: Should i16 be used always if legal? For now it would force VALU
3881
139k
  // shifts.
3882
139k
  return (VT == MVT::i16) ? 
MVT::i1612.4k
:
MVT::i32127k
;
3883
139k
}
3884
3885
// Answering this is somewhat tricky and depends on the specific device which
3886
// have different rates for fma or all f64 operations.
3887
//
3888
// v_fma_f64 and v_mul_f64 always take the same number of cycles as each other
3889
// regardless of which device (although the number of cycles differs between
3890
// devices), so it is always profitable for f64.
3891
//
3892
// v_fma_f32 takes 4 or 16 cycles depending on the device, so it is profitable
3893
// only on full rate devices. Normally, we should prefer selecting v_mad_f32
3894
// which we can always do even without fused FP ops since it returns the same
3895
// result as the separate operations and since it is always full
3896
// rate. Therefore, we lie and report that it is not faster for f32. v_mad_f32
3897
// however does not support denormals, so we do report fma as faster if we have
3898
// a fast fma device and require denormals.
3899
//
3900
13.5k
bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
3901
13.5k
  VT = VT.getScalarType();
3902
13.5k
3903
13.5k
  switch (VT.getSimpleVT().SimpleTy) {
3904
13.5k
  case MVT::f32: {
3905
10.1k
    // This is as fast on some subtargets. However, we always have full rate f32
3906
10.1k
    // mad available which returns the same result as the separate operations
3907
10.1k
    // which we should prefer over fma. We can't use this if we want to support
3908
10.1k
    // denormals, so only report this in these cases.
3909
10.1k
    if (Subtarget->hasFP32Denormals())
3910
628
      return Subtarget->hasFastFMAF32() || 
Subtarget->hasDLInsts()316
;
3911
9.49k
3912
9.49k
    // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32.
3913
9.49k
    return Subtarget->hasFastFMAF32() && 
Subtarget->hasDLInsts()3.32k
;
3914
9.49k
  }
3915
9.49k
  case MVT::f64:
3916
1.08k
    return true;
3917
9.49k
  case MVT::f16:
3918
2.32k
    return Subtarget->has16BitInsts() && 
Subtarget->hasFP16Denormals()2.03k
;
3919
9.49k
  default:
3920
0
    break;
3921
0
  }
3922
0
3923
0
  return false;
3924
0
}
3925
3926
//===----------------------------------------------------------------------===//
3927
// Custom DAG Lowering Operations
3928
//===----------------------------------------------------------------------===//
3929
3930
// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
3931
// wider vector type is legal.
3932
SDValue SITargetLowering::splitUnaryVectorOp(SDValue Op,
3933
41
                                             SelectionDAG &DAG) const {
3934
41
  unsigned Opc = Op.getOpcode();
3935
41
  EVT VT = Op.getValueType();
3936
41
  assert(VT == MVT::v4f16);
3937
41
3938
41
  SDValue Lo, Hi;
3939
41
  std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
3940
41
3941
41
  SDLoc SL(Op);
3942
41
  SDValue OpLo = DAG.getNode(Opc, SL, Lo.getValueType(), Lo,
3943
41
                             Op->getFlags());
3944
41
  SDValue OpHi = DAG.getNode(Opc, SL, Hi.getValueType(), Hi,
3945
41
                             Op->getFlags());
3946
41
3947
41
  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
3948
41
}
3949
3950
// Work around LegalizeDAG doing the wrong thing and fully scalarizing if the
3951
// wider vector type is legal.
3952
SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op,
3953
172
                                              SelectionDAG &DAG) const {
3954
172
  unsigned Opc = Op.getOpcode();
3955
172
  EVT VT = Op.getValueType();
3956
172
  assert(VT == MVT::v4i16 || VT == MVT::v4f16);
3957
172
3958
172
  SDValue Lo0, Hi0;
3959
172
  std::tie(Lo0, Hi0) = DAG.SplitVectorOperand(Op.getNode(), 0);
3960
172
  SDValue Lo1, Hi1;
3961
172
  std::tie(Lo1, Hi1) = DAG.SplitVectorOperand(Op.getNode(), 1);
3962
172
3963
172
  SDLoc SL(Op);
3964
172
3965
172
  SDValue OpLo = DAG.getNode(Opc, SL, Lo0.getValueType(), Lo0, Lo1,
3966
172
                             Op->getFlags());
3967
172
  SDValue OpHi = DAG.getNode(Opc, SL, Hi0.getValueType(), Hi0, Hi1,
3968
172
                             Op->getFlags());
3969
172
3970
172
  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op), VT, OpLo, OpHi);
3971
172
}
3972
3973
264k
SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
3974
264k
  switch (Op.getOpcode()) {
3975
264k
  
default: return AMDGPUTargetLowering::LowerOperation(Op, DAG)28.0k
;
3976
264k
  
case ISD::BRCOND: return LowerBRCOND(Op, DAG)2.20k
;
3977
264k
  
case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG)4
;
3978
264k
  case ISD::LOAD: {
3979
94.8k
    SDValue Result = LowerLOAD(Op, DAG);
3980
94.8k
    assert((!Result.getNode() ||
3981
94.8k
            Result.getNode()->getNumValues() == 2) &&
3982
94.8k
           "Load should return a value and a chain");
3983
94.8k
    return Result;
3984
264k
  }
3985
264k
3986
264k
  case ISD::FSIN:
3987
98
  case ISD::FCOS:
3988
98
    return LowerTrig(Op, DAG);
3989
895
  case ISD::SELECT: return LowerSELECT(Op, DAG);
3990
283
  case ISD::FDIV: return LowerFDIV(Op, DAG);
3991
845
  case ISD::ATOMIC_CMP_SWAP: return LowerATOMIC_CMP_SWAP(Op, DAG);
3992
91.9k
  case ISD::STORE: return LowerSTORE(Op, DAG);
3993
1.19k
  case ISD::GlobalAddress: {
3994
1.19k
    MachineFunction &MF = DAG.getMachineFunction();
3995
1.19k
    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
3996
1.19k
    return LowerGlobalAddress(MFI, Op, DAG);
3997
98
  }
3998
26.7k
  case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3999
2.42k
  case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, DAG);
4000
3.25k
  case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG);
4001
98
  
case ISD::ADDRSPACECAST: return lowerADDRSPACECAST(Op, DAG)58
;
4002
440
  case ISD::INSERT_SUBVECTOR:
4003
440
    return lowerINSERT_SUBVECTOR(Op, DAG);
4004
239
  case ISD::INSERT_VECTOR_ELT:
4005
239
    return lowerINSERT_VECTOR_ELT(Op, DAG);
4006
7.48k
  case ISD::EXTRACT_VECTOR_ELT:
4007
7.48k
    return lowerEXTRACT_VECTOR_ELT(Op, DAG);
4008
98
  case ISD::VECTOR_SHUFFLE:
4009
50
    return lowerVECTOR_SHUFFLE(Op, DAG);
4010
1.39k
  case ISD::BUILD_VECTOR:
4011
1.39k
    return lowerBUILD_VECTOR(Op, DAG);
4012
504
  case ISD::FP_ROUND:
4013
504
    return lowerFP_ROUND(Op, DAG);
4014
98
  case ISD::TRAP:
4015
28
    return lowerTRAP(Op, DAG);
4016
98
  case ISD::DEBUGTRAP:
4017
10
    return lowerDEBUGTRAP(Op, DAG);
4018
98
  case ISD::FABS:
4019
41
  case ISD::FNEG:
4020
41
  case ISD::FCANONICALIZE:
4021
41
    return splitUnaryVectorOp(Op, DAG);
4022
1.30k
  case ISD::FMINNUM:
4023
1.30k
  case ISD::FMAXNUM:
4024
1.30k
    return lowerFMINNUM_FMAXNUM(Op, DAG);
4025
1.30k
  case ISD::SHL:
4026
172
  case ISD::SRA:
4027
172
  case ISD::SRL:
4028
172
  case ISD::ADD:
4029
172
  case ISD::SUB:
4030
172
  case ISD::MUL:
4031
172
  case ISD::SMIN:
4032
172
  case ISD::SMAX:
4033
172
  case ISD::UMIN:
4034
172
  case ISD::UMAX:
4035
172
  case ISD::FADD:
4036
172
  case ISD::FMUL:
4037
172
  case ISD::FMINNUM_IEEE:
4038
172
  case ISD::FMAXNUM_IEEE:
4039
172
    return splitBinaryVectorOp(Op, DAG);
4040
0
  }
4041
0
  return SDValue();
4042
0
}
4043
4044
static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT,
4045
                                       const SDLoc &DL,
4046
27
                                       SelectionDAG &DAG, bool Unpacked) {
4047
27
  if (!LoadVT.isVector())
4048
6
    return Result;
4049
21
4050
21
  if (Unpacked) { // From v2i32/v4i32 back to v2f16/v4f16.
4051
21
    // Truncate to v2i16/v4i16.
4052
21
    EVT IntLoadVT = LoadVT.changeTypeToInteger();
4053
21
4054
21
    // Workaround legalizer not scalarizing truncate after vector op
4055
21
    // legalization byt not creating intermediate vector trunc.
4056
21
    SmallVector<SDValue, 4> Elts;
4057
21
    DAG.ExtractVectorElements(Result, Elts);
4058
21
    for (SDValue &Elt : Elts)
4059
64
      Elt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Elt);
4060
21
4061
21
    Result = DAG.getBuildVector(IntLoadVT, DL, Elts);
4062
21
4063
21
    // Bitcast to original type (v2f16/v4f16).
4064
21
    return DAG.getNode(ISD::BITCAST, DL, LoadVT, Result);
4065
21
  }
4066
0
4067
0
  // Cast back to the original packed type.
4068
0
  return DAG.getNode(ISD::BITCAST, DL, LoadVT, Result);
4069
0
}
4070
4071
SDValue SITargetLowering::adjustLoadValueType(unsigned Opcode,
4072
                                              MemSDNode *M,
4073
                                              SelectionDAG &DAG,
4074
                                              ArrayRef<SDValue> Ops,
4075
60
                                              bool IsIntrinsic) const {
4076
60
  SDLoc DL(M);
4077
60
4078
60
  bool Unpacked = Subtarget->hasUnpackedD16VMem();
4079
60
  EVT LoadVT = M->getValueType(0);
4080
60
4081
60
  EVT EquivLoadVT = LoadVT;
4082
60
  if (Unpacked && 
LoadVT.isVector()18
) {
4083
12
    EquivLoadVT = LoadVT.isVector() ?
4084
12
      EVT::getVectorVT(*DAG.getContext(), MVT::i32,
4085
12
                       LoadVT.getVectorNumElements()) : 
LoadVT0
;
4086
12
  }
4087
60
4088
60
  // Change from v4f16/v2f16 to EquivLoadVT.
4089
60
  SDVTList VTList = DAG.getVTList(EquivLoadVT, MVT::Other);
4090
60
4091
60
  SDValue Load
4092
60
    = DAG.getMemIntrinsicNode(
4093
60
      IsIntrinsic ? 
(unsigned)ISD::INTRINSIC_W_CHAIN0
: Opcode, DL,
4094
60
      VTList, Ops, M->getMemoryVT(),
4095
60
      M->getMemOperand());
4096
60
  if (!Unpacked) // Just adjusted the opcode.
4097
42
    return Load;
4098
18
4099
18
  SDValue Adjusted = adjustLoadValueTypeImpl(Load, LoadVT, DL, DAG, Unpacked);
4100
18
4101
18
  return DAG.getMergeValues({ Adjusted, Load.getValue(1) }, DL);
4102
18
}
4103
4104
static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI,
4105
225
                                  SDNode *N, SelectionDAG &DAG) {
4106
225
  EVT VT = N->getValueType(0);
4107
225
  const auto *CD = cast<ConstantSDNode>(N->getOperand(3));
4108
225
  int CondCode = CD->getSExtValue();
4109
225
  if (CondCode < ICmpInst::Predicate::FIRST_ICMP_PREDICATE ||
4110
225
      
CondCode > ICmpInst::Predicate::LAST_ICMP_PREDICATE219
)
4111
10
    return DAG.getUNDEF(VT);
4112
215
4113
215
  ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode);
4114
215
4115
215
  SDValue LHS = N->getOperand(1);
4116
215
  SDValue RHS = N->getOperand(2);
4117
215
4118
215
  SDLoc DL(N);
4119
215
4120
215
  EVT CmpVT = LHS.getValueType();
4121
215
  if (CmpVT == MVT::i16 && 
!TLI.isTypeLegal(MVT::i16)20
) {
4122
10
    unsigned PromoteOp = ICmpInst::isSigned(IcInput) ?
4123
6
      
ISD::SIGN_EXTEND4
: ISD::ZERO_EXTEND;
4124
10
    LHS = DAG.getNode(PromoteOp, DL, MVT::i32, LHS);
4125
10
    RHS = DAG.getNode(PromoteOp, DL, MVT::i32, RHS);
4126
10
  }
4127
215
4128
215
  ISD::CondCode CCOpcode = getICmpCondCode(IcInput);
4129
215
4130
215
  unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
4131
215
  EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize);
4132
215
4133
215
  SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, DL, CCVT, LHS, RHS,
4134
215
                              DAG.getCondCode(CCOpcode));
4135
215
  if (VT.bitsEq(CCVT))
4136
205
    return SetCC;
4137
10
  return DAG.getZExtOrTrunc(SetCC, DL, VT);
4138
10
}
4139
4140
static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI,
4141
104
                                  SDNode *N, SelectionDAG &DAG) {
4142
104
  EVT VT = N->getValueType(0);
4143
104
  const auto *CD = cast<ConstantSDNode>(N->getOperand(3));
4144
104
4145
104
  int CondCode = CD->getSExtValue();
4146
104
  if (CondCode < FCmpInst::Predicate::FIRST_FCMP_PREDICATE ||
4147
104
      
CondCode > FCmpInst::Predicate::LAST_FCMP_PREDICATE100
) {
4148
4
    return DAG.getUNDEF(VT);
4149
4
  }
4150
100
4151
100
  SDValue Src0 = N->getOperand(1);
4152
100
  SDValue Src1 = N->getOperand(2);
4153
100
  EVT CmpVT = Src0.getValueType();
4154
100
  SDLoc SL(N);
4155
100
4156
100
  if (CmpVT == MVT::f16 && 
!TLI.isTypeLegal(CmpVT)28
) {
4157
14
    Src0 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src0);
4158
14
    Src1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Src1);
4159
14
  }
4160
100
4161
100
  FCmpInst::Predicate IcInput = static_cast<FCmpInst::Predicate>(CondCode);
4162
100
  ISD::CondCode CCOpcode = getFCmpCondCode(IcInput);
4163
100
  unsigned WavefrontSize = TLI.getSubtarget()->getWavefrontSize();
4164
100
  EVT CCVT = EVT::getIntegerVT(*DAG.getContext(), WavefrontSize);
4165
100
  SDValue SetCC = DAG.getNode(AMDGPUISD::SETCC, SL, CCVT, Src0,
4166
100
                              Src1, DAG.getCondCode(CCOpcode));
4167
100
  if (VT.bitsEq(CCVT))
4168
90
    return SetCC;
4169
10
  return DAG.getZExtOrTrunc(SetCC, SL, VT);
4170
10
}
4171
4172
void SITargetLowering::ReplaceNodeResults(SDNode *N,
4173
                                          SmallVectorImpl<SDValue> &Results,
4174
613
                                          SelectionDAG &DAG) const {
4175
613
  switch (N->getOpcode()) {
4176
613
  case ISD::INSERT_VECTOR_ELT: {
4177
76
    if (SDValue Res = lowerINSERT_VECTOR_ELT(SDValue(N, 0), DAG))
4178
25
      Results.push_back(Res);
4179
76
    return;
4180
613
  }
4181
613
  case ISD::EXTRACT_VECTOR_ELT: {
4182
0
    if (SDValue Res = lowerEXTRACT_VECTOR_ELT(SDValue(N, 0), DAG))
4183
0
      Results.push_back(Res);
4184
0
    return;
4185
613
  }
4186
613
  case ISD::INTRINSIC_WO_CHAIN: {
4187
85
    unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
4188
85
    switch (IID) {
4189
85
    case Intrinsic::amdgcn_cvt_pkrtz: {
4190
29
      SDValue Src0 = N->getOperand(1);
4191
29
      SDValue Src1 = N->getOperand(2);
4192
29
      SDLoc SL(N);
4193
29
      SDValue Cvt = DAG.getNode(AMDGPUISD::CVT_PKRTZ_F16_F32, SL, MVT::i32,
4194
29
                                Src0, Src1);
4195
29
      Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Cvt));
4196
29
      return;
4197
85
    }
4198
85
    case Intrinsic::amdgcn_cvt_pknorm_i16:
4199
56
    case Intrinsic::amdgcn_cvt_pknorm_u16:
4200
56
    case Intrinsic::amdgcn_cvt_pk_i16:
4201
56
    case Intrinsic::amdgcn_cvt_pk_u16: {
4202
56
      SDValue Src0 = N->getOperand(1);
4203
56
      SDValue Src1 = N->getOperand(2);
4204
56
      SDLoc SL(N);
4205
56
      unsigned Opcode;
4206
56
4207
56
      if (IID == Intrinsic::amdgcn_cvt_pknorm_i16)
4208
18
        Opcode = AMDGPUISD::CVT_PKNORM_I16_F32;
4209
38
      else if (IID == Intrinsic::amdgcn_cvt_pknorm_u16)
4210
18
        Opcode = AMDGPUISD::CVT_PKNORM_U16_F32;
4211
20
      else if (IID == Intrinsic::amdgcn_cvt_pk_i16)
4212
10
        Opcode = AMDGPUISD::CVT_PK_I16_I32;
4213
10
      else
4214
10
        Opcode = AMDGPUISD::CVT_PK_U16_U32;
4215
56
4216
56
      EVT VT = N->getValueType(0);
4217
56
      if (isTypeLegal(VT))
4218
0
        Results.push_back(DAG.getNode(Opcode, SL, VT, Src0, Src1));
4219
56
      else {
4220
56
        SDValue Cvt = DAG.getNode(Opcode, SL, MVT::i32, Src0, Src1);
4221
56
        Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, Cvt));
4222
56
      }
4223
56
      return;
4224
0
    }
4225
0
    }
4226
0
    break;
4227
0
  }
4228
34
  case ISD::INTRINSIC_W_CHAIN: {
4229
34
    if (SDValue Res = LowerINTRINSIC_W_CHAIN(SDValue(N, 0), DAG)) {
4230
34
      Results.push_back(Res);
4231
34
      Results.push_back(Res.getValue(1));
4232
34
      return;
4233
34
    }
4234
0
4235
0
    break;
4236
0
  }
4237
26
  case ISD::SELECT: {
4238
26
    SDLoc SL(N);
4239
26
    EVT VT = N->getValueType(0);
4240
26
    EVT NewVT = getEquivalentMemType(*DAG.getContext(), VT);
4241
26
    SDValue LHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(1));
4242
26
    SDValue RHS = DAG.getNode(ISD::BITCAST, SL, NewVT, N->getOperand(2));
4243
26
4244
26
    EVT SelectVT = NewVT;
4245
26
    if (NewVT.bitsLT(MVT::i32)) {
4246
2
      LHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, LHS);
4247
2
      RHS = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, RHS);
4248
2
      SelectVT = MVT::i32;
4249
2
    }
4250
26
4251
26
    SDValue NewSelect = DAG.getNode(ISD::SELECT, SL, SelectVT,
4252
26
                                    N->getOperand(0), LHS, RHS);
4253
26
4254
26
    if (NewVT != SelectVT)
4255
2
      NewSelect = DAG.getNode(ISD::TRUNCATE, SL, NewVT, NewSelect);
4256
26
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, VT, NewSelect));
4257
26
    return;
4258
0
  }
4259
6
  case ISD::FNEG: {
4260
6
    if (N->getValueType(0) != MVT::v2f16)
4261
1
      break;
4262
5
4263
5
    SDLoc SL(N);
4264
5
    SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
4265
5
4266
5
    SDValue Op = DAG.getNode(ISD::XOR, SL, MVT::i32,
4267
5
                             BC,
4268
5
                             DAG.getConstant(0x80008000, SL, MVT::i32));
4269
5
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
4270
5
    return;
4271
5
  }
4272
8
  case ISD::FABS: {
4273
8
    if (N->getValueType(0) != MVT::v2f16)
4274
2
      break;
4275
6
4276
6
    SDLoc SL(N);
4277
6
    SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::i32, N->getOperand(0));
4278
6
4279
6
    SDValue Op = DAG.getNode(ISD::AND, SL, MVT::i32,
4280
6
                             BC,
4281
6
                             DAG.getConstant(0x7fff7fff, SL, MVT::i32));
4282
6
    Results.push_back(DAG.getNode(ISD::BITCAST, SL, MVT::v2f16, Op));
4283
6
    return;
4284
6
  }
4285
378
  default:
4286
378
    break;
4287
613
  }
4288
613
}
4289
4290
/// Helper function for LowerBRCOND
4291
1.08k
static SDNode *findUser(SDValue Value, unsigned Opcode) {
4292
1.08k
4293
1.08k
  SDNode *Parent = Value.getNode();
4294
1.08k
  for (SDNode::use_iterator I = Parent->use_begin(), E = Parent->use_end();
4295
2.45k
       I != E; 
++I1.36k
) {
4296
2.45k
4297
2.45k
    if (I.getUse().get() != Value)
4298
1.36k
      continue;
4299
1.08k
4300
1.08k
    if (I->getOpcode() == Opcode)
4301
1.08k
      return *I;
4302
1.08k
  }
4303
1.08k
  
return nullptr5
;
4304
1.08k
}
4305
4306
2.20k
unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
4307
2.20k
  if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
4308
823
    switch (cast<ConstantSDNode>(Intr->getOperand(1))->getZExtValue()) {
4309
823
    case Intrinsic::amdgcn_if:
4310
618
      return AMDGPUISD::IF;
4311
823
    case Intrinsic::amdgcn_else:
4312
64
      return AMDGPUISD::ELSE;
4313
823
    case Intrinsic::amdgcn_loop:
4314
139
      return AMDGPUISD::LOOP;
4315
823
    case Intrinsic::amdgcn_end_cf:
4316
0
      llvm_unreachable("should not occur");
4317
823
    default:
4318
2
      return 0;
4319
1.38k
    }
4320
1.38k
  }
4321
1.38k
4322
1.38k
  // break, if_break, else_break are all only used as inputs to loop, not
4323
1.38k
  // directly as branch conditions.
4324
1.38k
  return 0;
4325
1.38k
}
4326
4327
2.29k
bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
4328
2.29k
  const Triple &TT = getTargetMachine().getTargetTriple();
4329
2.29k
  return (GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
4330
2.29k
          
GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT2.19k
) &&
4331
2.29k
         
AMDGPU::shouldEmitConstantsToTextSection(TT)103
;
4332
2.29k
}
4333
4334
844
bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
4335
844
  // FIXME: Either avoid relying on address space here or change the default
4336
844
  // address space for functions to avoid the explicit check.
4337
844
  return (GV->getValueType()->isFunctionTy() ||
4338
844
          
GV->getType()->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS170
||
4339
844
          
GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS72
||
4340
844
          
GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT0
) &&
4341
844
         !shouldEmitFixup(GV) &&
4342
844
         
!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)783
;
4343
844
}
4344
4345
715
bool SITargetLowering::shouldEmitPCReloc(const GlobalValue *GV) const {
4346
715
  return !shouldEmitFixup(GV) && !shouldEmitGOTReloc(GV);
4347
715
}
4348
4349
/// This transforms the control flow intrinsics to get the branch destination as
4350
/// last parameter, also switches branch target with BR if the need arise
4351
SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND,
4352
2.20k
                                      SelectionDAG &DAG) const {
4353
2.20k
  SDLoc DL(BRCOND);
4354
2.20k
4355
2.20k
  SDNode *Intr = BRCOND.getOperand(1).getNode();
4356
2.20k
  SDValue Target = BRCOND.getOperand(2);
4357
2.20k
  SDNode *BR = nullptr;
4358
2.20k
  SDNode *SetCC = nullptr;
4359
2.20k
4360
2.20k
  if (Intr->getOpcode() == ISD::SETCC) {
4361
1.80k
    // As long as we negate the condition everything is fine
4362
1.80k
    SetCC = Intr;
4363
1.80k
    Intr = SetCC->getOperand(0).getNode();
4364
1.80k
4365
1.80k
  } else {
4366
406
    // Get the target from BR if we don't negate the condition
4367
406
    BR = findUser(BRCOND, ISD::BR);
4368
406
    Target = BR->getOperand(1);
4369
406
  }
4370
2.20k
4371
2.20k
  // FIXME: This changes the types of the intrinsics instead of introducing new
4372
2.20k
  // nodes with the correct types.
4373
2.20k
  // e.g. llvm.amdgcn.loop
4374
2.20k
4375
2.20k
  // eg: i1,ch = llvm.amdgcn.loop t0, TargetConstant:i32<6271>, t3
4376
2.20k
  // =>     t9: ch = llvm.amdgcn.loop t0, TargetConstant:i32<6271>, t3, BasicBlock:ch<bb1 0x7fee5286d088>
4377
2.20k
4378
2.20k
  unsigned CFNode = isCFIntrinsic(Intr);
4379
2.20k
  if (CFNode == 0) {
4380
1.38k
    // This is a uniform branch so we don't need to legalize.
4381
1.38k
    return BRCOND;
4382
1.38k
  }
4383
821
4384
821
  bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
4385
821
                   Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
4386
821
4387
821
  assert(!SetCC ||
4388
821
        (SetCC->getConstantOperandVal(1) == 1 &&
4389
821
         cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
4390
821
                                                             ISD::SETNE));
4391
821
4392
821
  // operands of the new intrinsic call
4393
821
  SmallVector<SDValue, 4> Ops;
4394
821
  if (HaveChain)
4395
821
    Ops.push_back(BRCOND.getOperand(0));
4396
821
4397
821
  Ops.append(Intr->op_begin() + (HaveChain ?  2 : 
10
), Intr->op_end());
4398
821
  Ops.push_back(Target);
4399
821
4400
821
  ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
4401
821
4402
821
  // build the new intrinsic call
4403
821
  SDNode *Result = DAG.getNode(CFNode, DL, DAG.getVTList(Res), Ops).getNode();
4404
821
4405
821
  if (!HaveChain) {
4406
0
    SDValue Ops[] =  {
4407
0
      SDValue(Result, 0),
4408
0
      BRCOND.getOperand(0)
4409
0
    };
4410
0
4411
0
    Result = DAG.getMergeValues(Ops, DL).getNode();
4412
0
  }
4413
821
4414
821
  if (BR) {
4415
115
    // Give the branch instruction our target
4416
115
    SDValue Ops[] = {
4417
115
      BR->getOperand(0),
4418
115
      BRCOND.getOperand(2)
4419
115
    };
4420
115
    SDValue NewBR = DAG.getNode(ISD::BR, DL, BR->getVTList(), Ops);
4421
115
    DAG.ReplaceAllUsesWith(BR, NewBR.getNode());
4422
115
    BR = NewBR.getNode();
4423
115
  }
4424
821
4425
821
  SDValue Chain = SDValue(Result, Result->getNumValues() - 1);
4426
821
4427
821
  // Copy the intrinsic results to registers
4428
1.50k
  for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; 
++i682
) {
4429
682
    SDNode *CopyToReg = findUser(SDValue(Intr, i), ISD::CopyToReg);
4430
682
    if (!CopyToReg)
4431
5
      continue;
4432
677
4433
677
    Chain = DAG.getCopyToReg(
4434
677
      Chain, DL,
4435
677
      CopyToReg->getOperand(1),
4436
677
      SDValue(Result, i - 1),
4437
677
      SDValue());
4438
677
4439
677
    DAG.ReplaceAllUsesWith(SDValue(CopyToReg, 0), CopyToReg->getOperand(0));
4440
677
  }
4441
821
4442
821
  // Remove the old intrinsic from the chain
4443
821
  DAG.ReplaceAllUsesOfValueWith(
4444
821
    SDValue(Intr, Intr->getNumValues() - 1),
4445
821
    Intr->getOperand(0));
4446
821
4447
821
  return Chain;
4448
821
}
4449
4450
SDValue SITargetLowering::LowerRETURNADDR(SDValue Op,
4451
4
                                          SelectionDAG &DAG) const {
4452
4
  MVT VT = Op.getSimpleValueType();
4453
4
  SDLoc DL(Op);
4454
4
  // Checking the depth
4455
4
  if (cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue() != 0)
4456
2
    return DAG.getConstant(0, DL, VT);
4457
2
4458
2
  MachineFunction &MF = DAG.getMachineFunction();
4459
2
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4460
2
  // Check for kernel and shader functions
4461
2
  if (Info->isEntryFunction())
4462
1
    return DAG.getConstant(0, DL, VT);
4463
1
4464
1
  MachineFrameInfo &MFI = MF.getFrameInfo();
4465
1
  // There is a call to @llvm.returnaddress in this function
4466
1
  MFI.setReturnAddressIsTaken(true);
4467
1
4468
1
  const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
4469
1
  // Get the return address reg and mark it as an implicit live-in
4470
1
  unsigned Reg = MF.addLiveIn(TRI->getReturnAddressReg(MF), getRegClassFor(VT, Op.getNode()->isDivergent()));
4471
1
4472
1
  return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
4473
1
}
4474
4475
SDValue SITargetLowering::getFPExtOrFPTrunc(SelectionDAG &DAG,
4476
                                            SDValue Op,
4477
                                            const SDLoc &DL,
4478
2.81k
                                            EVT VT) const {
4479
2.81k
  return Op.getValueType().bitsLE(VT) ?
4480
2.81k
      DAG.getNode(ISD::FP_EXTEND, DL, VT, Op) :
4481
2.81k
      
DAG.getNode(ISD::FTRUNC, DL, VT, Op)0
;
4482
2.81k
}
4483
4484
504
SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
4485
504
  assert(Op.getValueType() == MVT::f16 &&
4486
504
         "Do not know how to custom lower FP_ROUND for non-f16 type");
4487
504
4488
504
  SDValue Src = Op.getOperand(0);
4489
504
  EVT SrcVT = Src.getValueType();
4490
504
  if (SrcVT != MVT::f64)
4491
494
    return Op;
4492
10
4493
10
  SDLoc DL(Op);
4494
10
4495
10
  SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src);
4496
10
  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16);
4497
10
  return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc);
4498
10
}
4499
4500
SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op,
4501
1.30k
                                               SelectionDAG &DAG) const {
4502
1.30k
  EVT VT = Op.getValueType();
4503
1.30k
  const MachineFunction &MF = DAG.getMachineFunction();
4504
1.30k
  const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4505
1.30k
  bool IsIEEEMode = Info->getMode().IEEE;
4506
1.30k
4507
1.30k
  // FIXME: Assert during eslection that this is only selected for
4508
1.30k
  // ieee_mode. Currently a combine can produce the ieee version for non-ieee
4509
1.30k
  // mode functions, but this happens to be OK since it's only done in cases
4510
1.30k
  // where there is known no sNaN.
4511
1.30k
  if (IsIEEEMode)
4512
1.08k
    return expandFMINNUM_FMAXNUM(Op.getNode(), DAG);
4513
223
4514
223
  if (VT == MVT::v4f16)
4515
0
    return splitBinaryVectorOp(Op, DAG);
4516
223
  return Op;
4517
223
}
4518
4519
28
SDValue SITargetLowering::lowerTRAP(SDValue Op, SelectionDAG &DAG) const {
4520
28
  SDLoc SL(Op);
4521
28
  SDValue Chain = Op.getOperand(0);
4522
28
4523
28
  if (Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
4524
28
      
!Subtarget->isTrapHandlerEnabled()13
)
4525
21
    return DAG.getNode(AMDGPUISD::ENDPGM, SL, MVT::Other, Chain);
4526
7
4527
7
  MachineFunction &MF = DAG.getMachineFunction();
4528
7
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4529
7
  unsigned UserSGPR = Info->getQueuePtrUserSGPR();
4530
7
  assert(UserSGPR != AMDGPU::NoRegister);
4531
7
  SDValue QueuePtr = CreateLiveInRegister(
4532
7
    DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64);
4533
7
  SDValue SGPR01 = DAG.getRegister(AMDGPU::SGPR0_SGPR1, MVT::i64);
4534
7
  SDValue ToReg = DAG.getCopyToReg(Chain, SL, SGPR01,
4535
7
                                   QueuePtr, SDValue());
4536
7
  SDValue Ops[] = {
4537
7
    ToReg,
4538
7
    DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMTrap, SL, MVT::i16),
4539
7
    SGPR01,
4540
7
    ToReg.getValue(1)
4541
7
  };
4542
7
  return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
4543
7
}
4544
4545
10
SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const {
4546
10
  SDLoc SL(Op);
4547
10
  SDValue Chain = Op.getOperand(0);
4548
10
  MachineFunction &MF = DAG.getMachineFunction();
4549
10
4550
10
  if (Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
4551
10
      
!Subtarget->isTrapHandlerEnabled()5
) {
4552
7
    DiagnosticInfoUnsupported NoTrap(MF.getFunction(),
4553
7
                                     "debugtrap handler not supported",
4554
7
                                     Op.getDebugLoc(),
4555
7
                                     DS_Warning);
4556
7
    LLVMContext &Ctx = MF.getFunction().getContext();
4557
7
    Ctx.diagnose(NoTrap);
4558
7
    return Chain;
4559
7
  }
4560
3
4561
3
  SDValue Ops[] = {
4562
3
    Chain,
4563
3
    DAG.getTargetConstant(GCNSubtarget::TrapIDLLVMDebugTrap, SL, MVT::i16)
4564
3
  };
4565
3
  return DAG.getNode(AMDGPUISD::TRAP, SL, MVT::Other, Ops);
4566
3
}
4567
4568
SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL,
4569
44
                                             SelectionDAG &DAG) const {
4570
44
  // FIXME: Use inline constants (src_{shared, private}_base) instead.
4571
44
  if (Subtarget->hasApertureRegs()) {
4572
18
    unsigned Offset = AS == AMDGPUAS::LOCAL_ADDRESS ?
4573
5
        AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE :
4574
18
        
AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE13
;
4575
18
    unsigned WidthM1 = AS == AMDGPUAS::LOCAL_ADDRESS ?
4576
5
        AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE :
4577
18
        
AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE13
;
4578
18
    unsigned Encoding =
4579
18
        AMDGPU::Hwreg::ID_MEM_BASES << AMDGPU::Hwreg::ID_SHIFT_ |
4580
18
        Offset << AMDGPU::Hwreg::OFFSET_SHIFT_ |
4581
18
        WidthM1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_;
4582
18
4583
18
    SDValue EncodingImm = DAG.getTargetConstant(Encoding, DL, MVT::i16);
4584
18
    SDValue ApertureReg = SDValue(
4585
18
        DAG.getMachineNode(AMDGPU::S_GETREG_B32, DL, MVT::i32, EncodingImm), 0);
4586
18
    SDValue ShiftAmount = DAG.getTargetConstant(WidthM1 + 1, DL, MVT::i32);
4587
18
    return DAG.getNode(ISD::SHL, DL, MVT::i32, ApertureReg, ShiftAmount);
4588
18
  }
4589
26
4590
26
  MachineFunction &MF = DAG.getMachineFunction();
4591
26
  SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
4592
26
  unsigned UserSGPR = Info->getQueuePtrUserSGPR();
4593
26
  assert(UserSGPR != AMDGPU::NoRegister);
4594
26
4595
26
  SDValue QueuePtr = CreateLiveInRegister(
4596
26
    DAG, &AMDGPU::SReg_64RegClass, UserSGPR, MVT::i64);
4597
26
4598
26
  // Offset into amd_queue_t for group_segment_aperture_base_hi /
4599
26
  // private_segment_aperture_base_hi.
4600
26
  uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 
0x405
:
0x4421
;
4601
26
4602
26
  SDValue Ptr = DAG.getObjectPtrOffset(DL, QueuePtr, StructOffset);
4603
26
4604
26
  // TODO: Use custom target PseudoSourceValue.
4605
26
  // TODO: We should use the value from the IR intrinsic call, but it might not
4606
26
  // be available and how do we get it?
4607
26
  Value *V = UndefValue::get(PointerType::get(Type::getInt8Ty(*DAG.getContext()),
4608
26
                                              AMDGPUAS::CONSTANT_ADDRESS));
4609
26
4610
26
  MachinePointerInfo PtrInfo(V, StructOffset);
4611
26
  return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo,
4612
26
                     MinAlign(64, StructOffset),
4613
26
                     MachineMemOperand::MODereferenceable |
4614
26
                         MachineMemOperand::MOInvariant);
4615
26
}
4616
4617
SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op,
4618
58
                                             SelectionDAG &DAG) const {
4619
58
  SDLoc SL(Op);
4620
58
  const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Op);
4621
58
4622
58
  SDValue Src = ASC->getOperand(0);
4623
58
  SDValue FlatNullPtr = DAG.getConstant(0, SL, MVT::i64);
4624
58
4625
58
  const AMDGPUTargetMachine &TM =
4626
58
    static_cast<const AMDGPUTargetMachine &>(getTargetMachine());
4627
58
4628
58
  // flat -> local/private
4629
58
  if (ASC->getSrcAddressSpace() == AMDGPUAS::FLAT_ADDRESS) {
4630
12
    unsigned DestAS = ASC->getDestAddressSpace();
4631
12
4632
12
    if (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
4633
12
        
DestAS == AMDGPUAS::PRIVATE_ADDRESS5
) {
4634
12
      unsigned NullVal = TM.getNullPointerValue(DestAS);
4635
12
      SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
4636
12
      SDValue NonNull = DAG.getSetCC(SL, MVT::i1, Src, FlatNullPtr, ISD::SETNE);
4637
12
      SDValue Ptr = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, Src);
4638
12
4639
12
      return DAG.getNode(ISD::SELECT, SL, MVT::i32,
4640
12
                         NonNull, Ptr, SegmentNullPtr);
4641
12
    }
4642
46
  }
4643
46
4644
46
  // local/private -> flat
4645
46
  if (ASC->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS) {
4646
45
    unsigned SrcAS = ASC->getSrcAddressSpace();
4647
45
4648
45
    if (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
4649
45
        
SrcAS == AMDGPUAS::PRIVATE_ADDRESS35
) {
4650
44
      unsigned NullVal = TM.getNullPointerValue(SrcAS);
4651
44
      SDValue SegmentNullPtr = DAG.getConstant(NullVal, SL, MVT::i32);
4652
44
4653
44
      SDValue NonNull
4654
44
        = DAG.getSetCC(SL, MVT::i1, Src, SegmentNullPtr, ISD::SETNE);
4655
44
4656
44
      SDValue Aperture = getSegmentAperture(ASC->getSrcAddressSpace(), SL, DAG);
4657
44
      SDValue CvtPtr
4658
44
        = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Src, Aperture);
4659
44
4660
44
      return DAG.getNode(ISD::SELECT, SL, MVT::i64, NonNull,
4661
44
                         DAG.getNode(ISD::BITCAST, SL, MVT::i64, CvtPtr),
4662
44
                         FlatNullPtr);
4663
44
    }
4664
2
  }
4665
2
4666
2
  // global <-> flat are no-ops and never emitted.
4667
2
4668
2
  const MachineFunction &MF = DAG.getMachineFunction();
4669
2
  DiagnosticInfoUnsupported InvalidAddrSpaceCast(
4670
2
    MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc());
4671
2
  DAG.getContext()->diagnose(InvalidAddrSpaceCast);
4672
2
4673
2
  return DAG.getUNDEF(ASC->getValueType(0));
4674
2
}
4675
4676
// This lowers an INSERT_SUBVECTOR by extracting the individual elements from
4677
// the small vector and inserting them into the big vector. That is better than
4678
// the default expansion of doing it via a stack slot. Even though the use of
4679
// the stack slot would be optimized away afterwards, the stack slot itself
4680
// remains.
4681
SDValue SITargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
4682
440
                                                SelectionDAG &DAG) const {
4683
440
  SDValue Vec = Op.getOperand(0);
4684
440
  SDValue Ins = Op.getOperand(1);
4685
440
  SDValue Idx = Op.getOperand(2);
4686
440
  EVT VecVT = Vec.getValueType();
4687
440
  EVT InsVT = Ins.getValueType();
4688
440
  EVT EltVT = VecVT.getVectorElementType();
4689
440
  unsigned InsNumElts = InsVT.getVectorNumElements();
4690
440
  unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
4691
440
  SDLoc SL(Op);
4692
440
4693
1.38k
  for (unsigned I = 0; I != InsNumElts; 
++I942
) {
4694
942
    SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Ins,
4695
942
                              DAG.getConstant(I, SL, MVT::i32));
4696
942
    Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, VecVT, Vec, Elt,
4697
942
                      DAG.getConstant(IdxVal + I, SL, MVT::i32));
4698
942
  }
4699
440
  return Vec;
4700
440
}
4701
4702
SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4703
315
                                                 SelectionDAG &DAG) const {
4704
315
  SDValue Vec = Op.getOperand(0);
4705
315
  SDValue InsVal = Op.getOperand(1);
4706
315
  SDValue Idx = Op.getOperand(2);
4707
315
  EVT VecVT = Vec.getValueType();
4708
315
  EVT EltVT = VecVT.getVectorElementType();
4709
315
  unsigned VecSize = VecVT.getSizeInBits();
4710
315
  unsigned EltSize = EltVT.getSizeInBits();
4711
315
4712
315
4713
315
  assert(VecSize <= 64);
4714
315
4715
315
  unsigned NumElts = VecVT.getVectorNumElements();
4716
315
  SDLoc SL(Op);
4717
315
  auto KIdx = dyn_cast<ConstantSDNode>(Idx);
4718
315
4719
315
  if (NumElts == 4 && 
EltSize == 1647
&&
KIdx35
) {
4720
21
    SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Vec);
4721
21
4722
21
    SDValue LoHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
4723
21
                                 DAG.getConstant(0, SL, MVT::i32));
4724
21
    SDValue HiHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BCVec,
4725
21
                                 DAG.getConstant(1, SL, MVT::i32));
4726
21
4727
21
    SDValue LoVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, LoHalf);
4728
21
    SDValue HiVec = DAG.getNode(ISD::BITCAST, SL, MVT::v2i16, HiHalf);
4729
21
4730
21
    unsigned Idx = KIdx->getZExtValue();
4731
21
    bool InsertLo = Idx < 2;
4732
21
    SDValue InsHalf = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, MVT::v2i16,
4733
21
      InsertLo ? 
LoVec8
:
HiVec13
,
4734
21
      DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal),
4735
21
      DAG.getConstant(InsertLo ? 
Idx8
:
(Idx - 2)13
, SL, MVT::i32));
4736
21
4737
21
    InsHalf = DAG.getNode(ISD::BITCAST, SL, MVT::i32, InsHalf);
4738
21
4739
21
    SDValue Concat = InsertLo ?
4740
8
      DAG.getBuildVector(MVT::v2i32, SL, { InsHalf, HiHalf }) :
4741
21
      
DAG.getBuildVector(MVT::v2i32, SL, { LoHalf, InsHalf })13
;
4742
21
4743
21
    return DAG.getNode(ISD::BITCAST, SL, VecVT, Concat);
4744
21
  }
4745
294
4746
294
  if (isa<ConstantSDNode>(Idx))
4747
240
    return SDValue();
4748
54
4749
54
  MVT IntVT = MVT::getIntegerVT(VecSize);
4750
54
4751
54
  // Avoid stack access for dynamic indexing.
4752
54
  // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
4753
54
4754
54
  // Create a congruent vector with the target value in each element so that
4755
54
  // the required element can be masked and ORed into the target vector.
4756
54
  SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT,
4757
54
                               DAG.getSplatBuildVector(VecVT, SL, InsVal));
4758
54
4759
54
  assert(isPowerOf2_32(EltSize));
4760
54
  SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
4761
54
4762
54
  // Convert vector index to bit-index.
4763
54
  SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
4764
54
4765
54
  SDValue BCVec = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
4766
54
  SDValue BFM = DAG.getNode(ISD::SHL, SL, IntVT,
4767
54
                            DAG.getConstant(0xffff, SL, IntVT),
4768
54
                            ScaledIdx);
4769
54
4770
54
  SDValue LHS = DAG.getNode(ISD::AND, SL, IntVT, BFM, ExtVal);
4771
54
  SDValue RHS = DAG.getNode(ISD::AND, SL, IntVT,
4772
54
                            DAG.getNOT(SL, BFM, IntVT), BCVec);
4773
54
4774
54
  SDValue BFI = DAG.getNode(ISD::OR, SL, IntVT, LHS, RHS);
4775
54
  return DAG.getNode(ISD::BITCAST, SL, VecVT, BFI);
4776
54
}
4777
4778
SDValue SITargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4779
7.48k
                                                  SelectionDAG &DAG) const {
4780
7.48k
  SDLoc SL(Op);
4781
7.48k
4782
7.48k
  EVT ResultVT = Op.getValueType();
4783
7.48k
  SDValue Vec = Op.getOperand(0);
4784
7.48k
  SDValue Idx = Op.getOperand(1);
4785
7.48k
  EVT VecVT = Vec.getValueType();
4786
7.48k
  unsigned VecSize = VecVT.getSizeInBits();
4787
7.48k
  EVT EltVT = VecVT.getVectorElementType();
4788
7.48k
  assert(VecSize <= 64);
4789
7.48k
4790
7.48k
  DAGCombinerInfo DCI(DAG, AfterLegalizeVectorOps, true, nullptr);
4791
7.48k
4792
7.48k
  // Make sure we do any optimizations that will make it easier to fold
4793
7.48k
  // source modifiers before obscuring it with bit operations.
4794
7.48k
4795
7.48k
  // XXX - Why doesn't this get called when vector_shuffle is expanded?
4796
7.48k
  if (SDValue Combined = performExtractVectorEltCombine(Op.getNode(), DCI))
4797
7
    return Combined;
4798
7.47k
4799
7.47k
  unsigned EltSize = EltVT.getSizeInBits();
4800
7.47k
  assert(isPowerOf2_32(EltSize));
4801
7.47k
4802
7.47k
  MVT IntVT = MVT::getIntegerVT(VecSize);
4803
7.47k
  SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
4804
7.47k
4805
7.47k
  // Convert vector index to bit-index (* EltSize)
4806
7.47k
  SDValue ScaledIdx = DAG.getNode(ISD::SHL, SL, MVT::i32, Idx, ScaleFactor);
4807
7.47k
4808
7.47k
  SDValue BC = DAG.getNode(ISD::BITCAST, SL, IntVT, Vec);
4809
7.47k
  SDValue Elt = DAG.getNode(ISD::SRL, SL, IntVT, BC, ScaledIdx);
4810
7.47k
4811
7.47k
  if (ResultVT == MVT::f16) {
4812
1.82k
    SDValue Result = DAG.getNode(ISD::TRUNCATE, SL, MVT::i16, Elt);
4813
1.82k
    return DAG.getNode(ISD::BITCAST, SL, ResultVT, Result);
4814
1.82k
  }
4815
5.65k
4816
5.65k
  return DAG.getAnyExtOrTrunc(Elt, SL, ResultVT);
4817
5.65k
}
4818
4819
100
static bool elementPairIsContiguous(ArrayRef<int> Mask, int Elt) {
4820
100
  assert(Elt % 2 == 0);
4821
100
  return Mask[Elt + 1] == Mask[Elt] + 1 && 
(Mask[Elt] % 2 == 0)67
;
4822
100
}
4823
4824
SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
4825
50
                                              SelectionDAG &DAG) const {
4826
50
  SDLoc SL(Op);
4827
50
  EVT ResultVT = Op.getValueType();
4828
50
  ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
4829
50
4830
50
  EVT PackVT = ResultVT.isInteger() ? 
MVT::v2i165
:
MVT::v2f1645
;
4831
50
  EVT EltVT = PackVT.getVectorElementType();
4832
50
  int SrcNumElts = Op.getOperand(0).getValueType().getVectorNumElements();
4833
50
4834
50
  // vector_shuffle <0,1,6,7> lhs, rhs
4835
50
  // -> concat_vectors (extract_subvector lhs, 0), (extract_subvector rhs, 2)
4836
50
  //
4837
50
  // vector_shuffle <6,7,2,3> lhs, rhs
4838
50
  // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 2)
4839
50
  //
4840
50
  // vector_shuffle <6,7,0,1> lhs, rhs
4841
50
  // -> concat_vectors (extract_subvector rhs, 2), (extract_subvector lhs, 0)
4842
50
4843
50
  // Avoid scalarizing when both halves are reading from consecutive elements.
4844
50
  SmallVector<SDValue, 4> Pieces;
4845
150
  for (int I = 0, N = ResultVT.getVectorNumElements(); I != N; 
I += 2100
) {
4846
100
    if (elementPairIsContiguous(SVN->getMask(), I)) {
4847
57
      const int Idx = SVN->getMaskElt(I);
4848
57
      int VecIdx = Idx < SrcNumElts ? 
043
:
114
;
4849
57
      int EltIdx = Idx < SrcNumElts ? 
Idx43
:
Idx - SrcNumElts14
;
4850
57
      SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL,
4851
57
                                    PackVT, SVN->getOperand(VecIdx),
4852
57
                                    DAG.getConstant(EltIdx, SL, MVT::i32));
4853
57
      Pieces.push_back(SubVec);
4854
57
    } else {
4855
43
      const int Idx0 = SVN->getMaskElt(I);
4856
43
      const int Idx1 = SVN->getMaskElt(I + 1);
4857
43
      int VecIdx0 = Idx0 < SrcNumElts ? 
030
:
113
;
4858
43
      int VecIdx1 = Idx1 < SrcNumElts ? 
027
:
116
;
4859
43
      int EltIdx0 = Idx0 < SrcNumElts ? 
Idx030
:
Idx0 - SrcNumElts13
;
4860
43
      int EltIdx1 = Idx1 < SrcNumElts ? 
Idx127
:
Idx1 - SrcNumElts16
;
4861
43
4862
43
      SDValue Vec0 = SVN->getOperand(VecIdx0);
4863
43
      SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
4864
43
                                 Vec0, DAG.getConstant(EltIdx0, SL, MVT::i32));
4865
43
4866
43
      SDValue Vec1 = SVN->getOperand(VecIdx1);
4867
43
      SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
4868
43
                                 Vec1, DAG.getConstant(EltIdx1, SL, MVT::i32));
4869
43
      Pieces.push_back(DAG.getBuildVector(PackVT, SL, { Elt0, Elt1 }));
4870
43
    }
4871
100
  }
4872
50
4873
50
  return DAG.getNode(ISD::CONCAT_VECTORS, SL, ResultVT, Pieces);
4874
50
}
4875
4876
SDValue SITargetLowering::lowerBUILD_VECTOR(SDValue Op,
4877
1.39k
                                            SelectionDAG &DAG) const {
4878
1.39k
  SDLoc SL(Op);
4879
1.39k
  EVT VT = Op.getValueType();
4880
1.39k
4881
1.39k
  if (VT == MVT::v4i16 || 
VT == MVT::v4f161.14k
) {
4882
338
    EVT HalfVT = MVT::getVectorVT(VT.getVectorElementType().getSimpleVT(), 2);
4883
338
4884
338
    // Turn into pair of packed build_vectors.
4885
338
    // TODO: Special case for constants that can be materialized with s_mov_b64.
4886
338
    SDValue Lo = DAG.getBuildVector(HalfVT, SL,
4887
338
                                    { Op.getOperand(0), Op.getOperand(1) });
4888
338
    SDValue Hi = DAG.getBuildVector(HalfVT, SL,
4889
338
                                    { Op.getOperand(2), Op.getOperand(3) });
4890
338
4891
338
    SDValue CastLo = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Lo);
4892
338
    SDValue CastHi = DAG.getNode(ISD::BITCAST, SL, MVT::i32, Hi);
4893
338
4894
338
    SDValue Blend = DAG.getBuildVector(MVT::v2i32, SL, { CastLo, CastHi });
4895
338
    return DAG.getNode(ISD::BITCAST, SL, VT, Blend);
4896
338
  }
4897
1.05k
4898
1.05k
  assert(VT == MVT::v2f16 || VT == MVT::v2i16);
4899
1.05k
  assert(!Subtarget->hasVOP3PInsts() && "this should be legal");
4900
1.05k
4901
1.05k
  SDValue Lo = Op.getOperand(0);
4902
1.05k
  SDValue Hi = Op.getOperand(1);
4903
1.05k
4904
1.05k
  // Avoid adding defined bits with the zero_extend.
4905
1.05k
  if (Hi.isUndef()) {
4906
26
    Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
4907
26
    SDValue ExtLo = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Lo);
4908
26
    return DAG.getNode(ISD::BITCAST, SL, VT, ExtLo);
4909
26
  }
4910
1.03k
4911
1.03k
  Hi = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Hi);
4912
1.03k
  Hi = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Hi);
4913
1.03k
4914
1.03k
  SDValue ShlHi = DAG.getNode(ISD::SHL, SL, MVT::i32, Hi,
4915
1.03k
                              DAG.getConstant(16, SL, MVT::i32));
4916
1.03k
  if (Lo.isUndef())
4917
12
    return DAG.getNode(ISD::BITCAST, SL, VT, ShlHi);
4918
1.01k
4919
1.01k
  Lo = DAG.getNode(ISD::BITCAST, SL, MVT::i16, Lo);
4920
1.01k
  Lo = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Lo);
4921
1.01k
4922
1.01k
  SDValue Or = DAG.getNode(ISD::OR, SL, MVT::i32, Lo, ShlHi);
4923
1.01k
  return DAG.getNode(ISD::BITCAST, SL, VT, Or);
4924
1.01k
}
4925
4926
bool
4927
1.79k
SITargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
4928
1.79k
  // We can fold offsets for anything that doesn't require a GOT relocation.
4929
1.79k
  return (GA->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
4930
1.79k
          
GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS1.73k
||
4931
1.79k
          
GA->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT1.66k
) &&
4932
1.79k
         
!shouldEmitGOTReloc(GA->getGlobal())129
;
4933
1.79k
}
4934
4935
static SDValue
4936
buildPCRelGlobalAddress(SelectionDAG &DAG, const GlobalValue *GV,
4937
                        const SDLoc &DL, unsigned Offset, EVT PtrVT,
4938
740
                        unsigned GAFlags = SIInstrInfo::MO_NONE) {
4939
740
  // In order to support pc-relative addressing, the PC_ADD_REL_OFFSET SDNode is
4940
740
  // lowered to the following code sequence:
4941
740
  //
4942
740
  // For constant address space:
4943
740
  //   s_getpc_b64 s[0:1]
4944
740
  //   s_add_u32 s0, s0, $symbol
4945
740
  //   s_addc_u32 s1, s1, 0
4946
740
  //
4947
740
  //   s_getpc_b64 returns the address of the s_add_u32 instruction and then
4948
740
  //   a fixup or relocation is emitted to replace $symbol with a literal
4949
740
  //   constant, which is a pc-relative offset from the encoding of the $symbol
4950
740
  //   operand to the global variable.
4951
740
  //
4952
740
  // For global address space:
4953
740
  //   s_getpc_b64 s[0:1]
4954
740
  //   s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
4955
740
  //   s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
4956
740
  //
4957
740
  //   s_getpc_b64 returns the address of the s_add_u32 instruction and then
4958
740
  //   fixups or relocations are emitted to replace $symbol@*@lo and
4959
740
  //   $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
4960
740
  //   which is a 64-bit pc-relative offset from the encoding of the $symbol
4961
740
  //   operand to the global variable.
4962
740
  //
4963
740
  // What we want here is an offset from the value returned by s_getpc
4964
740
  // (which is the address of the s_add_u32 instruction) to the global
4965
740
  // variable, but since the encoding of $symbol starts 4 bytes after the start
4966
740
  // of the s_add_u32 instruction, we end up with an offset that is 4 bytes too
4967
740
  // small. This requires us to add 4 to the global variable offset in order to
4968
740
  // compute the correct address.
4969
740
  unsigned LoFlags = GAFlags;
4970
740
  if (LoFlags == SIInstrInfo::MO_NONE)
4971
25
    LoFlags = SIInstrInfo::MO_REL32;
4972
740
  SDValue PtrLo =
4973
740
      DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 4, LoFlags);
4974
740
  SDValue PtrHi;
4975
740
  if (GAFlags == SIInstrInfo::MO_NONE) {
4976
25
    PtrHi = DAG.getTargetConstant(0, DL, MVT::i32);
4977
715
  } else {
4978
715
    PtrHi =
4979
715
        DAG.getTargetGlobalAddress(GV, DL, MVT::i32, Offset + 4, GAFlags + 1);
4980
715
  }
4981
740
  return DAG.getNode(AMDGPUISD::PC_ADD_REL_OFFSET, DL, PtrVT, PtrLo, PtrHi);
4982
740
}
4983
4984
SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI,
4985
                                             SDValue Op,
4986
1.19k
                                             SelectionDAG &DAG) const {
4987
1.19k
  GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Op);
4988
1.19k
  const GlobalValue *GV = GSD->getGlobal();
4989
1.19k
  if ((GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
4990
1.19k
       
(453
!GV->hasExternalLinkage()453
||
4991
453
        
getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA310
||
4992
453
        
getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL295
)) ||
4993
1.19k
      
GSD->getAddressSpace() == AMDGPUAS::REGION_ADDRESS999
||
4994
1.19k
      
GSD->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS999
)
4995
194
    return AMDGPUTargetLowering::LowerGlobalAddress(MFI, Op, DAG);
4996
999
4997
999
  SDLoc DL(GSD);
4998
999
  EVT PtrVT = Op.getValueType();
4999
999
5000
999
  if (GSD->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
5001
259
    SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i32, GSD->getOffset(),
5002
259
                                            SIInstrInfo::MO_ABS32_LO);
5003
259
    return DAG.getNode(AMDGPUISD::LDS, DL, MVT::i32, GA);
5004
259
  }
5005
740
5006
740
  if (shouldEmitFixup(GV))
5007
25
    return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT);
5008
715
  else if (shouldEmitPCReloc(GV))
5009
385
    return buildPCRelGlobalAddress(DAG, GV, DL, GSD->getOffset(), PtrVT,
5010
385
                                   SIInstrInfo::MO_REL32);
5011
330
5012
330
  SDValue GOTAddr = buildPCRelGlobalAddress(DAG, GV, DL, 0, PtrVT,
5013
330
                                            SIInstrInfo::MO_GOTPCREL32);
5014
330
5015
330
  Type *Ty = PtrVT.getTypeForEVT(*DAG.getContext());
5016
330
  PointerType *PtrTy = PointerType::get(Ty, AMDGPUAS::CONSTANT_ADDRESS);
5017
330
  const DataLayout &DataLayout = DAG.getDataLayout();
5018
330
  unsigned Align = DataLayout.getABITypeAlignment(PtrTy);
5019
330
  MachinePointerInfo PtrInfo
5020
330
    = MachinePointerInfo::getGOT(DAG.getMachineFunction());
5021
330
5022
330
  return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Align,
5023
330
                     MachineMemOperand::MODereferenceable |
5024
330
                         MachineMemOperand::MOInvariant);
5025
330
}
5026
5027
SDValue SITargetLowering::copyToM0(SelectionDAG &DAG, SDValue Chain,
5028
9.73k
                                   const SDLoc &DL, SDValue V) const {
5029
9.73k
  // We can't use S_MOV_B32 directly, because there is no way to specify m0 as
5030
9.73k
  // the destination register.
5031
9.73k
  //
5032
9.73k
  // We can't use CopyToReg, because MachineCSE won't combine COPY instructions,
5033
9.73k
  // so we will end up with redundant moves to m0.
5034
9.73k
  //
5035
9.73k
  // We use a pseudo to ensure we emit s_mov_b32 with m0 as the direct result.
5036
9.73k
5037
9.73k
  // A Null SDValue creates a glue result.
5038
9.73k
  SDNode *M0 = DAG.getMachineNode(AMDGPU::SI_INIT_M0, DL, MVT::Other, MVT::Glue,
5039
9.73k
                                  V, Chain);
5040
9.73k
  return SDValue(M0, 0);
5041
9.73k
}
5042
5043
SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG,
5044
                                                 SDValue Op,
5045
                                                 MVT VT,
5046
91
                                                 unsigned Offset) const {
5047
91
  SDLoc SL(Op);
5048
91
  SDValue Param = lowerKernargMemParameter(DAG, MVT::i32, MVT::i32, SL,
5049
91
                                           DAG.getEntryNode(), Offset, 4, false);
5050
91
  // The local size values will have the hi 16-bits as zero.
5051
91
  return DAG.getNode(ISD::AssertZext, SL, MVT::i32, Param,
5052
91
                     DAG.getValueType(VT));
5053
91
}
5054
5055
static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
5056
2
                                        EVT VT) {
5057
2
  DiagnosticInfoUnsupported BadIntrin(DAG.getMachineFunction().getFunction(),
5058
2
                                      "non-hsa intrinsic with hsa target",
5059
2
                                      DL.getDebugLoc());
5060
2
  DAG.getContext()->diagnose(BadIntrin);
5061
2
  return DAG.getUNDEF(VT);
5062
2
}
5063
5064
static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
5065
5
                                         EVT VT) {
5066
5
  DiagnosticInfoUnsupported BadIntrin(DAG.getMachineFunction().getFunction(),
5067
5
                                      "intrinsic not supported on subtarget",
5068
5
                                      DL.getDebugLoc());
5069
5
  DAG.getContext()->diagnose(BadIntrin);
5070
5
  return DAG.getUNDEF(VT);
5071
5
}
5072
5073
static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL,
5074
1.16k
                                    ArrayRef<SDValue> Elts) {
5075
1.16k
  assert(!Elts.empty());
5076
1.16k
  MVT Type;
5077
1.16k
  unsigned NumElts;
5078
1.16k
5079
1.16k
  if (Elts.size() == 1) {
5080
451
    Type = MVT::f32;
5081
451
    NumElts = 1;
5082
715
  } else if (Elts.size() == 2) {
5083
274
    Type = MVT::v2f32;
5084
274
    NumElts = 2;
5085
441
  } else if (Elts.size() <= 4) {
5086
331
    Type = MVT::v4f32;
5087
331
    NumElts = 4;
5088
331
  } else 
if (110
Elts.size() <= 8110
) {
5089
97
    Type = MVT::v8f32;
5090
97
    NumElts = 8;
5091
97
  } else {
5092
13
    assert(Elts.size() <= 16);
5093
13
    Type = MVT::v16f32;
5094
13
    NumElts = 16;
5095
13
  }
5096
1.16k
5097
1.16k
  SmallVector<SDValue, 16> VecElts(NumElts);
5098
3.96k
  for (unsigned i = 0; i < Elts.size(); 
++i2.80k
) {
5099
2.80k
    SDValue Elt = Elts[i];
5100
2.80k
    if (Elt.getValueType() != MVT::f32)
5101
1.28k
      Elt = DAG.getBitcast(MVT::f32, Elt);
5102
2.80k
    VecElts[i] = Elt;
5103
2.80k
  }
5104
1.67k
  for (unsigned i = Elts.size(); i < NumElts; 
++i504
)
5105
504
    VecElts[i] = DAG.getUNDEF(MVT::f32);
5106
1.16k
5107
1.16k
  if (NumElts == 1)
5108
451
    return VecElts[0];
5109
715
  return DAG.getBuildVector(Type, DL, VecElts);
5110
715
}
5111
5112
static bool parseCachePolicy(SDValue CachePolicy, SelectionDAG &DAG,
5113
1.89k
                             SDValue *GLC, SDValue *SLC, SDValue *DLC) {
5114
1.89k
  auto CachePolicyConst = cast<ConstantSDNode>(CachePolicy.getNode());
5115
1.89k
5116
1.89k
  uint64_t Value = CachePolicyConst->getZExtValue();
5117
1.89k
  SDLoc DL(CachePolicy);
5118
1.89k
  if (GLC) {
5119
1.83k
    *GLC = DAG.getTargetConstant((Value & 0x1) ? 
138
:
01.79k
, DL, MVT::i32);
5120
1.83k
    Value &= ~(uint64_t)0x1;
5121
1.83k
  }
5122
1.89k
  if (SLC) {
5123
1.26k
    *SLC = DAG.getTargetConstant((Value & 0x2) ? 
133
:
01.23k
, DL, MVT::i32);
5124
1.26k
    Value &= ~(uint64_t)0x2;
5125
1.26k
  }
5126
1.89k
  if (DLC) {
5127
297
    *DLC = DAG.getTargetConstant((Value & 0x4) ? 
14