Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
9
#include "AMDGPU.h"
10
#include "AMDKernelCodeT.h"
11
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
12
#include "MCTargetDesc/AMDGPUTargetStreamer.h"
13
#include "SIDefines.h"
14
#include "SIInstrInfo.h"
15
#include "TargetInfo/AMDGPUTargetInfo.h"
16
#include "Utils/AMDGPUAsmUtils.h"
17
#include "Utils/AMDGPUBaseInfo.h"
18
#include "Utils/AMDKernelCodeTUtils.h"
19
#include "llvm/ADT/APFloat.h"
20
#include "llvm/ADT/APInt.h"
21
#include "llvm/ADT/ArrayRef.h"
22
#include "llvm/ADT/STLExtras.h"
23
#include "llvm/ADT/SmallBitVector.h"
24
#include "llvm/ADT/SmallString.h"
25
#include "llvm/ADT/StringRef.h"
26
#include "llvm/ADT/StringSwitch.h"
27
#include "llvm/ADT/Twine.h"
28
#include "llvm/BinaryFormat/ELF.h"
29
#include "llvm/MC/MCAsmInfo.h"
30
#include "llvm/MC/MCContext.h"
31
#include "llvm/MC/MCExpr.h"
32
#include "llvm/MC/MCInst.h"
33
#include "llvm/MC/MCInstrDesc.h"
34
#include "llvm/MC/MCInstrInfo.h"
35
#include "llvm/MC/MCParser/MCAsmLexer.h"
36
#include "llvm/MC/MCParser/MCAsmParser.h"
37
#include "llvm/MC/MCParser/MCAsmParserExtension.h"
38
#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
39
#include "llvm/MC/MCParser/MCTargetAsmParser.h"
40
#include "llvm/MC/MCRegisterInfo.h"
41
#include "llvm/MC/MCStreamer.h"
42
#include "llvm/MC/MCSubtargetInfo.h"
43
#include "llvm/MC/MCSymbol.h"
44
#include "llvm/Support/AMDGPUMetadata.h"
45
#include "llvm/Support/AMDHSAKernelDescriptor.h"
46
#include "llvm/Support/Casting.h"
47
#include "llvm/Support/Compiler.h"
48
#include "llvm/Support/ErrorHandling.h"
49
#include "llvm/Support/MachineValueType.h"
50
#include "llvm/Support/MathExtras.h"
51
#include "llvm/Support/SMLoc.h"
52
#include "llvm/Support/TargetParser.h"
53
#include "llvm/Support/TargetRegistry.h"
54
#include "llvm/Support/raw_ostream.h"
55
#include <algorithm>
56
#include <cassert>
57
#include <cstdint>
58
#include <cstring>
59
#include <iterator>
60
#include <map>
61
#include <memory>
62
#include <string>
63
64
using namespace llvm;
65
using namespace llvm::AMDGPU;
66
using namespace llvm::amdhsa;
67
68
namespace {
69
70
class AMDGPUAsmParser;
71
72
enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_AGPR, IS_TTMP, IS_SPECIAL };
73
74
//===----------------------------------------------------------------------===//
75
// Operand
76
//===----------------------------------------------------------------------===//
77
78
class AMDGPUOperand : public MCParsedAsmOperand {
79
  enum KindTy {
80
    Token,
81
    Immediate,
82
    Register,
83
    Expression
84
  } Kind;
85
86
  SMLoc StartLoc, EndLoc;
87
  const AMDGPUAsmParser *AsmParser;
88
89
public:
90
  AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
91
1.56M
    : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
92
93
  using Ptr = std::unique_ptr<AMDGPUOperand>;
94
95
  struct Modifiers {
96
    bool Abs = false;
97
    bool Neg = false;
98
    bool Sext = false;
99
100
2.18M
    bool hasFPModifiers() const { return Abs || 
Neg2.17M
; }
101
2.02M
    bool hasIntModifiers() const { return Sext; }
102
1.66M
    bool hasModifiers() const { return hasFPModifiers() || 
hasIntModifiers()1.65M
; }
103
104
11.9k
    int64_t getFPModifiersOperand() const {
105
11.9k
      int64_t Operand = 0;
106
11.9k
      Operand |= Abs ? 
SISrcMods::ABS4.37k
:
0u7.52k
;
107
11.9k
      Operand |= Neg ? 
SISrcMods::NEG7.65k
:
0u4.24k
;
108
11.9k
      return Operand;
109
11.9k
    }
110
111
1.38k
    int64_t getIntModifiersOperand() const {
112
1.38k
      int64_t Operand = 0;
113
1.38k
      Operand |= Sext ? SISrcMods::SEXT : 
0u0
;
114
1.38k
      return Operand;
115
1.38k
    }
116
117
234k
    int64_t getModifiersOperand() const {
118
234k
      assert(!(hasFPModifiers() && hasIntModifiers())
119
234k
           && "fp and int modifiers should not be used simultaneously");
120
234k
      if (hasFPModifiers()) {
121
11.9k
        return getFPModifiersOperand();
122
222k
      } else if (hasIntModifiers()) {
123
1.38k
        return getIntModifiersOperand();
124
221k
      } else {
125
221k
        return 0;
126
221k
      }
127
234k
    }
128
129
    friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
130
  };
131
132
  enum ImmTy {
133
    ImmTyNone,
134
    ImmTyGDS,
135
    ImmTyLDS,
136
    ImmTyOffen,
137
    ImmTyIdxen,
138
    ImmTyAddr64,
139
    ImmTyOffset,
140
    ImmTyInstOffset,
141
    ImmTyOffset0,
142
    ImmTyOffset1,
143
    ImmTyDLC,
144
    ImmTyGLC,
145
    ImmTySLC,
146
    ImmTyTFE,
147
    ImmTyD16,
148
    ImmTyClampSI,
149
    ImmTyOModSI,
150
    ImmTyDPP8,
151
    ImmTyDppCtrl,
152
    ImmTyDppRowMask,
153
    ImmTyDppBankMask,
154
    ImmTyDppBoundCtrl,
155
    ImmTyDppFi,
156
    ImmTySdwaDstSel,
157
    ImmTySdwaSrc0Sel,
158
    ImmTySdwaSrc1Sel,
159
    ImmTySdwaDstUnused,
160
    ImmTyDMask,
161
    ImmTyDim,
162
    ImmTyUNorm,
163
    ImmTyDA,
164
    ImmTyR128A16,
165
    ImmTyLWE,
166
    ImmTyExpTgt,
167
    ImmTyExpCompr,
168
    ImmTyExpVM,
169
    ImmTyFORMAT,
170
    ImmTyHwreg,
171
    ImmTyOff,
172
    ImmTySendMsg,
173
    ImmTyInterpSlot,
174
    ImmTyInterpAttr,
175
    ImmTyAttrChan,
176
    ImmTyOpSel,
177
    ImmTyOpSelHi,
178
    ImmTyNegLo,
179
    ImmTyNegHi,
180
    ImmTySwizzle,
181
    ImmTyGprIdxMode,
182
    ImmTyHigh,
183
    ImmTyBLGP,
184
    ImmTyCBSZ,
185
    ImmTyABID,
186
    ImmTyEndpgm,
187
  };
188
189
private:
190
  struct TokOp {
191
    const char *Data;
192
    unsigned Length;
193
  };
194
195
  struct ImmOp {
196
    int64_t Val;
197
    ImmTy Type;
198
    bool IsFPImm;
199
    Modifiers Mods;
200
  };
201
202
  struct RegOp {
203
    unsigned RegNo;
204
    Modifiers Mods;
205
  };
206
207
  union {
208
    TokOp Tok;
209
    ImmOp Imm;
210
    RegOp Reg;
211
    const MCExpr *Expr;
212
  };
213
214
public:
215
2.55M
  bool isToken() const override {
216
2.55M
    if (Kind == Token)
217
1.37k
      return true;
218
2.55M
219
2.55M
    // When parsing operands, we can't always tell if something was meant to be
220
2.55M
    // a token, like 'gds', or an expression that references a global variable.
221
2.55M
    // In this case, we assume the string is an expression, and if we need to
222
2.55M
    // interpret is a token, then we treat the symbol name as the token.
223
2.55M
    return isSymbolRefExpr();
224
2.55M
  }
225
226
2.55M
  bool isSymbolRefExpr() const {
227
2.55M
    return isExpr() && 
Expr28.0k
&&
isa<MCSymbolRefExpr>(Expr)28.0k
;
228
2.55M
  }
229
230
1.20M
  bool isImm() const override {
231
1.20M
    return Kind == Immediate;
232
1.20M
  }
233
234
  bool isInlinableImm(MVT type) const;
235
  bool isLiteralImm(MVT type) const;
236
237
5.14M
  bool isRegKind() const {
238
5.14M
    return Kind == Register;
239
5.14M
  }
240
241
1.36M
  bool isReg() const override {
242
1.36M
    return isRegKind() && 
!hasModifiers()1.12M
;
243
1.36M
  }
244
245
203k
  bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const {
246
203k
    return isRegClass(RCID) || 
isInlinableImm(type)26.3k
||
isLiteralImm(type)6.50k
;
247
203k
  }
248
249
0
  bool isRegOrImmWithInt16InputMods() const {
250
0
    return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i16);
251
0
  }
252
253
3.27k
  bool isRegOrImmWithInt32InputMods() const {
254
3.27k
    return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i32);
255
3.27k
  }
256
257
0
  bool isRegOrImmWithInt64InputMods() const {
258
0
    return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::i64);
259
0
  }
260
261
47.4k
  bool isRegOrImmWithFP16InputMods() const {
262
47.4k
    return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f16);
263
47.4k
  }
264
265
101k
  bool isRegOrImmWithFP32InputMods() const {
266
101k
    return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f32);
267
101k
  }
268
269
51.9k
  bool isRegOrImmWithFP64InputMods() const {
270
51.9k
    return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::f64);
271
51.9k
  }
272
273
0
  bool isVReg() const {
274
0
    return isRegClass(AMDGPU::VGPR_32RegClassID) ||
275
0
           isRegClass(AMDGPU::VReg_64RegClassID) ||
276
0
           isRegClass(AMDGPU::VReg_96RegClassID) ||
277
0
           isRegClass(AMDGPU::VReg_128RegClassID) ||
278
0
           isRegClass(AMDGPU::VReg_256RegClassID) ||
279
0
           isRegClass(AMDGPU::VReg_512RegClassID);
280
0
  }
281
282
38.8k
  bool isVReg32() const {
283
38.8k
    return isRegClass(AMDGPU::VGPR_32RegClassID);
284
38.8k
  }
285
286
3.11k
  bool isVReg32OrOff() const {
287
3.11k
    return isOff() || 
isVReg32()2.46k
;
288
3.11k
  }
289
290
  bool isSDWAOperand(MVT type) const;
291
  bool isSDWAFP16Operand() const;
292
  bool isSDWAFP32Operand() const;
293
  bool isSDWAInt16Operand() const;
294
  bool isSDWAInt32Operand() const;
295
296
941k
  bool isImmTy(ImmTy ImmT) const {
297
941k
    return isImm() && 
Imm.Type == ImmT705k
;
298
941k
  }
299
300
14.2k
  bool isImmModifier() const {
301
14.2k
    return isImm() && 
Imm.Type != ImmTyNone12.4k
;
302
14.2k
  }
303
304
87.0k
  bool isClampSI() const { return isImmTy(ImmTyClampSI); }
305
41.9k
  bool isOModSI() const { return isImmTy(ImmTyOModSI); }
306
4.93k
  bool isDMask() const { return isImmTy(ImmTyDMask); }
307
158
  bool isDim() const { return isImmTy(ImmTyDim); }
308
2.71k
  bool isUNorm() const { return isImmTy(ImmTyUNorm); }
309
678
  bool isDA() const { return isImmTy(ImmTyDA); }
310
1.11k
  bool isR128A16() const { return isImmTy(ImmTyR128A16); }
311
926
  bool isLWE() const { return isImmTy(ImmTyLWE); }
312
3.54k
  bool isOff() const { return isImmTy(ImmTyOff); }
313
833
  bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
314
135
  bool isExpVM() const { return isImmTy(ImmTyExpVM); }
315
161
  bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
316
2.08k
  bool isOffen() const { return isImmTy(ImmTyOffen); }
317
4.49k
  bool isIdxen() const { return isImmTy(ImmTyIdxen); }
318
582
  bool isAddr64() const { return isImmTy(ImmTyAddr64); }
319
32.0k
  bool isOffset() const { return isImmTy(ImmTyOffset) && 
isUInt<16>(getImm())30.2k
; }
320
2.76k
  bool isOffset0() const { return isImmTy(ImmTyOffset0) && 
isUInt<8>(getImm())2.49k
; }
321
2.55k
  bool isOffset1() const { return isImmTy(ImmTyOffset1) && 
isUInt<8>(getImm())2.50k
; }
322
323
5.65k
  bool isFlatOffset() const { return isImmTy(ImmTyOffset) || 
isImmTy(ImmTyInstOffset)1.63k
; }
324
5.59k
  bool isGDS() const { return isImmTy(ImmTyGDS); }
325
18.7k
  bool isLDS() const { return isImmTy(ImmTyLDS); }
326
3.47k
  bool isDLC() const { return isImmTy(ImmTyDLC); }
327
8.89k
  bool isGLC() const { return isImmTy(ImmTyGLC); }
328
6.79k
  bool isSLC() const { return isImmTy(ImmTySLC); }
329
1.78k
  bool isTFE() const { return isImmTy(ImmTyTFE); }
330
354
  bool isD16() const { return isImmTy(ImmTyD16); }
331
203
  bool isFORMAT() const { return isImmTy(ImmTyFORMAT) && isUInt<8>(getImm()); }
332
28.1k
  bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
333
28.9k
  bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
334
2.19k
  bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
335
2.11k
  bool isFI() const { return isImmTy(ImmTyDppFi); }
336
52.2k
  bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
337
84.1k
  bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
338
55.3k
  bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
339
52.2k
  bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
340
1.56k
  bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
341
2.95k
  bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
342
2.07k
  bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
343
3.09k
  bool isOpSel() const { return isImmTy(ImmTyOpSel); }
344
1.45k
  bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
345
524
  bool isNegLo() const { return isImmTy(ImmTyNegLo); }
346
426
  bool isNegHi() const { return isImmTy(ImmTyNegHi); }
347
182
  bool isHigh() const { return isImmTy(ImmTyHigh); }
348
349
20.3k
  bool isMod() const {
350
20.3k
    return isClampSI() || 
isOModSI()20.1k
;
351
20.3k
  }
352
353
1.72k
  bool isRegOrImm() const {
354
1.72k
    return isReg() || 
isImm()188
;
355
1.72k
  }
356
357
  bool isRegClass(unsigned RCID) const;
358
359
  bool isInlineValue() const;
360
361
673k
  bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
362
673k
    return (isRegClass(RCID) || 
isInlinableImm(type)195k
) &&
!hasModifiers()538k
;
363
673k
  }
364
365
0
  bool isSCSrcB16() const {
366
0
    return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
367
0
  }
368
369
0
  bool isSCSrcV2B16() const {
370
0
    return isSCSrcB16();
371
0
  }
372
373
128k
  bool isSCSrcB32() const {
374
128k
    return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
375
128k
  }
376
377
143k
  bool isSCSrcB64() const {
378
143k
    return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
379
143k
  }
380
381
  bool isBoolReg() const;
382
383
0
  bool isSCSrcF16() const {
384
0
    return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
385
0
  }
386
387
0
  bool isSCSrcV2F16() const {
388
0
    return isSCSrcF16();
389
0
  }
390
391
0
  bool isSCSrcF32() const {
392
0
    return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
393
0
  }
394
395
0
  bool isSCSrcF64() const {
396
0
    return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
397
0
  }
398
399
61.5k
  bool isSSrcB32() const {
400
61.5k
    return isSCSrcB32() || 
isLiteralImm(MVT::i32)4.01k
||
isExpr()314
;
401
61.5k
  }
402
403
0
  bool isSSrcB16() const {
404
0
    return isSCSrcB16() || isLiteralImm(MVT::i16);
405
0
  }
406
407
0
  bool isSSrcV2B16() const {
408
0
    llvm_unreachable("cannot happen");
409
0
    return isSSrcB16();
410
0
  }
411
412
20.5k
  bool isSSrcB64() const {
413
20.5k
    // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
414
20.5k
    // See isVSrc64().
415
20.5k
    return isSCSrcB64() || 
isLiteralImm(MVT::i64)1.94k
;
416
20.5k
  }
417
418
0
  bool isSSrcF32() const {
419
0
    return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
420
0
  }
421
422
0
  bool isSSrcF64() const {
423
0
    return isSCSrcB64() || isLiteralImm(MVT::f64);
424
0
  }
425
426
0
  bool isSSrcF16() const {
427
0
    return isSCSrcB16() || isLiteralImm(MVT::f16);
428
0
  }
429
430
0
  bool isSSrcV2F16() const {
431
0
    llvm_unreachable("cannot happen");
432
0
    return isSSrcF16();
433
0
  }
434
435
210
  bool isSSrcOrLdsB32() const {
436
210
    return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
437
210
           
isLiteralImm(MVT::i32)16
||
isExpr()0
;
438
210
  }
439
440
0
  bool isVCSrcB32() const {
441
0
    return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
442
0
  }
443
444
0
  bool isVCSrcB64() const {
445
0
    return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
446
0
  }
447
448
0
  bool isVCSrcB16() const {
449
0
    return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
450
0
  }
451
452
0
  bool isVCSrcV2B16() const {
453
0
    return isVCSrcB16();
454
0
  }
455
456
228k
  bool isVCSrcF32() const {
457
228k
    return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
458
228k
  }
459
460
89.7k
  bool isVCSrcF64() const {
461
89.7k
    return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
462
89.7k
  }
463
464
81.5k
  bool isVCSrcF16() const {
465
81.5k
    return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
466
81.5k
  }
467
468
0
  bool isVCSrcV2F16() const {
469
0
    return isVCSrcF16();
470
0
  }
471
472
178k
  bool isVSrcB32() const {
473
178k
    return isVCSrcF32() || 
isLiteralImm(MVT::i32)17.7k
||
isExpr()14.5k
;
474
178k
  }
475
476
59.0k
  bool isVSrcB64() const {
477
59.0k
    return isVCSrcF64() || 
isLiteralImm(MVT::i64)4.32k
;
478
59.0k
  }
479
480
56.2k
  bool isVSrcB16() const {
481
56.2k
    return isVCSrcF16() || 
isLiteralImm(MVT::i16)7.66k
;
482
56.2k
  }
483
484
4.02k
  bool isVSrcV2B16() const {
485
4.02k
    return isVSrcB16() || 
isLiteralImm(MVT::v2i16)58
;
486
4.02k
  }
487
488
47.5k
  bool isVSrcF32() const {
489
47.5k
    return isVCSrcF32() || 
isLiteralImm(MVT::f32)16.5k
||
isExpr()13.5k
;
490
47.5k
  }
491
492
30.7k
  bool isVSrcF64() const {
493
30.7k
    return isVCSrcF64() || 
isLiteralImm(MVT::f64)5.36k
;
494
30.7k
  }
495
496
25.2k
  bool isVSrcF16() const {
497
25.2k
    return isVCSrcF16() || 
isLiteralImm(MVT::f16)8.04k
;
498
25.2k
  }
499
500
3.12k
  bool isVSrcV2F16() const {
501
3.12k
    return isVSrcF16() || 
isLiteralImm(MVT::v2f16)86
;
502
3.12k
  }
503
504
13
  bool isVISrcB32() const {
505
13
    return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
506
13
  }
507
508
0
  bool isVISrcB16() const {
509
0
    return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
510
0
  }
511
512
0
  bool isVISrcV2B16() const {
513
0
    return isVISrcB16();
514
0
  }
515
516
0
  bool isVISrcF32() const {
517
0
    return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
518
0
  }
519
520
0
  bool isVISrcF16() const {
521
0
    return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
522
0
  }
523
524
0
  bool isVISrcV2F16() const {
525
0
    return isVISrcF16() || isVISrcB32();
526
0
  }
527
528
0
  bool isAISrcB32() const {
529
0
    return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
530
0
  }
531
532
0
  bool isAISrcB16() const {
533
0
    return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
534
0
  }
535
536
0
  bool isAISrcV2B16() const {
537
0
    return isAISrcB16();
538
0
  }
539
540
0
  bool isAISrcF32() const {
541
0
    return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
542
0
  }
543
544
0
  bool isAISrcF16() const {
545
0
    return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
546
0
  }
547
548
0
  bool isAISrcV2F16() const {
549
0
    return isAISrcF16() || isAISrcB32();
550
0
  }
551
552
96
  bool isAISrc_128B32() const {
553
96
    return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
554
96
  }
555
556
0
  bool isAISrc_128B16() const {
557
0
    return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
558
0
  }
559
560
0
  bool isAISrc_128V2B16() const {
561
0
    return isAISrc_128B16();
562
0
  }
563
564
32
  bool isAISrc_128F32() const {
565
32
    return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
566
32
  }
567
568
0
  bool isAISrc_128F16() const {
569
0
    return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
570
0
  }
571
572
0
  bool isAISrc_128V2F16() const {
573
0
    return isAISrc_128F16() || isAISrc_128B32();
574
0
  }
575
576
96
  bool isAISrc_512B32() const {
577
96
    return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
578
96
  }
579
580
0
  bool isAISrc_512B16() const {
581
0
    return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
582
0
  }
583
584
0
  bool isAISrc_512V2B16() const {
585
0
    return isAISrc_512B16();
586
0
  }
587
588
32
  bool isAISrc_512F32() const {
589
32
    return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
590
32
  }
591
592
0
  bool isAISrc_512F16() const {
593
0
    return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
594
0
  }
595
596
0
  bool isAISrc_512V2F16() const {
597
0
    return isAISrc_512F16() || isAISrc_512B32();
598
0
  }
599
600
48
  bool isAISrc_1024B32() const {
601
48
    return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
602
48
  }
603
604
0
  bool isAISrc_1024B16() const {
605
0
    return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
606
0
  }
607
608
0
  bool isAISrc_1024V2B16() const {
609
0
    return isAISrc_1024B16();
610
0
  }
611
612
21
  bool isAISrc_1024F32() const {
613
21
    return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
614
21
  }
615
616
0
  bool isAISrc_1024F16() const {
617
0
    return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
618
0
  }
619
620
0
  bool isAISrc_1024V2F16() const {
621
0
    return isAISrc_1024F16() || isAISrc_1024B32();
622
0
  }
623
624
517
  bool isKImmFP32() const {
625
517
    return isLiteralImm(MVT::f32);
626
517
  }
627
628
78
  bool isKImmFP16() const {
629
78
    return isLiteralImm(MVT::f16);
630
78
  }
631
632
0
  bool isMem() const override {
633
0
    return false;
634
0
  }
635
636
2.63M
  bool isExpr() const {
637
2.63M
    return Kind == Expression;
638
2.63M
  }
639
640
584
  bool isSoppBrTarget() const {
641
584
    return isExpr() || 
isImm()551
;
642
584
  }
643
644
  bool isSWaitCnt() const;
645
  bool isHwreg() const;
646
  bool isSendMsg() const;
647
  bool isSwizzle() const;
648
  bool isSMRDOffset8() const;
649
  bool isSMRDOffset20() const;
650
  bool isSMRDLiteralOffset() const;
651
  bool isDPP8() const;
652
  bool isDPPCtrl() const;
653
  bool isBLGP() const;
654
  bool isCBSZ() const;
655
  bool isABID() const;
656
  bool isGPRIdxMode() const;
657
  bool isS16Imm() const;
658
  bool isU16Imm() const;
659
  bool isEndpgm() const;
660
661
17.7k
  StringRef getExpressionAsToken() const {
662
17.7k
    assert(isExpr());
663
17.7k
    const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
664
17.7k
    return S->getSymbol().getName();
665
17.7k
  }
666
667
481k
  StringRef getToken() const {
668
481k
    assert(isToken());
669
481k
670
481k
    if (Kind == Expression)
671
17.7k
      return getExpressionAsToken();
672
463k
673
463k
    return StringRef(Tok.Data, Tok.Length);
674
463k
  }
675
676
124k
  int64_t getImm() const {
677
124k
    assert(isImm());
678
124k
    return Imm.Val;
679
124k
  }
680
681
278k
  ImmTy getImmTy() const {
682
278k
    assert(isImm());
683
278k
    return Imm.Type;
684
278k
  }
685
686
2.87M
  unsigned getReg() const override {
687
2.87M
    assert(isRegKind());
688
2.87M
    return Reg.RegNo;
689
2.87M
  }
690
691
21.5k
  SMLoc getStartLoc() const override {
692
21.5k
    return StartLoc;
693
21.5k
  }
694
695
0
  SMLoc getEndLoc() const override {
696
0
    return EndLoc;
697
0
  }
698
699
36
  SMRange getLocRange() const {
700
36
    return SMRange(StartLoc, EndLoc);
701
36
  }
702
703
1.89M
  Modifiers getModifiers() const {
704
1.89M
    assert(isRegKind() || isImmTy(ImmTyNone));
705
1.89M
    return isRegKind() ? 
Reg.Mods1.81M
:
Imm.Mods78.6k
;
706
1.89M
  }
707
708
15.0k
  void setModifiers(Modifiers Mods) {
709
15.0k
    assert(isRegKind() || isImmTy(ImmTyNone));
710
15.0k
    if (isRegKind())
711
14.7k
      Reg.Mods = Mods;
712
309
    else
713
309
      Imm.Mods = Mods;
714
15.0k
  }
715
716
1.66M
  bool hasModifiers() const {
717
1.66M
    return getModifiers().hasModifiers();
718
1.66M
  }
719
720
2.47k
  bool hasFPModifiers() const {
721
2.47k
    return getModifiers().hasFPModifiers();
722
2.47k
  }
723
724
0
  bool hasIntModifiers() const {
725
0
    return getModifiers().hasIntModifiers();
726
0
  }
727
728
  uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
729
730
  void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
731
732
  void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
733
734
  template <unsigned Bitwidth>
735
  void addKImmFPOperands(MCInst &Inst, unsigned N) const;
736
737
66
  void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
738
66
    addKImmFPOperands<16>(Inst, N);
739
66
  }
740
741
332
  void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
742
332
    addKImmFPOperands<32>(Inst, N);
743
332
  }
744
745
  void addRegOperands(MCInst &Inst, unsigned N) const;
746
747
0
  void addBoolRegOperands(MCInst &Inst, unsigned N) const {
748
0
    addRegOperands(Inst, N);
749
0
  }
750
751
204k
  void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
752
204k
    if (isRegKind())
753
166k
      addRegOperands(Inst, N);
754
38.3k
    else if (isExpr())
755
46
      Inst.addOperand(MCOperand::createExpr(Expr));
756
38.2k
    else
757
38.2k
      addImmOperands(Inst, N);
758
204k
  }
759
760
219k
  void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
761
219k
    Modifiers Mods = getModifiers();
762
219k
    Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
763
219k
    if (isRegKind()) {
764
203k
      addRegOperands(Inst, N);
765
203k
    } else {
766
15.7k
      addImmOperands(Inst, N, false);
767
15.7k
    }
768
219k
  }
769
770
129k
  void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
771
129k
    assert(!hasIntModifiers());
772
129k
    addRegOrImmWithInputModsOperands(Inst, N);
773
129k
  }
774
775
0
  void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
776
0
    assert(!hasFPModifiers());
777
0
    addRegOrImmWithInputModsOperands(Inst, N);
778
0
  }
779
780
15.1k
  void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
781
15.1k
    Modifiers Mods = getModifiers();
782
15.1k
    Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
783
15.1k
    assert(isRegKind());
784
15.1k
    addRegOperands(Inst, N);
785
15.1k
  }
786
787
15.1k
  void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
788
15.1k
    assert(!hasIntModifiers());
789
15.1k
    addRegWithInputModsOperands(Inst, N);
790
15.1k
  }
791
792
0
  void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
793
0
    assert(!hasFPModifiers());
794
0
    addRegWithInputModsOperands(Inst, N);
795
0
  }
796
797
444
  void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
798
444
    if (isImm())
799
415
      addImmOperands(Inst, N);
800
29
    else {
801
29
      assert(isExpr());
802
29
      Inst.addOperand(MCOperand::createExpr(Expr));
803
29
    }
804
444
  }
805
806
  static void printImmTy(raw_ostream& OS, ImmTy Type) {
807
    switch (Type) {
808
    case ImmTyNone: OS << "None"; break;
809
    case ImmTyGDS: OS << "GDS"; break;
810
    case ImmTyLDS: OS << "LDS"; break;
811
    case ImmTyOffen: OS << "Offen"; break;
812
    case ImmTyIdxen: OS << "Idxen"; break;
813
    case ImmTyAddr64: OS << "Addr64"; break;
814
    case ImmTyOffset: OS << "Offset"; break;
815
    case ImmTyInstOffset: OS << "InstOffset"; break;
816
    case ImmTyOffset0: OS << "Offset0"; break;
817
    case ImmTyOffset1: OS << "Offset1"; break;
818
    case ImmTyDLC: OS << "DLC"; break;
819
    case ImmTyGLC: OS << "GLC"; break;
820
    case ImmTySLC: OS << "SLC"; break;
821
    case ImmTyTFE: OS << "TFE"; break;
822
    case ImmTyD16: OS << "D16"; break;
823
    case ImmTyFORMAT: OS << "FORMAT"; break;
824
    case ImmTyClampSI: OS << "ClampSI"; break;
825
    case ImmTyOModSI: OS << "OModSI"; break;
826
    case ImmTyDPP8: OS << "DPP8"; break;
827
    case ImmTyDppCtrl: OS << "DppCtrl"; break;
828
    case ImmTyDppRowMask: OS << "DppRowMask"; break;
829
    case ImmTyDppBankMask: OS << "DppBankMask"; break;
830
    case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
831
    case ImmTyDppFi: OS << "FI"; break;
832
    case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
833
    case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
834
    case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
835
    case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
836
    case ImmTyDMask: OS << "DMask"; break;
837
    case ImmTyDim: OS << "Dim"; break;
838
    case ImmTyUNorm: OS << "UNorm"; break;
839
    case ImmTyDA: OS << "DA"; break;
840
    case ImmTyR128A16: OS << "R128A16"; break;
841
    case ImmTyLWE: OS << "LWE"; break;
842
    case ImmTyOff: OS << "Off"; break;
843
    case ImmTyExpTgt: OS << "ExpTgt"; break;
844
    case ImmTyExpCompr: OS << "ExpCompr"; break;
845
    case ImmTyExpVM: OS << "ExpVM"; break;
846
    case ImmTyHwreg: OS << "Hwreg"; break;
847
    case ImmTySendMsg: OS << "SendMsg"; break;
848
    case ImmTyInterpSlot: OS << "InterpSlot"; break;
849
    case ImmTyInterpAttr: OS << "InterpAttr"; break;
850
    case ImmTyAttrChan: OS << "AttrChan"; break;
851
    case ImmTyOpSel: OS << "OpSel"; break;
852
    case ImmTyOpSelHi: OS << "OpSelHi"; break;
853
    case ImmTyNegLo: OS << "NegLo"; break;
854
    case ImmTyNegHi: OS << "NegHi"; break;
855
    case ImmTySwizzle: OS << "Swizzle"; break;
856
    case ImmTyGprIdxMode: OS << "GprIdxMode"; break;
857
    case ImmTyHigh: OS << "High"; break;
858
    case ImmTyBLGP: OS << "BLGP"; break;
859
    case ImmTyCBSZ: OS << "CBSZ"; break;
860
    case ImmTyABID: OS << "ABID"; break;
861
    case ImmTyEndpgm: OS << "Endpgm"; break;
862
    }
863
  }
864
865
0
  void print(raw_ostream &OS) const override {
866
0
    switch (Kind) {
867
0
    case Register:
868
0
      OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
869
0
      break;
870
0
    case Immediate:
871
0
      OS << '<' << getImm();
872
0
      if (getImmTy() != ImmTyNone) {
873
0
        OS << " type: "; printImmTy(OS, getImmTy());
874
0
      }
875
0
      OS << " mods: " << Imm.Mods << '>';
876
0
      break;
877
0
    case Token:
878
0
      OS << '\'' << getToken() << '\'';
879
0
      break;
880
0
    case Expression:
881
0
      OS << "<expr " << *Expr << '>';
882
0
      break;
883
0
    }
884
0
  }
885
886
  static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
887
                                      int64_t Val, SMLoc Loc,
888
                                      ImmTy Type = ImmTyNone,
889
356k
                                      bool IsFPImm = false) {
890
356k
    auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
891
356k
    Op->Imm.Val = Val;
892
356k
    Op->Imm.IsFPImm = IsFPImm;
893
356k
    Op->Imm.Type = Type;
894
356k
    Op->Imm.Mods = Modifiers();
895
356k
    Op->StartLoc = Loc;
896
356k
    Op->EndLoc = Loc;
897
356k
    return Op;
898
356k
  }
899
900
  static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
901
                                        StringRef Str, SMLoc Loc,
902
333k
                                        bool HasExplicitEncodingSize = true) {
903
333k
    auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
904
333k
    Res->Tok.Data = Str.data();
905
333k
    Res->Tok.Length = Str.size();
906
333k
    Res->StartLoc = Loc;
907
333k
    Res->EndLoc = Loc;
908
333k
    return Res;
909
333k
  }
910
911
  static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
912
                                      unsigned RegNo, SMLoc S,
913
863k
                                      SMLoc E) {
914
863k
    auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
915
863k
    Op->Reg.RegNo = RegNo;
916
863k
    Op->Reg.Mods = Modifiers();
917
863k
    Op->StartLoc = S;
918
863k
    Op->EndLoc = E;
919
863k
    return Op;
920
863k
  }
921
922
  static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
923
9.41k
                                       const class MCExpr *Expr, SMLoc S) {
924
9.41k
    auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
925
9.41k
    Op->Expr = Expr;
926
9.41k
    Op->StartLoc = S;
927
9.41k
    Op->EndLoc = S;
928
9.41k
    return Op;
929
9.41k
  }
930
};
931
932
0
raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
933
0
  OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
934
0
  return OS;
935
0
}
936
937
//===----------------------------------------------------------------------===//
938
// AsmParser
939
//===----------------------------------------------------------------------===//
940
941
// Holds info related to the current kernel, e.g. count of SGPRs used.
942
// Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
943
// .amdgpu_hsa_kernel or at EOF.
944
class KernelScopeInfo {
945
  int SgprIndexUnusedMin = -1;
946
  int VgprIndexUnusedMin = -1;
947
  MCContext *Ctx = nullptr;
948
949
187k
  void usesSgprAt(int i) {
950
187k
    if (i >= SgprIndexUnusedMin) {
951
1.25k
      SgprIndexUnusedMin = ++i;
952
1.25k
      if (Ctx) {
953
1.25k
        MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
954
1.25k
        Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
955
1.25k
      }
956
1.25k
    }
957
187k
  }
958
959
548k
  void usesVgprAt(int i) {
960
548k
    if (i >= VgprIndexUnusedMin) {
961
1.55k
      VgprIndexUnusedMin = ++i;
962
1.55k
      if (Ctx) {
963
1.55k
        MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
964
1.55k
        Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
965
1.55k
      }
966
1.55k
    }
967
548k
  }
968
969
public:
970
521
  KernelScopeInfo() = default;
971
972
555
  void initialize(MCContext &Context) {
973
555
    Ctx = &Context;
974
555
    usesSgprAt(SgprIndexUnusedMin = -1);
975
555
    usesVgprAt(VgprIndexUnusedMin = -1);
976
555
  }
977
978
863k
  void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
979
863k
    switch (RegKind) {
980
863k
      
case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break186k
;
981
863k
      case IS_AGPR: // fall through
982
548k
      case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
983
548k
      
default: break128k
;
984
863k
    }
985
863k
  }
986
};
987
988
class AMDGPUAsmParser : public MCTargetAsmParser {
989
  MCAsmParser &Parser;
990
991
  // Number of extra operands parsed after the first optional operand.
992
  // This may be necessary to skip hardcoded mandatory operands.
993
  static const unsigned MAX_OPR_LOOKAHEAD = 8;
994
995
  unsigned ForcedEncodingSize = 0;
996
  bool ForcedDPP = false;
997
  bool ForcedSDWA = false;
998
  KernelScopeInfo KernelScope;
999
1000
  /// @name Auto-generated Match Functions
1001
  /// {
1002
1003
#define GET_ASSEMBLER_HEADER
1004
#include "AMDGPUGenAsmMatcher.inc"
1005
1006
  /// }
1007
1008
private:
1009
  bool ParseAsAbsoluteExpression(uint32_t &Ret);
1010
  bool OutOfRangeError(SMRange Range);
1011
  /// Calculate VGPR/SGPR blocks required for given target, reserved
1012
  /// registers, and user-specified NextFreeXGPR values.
1013
  ///
1014
  /// \param Features [in] Target features, used for bug corrections.
1015
  /// \param VCCUsed [in] Whether VCC special SGPR is reserved.
1016
  /// \param FlatScrUsed [in] Whether FLAT_SCRATCH special SGPR is reserved.
1017
  /// \param XNACKUsed [in] Whether XNACK_MASK special SGPR is reserved.
1018
  /// \param EnableWavefrontSize32 [in] Value of ENABLE_WAVEFRONT_SIZE32 kernel
1019
  /// descriptor field, if valid.
1020
  /// \param NextFreeVGPR [in] Max VGPR number referenced, plus one.
1021
  /// \param VGPRRange [in] Token range, used for VGPR diagnostics.
1022
  /// \param NextFreeSGPR [in] Max SGPR number referenced, plus one.
1023
  /// \param SGPRRange [in] Token range, used for SGPR diagnostics.
1024
  /// \param VGPRBlocks [out] Result VGPR block count.
1025
  /// \param SGPRBlocks [out] Result SGPR block count.
1026
  bool calculateGPRBlocks(const FeatureBitset &Features, bool VCCUsed,
1027
                          bool FlatScrUsed, bool XNACKUsed,
1028
                          Optional<bool> EnableWavefrontSize32, unsigned NextFreeVGPR,
1029
                          SMRange VGPRRange, unsigned NextFreeSGPR,
1030
                          SMRange SGPRRange, unsigned &VGPRBlocks,
1031
                          unsigned &SGPRBlocks);
1032
  bool ParseDirectiveAMDGCNTarget();
1033
  bool ParseDirectiveAMDHSAKernel();
1034
  bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
1035
  bool ParseDirectiveHSACodeObjectVersion();
1036
  bool ParseDirectiveHSACodeObjectISA();
1037
  bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
1038
  bool ParseDirectiveAMDKernelCodeT();
1039
  bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
1040
  bool ParseDirectiveAMDGPUHsaKernel();
1041
1042
  bool ParseDirectiveISAVersion();
1043
  bool ParseDirectiveHSAMetadata();
1044
  bool ParseDirectivePALMetadataBegin();
1045
  bool ParseDirectivePALMetadata();
1046
  bool ParseDirectiveAMDGPULDS();
1047
1048
  /// Common code to parse out a block of text (typically YAML) between start and
1049
  /// end directives.
1050
  bool ParseToEndDirective(const char *AssemblerDirectiveBegin,
1051
                           const char *AssemblerDirectiveEnd,
1052
                           std::string &CollectString);
1053
1054
  bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
1055
                             RegisterKind RegKind, unsigned Reg1,
1056
                             unsigned RegNum);
1057
  bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
1058
                           unsigned& RegNum, unsigned& RegWidth,
1059
                           unsigned *DwordRegIndex);
1060
  bool isRegister();
1061
  bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const;
1062
  Optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1063
  void initializeGprCountSymbol(RegisterKind RegKind);
1064
  bool updateGprCountSymbols(RegisterKind RegKind, unsigned DwordRegIndex,
1065
                             unsigned RegWidth);
1066
  void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
1067
                    bool IsAtomic, bool IsAtomicReturn, bool IsLds = false);
1068
  void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
1069
                 bool IsGdsHardcoded);
1070
1071
public:
1072
  enum AMDGPUMatchResultTy {
1073
    Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
1074
  };
1075
  enum OperandMode {
1076
    OperandMode_Default,
1077
    OperandMode_NSA,
1078
  };
1079
1080
  using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1081
1082
  AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1083
               const MCInstrInfo &MII,
1084
               const MCTargetOptions &Options)
1085
521
      : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
1086
521
    MCAsmParserExtension::Initialize(Parser);
1087
521
1088
521
    if (getFeatureBits().none()) {
1089
62
      // Set default features.
1090
62
      copySTI().ToggleFeature("southern-islands");
1091
62
    }
1092
521
1093
521
    setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1094
521
1095
521
    {
1096
521
      // TODO: make those pre-defined variables read-only.
1097
521
      // Currently there is none suitable machinery in the core llvm-mc for this.
1098
521
      // MCSymbol::isRedefinable is intended for another purpose, and
1099
521
      // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
1100
521
      AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
1101
521
      MCContext &Ctx = getContext();
1102
521
      if (ISA.Major >= 6 && 
AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())456
) {
1103
19
        MCSymbol *Sym =
1104
19
            Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_number"));
1105
19
        Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
1106
19
        Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_minor"));
1107
19
        Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
1108
19
        Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_stepping"));
1109
19
        Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
1110
502
      } else {
1111
502
        MCSymbol *Sym =
1112
502
            Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
1113
502
        Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
1114
502
        Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
1115
502
        Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
1116
502
        Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
1117
502
        Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
1118
502
      }
1119
521
      if (ISA.Major >= 6 && 
AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())456
) {
1120
19
        initializeGprCountSymbol(IS_VGPR);
1121
19
        initializeGprCountSymbol(IS_SGPR);
1122
19
      } else
1123
502
        KernelScope.initialize(getContext());
1124
521
    }
1125
521
  }
1126
1127
65
  bool hasXNACK() const {
1128
65
    return AMDGPU::hasXNACK(getSTI());
1129
65
  }
1130
1131
0
  bool hasMIMG_R128() const {
1132
0
    return AMDGPU::hasMIMG_R128(getSTI());
1133
0
  }
1134
1135
4.36k
  bool hasPackedD16() const {
1136
4.36k
    return AMDGPU::hasPackedD16(getSTI());
1137
4.36k
  }
1138
1139
767k
  bool isSI() const {
1140
767k
    return AMDGPU::isSI(getSTI());
1141
767k
  }
1142
1143
851k
  bool isCI() const {
1144
851k
    return AMDGPU::isCI(getSTI());
1145
851k
  }
1146
1147
199k
  bool isVI() const {
1148
199k
    return AMDGPU::isVI(getSTI());
1149
199k
  }
1150
1151
170k
  bool isGFX9() const {
1152
170k
    return AMDGPU::isGFX9(getSTI());
1153
170k
  }
1154
1155
1.65M
  bool isGFX10() const {
1156
1.65M
    return AMDGPU::isGFX10(getSTI());
1157
1.65M
  }
1158
1159
220k
  bool hasInv2PiInlineImm() const {
1160
220k
    return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1161
220k
  }
1162
1163
3.24k
  bool hasFlatOffsets() const {
1164
3.24k
    return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1165
3.24k
  }
1166
1167
26
  bool hasSGPR102_SGPR103() const {
1168
26
    return !isVI() && 
!isGFX9()4
;
1169
26
  }
1170
1171
2.56k
  bool hasSGPR104_SGPR105() const {
1172
2.56k
    return isGFX10();
1173
2.56k
  }
1174
1175
43.0k
  bool hasIntClamp() const {
1176
43.0k
    return getFeatureBits()[AMDGPU::FeatureIntClamp];
1177
43.0k
  }
1178
1179
200
  AMDGPUTargetStreamer &getTargetStreamer() {
1180
200
    MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1181
200
    return static_cast<AMDGPUTargetStreamer &>(TS);
1182
200
  }
1183
1184
953k
  const MCRegisterInfo *getMRI() const {
1185
953k
    // We need this const_cast because for some reason getContext() is not const
1186
953k
    // in MCAsmParser.
1187
953k
    return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
1188
953k
  }
1189
1190
378k
  const MCInstrInfo *getMII() const {
1191
378k
    return &MII;
1192
378k
  }
1193
1194
451k
  const FeatureBitset &getFeatureBits() const {
1195
451k
    return getSTI().getFeatureBits();
1196
451k
  }
1197
1198
428k
  void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
1199
353k
  void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1200
392k
  void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1201
1202
927k
  unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
1203
323k
  bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
1204
474k
  bool isForcedDPP() const { return ForcedDPP; }
1205
533k
  bool isForcedSDWA() const { return ForcedSDWA; }
1206
  ArrayRef<unsigned> getMatchedVariants() const;
1207
1208
  std::unique_ptr<AMDGPUOperand> parseRegister();
1209
  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
1210
  unsigned checkTargetMatchPredicate(MCInst &Inst) override;
1211
  unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
1212
                                      unsigned Kind) override;
1213
  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1214
                               OperandVector &Operands, MCStreamer &Out,
1215
                               uint64_t &ErrorInfo,
1216
                               bool MatchingInlineAsm) override;
1217
  bool ParseDirective(AsmToken DirectiveID) override;
1218
  OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic,
1219
                                    OperandMode Mode = OperandMode_Default);
1220
  StringRef parseMnemonicSuffix(StringRef Name);
1221
  bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
1222
                        SMLoc NameLoc, OperandVector &Operands) override;
1223
  //bool ProcessInstruction(MCInst &Inst);
1224
1225
  OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
1226
1227
  OperandMatchResultTy
1228
  parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1229
                     AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1230
                     bool (*ConvertResult)(int64_t &) = nullptr);
1231
1232
  OperandMatchResultTy
1233
  parseOperandArrayWithPrefix(const char *Prefix,
1234
                              OperandVector &Operands,
1235
                              AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1236
                              bool (*ConvertResult)(int64_t&) = nullptr);
1237
1238
  OperandMatchResultTy
1239
  parseNamedBit(const char *Name, OperandVector &Operands,
1240
                AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1241
  OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
1242
                                             StringRef &Value);
1243
1244
  bool isModifier();
1245
  bool isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1246
  bool isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1247
  bool isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1248
  bool isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const;
1249
  bool parseSP3NegModifier();
1250
  OperandMatchResultTy parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false);
1251
  OperandMatchResultTy parseReg(OperandVector &Operands);
1252
  OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false);
1253
  OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
1254
  OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
1255
  OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
1256
  OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
1257
  OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
1258
  OperandMatchResultTy parseDfmtNfmt(OperandVector &Operands);
1259
1260
  void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
1261
12.5k
  void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
1262
358
  void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
1263
  void cvtExp(MCInst &Inst, const OperandVector &Operands);
1264
1265
  bool parseCnt(int64_t &IntVal);
1266
  OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1267
  OperandMatchResultTy parseHwreg(OperandVector &Operands);
1268
1269
private:
1270
  struct OperandInfoTy {
1271
    int64_t Id;
1272
    bool IsSymbolic = false;
1273
    bool IsDefined = false;
1274
1275
1.05k
    OperandInfoTy(int64_t Id_) : Id(Id_) {}
1276
  };
1277
1278
  bool parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op, OperandInfoTy &Stream);
1279
  bool validateSendMsg(const OperandInfoTy &Msg,
1280
                       const OperandInfoTy &Op,
1281
                       const OperandInfoTy &Stream,
1282
                       const SMLoc Loc);
1283
1284
  bool parseHwregBody(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1285
  bool validateHwreg(const OperandInfoTy &HwReg,
1286
                     const int64_t Offset,
1287
                     const int64_t Width,
1288
                     const SMLoc Loc);
1289
1290
  void errorExpTgt();
1291
  OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1292
  SMLoc getFlatOffsetLoc(const OperandVector &Operands) const;
1293
1294
  bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc, const OperandVector &Operands);
1295
  bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands);
1296
  bool validateSOPLiteral(const MCInst &Inst) const;
1297
  bool validateConstantBusLimitations(const MCInst &Inst);
1298
  bool validateEarlyClobberLimitations(const MCInst &Inst);
1299
  bool validateIntClampSupported(const MCInst &Inst);
1300
  bool validateMIMGAtomicDMask(const MCInst &Inst);
1301
  bool validateMIMGGatherDMask(const MCInst &Inst);
1302
  bool validateMIMGDataSize(const MCInst &Inst);
1303
  bool validateMIMGAddrSize(const MCInst &Inst);
1304
  bool validateMIMGD16(const MCInst &Inst);
1305
  bool validateMIMGDim(const MCInst &Inst);
1306
  bool validateLdsDirect(const MCInst &Inst);
1307
  bool validateOpSel(const MCInst &Inst);
1308
  bool validateVccOperand(unsigned Reg) const;
1309
  bool validateVOP3Literal(const MCInst &Inst) const;
1310
  bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1311
  bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1312
  unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1313
1314
  bool isId(const StringRef Id) const;
1315
  bool isId(const AsmToken &Token, const StringRef Id) const;
1316
  bool isToken(const AsmToken::TokenKind Kind) const;
1317
  bool trySkipId(const StringRef Id);
1318
  bool trySkipId(const StringRef Id, const AsmToken::TokenKind Kind);
1319
  bool trySkipToken(const AsmToken::TokenKind Kind);
1320
  bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1321
  bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1322
  void peekTokens(MutableArrayRef<AsmToken> Tokens);
1323
  AsmToken::TokenKind getTokenKind() const;
1324
  bool parseExpr(int64_t &Imm);
1325
  bool parseExpr(OperandVector &Operands);
1326
  StringRef getTokenStr() const;
1327
  AsmToken peekToken();
1328
  AsmToken getToken() const;
1329
  SMLoc getLoc() const;
1330
  void lex();
1331
1332
public:
1333
  OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1334
  OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
1335
1336
  OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1337
  OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1338
  OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1339
  OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1340
  OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1341
  OperandMatchResultTy parseBoolReg(OperandVector &Operands);
1342
1343
  bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1344
                            const unsigned MinVal,
1345
                            const unsigned MaxVal,
1346
                            const StringRef ErrMsg);
1347
  OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1348
  bool parseSwizzleOffset(int64_t &Imm);
1349
  bool parseSwizzleMacro(int64_t &Imm);
1350
  bool parseSwizzleQuadPerm(int64_t &Imm);
1351
  bool parseSwizzleBitmaskPerm(int64_t &Imm);
1352
  bool parseSwizzleBroadcast(int64_t &Imm);
1353
  bool parseSwizzleSwap(int64_t &Imm);
1354
  bool parseSwizzleReverse(int64_t &Imm);
1355
1356
  OperandMatchResultTy parseGPRIdxMode(OperandVector &Operands);
1357
  int64_t parseGPRIdxMacro();
1358
1359
3.52k
  void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1360
3.19k
  void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1361
368
  void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1362
1.21k
  void cvtMubufLds(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false, true); }
1363
  void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1364
1365
  AMDGPUOperand::Ptr defaultDLC() const;
1366
  AMDGPUOperand::Ptr defaultGLC() const;
1367
  AMDGPUOperand::Ptr defaultSLC() const;
1368
1369
  AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1370
  AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1371
  AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1372
  AMDGPUOperand::Ptr defaultFlatOffset() const;
1373
1374
  OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1375
1376
  void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1377
               OptionalImmIndexMap &OptionalIdx);
1378
  void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1379
  void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1380
  void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1381
1382
  void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1383
1384
  void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1385
               bool IsAtomic = false);
1386
  void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1387
1388
  OperandMatchResultTy parseDim(OperandVector &Operands);
1389
  OperandMatchResultTy parseDPP8(OperandVector &Operands);
1390
  OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1391
  AMDGPUOperand::Ptr defaultRowMask() const;
1392
  AMDGPUOperand::Ptr defaultBankMask() const;
1393
  AMDGPUOperand::Ptr defaultBoundCtrl() const;
1394
  AMDGPUOperand::Ptr defaultFI() const;
1395
  void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8 = false);
1396
730
  void cvtDPP8(MCInst &Inst, const OperandVector &Operands) { cvtDPP(Inst, Operands, true); }
1397
1398
  OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1399
                                    AMDGPUOperand::ImmTy Type);
1400
  OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1401
  void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1402
  void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1403
  void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1404
  void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1405
  void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1406
                uint64_t BasicInstType, bool skipVcc = false);
1407
1408
  AMDGPUOperand::Ptr defaultBLGP() const;
1409
  AMDGPUOperand::Ptr defaultCBSZ() const;
1410
  AMDGPUOperand::Ptr defaultABID() const;
1411
1412
  OperandMatchResultTy parseEndpgmOp(OperandVector &Operands);
1413
  AMDGPUOperand::Ptr defaultEndpgmImmOperands() const;
1414
};
1415
1416
struct OptionalOperand {
1417
  const char *Name;
1418
  AMDGPUOperand::ImmTy Type;
1419
  bool IsBit;
1420
  bool (*ConvertResult)(int64_t&);
1421
};
1422
1423
} // end anonymous namespace
1424
1425
// May be called with integer type with equivalent bitwidth.
1426
31.9k
static const fltSemantics *getFltSemantics(unsigned Size) {
1427
31.9k
  switch (Size) {
1428
31.9k
  case 4:
1429
23.9k
    return &APFloat::IEEEsingle();
1430
31.9k
  case 8:
1431
0
    return &APFloat::IEEEdouble();
1432
31.9k
  case 2:
1433
7.98k
    return &APFloat::IEEEhalf();
1434
31.9k
  default:
1435
0
    llvm_unreachable("unsupported fp type");
1436
31.9k
  }
1437
31.9k
}
1438
1439
31.9k
static const fltSemantics *getFltSemantics(MVT VT) {
1440
31.9k
  return getFltSemantics(VT.getSizeInBits() / 8);
1441
31.9k
}
1442
1443
17.5k
static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1444
17.5k
  switch (OperandType) {
1445
17.5k
  case AMDGPU::OPERAND_REG_IMM_INT32:
1446
12.4k
  case AMDGPU::OPERAND_REG_IMM_FP32:
1447
12.4k
  case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1448
12.4k
  case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1449
12.4k
  case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1450
12.4k
  case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1451
12.4k
    return &APFloat::IEEEsingle();
1452
12.4k
  case AMDGPU::OPERAND_REG_IMM_INT64:
1453
0
  case AMDGPU::OPERAND_REG_IMM_FP64:
1454
0
  case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1455
0
  case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1456
0
    return &APFloat::IEEEdouble();
1457
5.04k
  case AMDGPU::OPERAND_REG_IMM_INT16:
1458
5.04k
  case AMDGPU::OPERAND_REG_IMM_FP16:
1459
5.04k
  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1460
5.04k
  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1461
5.04k
  case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1462
5.04k
  case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1463
5.04k
  case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
1464
5.04k
  case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1465
5.04k
  case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1466
5.04k
  case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
1467
5.04k
  case AMDGPU::OPERAND_REG_IMM_V2INT16:
1468
5.04k
  case AMDGPU::OPERAND_REG_IMM_V2FP16:
1469
5.04k
    return &APFloat::IEEEhalf();
1470
5.04k
  default:
1471
0
    llvm_unreachable("unsupported fp type");
1472
17.5k
  }
1473
17.5k
}
1474
1475
//===----------------------------------------------------------------------===//
1476
// Operand
1477
//===----------------------------------------------------------------------===//
1478
1479
31.9k
static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1480
31.9k
  bool Lost;
1481
31.9k
1482
31.9k
  // Convert literal to single precision
1483
31.9k
  APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1484
31.9k
                                               APFloat::rmNearestTiesToEven,
1485
31.9k
                                               &Lost);
1486
31.9k
  // We allow precision lost but not overflow or underflow
1487
31.9k
  if (Status != APFloat::opOK &&
1488
31.9k
      
Lost956
&&
1489
31.9k
      
(956
(Status & APFloat::opOverflow) != 0956
||
1490
956
       
(Status & APFloat::opUnderflow) != 0908
)) {
1491
234
    return false;
1492
234
  }
1493
31.6k
1494
31.6k
  return true;
1495
31.6k
}
1496
1497
90.6k
static bool isSafeTruncation(int64_t Val, unsigned Size) {
1498
90.6k
  return isUIntN(Size, Val) || 
isIntN(Size, Val)26.7k
;
1499
90.6k
}
1500
1501
231k
bool AMDGPUOperand::isInlinableImm(MVT type) const {
1502
231k
1503
231k
  // This is a hack to enable named inline values like
1504
231k
  // shared_base with both 32-bit and 64-bit operands.
1505
231k
  // Note that these values are defined as
1506
231k
  // 32-bit operands only.
1507
231k
  if (isInlineValue()) {
1508
238
    return true;
1509
238
  }
1510
231k
1511
231k
  if (!isImmTy(ImmTyNone)) {
1512
126k
    // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1513
126k
    return false;
1514
126k
  }
1515
105k
  // TODO: We should avoid using host float here. It would be better to
1516
105k
  // check the float bit values which is what a few other places do.
1517
105k
  // We've had bot failures before due to weird NaN support on mips hosts.
1518
105k
1519
105k
  APInt Literal(64, Imm.Val);
1520
105k
1521
105k
  if (Imm.IsFPImm) { // We got fp literal token
1522
41.9k
    if (type == MVT::f64 || 
type == MVT::i6432.7k
) { // Expected 64-bit operand
1523
10.9k
      return AMDGPU::isInlinableLiteral64(Imm.Val,
1524
10.9k
                                          AsmParser->hasInv2PiInlineImm());
1525
10.9k
    }
1526
31.0k
1527
31.0k
    APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1528
31.0k
    if (!canLosslesslyConvertToFPType(FPLiteral, type))
1529
112
      return false;
1530
30.9k
1531
30.9k
    if (type.getScalarSizeInBits() == 16) {
1532
7.64k
      return AMDGPU::isInlinableLiteral16(
1533
7.64k
        static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1534
7.64k
        AsmParser->hasInv2PiInlineImm());
1535
7.64k
    }
1536
23.2k
1537
23.2k
    // Check if single precision literal is inlinable
1538
23.2k
    return AMDGPU::isInlinableLiteral32(
1539
23.2k
      static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1540
23.2k
      AsmParser->hasInv2PiInlineImm());
1541
23.2k
  }
1542
63.1k
1543
63.1k
  // We got int literal token.
1544
63.1k
  if (type == MVT::f64 || 
type == MVT::i6449.9k
) { // Expected 64-bit operand
1545
16.5k
    return AMDGPU::isInlinableLiteral64(Imm.Val,
1546
16.5k
                                        AsmParser->hasInv2PiInlineImm());
1547
16.5k
  }
1548
46.5k
1549
46.5k
  if (!isSafeTruncation(Imm.Val, type.getScalarSizeInBits())) {
1550
668
    return false;
1551
668
  }
1552
45.9k
1553
45.9k
  if (type.getScalarSizeInBits() == 16) {
1554
10.1k
    return AMDGPU::isInlinableLiteral16(
1555
10.1k
      static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1556
10.1k
      AsmParser->hasInv2PiInlineImm());
1557
10.1k
  }
1558
35.7k
1559
35.7k
  return AMDGPU::isInlinableLiteral32(
1560
35.7k
    static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1561
35.7k
    AsmParser->hasInv2PiInlineImm());
1562
35.7k
}
1563
1564
72.8k
bool AMDGPUOperand::isLiteralImm(MVT type) const {
1565
72.8k
  // Check that this immediate can be added as literal
1566
72.8k
  if (!isImmTy(ImmTyNone)) {
1567
52.1k
    return false;
1568
52.1k
  }
1569
20.7k
1570
20.7k
  if (!Imm.IsFPImm) {
1571
19.3k
    // We got int literal token.
1572
19.3k
1573
19.3k
    if (type == MVT::f64 && 
hasFPModifiers()2.47k
) {
1574
15
      // Cannot apply fp modifiers to int literals preserving the same semantics
1575
15
      // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1576
15
      // disable these cases.
1577
15
      return false;
1578
15
    }
1579
19.3k
1580
19.3k
    unsigned Size = type.getSizeInBits();
1581
19.3k
    if (Size == 64)
1582
5.80k
      Size = 32;
1583
19.3k
1584
19.3k
    // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1585
19.3k
    // types.
1586
19.3k
    return isSafeTruncation(Imm.Val, Size);
1587
19.3k
  }
1588
1.38k
1589
1.38k
  // We got fp literal token
1590
1.38k
  if (type == MVT::f64) { // Expected 64-bit fp operand
1591
258
    // We would set low 64-bits of literal to zeroes but we accept this literals
1592
258
    return true;
1593
258
  }
1594
1.12k
1595
1.12k
  if (type == MVT::i64) { // Expected 64-bit int operand
1596
228
    // We don't allow fp literals in 64-bit integer instructions. It is
1597
228
    // unclear how we should encode them.
1598
228
    return false;
1599
228
  }
1600
894
1601
894
  // We allow fp literals with f16x2 operands assuming that the specified
1602
894
  // literal goes into the lower half and the upper half is zero. We also
1603
894
  // require that the literal may be losslesly converted to f16.
1604
894
  MVT ExpectedType = (type == MVT::v2f16)? 
MVT::f168
:
1605
894
                     
(type == MVT::v2i16)886
?
MVT::i168
:
type878
;
1606
894
1607
894
  APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1608
894
  return canLosslesslyConvertToFPType(FPLiteral, ExpectedType);
1609
894
}
1610
1611
1.05M
bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1612
1.05M
  return isRegKind() && 
AsmParser->getMRI()->getRegClass(RCID).contains(getReg())949k
;
1613
1.05M
}
1614
1615
153k
bool AMDGPUOperand::isSDWAOperand(MVT type) const {
1616
153k
  if (AsmParser->isVI())
1617
11.8k
    return isVReg32();
1618
141k
  else if (AsmParser->isGFX9() || 
AsmParser->isGFX10()71.5k
)
1619
139k
    return isRegClass(AMDGPU::VS_32RegClassID) || 
isInlinableImm(type)10.0k
;
1620
2.17k
  else
1621
2.17k
    return false;
1622
153k
}
1623
1624
38.2k
bool AMDGPUOperand::isSDWAFP16Operand() const {
1625
38.2k
  return isSDWAOperand(MVT::f16);
1626
38.2k
}
1627
1628
42.5k
bool AMDGPUOperand::isSDWAFP32Operand() const {
1629
42.5k
  return isSDWAOperand(MVT::f32);
1630
42.5k
}
1631
1632
21.2k
bool AMDGPUOperand::isSDWAInt16Operand() const {
1633
21.2k
  return isSDWAOperand(MVT::i16);
1634
21.2k
}
1635
1636
51.4k
bool AMDGPUOperand::isSDWAInt32Operand() const {
1637
51.4k
  return isSDWAOperand(MVT::i32);
1638
51.4k
}
1639
1640
167k
bool AMDGPUOperand::isBoolReg() const {
1641
167k
  return AsmParser->getFeatureBits()[AMDGPU::FeatureWavefrontSize64] ?
1642
123k
    isSCSrcB64() : 
isSCSrcB32()44.8k
;
1643
167k
}
1644
1645
uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1646
39
{
1647
39
  assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1648
39
  assert(Size == 2 || Size == 4 || Size == 8);
1649
39
1650
39
  const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1651
39
1652
39
  if (Imm.Mods.Abs) {
1653
22
    Val &= ~FpSignMask;
1654
22
  }
1655
39
  if (Imm.Mods.Neg) {
1656
19
    Val ^= FpSignMask;
1657
19
  }
1658
39
1659
39
  return Val;
1660
39
}
1661
1662
323k
void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1663
323k
  if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1664
323k
                             Inst.getNumOperands())) {
1665
55.5k
    addLiteralImmOperand(Inst, Imm.Val,
1666
55.5k
                         ApplyModifiers &
1667
55.5k
                         isImmTy(ImmTyNone) && 
Imm.Mods.hasFPModifiers()39.8k
);
1668
267k
  } else {
1669
267k
    assert(!isImmTy(ImmTyNone) || !hasModifiers());
1670
267k
    Inst.addOperand(MCOperand::createImm(Imm.Val));
1671
267k
  }
1672
323k
}
1673
1674
55.5k
void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1675
55.5k
  const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1676
55.5k
  auto OpNum = Inst.getNumOperands();
1677
55.5k
  // Check that this operand accepts literals
1678
55.5k
  assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1679
55.5k
1680
55.5k
  if (ApplyModifiers) {
1681
39
    assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1682
39
    const unsigned Size = Imm.IsFPImm ? 
sizeof(double)2
:
getOperandSize(InstDesc, OpNum)37
;
1683
39
    Val = applyInputFPModifiers(Val, Size);
1684
39
  }
1685
55.5k
1686
55.5k
  APInt Literal(64, Val);
1687
55.5k
  uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1688
55.5k
1689
55.5k
  if (Imm.IsFPImm) { // We got fp literal token
1690
22.9k
    switch (OpTy) {
1691
22.9k
    case AMDGPU::OPERAND_REG_IMM_INT64:
1692
5.43k
    case AMDGPU::OPERAND_REG_IMM_FP64:
1693
5.43k
    case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1694
5.43k
    case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1695
5.43k
      if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1696
5.43k
                                       AsmParser->hasInv2PiInlineImm())) {
1697
5.31k
        Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1698
5.31k
        return;
1699
5.31k
      }
1700
112
1701
112
      // Non-inlineable
1702
112
      if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1703
112
        // For fp operands we check if low 32 bits are zeros
1704
112
        if (Literal.getLoBits(32) != 0) {
1705
92
          const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1706
92
          "Can't encode literal as exact 64-bit floating-point operand. "
1707
92
          "Low 32-bits will be set to zero");
1708
92
        }
1709
112
1710
112
        Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1711
112
        return;
1712
112
      }
1713
0
1714
0
      // We don't allow fp literals in 64-bit integer instructions. It is
1715
0
      // unclear how we should encode them. This case should be checked earlier
1716
0
      // in predicate methods (isLiteralImm())
1717
0
      llvm_unreachable("fp literal in 64-bit integer instruction.");
1718
0
1719
17.5k
    case AMDGPU::OPERAND_REG_IMM_INT32:
1720
17.5k
    case AMDGPU::OPERAND_REG_IMM_FP32:
1721
17.5k
    case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1722
17.5k
    case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1723
17.5k
    case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1724
17.5k
    case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1725
17.5k
    case AMDGPU::OPERAND_REG_IMM_INT16:
1726
17.5k
    case AMDGPU::OPERAND_REG_IMM_FP16:
1727
17.5k
    case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1728
17.5k
    case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1729
17.5k
    case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1730
17.5k
    case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1731
17.5k
    case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
1732
17.5k
    case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1733
17.5k
    case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1734
17.5k
    case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
1735
17.5k
    case AMDGPU::OPERAND_REG_IMM_V2INT16:
1736
17.5k
    case AMDGPU::OPERAND_REG_IMM_V2FP16: {
1737
17.5k
      bool lost;
1738
17.5k
      APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1739
17.5k
      // Convert literal to single precision
1740
17.5k
      FPLiteral.convert(*getOpFltSemantics(OpTy),
1741
17.5k
                        APFloat::rmNearestTiesToEven, &lost);
1742
17.5k
      // We allow precision lost but not overflow or underflow. This should be
1743
17.5k
      // checked earlier in isLiteralImm()
1744
17.5k
1745
17.5k
      uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1746
17.5k
      Inst.addOperand(MCOperand::createImm(ImmVal));
1747
17.5k
      return;
1748
17.5k
    }
1749
17.5k
    default:
1750
0
      llvm_unreachable("invalid operand size");
1751
0
    }
1752
0
1753
0
    return;
1754
0
  }
1755
32.6k
1756
32.6k
  // We got int literal token.
1757
32.6k
  // Only sign extend inline immediates.
1758
32.6k
  switch (OpTy) {
1759
32.6k
  case AMDGPU::OPERAND_REG_IMM_INT32:
1760
18.3k
  case AMDGPU::OPERAND_REG_IMM_FP32:
1761
18.3k
  case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1762
18.3k
  case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1763
18.3k
  case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1764
18.3k
  case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1765
18.3k
  case AMDGPU::OPERAND_REG_IMM_V2INT16:
1766
18.3k
  case AMDGPU::OPERAND_REG_IMM_V2FP16:
1767
18.3k
    if (isSafeTruncation(Val, 32) &&
1768
18.3k
        AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1769
18.3k
                                     AsmParser->hasInv2PiInlineImm())) {
1770
13.4k
      Inst.addOperand(MCOperand::createImm(Val));
1771
13.4k
      return;
1772
13.4k
    }
1773
4.97k
1774
4.97k
    Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1775
4.97k
    return;
1776
4.97k
1777
7.84k
  case AMDGPU::OPERAND_REG_IMM_INT64:
1778
7.84k
  case AMDGPU::OPERAND_REG_IMM_FP64:
1779
7.84k
  case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1780
7.84k
  case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1781
7.84k
    if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1782
5.43k
      Inst.addOperand(MCOperand::createImm(Val));
1783
5.43k
      return;
1784
5.43k
    }
1785
2.40k
1786
2.40k
    Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
1787
2.40k
    return;
1788
2.40k
1789
6.39k
  case AMDGPU::OPERAND_REG_IMM_INT16:
1790
6.39k
  case AMDGPU::OPERAND_REG_IMM_FP16:
1791
6.39k
  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1792
6.39k
  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1793
6.39k
  case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
1794
6.39k
  case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
1795
6.39k
    if (isSafeTruncation(Val, 16) &&
1796
6.39k
        AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1797
6.39k
                                     AsmParser->hasInv2PiInlineImm())) {
1798
4.92k
      Inst.addOperand(MCOperand::createImm(Val));
1799
4.92k
      return;
1800
4.92k
    }
1801
1.46k
1802
1.46k
    Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1803
1.46k
    return;
1804
1.46k
1805
1.46k
  case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1806
0
  case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1807
0
  case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
1808
0
  case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
1809
0
    assert(isSafeTruncation(Val, 16));
1810
0
    assert(AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1811
0
                                        AsmParser->hasInv2PiInlineImm()));
1812
0
1813
0
    Inst.addOperand(MCOperand::createImm(Val));
1814
0
    return;
1815
0
  }
1816
0
  default:
1817
0
    llvm_unreachable("invalid operand size");
1818
32.6k
  }
1819
32.6k
}
1820
1821
template <unsigned Bitwidth>
1822
398
void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1823
398
  APInt Literal(64, Imm.Val);
1824
398
1825
398
  if (!Imm.IsFPImm) {
1826
374
    // We got int literal token.
1827
374
    Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1828
374
    return;
1829
374
  }
1830
24
1831
24
  bool Lost;
1832
24
  APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1833
24
  FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1834
24
                    APFloat::rmNearestTiesToEven, &Lost);
1835
24
  Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1836
24
}
AMDGPUAsmParser.cpp:void (anonymous namespace)::AMDGPUOperand::addKImmFPOperands<16u>(llvm::MCInst&, unsigned int) const
Line
Count
Source
1822
66
void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1823
66
  APInt Literal(64, Imm.Val);
1824
66
1825
66
  if (!Imm.IsFPImm) {
1826
60
    // We got int literal token.
1827
60
    Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1828
60
    return;
1829
60
  }
1830
6
1831
6
  bool Lost;
1832
6
  APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1833
6
  FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1834
6
                    APFloat::rmNearestTiesToEven, &Lost);
1835
6
  Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1836
6
}
AMDGPUAsmParser.cpp:void (anonymous namespace)::AMDGPUOperand::addKImmFPOperands<32u>(llvm::MCInst&, unsigned int) const
Line
Count
Source
1822
332
void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1823
332
  APInt Literal(64, Imm.Val);
1824
332
1825
332
  if (!Imm.IsFPImm) {
1826
314
    // We got int literal token.
1827
314
    Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1828
314
    return;
1829
314
  }
1830
18
1831
18
  bool Lost;
1832
18
  APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1833
18
  FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1834
18
                    APFloat::rmNearestTiesToEven, &Lost);
1835
18
  Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1836
18
}
1837
1838
745k
void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1839
745k
  Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1840
745k
}
1841
1842
126k
static bool isInlineValue(unsigned Reg) {
1843
126k
  switch (Reg) {
1844
126k
  case AMDGPU::SRC_SHARED_BASE:
1845
66
  case AMDGPU::SRC_SHARED_LIMIT:
1846
66
  case AMDGPU::SRC_PRIVATE_BASE:
1847
66
  case AMDGPU::SRC_PRIVATE_LIMIT:
1848
66
  case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
1849
66
    return true;
1850
172
  case AMDGPU::SRC_VCCZ:
1851
172
  case AMDGPU::SRC_EXECZ:
1852
172
  case AMDGPU::SRC_SCC:
1853
172
    return true;
1854
126k
  default:
1855
126k
    return false;
1856
126k
  }
1857
126k
}
1858
1859
231k
bool AMDGPUOperand::isInlineValue() const {
1860
231k
  return isRegKind() && 
::isInlineValue(getReg())126k
;
1861
231k
}
1862
1863
//===----------------------------------------------------------------------===//
1864
// AsmParser
1865
//===----------------------------------------------------------------------===//
1866
1867
741k
static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1868
741k
  if (Is == IS_VGPR) {
1869
547k
    switch (RegWidth) {
1870
547k
      
default: return -14
;
1871
547k
      
case 1: return AMDGPU::VGPR_32RegClassID466k
;
1872
547k
      
case 2: return AMDGPU::VReg_64RegClassID72.7k
;
1873
547k
      
case 3: return AMDGPU::VReg_96RegClassID1.51k
;
1874
547k
      
case 4: return AMDGPU::VReg_128RegClassID6.92k
;
1875
547k
      
case 8: return AMDGPU::VReg_256RegClassID39
;
1876
547k
      
case 16: return AMDGPU::VReg_512RegClassID7
;
1877
193k
    }
1878
193k
  } else if (Is == IS_TTMP) {
1879
5.73k
    switch (RegWidth) {
1880
5.73k
      
default: return -10
;
1881
5.73k
      
case 1: return AMDGPU::TTMP_32RegClassID3.63k
;
1882
5.73k
      
case 2: return AMDGPU::TTMP_64RegClassID1.50k
;
1883
5.73k
      
case 4: return AMDGPU::TTMP_128RegClassID415
;
1884
5.73k
      
case 8: return AMDGPU::TTMP_256RegClassID162
;
1885
5.73k
      
case 16: return AMDGPU::TTMP_512RegClassID16
;
1886
187k
    }
1887
187k
  } else if (Is == IS_SGPR) {
1888
186k
    switch (RegWidth) {
1889
186k
      
default: return -126
;
1890
186k
      
case 1: return AMDGPU::SGPR_32RegClassID85.9k
;
1891
186k
      
case 2: return AMDGPU::SGPR_64RegClassID82.1k
;
1892
186k
      
case 4: return AMDGPU::SGPR_128RegClassID13.5k
;
1893
186k
      
case 8: return AMDGPU::SGPR_256RegClassID4.97k
;
1894
186k
      
case 16: return AMDGPU::SGPR_512RegClassID340
;
1895
848
    }
1896
848
  } else if (Is == IS_AGPR) {
1897
848
    switch (RegWidth) {
1898
848
      
default: return -10
;
1899
848
      
case 1: return AMDGPU::AGPR_32RegClassID272
;
1900
848
      
case 2: return AMDGPU::AReg_64RegClassID80
;
1901
848
      
case 4: return AMDGPU::AReg_128RegClassID192
;
1902
848
      
case 16: return AMDGPU::AReg_512RegClassID192
;
1903
848
      
case 32: return AMDGPU::AReg_1024RegClassID112
;
1904
0
    }
1905
0
  }
1906
0
  return -1;
1907
0
}
1908
1909
998k
static unsigned getSpecialRegForName(StringRef RegName) {
1910
998k
  return StringSwitch<unsigned>(RegName)
1911
998k
    .Case("exec", AMDGPU::EXEC)
1912
998k
    .Case("vcc", AMDGPU::VCC)
1913
998k
    .Case("flat_scratch", AMDGPU::FLAT_SCR)
1914
998k
    .Case("xnack_mask", AMDGPU::XNACK_MASK)
1915
998k
    .Case("shared_base", AMDGPU::SRC_SHARED_BASE)
1916
998k
    .Case("src_shared_base", AMDGPU::SRC_SHARED_BASE)
1917
998k
    .Case("shared_limit", AMDGPU::SRC_SHARED_LIMIT)
1918
998k
    .Case("src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
1919
998k
    .Case("private_base", AMDGPU::SRC_PRIVATE_BASE)
1920
998k
    .Case("src_private_base", AMDGPU::SRC_PRIVATE_BASE)
1921
998k
    .Case("private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
1922
998k
    .Case("src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
1923
998k
    .Case("pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
1924
998k
    .Case("src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
1925
998k
    .Case("lds_direct", AMDGPU::LDS_DIRECT)
1926
998k
    .Case("src_lds_direct", AMDGPU::LDS_DIRECT)
1927
998k
    .Case("m0", AMDGPU::M0)
1928
998k
    .Case("vccz", AMDGPU::SRC_VCCZ)
1929
998k
    .Case("src_vccz", AMDGPU::SRC_VCCZ)
1930
998k
    .Case("execz", AMDGPU::SRC_EXECZ)
1931
998k
    .Case("src_execz", AMDGPU::SRC_EXECZ)
1932
998k
    .Case("scc", AMDGPU::SRC_SCC)
1933
998k
    .Case("src_scc", AMDGPU::SRC_SCC)
1934
998k
    .Case("tba", AMDGPU::TBA)
1935
998k
    .Case("tma", AMDGPU::TMA)
1936
998k
    .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1937
998k
    .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1938
998k
    .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
1939
998k
    .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
1940
998k
    .Case("vcc_lo", AMDGPU::VCC_LO)
1941
998k
    .Case("vcc_hi", AMDGPU::VCC_HI)
1942
998k
    .Case("exec_lo", AMDGPU::EXEC_LO)
1943
998k
    .Case("exec_hi", AMDGPU::EXEC_HI)
1944
998k
    .Case("tma_lo", AMDGPU::TMA_LO)
1945
998k
    .Case("tma_hi", AMDGPU::TMA_HI)
1946
998k
    .Case("tba_lo", AMDGPU::TBA_LO)
1947
998k
    .Case("tba_hi", AMDGPU::TBA_HI)
1948
998k
    .Case("null", AMDGPU::SGPR_NULL)
1949
998k
    .Default(0);
1950
998k
}
1951
1952
bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1953
0
                                    SMLoc &EndLoc) {
1954
0
  auto R = parseRegister();
1955
0
  if (!R) return true;
1956
0
  assert(R->isReg());
1957
0
  RegNo = R->getReg();
1958
0
  StartLoc = R->getStartLoc();
1959
0
  EndLoc = R->getEndLoc();
1960
0
  return false;
1961
0
}
1962
1963
bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1964
                                            RegisterKind RegKind, unsigned Reg1,
1965
274
                                            unsigned RegNum) {
1966
274
  switch (RegKind) {
1967
274
  case IS_SPECIAL:
1968
19
    if (Reg == AMDGPU::EXEC_LO && 
Reg1 == AMDGPU::EXEC_HI5
) {
1969
5
      Reg = AMDGPU::EXEC;
1970
5
      RegWidth = 2;
1971
5
      return true;
1972
5
    }
1973
14
    if (Reg == AMDGPU::FLAT_SCR_LO && 
Reg1 == AMDGPU::FLAT_SCR_HI1
) {
1974
1
      Reg = AMDGPU::FLAT_SCR;
1975
1
      RegWidth = 2;
1976
1
      return true;
1977
1
    }
1978
13
    if (Reg == AMDGPU::XNACK_MASK_LO && 
Reg1 == AMDGPU::XNACK_MASK_HI0
) {
1979
0
      Reg = AMDGPU::XNACK_MASK;
1980
0
      RegWidth = 2;
1981
0
      return true;
1982
0
    }
1983
13
    if (Reg == AMDGPU::VCC_LO && 
Reg1 == AMDGPU::VCC_HI5
) {
1984
5
      Reg = AMDGPU::VCC;
1985
5
      RegWidth = 2;
1986
5
      return true;
1987
5
    }
1988
8
    if (Reg == AMDGPU::TBA_LO && 
Reg1 == AMDGPU::TBA_HI4
) {
1989
4
      Reg = AMDGPU::TBA;
1990
4
      RegWidth = 2;
1991
4
      return true;
1992
4
    }
1993
4
    if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1994
4
      Reg = AMDGPU::TMA;
1995
4
      RegWidth = 2;
1996
4
      return true;
1997
4
    }
1998
0
    return false;
1999
255
  case IS_VGPR:
2000
255
  case IS_SGPR:
2001
255
  case IS_AGPR:
2002
255
  case IS_TTMP:
2003
255
    if (Reg1 != Reg + RegWidth) {
2004
1
      return false;
2005
1
    }
2006
254
    RegWidth++;
2007
254
    return true;
2008
254
  default:
2009
0
    llvm_unreachable("unexpected register kind");
2010
274
  }
2011
274
}
2012
2013
static const StringRef Registers[] = {
2014
  { "v" },
2015
  { "s" },
2016
  { "ttmp" },
2017
  { "acc" },
2018
  { "a" },
2019
};
2020
2021
bool
2022
AMDGPUAsmParser::isRegister(const AsmToken &Token,
2023
987k
                            const AsmToken &NextToken) const {
2024
987k
2025
987k
  // A list of consecutive registers: [s0,s1,s2,s3]
2026
987k
  if (Token.is(AsmToken::LBrac))
2027
195
    return true;
2028
986k
2029
986k
  if (!Token.is(AsmToken::Identifier))
2030
103k
    return false;
2031
883k
2032
883k
  // A single register like s0 or a range of registers like s[0:1]
2033
883k
2034
883k
  StringRef RegName = Token.getString();
2035
883k
2036
1.62M
  for (StringRef Reg : Registers) {
2037
1.62M
    if (RegName.startswith(Reg)) {
2038
813k
      if (Reg.size() < RegName.size()) {
2039
626k
        unsigned RegNum;
2040
626k
        // A single register with an index: rXX
2041
626k
        if (!RegName.substr(Reg.size()).getAsInteger(10, RegNum))
2042
562k
          return true;
2043
186k
      } else {
2044
186k
        // A range of registers: r[XX:YY].
2045
186k
        if (NextToken.is(AsmToken::LBrac))
2046
186k
          return true;
2047
186k
      }
2048
813k
    }
2049
1.62M
  }
2050
883k
2051
883k
  
return getSpecialRegForName(RegName)134k
;
2052
883k
}
2053
2054
bool
2055
AMDGPUAsmParser::isRegister()
2056
941k
{
2057
941k
  return isRegister(getToken(), peekToken());
2058
941k
}
2059
2060
bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
2061
                                          unsigned &RegNum, unsigned &RegWidth,
2062
864k
                                          unsigned *DwordRegIndex) {
2063
864k
  if (DwordRegIndex) 
{ *DwordRegIndex = 0; }863k
2064
864k
  const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2065
864k
  if (getLexer().is(AsmToken::Identifier)) {
2066
864k
    StringRef RegName = Parser.getTok().getString();
2067
864k
    if ((Reg = getSpecialRegForName(RegName))) {
2068
123k
      Parser.Lex();
2069
123k
      RegKind = IS_SPECIAL;
2070
740k
    } else {
2071
740k
      unsigned RegNumIndex = 0;
2072
740k
      if (RegName[0] == 'v') {
2073
547k
        RegNumIndex = 1;
2074
547k
        RegKind = IS_VGPR;
2075
547k
      } else 
if (193k
RegName[0] == 's'193k
) {
2076
187k
        RegNumIndex = 1;
2077
187k
        RegKind = IS_SGPR;
2078
187k
      } else 
if (6.48k
RegName[0] == 'a'6.48k
) {
2079
848
        RegNumIndex = RegName.startswith("acc") ? 
36
:
1842
;
2080
848
        RegKind = IS_AGPR;
2081
5.63k
      } else if (RegName.startswith("ttmp")) {
2082
5.63k
        RegNumIndex = strlen("ttmp");
2083
5.63k
        RegKind = IS_TTMP;
2084
5.63k
      } else {
2085
0
        return false;
2086
0
      }
2087
740k
      if (RegName.size() > RegNumIndex) {
2088
556k
        // Single 32-bit register: vXX.
2089
556k
        if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
2090
0
          return false;
2091
556k
        Parser.Lex();
2092
556k
        RegWidth = 1;
2093
556k
      } else {
2094
184k
        // Range of registers: v[XX:YY]. ":YY" is optional.
2095
184k
        Parser.Lex();
2096
184k
        int64_t RegLo, RegHi;
2097
184k
        if (getLexer().isNot(AsmToken::LBrac))
2098
0
          return false;
2099
184k
        Parser.Lex();
2100
184k
2101
184k
        if (getParser().parseAbsoluteExpression(RegLo))
2102
0
          return false;
2103
184k
2104
184k
        const bool isRBrace = getLexer().is(AsmToken::RBrac);
2105
184k
        if (!isRBrace && 
getLexer().isNot(AsmToken::Colon)184k
)
2106
4
          return false;
2107
184k
        Parser.Lex();
2108
184k
2109
184k
        if (isRBrace) {
2110
68
          RegHi = RegLo;
2111
184k
        } else {
2112
184k
          if (getParser().parseAbsoluteExpression(RegHi))
2113
0
            return false;
2114
184k
2115
184k
          if (getLexer().isNot(AsmToken::RBrac))
2116
2
            return false;
2117
184k
          Parser.Lex();
2118
184k
        }
2119
184k
        RegNum = (unsigned) RegLo;
2120
184k
        RegWidth = (RegHi - RegLo) + 1;
2121
184k
      }
2122
740k
    }
2123
864k
  } else 
if (194
getLexer().is(AsmToken::LBrac)194
) {
2124
193
    // List of consecutive registers: [s0,s1,s2,s3]
2125
193
    Parser.Lex();
2126
193
    if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
2127
8
      return false;
2128
185
    if (RegWidth != 1)
2129
2
      return false;
2130
183
    RegisterKind RegKind1;
2131
183
    unsigned Reg1, RegNum1, RegWidth1;
2132
731
    do {
2133
731
      if (getLexer().is(AsmToken::Comma)) {
2134
275
        Parser.Lex();
2135
456
      } else if (getLexer().is(AsmToken::RBrac)) {
2136
181
        Parser.Lex();
2137
181
        break;
2138
275
      } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
2139
274
        if (RegWidth1 != 1) {
2140
0
          return false;
2141
0
        }
2142
274
        if (RegKind1 != RegKind) {
2143
0
          return false;
2144
0
        }
2145
274
        if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
2146
1
          return false;
2147
1
        }
2148
1
      } else {
2149
1
        return false;
2150
1
      }
2151
548
    } while (true);
2152
183
  } else {
2153
1
    return false;
2154
1
  }
2155
864k
  switch (RegKind) {
2156
864k
  case IS_SPECIAL:
2157
123k
    RegNum = 0;
2158
123k
    RegWidth = 1;
2159
123k
    break;
2160
864k
  case IS_VGPR:
2161
741k
  case IS_SGPR:
2162
741k
  case IS_AGPR:
2163
741k
  case IS_TTMP:
2164
741k
  {
2165
741k
    unsigned Size = 1;
2166
741k
    if (RegKind == IS_SGPR || 
RegKind == IS_TTMP554k
) {
2167
192k
      // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
2168
192k
      Size = std::min(RegWidth, 4u);
2169
192k
    }
2170
741k
    if (RegNum % Size != 0)
2171
48
      return false;
2172
741k
    if (DwordRegIndex) 
{ *DwordRegIndex = RegNum; }740k
2173
741k
    RegNum = RegNum / Size;
2174
741k
    int RCID = getRegClass(RegKind, RegWidth);
2175
741k
    if (RCID == -1)
2176
30
      return false;
2177
741k
    const MCRegisterClass RC = TRI->getRegClass(RCID);
2178
741k
    if (RegNum >= RC.getNumRegs())
2179
107
      return false;
2180
740k
    Reg = RC.getRegister(RegNum);
2181
740k
    break;
2182
740k
  }
2183
740k
2184
740k
  default:
2185
0
    llvm_unreachable("unexpected register kind");
2186
864k
  }
2187
864k
2188
864k
  if (!subtargetHasRegister(*TRI, Reg))
2189
583
    return false;
2190
863k
  return true;
2191
863k
}
2192
2193
Optional<StringRef>
2194
74
AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
2195
74
  switch (RegKind) {
2196
74
  case IS_VGPR:
2197
29
    return StringRef(".amdgcn.next_free_vgpr");
2198
74
  case IS_SGPR:
2199
45
    return StringRef(".amdgcn.next_free_sgpr");
2200
74
  default:
2201
0
    return None;
2202
74
  }
2203
74
}
2204
2205
38
void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
2206
38
  auto SymbolName = getGprCountSymbolName(RegKind);
2207
38
  assert(SymbolName && "initializing invalid register kind");
2208
38
  MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
2209
38
  Sym->setVariableValue(MCConstantExpr::create(0, getContext()));
2210
38
}
2211
2212
bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
2213
                                            unsigned DwordRegIndex,
2214
36
                                            unsigned RegWidth) {
2215
36
  // Symbols are only defined for GCN targets
2216
36
  if (AMDGPU::getIsaVersion(getSTI().getCPU()).Major < 6)
2217
0
    return true;
2218
36
2219
36
  auto SymbolName = getGprCountSymbolName(RegKind);
2220
36
  if (!SymbolName)
2221
0
    return true;
2222
36
  MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
2223
36
2224
36
  int64_t NewMax = DwordRegIndex + RegWidth - 1;
2225
36
  int64_t OldCount;
2226
36
2227
36
  if (!Sym->isVariable())
2228
0
    return !Error(getParser().getTok().getLoc(),
2229
0
                  ".amdgcn.next_free_{v,s}gpr symbols must be variable");
2230
36
  if (!Sym->getVariableValue(false)->evaluateAsAbsolute(OldCount))
2231
2
    return !Error(
2232
2
        getParser().getTok().getLoc(),
2233
2
        ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
2234
34
2235
34
  if (OldCount <= NewMax)
2236
17
    Sym->setVariableValue(MCConstantExpr::create(NewMax + 1, getContext()));
2237
34
2238
34
  return true;
2239
34
}
2240
2241
863k
std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
2242
863k
  const auto &Tok = Parser.getTok();
2243
863k
  SMLoc StartLoc = Tok.getLoc();
2244
863k
  SMLoc EndLoc = Tok.getEndLoc();
2245
863k
  RegisterKind RegKind;
2246
863k
  unsigned Reg, RegNum, RegWidth, DwordRegIndex;
2247
863k
2248
863k
  if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
2249
778
    //FIXME: improve error messages (bug 41303).
2250
778
    Error(StartLoc, "not a valid operand.");
2251
778
    return nullptr;
2252
778
  }
2253
863k
  if (AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())) {
2254
36
    if (!updateGprCountSymbols(RegKind, DwordRegIndex, RegWidth))
2255
2
      return nullptr;
2256
863k
  } else
2257
863k
    KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
2258
863k
  
return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc)863k
;
2259
863k
}
2260
2261
OperandMatchResultTy
2262
75.6k
AMDGPUAsmParser::parseImm(OperandVector &Operands, bool HasSP3AbsModifier) {
2263
75.6k
  // TODO: add syntactic sugar for 1/(2*PI)
2264
75.6k
2265
75.6k
  assert(!isRegister());
2266
75.6k
  assert(!isModifier());
2267
75.6k
2268
75.6k
  const auto& Tok = getToken();
2269
75.6k
  const auto& NextTok = peekToken();
2270
75.6k
  bool IsReal = Tok.is(AsmToken::Real);
2271
75.6k
  SMLoc S = getLoc();
2272
75.6k
  bool Negate = false;
2273
75.6k
2274
75.6k
  if (!IsReal && 
Tok.is(AsmToken::Minus)62.4k
&&
NextTok.is(AsmToken::Real)25.7k
) {
2275
12.6k
    lex();
2276
12.6k
    IsReal = true;
2277
12.6k
    Negate = true;
2278
12.6k
  }
2279
75.6k
2280
75.6k
  if (IsReal) {
2281
25.8k
    // Floating-point expressions are not supported.
2282
25.8k
    // Can only allow floating-point literals with an
2283
25.8k
    // optional sign.
2284
25.8k
2285
25.8k
    StringRef Num = getTokenStr();
2286
25.8k
    lex();
2287
25.8k
2288
25.8k
    APFloat RealVal(APFloat::IEEEdouble());
2289
25.8k
    auto roundMode = APFloat::rmNearestTiesToEven;
2290
25.8k
    if (RealVal.convertFromString(Num, roundMode) == APFloat::opInvalidOp) {
2291
0
      return MatchOperand_ParseFail;
2292
0
    }
2293
25.8k
    if (Negate)
2294
12.6k
      RealVal.changeSign();
2295
25.8k
2296
25.8k
    Operands.push_back(
2297
25.8k
      AMDGPUOperand::CreateImm(this, RealVal.bitcastToAPInt().getZExtValue(), S,
2298
25.8k
                               AMDGPUOperand::ImmTyNone, true));
2299
25.8k
2300
25.8k
    return MatchOperand_Success;
2301
25.8k
2302
49.7k
  } else {
2303
49.7k
    int64_t IntVal;
2304
49.7k
    const MCExpr *Expr;
2305
49.7k
    SMLoc S = getLoc();
2306
49.7k
2307
49.7k
    if (HasSP3AbsModifier) {
2308
88
      // This is a workaround for handling expressions
2309
88
      // as arguments of SP3 'abs' modifier, for example:
2310
88
      //     |1.0|
2311
88
      //     |-1|
2312
88
      //     |1+x|
2313
88
      // This syntax is not compatible with syntax of standard
2314
88
      // MC expressions (due to the trailing '|').
2315
88
      SMLoc EndLoc;
2316
88
      if (getParser().parsePrimaryExpr(Expr, EndLoc))
2317
0
        return MatchOperand_ParseFail;
2318
49.6k
    } else {
2319
49.6k
      if (Parser.parseExpression(Expr))
2320
106
        return MatchOperand_ParseFail;
2321
49.6k
    }
2322
49.6k
2323
49.6k
    if (Expr->evaluateAsAbsolute(IntVal)) {
2324
40.2k
      Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
2325
40.2k
    } else {
2326
9.38k
      Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2327
9.38k
    }
2328
49.6k
2329
49.6k
    return MatchOperand_Success;
2330
49.6k
  }
2331
0
2332
0
  return MatchOperand_NoMatch;
2333
0
}
2334
2335
OperandMatchResultTy
2336
940k
AMDGPUAsmParser::parseReg(OperandVector &Operands) {
2337
940k
  if (!isRegister())
2338
77.2k
    return MatchOperand_NoMatch;
2339
863k
2340
863k
  if (auto R = parseRegister()) {
2341
862k
    assert(R->isReg());
2342
862k
    Operands.push_back(std::move(R));
2343
862k
    return MatchOperand_Success;
2344
862k
  }
2345
779
  return MatchOperand_ParseFail;
2346
779
}
2347
2348
OperandMatchResultTy
2349
822k
AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod) {
2350
822k
  auto res = parseReg(Operands);
2351
822k
  if (res != MatchOperand_NoMatch) {
2352
745k
    return res;
2353
745k
  } else 
if (77.2k
isModifier()77.2k
) {
2354
2.03k
    return MatchOperand_NoMatch;
2355
75.2k
  } else {
2356
75.2k
    return parseImm(Operands, HasSP3AbsMod);
2357
75.2k
  }
2358
822k
}
2359
2360
bool
2361
103k
AMDGPUAsmParser::isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
2362
103k
  if (Token.is(AsmToken::Identifier) && 
NextToken.is(AsmToken::LParen)11.1k
) {
2363
182
    const auto &str = Token.getString();
2364
182
    return str == "abs" || 
str == "neg"74
||
str == "sext"55
;
2365
182
  }
2366
103k
  return false;
2367
103k
}
2368
2369
bool
2370
77.1k
AMDGPUAsmParser::isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const {
2371
77.1k
  return Token.is(AsmToken::Identifier) && 
NextToken.is(AsmToken::Colon)10.9k
;
2372
77.1k
}
2373
2374
bool
2375
103k
AMDGPUAsmParser::isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
2376
103k
  return isNamedOperandModifier(Token, NextToken) || 
Token.is(AsmToken::Pipe)103k
;
2377
103k
}
2378
2379
bool
2380
25.9k
AMDGPUAsmParser::isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
2381
25.9k
  return isRegister(Token, NextToken) || 
isOperandModifier(Token, NextToken)25.7k
;
2382
25.9k
}
2383
2384
// Check if this is an operand modifier or an opcode modifier
2385
// which may look like an expression but it is not. We should
2386
// avoid parsing these modifiers as expressions. Currently
2387
// recognized sequences are:
2388
//   |...|
2389
//   abs(...)
2390
//   neg(...)
2391
//   sext(...)
2392
//   -reg
2393
//   -|...|
2394
//   -abs(...)
2395
//   name:...
2396
// Note that simple opcode modifiers like 'gds' may be parsed as
2397
// expressions; this is a special case. See getExpressionAsToken.
2398
//
2399
bool
2400
77.7k
AMDGPUAsmParser::isModifier() {
2401
77.7k
2402
77.7k
  AsmToken Tok = getToken();
2403
77.7k
  AsmToken NextToken[2];
2404
77.7k
  peekTokens(NextToken);
2405
77.7k
2406
77.7k
  return isOperandModifier(Tok, NextToken[0]) ||
2407
77.7k
         
(77.3k
Tok.is(AsmToken::Minus)77.3k
&&
isRegOrOperandModifier(NextToken[0], NextToken[1])25.9k
) ||
2408
77.7k
         
isOpcodeModifierWithVal(Tok, NextToken[0])77.1k
;
2409
77.7k
}
2410
2411
// Check if the current token is an SP3 'neg' modifier.
2412
// Currently this modifier is allowed in the following context:
2413
//
2414
// 1. Before a register, e.g. "-v0", "-v[...]" or "-[v0,v1]".
2415
// 2. Before an 'abs' modifier: -abs(...)
2416
// 3. Before an SP3 'abs' modifier: -|...|
2417
//
2418
// In all other cases "-" is handled as a part
2419
// of an expression that follows the sign.
2420
//
2421
// Note: When "-" is followed by an integer literal,
2422
// this is interpreted as integer negation rather
2423
// than a floating-point NEG modifier applied to N.
2424
// Beside being contr-intuitive, such use of floating-point
2425
// NEG modifier would have resulted in different meaning
2426
// of integer literals used with VOP1/2/C and VOP3,
2427
// for example:
2428
//    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
2429
//    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
2430
// Negative fp literals with preceding "-" are
2431
// handled likewise for unifomtity
2432
//
2433
bool
2434
247k
AMDGPUAsmParser::parseSP3NegModifier() {
2435
247k
2436
247k
  AsmToken NextToken[2];
2437
247k
  peekTokens(NextToken);
2438
247k
2439
247k
  if (isToken(AsmToken::Minus) &&
2440
247k
      
(19.4k
isRegister(NextToken[0], NextToken[1])19.4k
||
2441
19.4k
       
NextToken[0].is(AsmToken::Pipe)10.8k
||
2442
19.4k
       
isId(NextToken[0], "abs")10.7k
)) {
2443
8.72k
    lex();
2444
8.72k
    return true;
2445
8.72k
  }
2446
238k
2447
238k
  return false;
2448
238k
}
2449
2450
OperandMatchResultTy
2451
AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
2452
247k
                                              bool AllowImm) {
2453
247k
  bool Neg, SP3Neg;
2454
247k
  bool Abs, SP3Abs;
2455
247k
  SMLoc Loc;
2456
247k
2457
247k
  // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
2458
247k
  if (isToken(AsmToken::Minus) && 
peekToken().is(AsmToken::Minus)19.4k
) {
2459
1
    Error(getLoc(), "invalid syntax, expected 'neg' modifier");
2460
1
    return MatchOperand_ParseFail;
2461
1
  }
2462
247k
2463
247k
  SP3Neg = parseSP3NegModifier();
2464
247k
2465
247k
  Loc = getLoc();
2466
247k
  Neg = trySkipId("neg");
2467
247k
  if (Neg && 
SP3Neg116
) {
2468
0
    Error(Loc, "expected register or immediate");
2469
0
    return MatchOperand_ParseFail;
2470
0
  }
2471
247k
  if (Neg && 
!skipToken(AsmToken::LParen, "expected left paren after neg")116
)
2472
0
    return MatchOperand_ParseFail;
2473
247k
2474
247k
  Abs = trySkipId("abs");
2475
247k
  if (Abs && 
!skipToken(AsmToken::LParen, "expected left paren after abs")180
)
2476
0
    return MatchOperand_ParseFail;
2477
247k
2478
247k
  Loc = getLoc();
2479
247k
  SP3Abs = trySkipToken(AsmToken::Pipe);
2480
247k
  if (Abs && 
SP3Abs180
) {
2481
0
    Error(Loc, "expected register or immediate");
2482
0
    return MatchOperand_ParseFail;
2483
0
  }
2484
247k
2485
247k
  OperandMatchResultTy Res;
2486
247k
  if (AllowImm) {
2487
246k
    Res = parseRegOrImm(Operands, SP3Abs);
2488
246k
  } else {
2489
206
    Res = parseReg(Operands);
2490
206
  }
2491
247k
  if (Res != MatchOperand_Success) {
2492
131
    return (SP3Neg || 
Neg115
||
SP3Abs113
||
Abs91
)?
MatchOperand_ParseFail43
:
Res88
;
2493
131
  }
2494
246k
2495
246k
  if (SP3Abs && 
!skipToken(AsmToken::Pipe, "expected vertical bar")4.59k
)
2496
4
    return MatchOperand_ParseFail;
2497
246k
  if (Abs && 
!skipToken(AsmToken::RParen, "expected closing parentheses")177
)
2498
0
    return MatchOperand_ParseFail;
2499
246k
  if (Neg && 
!skipToken(AsmToken::RParen, "expected closing parentheses")114
)
2500
0
    return MatchOperand_ParseFail;
2501
246k
2502
246k
  AMDGPUOperand::Modifiers Mods;
2503
246k
  Mods.Abs = Abs || 
SP3Abs246k
;
2504
246k
  Mods.Neg = Neg || 
SP3Neg246k
;
2505
246k
2506
246k
  if (Mods.hasFPModifiers()) {
2507
13.4k
    AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2508
13.4k
    if (Op.isExpr()) {
2509
6
      Error(Op.getStartLoc(), "expected an absolute expression");
2510
6
      return MatchOperand_ParseFail;
2511
6
    }
2512
13.4k
    Op.setModifiers(Mods);
2513
13.4k
  }
2514
246k
  
return MatchOperand_Success246k
;
2515
246k
}
2516
2517
OperandMatchResultTy
2518
AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
2519
144k
                                               bool AllowImm) {
2520
144k
  bool Sext = trySkipId("sext");
2521
144k
  if (Sext && 
!skipToken(AsmToken::LParen, "expected left paren after sext")1.66k
)
2522
0
    return MatchOperand_ParseFail;
2523
144k
2524
144k
  OperandMatchResultTy Res;
2525
144k
  if (AllowImm) {
2526
144k
    Res = parseRegOrImm(Operands);
2527
144k
  } else {
2528
314
    Res = parseReg(Operands);
2529
314
  }
2530
144k
  if (Res != MatchOperand_Success) {
2531
16
    return Sext? 
MatchOperand_ParseFail0
: Res;
2532
16
  }
2533
144k
2534
144k
  if (Sext && 
!skipToken(AsmToken::RParen, "expected closing parentheses")1.66k
)
2535
0
    return MatchOperand_ParseFail;
2536
144k
2537
144k
  AMDGPUOperand::Modifiers Mods;
2538
144k
  Mods.Sext = Sext;
2539
144k
2540
144k
  if (Mods.hasIntModifiers()) {
2541
1.66k
    AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2542
1.66k
    if (Op.isExpr()) {
2543
2
      Error(Op.getStartLoc(), "expected an absolute expression");
2544
2
      return MatchOperand_ParseFail;
2545
2
    }
2546
1.65k
    Op.setModifiers(Mods);
2547
1.65k
  }
2548
144k
2549
144k
  
return MatchOperand_Success144k
;
2550
144k
}
2551
2552
OperandMatchResultTy
2553
206
AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
2554
206
  return parseRegOrImmWithFPInputMods(Operands, false);
2555
206
}
2556
2557
OperandMatchResultTy
2558
314
AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
2559
314
  return parseRegOrImmWithIntInputMods(Operands, false);
2560
314
}
2561
2562
960
OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
2563
960
  auto Loc = getLoc();
2564
960
  if (trySkipId("off")) {
2565
207
    Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Loc,
2566
207
                                                AMDGPUOperand::ImmTyOff, false));
2567
207
    return MatchOperand_Success;
2568
207
  }
2569
753
2570
753
  if (!isRegister())
2571
48
    return MatchOperand_NoMatch;
2572
705
2573
705
  std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
2574
705
  if (Reg) {
2575
704
    Operands.push_back(std::move(Reg));
2576
704
    return MatchOperand_Success;
2577
704
  }
2578
1
2579
1
  return MatchOperand_ParseFail;
2580
1
2581
1
}
2582
2583
299k
unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
2584
299k
  uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2585
299k
2586
299k
  if ((getForcedEncodingSize() == 32 && 
(TSFlags & SIInstrFlags::VOP3)5.86k
) ||
2587
299k
      (getForcedEncodingSize() == 64 && 
!(TSFlags & SIInstrFlags::VOP3)77.6k
) ||
2588
299k
      
(299k
isForcedDPP()299k
&&
!(TSFlags & SIInstrFlags::DPP)19.5k
) ||
2589
299k
      
(299k
isForcedSDWA()299k
&&
!(TSFlags & SIInstrFlags::SDWA)50.8k
) )
2590
6
    return Match_InvalidOperand;
2591
299k
2592
299k
  if ((TSFlags & SIInstrFlags::VOP3) &&
2593
299k
      
(TSFlags & SIInstrFlags::VOPAsmPrefer32Bit)113k
&&
2594
299k
      
getForcedEncodingSize() != 6474
)
2595
0
    return Match_PreferE32;
2596
299k
2597
299k
  if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
2598
299k
      
Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi298k
) {
2599
62
    // v_mac_f32/16 allow only dst_sel == DWORD;
2600
62
    auto OpNum =
2601
62
        AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2602
62
    const auto &Op = Inst.getOperand(OpNum);
2603
62
    if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2604
2
      return Match_InvalidOperand;
2605
2
    }
2606
299k
  }
2607
299k
2608
299k
  return Match_Success;
2609
299k
}
2610
2611
// What asm variants we should check
2612
329k
ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2613
329k
  if (getForcedEncodingSize() == 32) {
2614
6.47k
    static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2615
6.47k
    return makeArrayRef(Variants);
2616
6.47k
  }
2617
323k
2618
323k
  if (isForcedVOP3()) {
2619
88.9k
    static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2620
88.9k
    return makeArrayRef(Variants);
2621
88.9k
  }
2622
234k
2623
234k
  if (isForcedSDWA()) {
2624
58.8k
    static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2625
58.8k
                                        AMDGPUAsmVariants::SDWA9};
2626
58.8k
    return makeArrayRef(Variants);
2627
58.8k
  }
2628
175k
2629
175k
  if (isForcedDPP()) {
2630
19.6k
    static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2631
19.6k
    return makeArrayRef(Variants);
2632
19.6k
  }
2633
155k
2634
155k
  static const unsigned Variants[] = {
2635
155k
    AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
2636
155k
    AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
2637
155k
  };
2638
155k
2639
155k
  return makeArrayRef(Variants);
2640
155k
}
2641
2642
211k
unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2643
211k
  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2644
211k
  const unsigned Num = Desc.getNumImplicitUses();
2645
420k
  for (unsigned i = 0; i < Num; 
++i209k
) {
2646
211k
    unsigned Reg = Desc.ImplicitUses[i];
2647
211k
    switch (Reg) {
2648
211k
    case AMDGPU::FLAT_SCR:
2649
2.25k
    case AMDGPU::VCC:
2650
2.25k
    case AMDGPU::VCC_LO:
2651
2.25k
    case AMDGPU::VCC_HI:
2652
2.25k
    case AMDGPU::M0:
2653
2.25k
    case AMDGPU::SGPR_NULL:
2654
2.25k
      return Reg;
2655
209k
    default:
2656
209k
      break;
2657
211k
    }
2658
211k
  }
2659
211k
  
return AMDGPU::NoRegister209k
;
2660
211k
}
2661
2662
// NB: This code is correct only when used to check constant
2663
// bus limitations because GFX7 support no f16 inline constants.
2664
// Note that there are no cases when a GFX7 opcode violates
2665
// constant bus limitations due to the use of an f16 constant.
2666
bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2667
77.8k
                                       unsigned OpIdx) const {
2668
77.8k
  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2669
77.8k
2670
77.8k
  if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2671
62
    return false;
2672
62
  }
2673
77.8k
2674
77.8k
  const MCOperand &MO = Inst.getOperand(OpIdx);
2675
77.8k
2676
77.8k
  int64_t Val = MO.getImm();
2677
77.8k
  auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2678
77.8k
2679
77.8k
  switch (OpSize) { // expected operand size
2680
77.8k
  case 8:
2681
18.1k
    return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2682
77.8k
  case 4:
2683
41.3k
    return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2684
77.8k
  case 2: {
2685
18.3k
    const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2686
18.3k
    if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2687
18.3k
        OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16 ||
2688
18.3k
        OperandType == AMDGPU::OPERAND_REG_INLINE_AC_V2INT16 ||
2689
18.3k
        OperandType == AMDGPU::OPERAND_REG_INLINE_AC_V2FP16 ||
2690
18.3k
        OperandType == AMDGPU::OPERAND_REG_IMM_V2INT16 ||
2691
18.3k
        
OperandType == AMDGPU::OPERAND_REG_IMM_V2FP1618.1k
) {
2692
619
      return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2693
17.7k
    } else {
2694
17.7k
      return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2695
17.7k
    }
2696
0
  }
2697
0
  default:
2698
0
    llvm_unreachable("invalid operand size");
2699
77.8k
  }
2700
77.8k
}
2701
2702
414k
bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2703
414k
  const MCOperand &MO = Inst.getOperand(OpIdx);
2704
414k
  if (MO.isImm()) {
2705
44.7k
    return !isInlineConstant(Inst, OpIdx);
2706
44.7k
  }
2707
369k
  return !MO.isReg() ||
2708
369k
         
isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo())369k
;
2709
369k
}
2710
2711
298k
bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2712
298k
  const unsigned Opcode = Inst.getOpcode();
2713
298k
  const MCInstrDesc &Desc = MII.get(Opcode);
2714
298k
  unsigned ConstantBusUseCount = 0;
2715
298k
  unsigned NumLiterals = 0;
2716
298k
  unsigned LiteralSize;
2717
298k
2718
298k
  if (Desc.TSFlags &
2719
298k
      (SIInstrFlags::VOPC |
2720
298k
       SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
2721
298k
       SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
2722
298k
       SIInstrFlags::SDWA)) {
2723
211k
    // Check special imm operands (used by madmk, etc)
2724
211k
    if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2725
398
      ++ConstantBusUseCount;
2726
398
    }
2727
211k
2728
211k
    SmallDenseSet<unsigned> SGPRsUsed;
2729
211k
    unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2730
211k
    if (SGPRUsed != AMDGPU::NoRegister) {
2731
2.25k
      SGPRsUsed.insert(SGPRUsed);
2732
2.25k
      ++ConstantBusUseCount;
2733
2.25k
    }
2734
211k
2735
211k
    const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2736
211k
    const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2737
211k
    const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2738
211k
2739
211k
    const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2740
211k
2741
598k
    for (int OpIdx : OpIndices) {
2742
598k
      if (OpIdx == -1) 
break184k
;
2743
414k
2744
414k
      const MCOperand &MO = Inst.getOperand(OpIdx);
2745
414k
      if (usesConstantBus(Inst, OpIdx)) {
2746
93.4k
        if (MO.isReg()) {
2747
87.7k
          const unsigned Reg = mc2PseudoReg(MO.getReg());
2748
87.7k
          // Pairs of registers with a partial intersections like these
2749
87.7k
          //   s0, s[0:1]
2750
87.7k
          //   flat_scratch_lo, flat_scratch
2751
87.7k
          //   flat_scratch_lo, flat_scratch_hi
2752
87.7k
          // are theoretically valid but they are disabled anyway.
2753
87.7k
          // Note that this code mimics SIInstrInfo::verifyInstruction
2754
87.7k
          if (!SGPRsUsed.count(Reg)) {
2755
87.6k
            SGPRsUsed.insert(Reg);
2756
87.6k
            ++ConstantBusUseCount;
2757
87.6k
          }
2758
87.7k
        } else { // Expression or a literal
2759
5.75k
2760
5.75k
          if (Desc.OpInfo[OpIdx].OperandType == MCOI::OPERAND_IMMEDIATE)
2761
62
            continue; // special operand like VINTERP attr_chan
2762
5.69k
2763
5.69k
          // An instruction may use only one literal.
2764
5.69k
          // This has been validated on the previous step.
2765
5.69k
          // See validateVOP3Literal.
2766
5.69k
          // This literal may be used as more than one operand.
2767
5.69k
          // If all these operands are of the same size,
2768
5.69k
          // this literal counts as one scalar value.
2769
5.69k
          // Otherwise it counts as 2 scalar values.
2770
5.69k
          // See "GFX10 Shader Programming", section 3.6.2.3.
2771
5.69k
2772
5.69k
          unsigned Size = AMDGPU::getOperandSize(Desc, OpIdx);
2773
5.69k
          if (Size < 4) 
Size = 41.48k
;
2774
5.69k
2775
5.69k
          if (NumLiterals == 0) {
2776
5.61k
            NumLiterals = 1;
2777
5.61k
            LiteralSize = Size;
2778
5.61k
          } else 
if (80
LiteralSize != Size80
) {
2779
8
            NumLiterals = 2;
2780
8
          }
2781
5.69k
        }
2782
93.4k
      }
2783
414k
    }
2784
211k
  }
2785
298k
  ConstantBusUseCount += NumLiterals;
2786
298k
2787
298k
  if (isGFX10())
2788
135k
    return ConstantBusUseCount <= 2;
2789
162k
2790
162k
  return ConstantBusUseCount <= 1;
2791
162k
}
2792
2793
298k
bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2794
298k
  const unsigned Opcode = Inst.getOpcode();
2795
298k
  const MCInstrDesc &Desc = MII.get(Opcode);
2796
298k
2797
298k
  const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2798
298k
  if (DstIdx == -1 ||
2799
298k
      
Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1125k
) {
2800
297k
    return true;
2801
297k
  }
2802
833
2803
833
  const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2804
833
2805
833
  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2806
833
  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2807
833
  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2808
833
2809
833
  assert(DstIdx != -1);
2810
833
  const MCOperand &Dst = Inst.getOperand(DstIdx);
2811
833
  assert(Dst.isReg());
2812
833
  const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2813
833
2814
833
  const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2815
833
2816
2.46k
  for (int SrcIdx : SrcIndices) {
2817
2.46k
    if (SrcIdx == -1) 
break0
;
2818
2.46k
    const MCOperand &Src = Inst.getOperand(SrcIdx);
2819
2.46k
    if (Src.isReg()) {
2820
2.28k
      const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2821
2.28k
      if (isRegIntersect(DstReg, SrcReg, TRI)) {
2822
34
        return false;
2823
34
      }
2824
2.28k
    }
2825
2.46k
  }
2826
833
2827
833
  
return true799
;
2828
833
}
2829
2830
298k
bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
2831
298k
2832
298k
  const unsigned Opc = Inst.getOpcode();
2833
298k
  const MCInstrDesc &Desc = MII.get(Opc);
2834
298k
2835
298k
  if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && 
!hasIntClamp()43.0k
) {
2836
7.05k
    int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
2837
7.05k
    assert(ClampIdx != -1);
2838
7.05k
    return Inst.getOperand(ClampIdx).getImm() == 0;
2839
7.05k
  }
2840
291k
2841
291k
  return true;
2842
291k
}
2843
2844
298k
bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst) {
2845
298k
2846
298k
  const unsigned Opc = Inst.getOpcode();
2847
298k
  const MCInstrDesc &Desc = MII.get(Opc);
2848
298k
2849
298k
  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2850
293k
    return true;
2851
4.36k
2852
4.36k
  int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
2853
4.36k
  int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2854
4.36k
  int TFEIdx   = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
2855
4.36k
2856
4.36k
  assert(VDataIdx != -1);
2857
4.36k
  assert(DMaskIdx != -1);
2858
4.36k
  assert(TFEIdx != -1);
2859
4.36k
2860
4.36k
  unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
2861
4.36k
  unsigned TFESize = Inst.getOperand(TFEIdx).getImm()? 
1157
:
04.21k
;
2862
4.36k
  unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2863
4.36k
  if (DMask == 0)
2864
288
    DMask = 1;
2865
4.36k
2866
4.36k
  unsigned DataSize =
2867
4.36k
    (Desc.TSFlags & SIInstrFlags::Gather4) ? 
41.19k
:
countPopulation(DMask)3.17k
;
2868
4.36k
  if (hasPackedD16()) {
2869
3.09k
    int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
2870
3.09k
    if (D16Idx >= 0 && 
Inst.getOperand(D16Idx).getImm()2.25k
)
2871
125
      DataSize = (DataSize + 1) / 2;
2872
3.09k
  }
2873
4.36k
2874
4.36k
  return (VDataSize / 4) == DataSize + TFESize;
2875
4.36k
}
2876
2877
298k
bool AMDGPUAsmParser::validateMIMGAddrSize(const MCInst &Inst) {
2878
298k
  const unsigned Opc = Inst.getOpcode();
2879
298k
  const MCInstrDesc &Desc = MII.get(Opc);
2880
298k
2881
298k
  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0 || 
!isGFX10()4.27k
)
2882
297k
    return true;
2883
130
2884
130
  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
2885
130
  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2886
130
      AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
2887
130
  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
2888
130
  int SrsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
2889
130
  int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
2890
130
2891
130
  assert(VAddr0Idx != -1);
2892
130
  assert(SrsrcIdx != -1);
2893
130
  assert(DimIdx != -1);
2894
130
  assert(SrsrcIdx > VAddr0Idx);
2895
130
2896
130
  unsigned Dim = Inst.getOperand(DimIdx).getImm();
2897
130
  const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(Dim);
2898
130
  bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
2899
130
  unsigned VAddrSize =
2900
130
      IsNSA ? 
SrsrcIdx - VAddr0Idx96
2901
130
            : 
AMDGPU::getRegOperandSize(getMRI(), Desc, VAddr0Idx) / 434
;
2902
130
2903
130
  unsigned AddrSize = BaseOpcode->NumExtraArgs +
2904
130
                      (BaseOpcode->Gradients ? 
DimInfo->NumGradients27
:
0103
) +
2905
130
                      (BaseOpcode->Coordinates ? 
DimInfo->NumCoords129
:
01
) +
2906
130
                      (BaseOpcode->LodOrClampOrMip ? 
150
:
080
);
2907
130
  if (!IsNSA) {
2908
34
    if (AddrSize > 8)
2909
2
      AddrSize = 16;
2910
32
    else if (AddrSize > 4)
2911
2
      AddrSize = 8;
2912
34
  }
2913
130
2914
130
  return VAddrSize == AddrSize;
2915
130
}
2916
2917
298k
bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
2918
298k
2919
298k
  const unsigned Opc = Inst.getOpcode();
2920
298k
  const MCInstrDesc &Desc = MII.get(Opc);
2921
298k
2922
298k
  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2923
293k
    return true;
2924
4.26k
  if (!Desc.mayLoad() || 
!Desc.mayStore()3.71k
)
2925
3.52k
    return true; // Not atomic
2926
746
2927
746
  int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2928
746
  unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2929
746
2930
746
  // This is an incomplete check because image_atomic_cmpswap
2931
746
  // may only use 0x3 and 0xf while other atomic operations
2932
746
  // may use 0x1 and 0x3. However these limitations are
2933
746
  // verified when we check that dmask matches dst size.
2934
746
  return DMask == 0x1 || 
DMask == 0x3137
||
DMask == 0xf25
;
2935
746
}
2936
2937
298k
bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {
2938
298k
2939
298k
  const unsigned Opc = Inst.getOpcode();
2940
298k
  const MCInstrDesc &Desc = MII.get(Opc);
2941
298k
2942
298k
  if ((Desc.TSFlags & SIInstrFlags::Gather4) == 0)
2943
296k
    return true;
2944
1.17k
2945
1.17k
  int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2946
1.17k
  unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2947
1.17k
2948
1.17k
  // GATHER4 instructions use dmask in a different fashion compared to
2949
1.17k
  // other MIMG instructions. The only useful DMASK values are
2950
1.17k
  // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
2951
1.17k
  // (red,red,red,red) etc.) The ISA document doesn't mention
2952
1.17k
  // this.
2953
1.17k
  return DMask == 0x1 || 
DMask == 0x2228
||
DMask == 0x4153
||
DMask == 0x878
;
2954
1.17k
}
2955
2956
298k
bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
2957
298k
2958
298k
  const unsigned Opc = Inst.getOpcode();
2959
298k
  const MCInstrDesc &Desc = MII.get(Opc);
2960
298k
2961
298k
  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2962
293k
    return true;
2963
4.48k
2964
4.48k
  int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
2965
4.48k
  if (D16Idx >= 0 && 
Inst.getOperand(D16Idx).getImm()3.38k
) {
2966
325
    if (isCI() || 
isSI()285
)
2967
120
      return false;
2968
4.36k
  }
2969
4.36k
2970
4.36k
  return true;
2971
4.36k
}
2972
2973
298k
bool AMDGPUAsmParser::validateMIMGDim(const MCInst &Inst) {
2974
298k
  const unsigned Opc = Inst.getOpcode();
2975
298k
  const MCInstrDesc &Desc = MII.get(Opc);
2976
298k
2977
298k
  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2978
293k
    return true;
2979
4.36k
2980
4.36k
  int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
2981
4.36k
  if (DimIdx < 0)
2982
4.23k
    return true;
2983
131
2984
131
  long Imm = Inst.getOperand(DimIdx).getImm();
2985
131
  if (Imm < 0 || Imm >= 8)
2986
0
    return false;
2987
131
2988
131
  return true;
2989
131
}
2990
2991
static bool IsRevOpcode(const unsigned Opcode)
2992
81
{
2993
81
  switch (Opcode) {
2994
81
  case AMDGPU::V_SUBREV_F32_e32:
2995
42
  case AMDGPU::V_SUBREV_F32_e64:
2996
42
  case AMDGPU::V_SUBREV_F32_e32_gfx10:
2997
42
  case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
2998
42
  case AMDGPU::V_SUBREV_F32_e32_vi:
2999
42
  case AMDGPU::V_SUBREV_F32_e64_gfx10:
3000
42
  case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
3001
42
  case AMDGPU::V_SUBREV_F32_e64_vi:
3002
42
3003
42
  case AMDGPU::V_SUBREV_I32_e32:
3004
42
  case AMDGPU::V_SUBREV_I32_e64:
3005
42
  case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
3006
42
  case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
3007
42
3008
42
  case AMDGPU::V_SUBBREV_U32_e32:
3009
42
  case AMDGPU::V_SUBBREV_U32_e64:
3010
42
  case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
3011
42
  case AMDGPU::V_SUBBREV_U32_e32_vi:
3012
42
  case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
3013
42
  case AMDGPU::V_SUBBREV_U32_e64_vi:
3014
42
3015
42
  case AMDGPU::V_SUBREV_U32_e32:
3016
42
  case AMDGPU::V_SUBREV_U32_e64:
3017
42
  case AMDGPU::V_SUBREV_U32_e32_gfx9:
3018
42
  case AMDGPU::V_SUBREV_U32_e32_vi:
3019
42
  case AMDGPU::V_SUBREV_U32_e64_gfx9:
3020
42
  case AMDGPU::V_SUBREV_U32_e64_vi:
3021
42
3022
42
  case AMDGPU::V_SUBREV_F16_e32:
3023
42
  case AMDGPU::V_SUBREV_F16_e64:
3024
42
  case AMDGPU::V_SUBREV_F16_e32_gfx10:
3025
42
  case AMDGPU::V_SUBREV_F16_e32_vi:
3026
42
  case AMDGPU::V_SUBREV_F16_e64_gfx10:
3027
42
  case AMDGPU::V_SUBREV_F16_e64_vi:
3028
42
3029
42
  case AMDGPU::V_SUBREV_U16_e32:
3030
42
  case AMDGPU::V_SUBREV_U16_e64:
3031
42
  case AMDGPU::V_SUBREV_U16_e32_vi:
3032
42
  case AMDGPU::V_SUBREV_U16_e64_vi:
3033
42
3034
42
  case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
3035
42
  case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
3036
42
  case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
3037
42
3038
42
  case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
3039
42
  case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
3040
42
3041
42
  case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
3042
42
  case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
3043
42
3044
42
  case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
3045
42
  case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
3046
42
3047
42
  case AMDGPU::V_LSHRREV_B32_e32:
3048
42
  case AMDGPU::V_LSHRREV_B32_e64:
3049
42
  case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
3050
42
  case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
3051
42
  case AMDGPU::V_LSHRREV_B32_e32_vi:
3052
42
  case AMDGPU::V_LSHRREV_B32_e64_vi:
3053
42
  case AMDGPU::V_LSHRREV_B32_e32_gfx10:
3054
42
  case AMDGPU::V_LSHRREV_B32_e64_gfx10:
3055
42
3056
42
  case AMDGPU::V_ASHRREV_I32_e32:
3057
42
  case AMDGPU::V_ASHRREV_I32_e64:
3058
42
  case AMDGPU::V_ASHRREV_I32_e32_gfx10:
3059
42
  case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
3060
42
  case AMDGPU::V_ASHRREV_I32_e32_vi:
3061
42
  case AMDGPU::V_ASHRREV_I32_e64_gfx10:
3062
42
  case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
3063
42
  case AMDGPU::V_ASHRREV_I32_e64_vi:
3064
42
3065
42
  case AMDGPU::V_LSHLREV_B32_e32:
3066
42
  case AMDGPU::V_LSHLREV_B32_e64:
3067
42
  case AMDGPU::V_LSHLREV_B32_e32_gfx10:
3068
42
  case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
3069
42
  case AMDGPU::V_LSHLREV_B32_e32_vi:
3070
42
  case AMDGPU::V_LSHLREV_B32_e64_gfx10:
3071
42
  case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
3072
42
  case AMDGPU::V_LSHLREV_B32_e64_vi:
3073
42
3074
42
  case AMDGPU::V_LSHLREV_B16_e32:
3075
42
  case AMDGPU::V_LSHLREV_B16_e64:
3076
42
  case AMDGPU::V_LSHLREV_B16_e32_vi:
3077
42
  case AMDGPU::V_LSHLREV_B16_e64_vi:
3078
42
  case AMDGPU::V_LSHLREV_B16_gfx10:
3079
42
3080
42
  case AMDGPU::V_LSHRREV_B16_e32:
3081
42
  case AMDGPU::V_LSHRREV_B16_e64:
3082
42
  case AMDGPU::V_LSHRREV_B16_e32_vi:
3083
42
  case AMDGPU::V_LSHRREV_B16_e64_vi:
3084
42
  case AMDGPU::V_LSHRREV_B16_gfx10:
3085
42
3086
42
  case AMDGPU::V_ASHRREV_I16_e32:
3087
42
  case AMDGPU::V_ASHRREV_I16_e64:
3088
42
  case AMDGPU::V_ASHRREV_I16_e32_vi:
3089
42
  case AMDGPU::V_ASHRREV_I16_e64_vi:
3090
42
  case AMDGPU::V_ASHRREV_I16_gfx10:
3091
42
3092
42
  case AMDGPU::V_LSHLREV_B64:
3093
42
  case AMDGPU::V_LSHLREV_B64_gfx10:
3094
42
  case AMDGPU::V_LSHLREV_B64_vi:
3095
42
3096
42
  case AMDGPU::V_LSHRREV_B64:
3097
42
  case AMDGPU::V_LSHRREV_B64_gfx10:
3098
42
  case AMDGPU::V_LSHRREV_B64_vi:
3099
42
3100
42
  case AMDGPU::V_ASHRREV_I64:
3101
42
  case AMDGPU::V_ASHRREV_I64_gfx10:
3102
42
  case AMDGPU::V_ASHRREV_I64_vi:
3103
42
3104
42
  case AMDGPU::V_PK_LSHLREV_B16:
3105
42
  case AMDGPU::V_PK_LSHLREV_B16_gfx10:
3106
42
  case AMDGPU::V_PK_LSHLREV_B16_vi:
3107
42
3108
42
  case AMDGPU::V_PK_LSHRREV_B16:
3109
42
  case AMDGPU::V_PK_LSHRREV_B16_gfx10:
3110
42
  case AMDGPU::V_PK_LSHRREV_B16_vi:
3111
42
  case AMDGPU::V_PK_ASHRREV_I16:
3112
42
  case AMDGPU::V_PK_ASHRREV_I16_gfx10:
3113
42
  case AMDGPU::V_PK_ASHRREV_I16_vi:
3114
42
    return true;
3115
42
  default:
3116
39
    return false;
3117
81
  }
3118
81
}
3119
3120
299k
bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
3121
299k
3122
299k
  using namespace SIInstrFlags;
3123
299k
  const unsigned Opcode = Inst.getOpcode();
3124
299k
  const MCInstrDesc &Desc = MII.get(Opcode);
3125
299k
3126
299k
  // lds_direct register is defined so that it can be used
3127
299k
  // with 9-bit operands only. Ignore encodings which do not accept these.
3128
299k
  if ((Desc.TSFlags & (VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA)) == 0)
3129
87.0k
    return true;
3130
211k
3131
211k
  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3132
211k
  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3133
211k
  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3134
211k
3135
211k
  const int SrcIndices[] = { Src1Idx, Src2Idx };
3136
211k
3137
211k
  // lds_direct cannot be specified as either src1 or src2.
3138
388k
  for (int SrcIdx : SrcIndices) {
3139
388k
    if (SrcIdx == -1) 
break184k
;
3140
203k
    const MCOperand &Src = Inst.getOperand(SrcIdx);
3141
203k
    if (Src.isReg() && 
Src.getReg() == LDS_DIRECT190k
) {
3142
2
      return false;
3143
2
    }
3144
203k
  }
3145
211k
3146
211k
  
if (211k
Src0Idx == -1211k
)
3147
144
    return true;
3148
211k
3149
211k
  const MCOperand &Src = Inst.getOperand(Src0Idx);
3150
211k
  if (!Src.isReg() || 
Src.getReg() != LDS_DIRECT179k
)
3151
211k
    return true;
3152
81
3153
81
  // lds_direct is specified as src0. Check additional limitations.
3154
81
  return (Desc.TSFlags & SIInstrFlags::SDWA) == 0 && !IsRevOpcode(Opcode);
3155
81
}
3156
3157
58
SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const {
3158
202
  for (unsigned i = 1, e = Operands.size(); i != e; 
++i144
) {
3159
202
    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
3160
202
    if (Op.isFlatOffset())
3161
58
      return Op.getStartLoc();
3162
202
  }
3163
58
  
return getLoc()0
;
3164
58
}
3165
3166
bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst,
3167
298k
                                         const OperandVector &Operands) {
3168
298k
  uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3169
298k
  if ((TSFlags & SIInstrFlags::FLAT) == 0)
3170
294k
    return true;
3171
3.24k
3172
3.24k
  auto Opcode = Inst.getOpcode();
3173
3.24k
  auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
3174
3.24k
  assert(OpNum != -1);
3175
3.24k
3176
3.24k
  const auto &Op = Inst.getOperand(OpNum);
3177
3.24k
  if (!hasFlatOffsets() && 
Op.getImm() != 0725
) {
3178
26
    Error(getFlatOffsetLoc(Operands),
3179
26
          "flat offset modifier is not supported on this GPU");
3180
26
    return false;
3181
26
  }
3182
3.21k
3183
3.21k
  // Address offset is 12-bit signed for GFX10, 13-bit for GFX9.
3184
3.21k
  // For FLAT segment the offset must be positive;
3185
3.21k
  // MSB is ignored and forced to zero.
3186
3.21k
  unsigned OffsetSize = isGFX9() ? 
131.98k
:
121.22k
;
3187
3.21k
  if (TSFlags & SIInstrFlags::IsNonFlatSeg) {
3188
1.47k
    if (!isIntN(OffsetSize, Op.getImm())) {
3189
24
      Error(getFlatOffsetLoc(Operands),
3190
24
            isGFX9() ? 
"expected a 13-bit signed offset"8
:
3191
24
                       
"expected a 12-bit signed offset"16
);
3192
24
      return false;
3193
24
    }
3194
1.74k
  } else {
3195
1.74k
    if (!isUIntN(OffsetSize - 1, Op.getImm())) {
3196
8
      Error(getFlatOffsetLoc(Operands),
3197
8
            isGFX9() ? 
"expected a 12-bit unsigned offset"4
:
3198
8
                       
"expected an 11-bit unsigned offset"4
);
3199
8
      return false;
3200
8
    }
3201
3.18k
  }
3202
3.18k
3203
3.18k
  return true;
3204
3.18k
}
3205
3206
298k
bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
3207
298k
  unsigned Opcode = Inst.getOpcode();
3208
298k
  const MCInstrDesc &Desc = MII.get(Opcode);
3209
298k
  if (!(Desc.TSFlags & (SIInstrFlags::SOP2 | SIInstrFlags::SOPC)))
3210
280k
    return true;
3211
18.6k
3212
18.6k
  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3213
18.6k
  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3214
18.6k
3215
18.6k
  const int OpIndices[] = { Src0Idx, Src1Idx };
3216
18.6k
3217
18.6k
  unsigned NumLiterals = 0;
3218
18.6k
  uint32_t LiteralValue;
3219
18.6k
3220
37.3k
  for (int OpIdx : OpIndices) {
3221
37.3k
    if (OpIdx == -1) 
break0
;
3222
37.3k
3223
37.3k
    const MCOperand &MO = Inst.getOperand(OpIdx);
3224
37.3k
    if (MO.isImm() &&
3225
37.3k
        // Exclude special imm operands (like that used by s_set_gpr_idx_on)
3226
37.3k
        
AMDGPU::isSISrcOperand(Desc, OpIdx)6.27k
&&
3227
37.3k
        
!isInlineConstant(Inst, OpIdx)6.21k
) {
3228
2.14k
      uint32_t Value = static_cast<uint32_t>(MO.getImm());
3229
2.14k
      if (NumLiterals == 0 || 
LiteralValue != Value45
) {
3230
2.10k
        LiteralValue = Value;
3231
2.10k
        ++NumLiterals;
3232
2.10k
      }
3233
2.14k
    }
3234
37.3k
  }
3235
18.6k
3236
18.6k
  return NumLiterals <= 1;
3237
18.6k
}
3238
3239
298k
bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
3240
298k
  const unsigned Opc = Inst.getOpcode();
3241
298k
  if (Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
3242
298k
      
Opc == AMDGPU::V_PERMLANEX16_B32_gfx10298k
) {
3243
312
    int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
3244
312
    unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
3245
312
3246
312
    if (OpSel & ~3)
3247
16
      return false;
3248
298k
  }
3249
298k
  return true;
3250
298k
}
3251
3252
// Check if VCC register matches wavefront size
3253
15.5k
bool AMDGPUAsmParser::validateVccOperand(unsigned Reg) const {
3254
15.5k
  auto FB = getFeatureBits();
3255
15.5k
  return (FB[AMDGPU::FeatureWavefrontSize64] && 
Reg == AMDGPU::VCC11.0k
) ||
3256
15.5k
    
(15.5k
FB[AMDGPU::FeatureWavefrontSize32]15.5k
&&
Reg == AMDGPU::VCC_LO4.51k
);
3257
15.5k
}
3258
3259
// VOP3 literal is only allowed in GFX10+ and only one can be used
3260
298k
bool AMDGPUAsmParser::validateVOP3Literal(const MCInst &Inst) const {
3261
298k
  unsigned Opcode = Inst.getOpcode();
3262
298k
  const MCInstrDesc &Desc = MII.get(Opcode);
3263
298k
  if (!(Desc.TSFlags & (SIInstrFlags::VOP3 | SIInstrFlags::VOP3P)))
3264
185k
    return true;
3265
113k
3266
113k
  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3267
113k
  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3268
113k
  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3269
113k
3270
113k
  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3271
113k
3272
113k
  unsigned NumLiterals = 0;
3273
113k
  uint32_t LiteralValue;
3274
113k
3275
327k
  for (int OpIdx : OpIndices) {
3276
327k
    if (OpIdx == -1) 
break86.4k
;
3277
241k
3278
241k
    const MCOperand &MO = Inst.getOperand(OpIdx);
3279
241k
    if (!MO.isImm() || 
!AMDGPU::isSISrcOperand(Desc, OpIdx)26.9k
)
3280
214k
      continue;
3281
26.9k
3282
26.9k
    if (!isInlineConstant(Inst, OpIdx)) {
3283
731
      uint32_t Value = static_cast<uint32_t>(MO.getImm());
3284
731
      if (NumLiterals == 0 || 
LiteralValue != Value192
) {
3285
589
        LiteralValue = Value;
3286
589
        ++NumLiterals;
3287
589
      }
3288
731
    }
3289
26.9k
  }
3290
113k
3291
113k
  return !NumLiterals ||
3292
113k
         
(539
NumLiterals == 1539
&&
getFeatureBits()[AMDGPU::FeatureVOP3Literal]493
);
3293
113k
}
3294
3295
bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
3296
                                          const SMLoc &IDLoc,
3297
299k
                                          const OperandVector &Operands) {
3298
299k
  if (!validateLdsDirect(Inst)) {
3299
44
    Error(IDLoc,
3300
44
      "invalid use of lds_direct");
3301
44
    return false;
3302
44
  }
3303
298k
  if (!validateSOPLiteral(Inst)) {
3304
4
    Error(IDLoc,
3305
4
      "only one literal operand is allowed");
3306
4
    return false;
3307
4
  }
3308
298k
  if (!validateVOP3Literal(Inst)) {
3309
303
    Error(IDLoc,
3310
303
      "invalid literal operand");
3311
303
    return false;
3312
303
  }
3313
298k
  if (!validateConstantBusLimitations(Inst)) {
3314
261
    Error(IDLoc,
3315
261
      "invalid operand (violates constant bus restrictions)");
3316
261
    return false;
3317
261
  }
3318
298k
  if (!validateEarlyClobberLimitations(Inst)) {
3319
34
    Error(IDLoc,
3320
34
      "destination must be different than all sources");
3321
34
    return false;
3322
34
  }
3323
298k
  if (!validateIntClampSupported(Inst)) {
3324
48
    Error(IDLoc,
3325
48
      "integer clamping is not supported on this GPU");
3326
48
    return false;
3327
48
  }
3328
298k
  if (!validateOpSel(Inst)) {
3329
16
    Error(IDLoc,
3330
16
      "invalid op_sel operand");
3331
16
    return false;
3332
16
  }
3333
298k
  // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
3334
298k
  if (!validateMIMGD16(Inst)) {
3335
120
    Error(IDLoc,
3336
120
      "d16 modifier is not supported on this GPU");
3337
120
    return false;
3338
120
  }
3339
298k
  if (!validateMIMGDim(Inst)) {
3340
0
    Error(IDLoc, "dim modifier is required on this GPU");
3341
0
    return false;
3342
0
  }
3343
298k
  if (!validateMIMGDataSize(Inst)) {
3344
93
    Error(IDLoc,
3345
93
      "image data size does not match dmask and tfe");
3346
93
    return false;
3347
93
  }
3348
298k
  if (!validateMIMGAddrSize(Inst)) {
3349
7
    Error(IDLoc,
3350
7
      "image address size does not match dim and a16");
3351
7
    return false;
3352
7
  }
3353
298k
  if (!validateMIMGAtomicDMask(Inst)) {
3354
9
    Error(IDLoc,
3355
9
      "invalid atomic image dmask");
3356
9
    return false;
3357
9
  }
3358
298k
  if (!validateMIMGGatherDMask(Inst)) {
3359
3
    Error(IDLoc,
3360
3
      "invalid image_gather dmask: only one bit must be set");
3361
3
    return false;
3362
3
  }
3363
298k
  if (!validateFlatOffset(Inst, Operands)) {
3364
58
    return false;
3365
58
  }
3366
298k
3367
298k
  return true;
3368
298k
}
3369
3370
static std::string AMDGPUMnemonicSpellCheck(StringRef S,
3371
                                            const FeatureBitset &FBS,
3372
                                            unsigned VariantID = 0);
3373
3374
bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
3375
                                              OperandVector &Operands,
3376
                                              MCStreamer &Out,
3377
                                              uint64_t &ErrorInfo,
3378
329k
                                              bool MatchingInlineAsm) {
3379
329k
  MCInst Inst;
3380
329k
  unsigned Result = Match_Success;
3381
463k
  for (auto Variant : getMatchedVariants()) {
3382
463k
    uint64_t EI;
3383
463k
    auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
3384
463k
                                  Variant);
3385
463k
    // We order match statuses from least to most specific. We use most specific
3386
463k
    // status as resulting
3387
463k
    // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
3388
463k
    if ((R == Match_Success) ||
3389
463k
        
(R == Match_PreferE32)164k
||
3390
463k
        
(164k
R == Match_MissingFeature164k
&&
Result != Match_PreferE3232.5k
) ||
3391
463k
        
(131k
R == Match_InvalidOperand131k
&&
Result != Match_MissingFeature77.0k
3392
131k
                                   && 
Result != Match_PreferE3263.6k
) ||
3393
463k
        
(68.0k
R == Match_MnemonicFail68.0k
&&
Result != Match_InvalidOperand54.6k
3394
68.0k
                                   && 
Result != Match_MissingFeature49.9k
3395
428k
                                   && 
Result != Match_PreferE3233.2k
)) {
3396
428k
      Result = R;
3397
428k
      ErrorInfo = EI;
3398
428k
    }
3399
463k
    if (R == Match_Success)
3400
299k
      break;
3401
463k
  }
3402
329k
3403
329k
  switch (Result) {
3404
329k
  
default: break0
;
3405
329k
  case Match_Success:
3406
299k
    if (!validateInstruction(Inst, IDLoc, Operands)) {
3407
1.00k
      return true;
3408
1.00k
    }
3409
298k
    Inst.setLoc(IDLoc);
3410
298k
    Out.EmitInstruction(Inst, getSTI());
3411
298k
    return false;
3412
298k
3413
298k
  case Match_MissingFeature:
3414
9.33k
    return Error(IDLoc, "instruction not supported on this GPU");
3415
298k
3416
298k
  case Match_MnemonicFail: {
3417
36
    FeatureBitset FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
3418
36
    std::string Suggestion = AMDGPUMnemonicSpellCheck(
3419
36
        ((AMDGPUOperand &)*Operands[0]).getToken(), FBS);
3420
36
    return Error(IDLoc, "invalid instruction" + Suggestion,
3421
36
                 ((AMDGPUOperand &)*Operands[0]).getLocRange());
3422
298k
  }
3423
298k
3424
298k
  case Match_InvalidOperand: {
3425
21.0k
    SMLoc ErrorLoc = IDLoc;
3426
21.0k
    if (ErrorInfo != ~0ULL) {
3427
21.0k
      if (ErrorInfo >= Operands.size()) {
3428
10
        return Error(IDLoc, "too few operands for instruction");
3429
10
      }
3430
21.0k
      ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
3431
21.0k
      if (ErrorLoc == SMLoc())
3432
0
        ErrorLoc = IDLoc;
3433
21.0k
    }
3434
21.0k
    
return Error(ErrorLoc, "invalid operand for instruction")21.0k
;
3435
21.0k
  }
3436
21.0k
3437
21.0k
  case Match_PreferE32:
3438
0
    return Error(IDLoc, "internal error: instruction without _e64 suffix "
3439
0
                        "should be encoded as e32");
3440
0
  }
3441
0
  llvm_unreachable("Implement any new match types added!");
3442
0
}
3443
3444
93
bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
3445
93
  int64_t Tmp = -1;
3446
93
  if (getLexer().isNot(AsmToken::Integer) && 
getLexer().isNot(AsmToken::Identifier)18
) {
3447
0
    return true;
3448
0
  }
3449
93
  if (getParser().parseAbsoluteExpression(Tmp)) {
3450
0
    return true;
3451
0
  }
3452
93
  Ret = static_cast<uint32_t>(Tmp);
3453
93
  return false;
3454
93
}
3455
3456
bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
3457
34
                                               uint32_t &Minor) {
3458
34
  if (ParseAsAbsoluteExpression(Major))
3459
0
    return TokError("invalid major version");
3460
34
3461
34
  if (getLexer().isNot(AsmToken::Comma))
3462
0
    return TokError("minor version number required, comma expected");
3463
34
  Lex();
3464
34
3465
34
  if (ParseAsAbsoluteExpression(Minor))
3466
0
    return TokError("invalid minor version");
3467
34
3468
34
  return false;
3469
34
}
3470
3471
7
bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
3472
7
  if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
3473
0
    return TokError("directive only supported for amdgcn architecture");
3474
7
3475
7
  std::string Target;
3476
7
3477
7
  SMLoc TargetStart = getTok().getLoc();
3478
7
  if (getParser().parseEscapedString(Target))
3479
0
    return true;
3480
7
  SMRange TargetRange = SMRange(TargetStart, getTok().getLoc());
3481
7
3482
7
  std::string ExpectedTarget;
3483
7
  raw_string_ostream ExpectedTargetOS(ExpectedTarget);
3484
7
  IsaInfo::streamIsaVersion(&getSTI(), ExpectedTargetOS);
3485
7
3486
7
  if (Target != ExpectedTargetOS.str())
3487
2
    return getParser().Error(TargetRange.Start, "target must match options",
3488
2
                             TargetRange);
3489
5
3490
5
  getTargetStreamer().EmitDirectiveAMDGCNTarget(Target);
3491
5
  return false;
3492
5
}
3493
3494
8
bool AMDGPUAsmParser::OutOfRangeError(SMRange Range) {
3495
8
  return getParser().Error(Range.Start, "value out of range", Range);
3496
8
}
3497
3498
bool AMDGPUAsmParser::calculateGPRBlocks(
3499
    const FeatureBitset &Features, bool VCCUsed, bool FlatScrUsed,
3500
    bool XNACKUsed, Optional<bool> EnableWavefrontSize32, unsigned NextFreeVGPR,
3501
    SMRange VGPRRange, unsigned NextFreeSGPR, SMRange SGPRRange,
3502
13
    unsigned &VGPRBlocks, unsigned &SGPRBlocks) {
3503
13
  // TODO(scott.linder): These calculations are duplicated from
3504
13
  // AMDGPUAsmPrinter::getSIProgramInfo and could be unified.
3505
13
  IsaVersion Version = getIsaVersion(getSTI().getCPU());
3506
13
3507
13
  unsigned NumVGPRs = NextFreeVGPR;
3508
13
  unsigned NumSGPRs = NextFreeSGPR;
3509
13
3510
13
  if (Version.Major >= 10)
3511
6
    NumSGPRs = 0;
3512
7
  else {
3513
7
    unsigned MaxAddressableNumSGPRs =
3514
7
        IsaInfo::getAddressableNumSGPRs(&getSTI());
3515
7
3516
7
    if (Version.Major >= 8 && !Features.test(FeatureSGPRInitBug) &&
3517
7
        
NumSGPRs > MaxAddressableNumSGPRs6
)
3518
0
      return OutOfRangeError(SGPRRange);
3519
7
3520
7
    NumSGPRs +=
3521
7
        IsaInfo::getNumExtraSGPRs(&getSTI(), VCCUsed, FlatScrUsed, XNACKUsed);
3522
7
3523
7
    if ((Version.Major <= 7 || Features.test(FeatureSGPRInitBug)) &&
3524
7
        
NumSGPRs > MaxAddressableNumSGPRs1
)
3525
0
      return OutOfRangeError(SGPRRange);
3526
7
3527
7
    if (Features.test(FeatureSGPRInitBug))
3528
1
      NumSGPRs = IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
3529
7
  }
3530
13
3531
13
  VGPRBlocks =
3532
13
      IsaInfo::getNumVGPRBlocks(&getSTI(), NumVGPRs, EnableWavefrontSize32);
3533
13
  SGPRBlocks = IsaInfo::getNumSGPRBlocks(&getSTI(), NumSGPRs);
3534
13
3535
13
  return false;
3536
13
}
3537
3538
45
bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
3539
45
  if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
3540
0
    return TokError("directive only supported for amdgcn architecture");
3541
45
3542
45
  if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA)
3543
0
    return TokError("directive only supported for amdhsa OS");
3544
45
3545
45
  StringRef KernelName;
3546
45
  if (getParser().parseIdentifier(KernelName))
3547
2
    return true;
3548
43
3549
43
  kernel_descriptor_t KD = getDefaultAmdhsaKernelDescriptor(&getSTI());
3550
43
3551
43
  StringSet<> Seen;
3552
43
3553
43
  IsaVersion IVersion = getIsaVersion(getSTI().getCPU());
3554
43
3555
43
  SMRange VGPRRange;
3556
43
  uint64_t NextFreeVGPR = 0;
3557
43
  SMRange SGPRRange;
3558
43
  uint64_t NextFreeSGPR = 0;
3559
43
  unsigned UserSGPRCount = 0;
3560
43
  bool ReserveVCC = true;
3561
43
  bool ReserveFlatScr = true;
3562
43
  bool ReserveXNACK = hasXNACK();
3563
43
  Optional<bool> EnableWavefrontSize32;
3564
43
3565
237
  while (true) {
3566
504
    while (getLexer().is(AsmToken::EndOfStatement))
3567
267
      Lex();
3568
237
3569
237
    if (getLexer().isNot(AsmToken::Identifier))
3570
2
      return TokError("expected .amdhsa_ directive or .end_amdhsa_kernel");
3571
235
3572
235
    StringRef ID = getTok().getIdentifier();
3573
235
    SMRange IDRange = getTok().getLocRange();
3574
235
    Lex();
3575
235
3576
235
    if (ID == ".end_amdhsa_kernel")
3577
23
      break;
3578
212
3579
212
    if (Seen.find(ID) != Seen.end())
3580
2
      return TokError(".amdhsa_ directives cannot be repeated");
3581
210
    Seen.insert(ID);
3582
210
3583
210
    SMLoc ValStart = getTok().getLoc();
3584
210
    int64_t IVal;
3585
210
    if (getParser().parseAbsoluteExpression(IVal))
3586
0
      return true;
3587
210
    SMLoc ValEnd = getTok().getLoc();
3588
210
    SMRange ValRange = SMRange(ValStart, ValEnd);
3589
210
3590
210
    if (IVal < 0)
3591
2
      return OutOfRangeError(ValRange);
3592
208
3593
208
    uint64_t Val = IVal;
3594
208
3595
208
#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)                           \
3596
208
  
if (152
!isUInt<ENTRY##_WIDTH>(VALUE)136
) \
3597
152
    
return OutOfRangeError(RANGE)4
; \
3598
152
  
AMDHSA_BITS_SET132
(FIELD, ENTRY, VALUE);
3599
208
3600
208
    if (ID == ".amdhsa_group_segment_fixed_size") {
3601
8
      if (!isUInt<sizeof(KD.group_segment_fixed_size) * CHAR_BIT>(Val))
3602
2
        return OutOfRangeError(ValRange);
3603
6
      KD.group_segment_fixed_size = Val;
3604
200
    } else if (ID == ".amdhsa_private_segment_fixed_size") {
3605
4
      if (!isUInt<sizeof(KD.private_segment_fixed_size) * CHAR_BIT>(Val))
3606
0
        return OutOfRangeError(ValRange);
3607
4
      KD.private_segment_fixed_size = Val;
3608
196
    } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") {
3609
4
      PARSE_BITS_ENTRY(KD.kernel_code_properties,
3610
4
                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
3611
4
                       Val, ValRange);
3612
4
      UserSGPRCount += 4;
3613
192
    } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") {
3614
4
      PARSE_BITS_ENTRY(KD.kernel_code_properties,
3615
4
                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val,
3616
4
                       ValRange);
3617
4
      UserSGPRCount += 2;
3618
188
    } else if (ID == ".amdhsa_user_sgpr_queue_ptr") {
3619
4
      PARSE_BITS_ENTRY(KD.kernel_code_properties,
3620
4
                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val,
3621
4
                       ValRange);
3622
4
      UserSGPRCount += 2;
3623
184
    } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") {
3624
4
      PARSE_BITS_ENTRY(KD.kernel_code_properties,
3625
4
                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
3626
4
                       Val, ValRange);
3627
4
      UserSGPRCount += 2;
3628
180
    } else if (ID == ".amdhsa_user_sgpr_dispatch_id") {
3629
4
      PARSE_BITS_ENTRY(KD.kernel_code_properties,
3630
4
                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val,
3631
4
                       ValRange);
3632
4
      UserSGPRCount += 2;
3633
176
    } else if (ID == ".amdhsa_user_sgpr_flat_scratch_init") {
3634
4
      PARSE_BITS_ENTRY(KD.kernel_code_properties,
3635
4
                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val,
3636
4
                       ValRange);
3637
4
      UserSGPRCount += 2;
3638
172
    } else if (ID == ".amdhsa_user_sgpr_private_segment_size") {
3639
4
      PARSE_BITS_ENTRY(KD.kernel_code_properties,
3640
4
                       KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
3641
4
                       Val, ValRange);
3642
4
      UserSGPRCount += 1;
3643
168
    } else if (ID == ".amdhsa_wavefront_size32") {
3644
6
      if (IVersion.Major < 10)
3645
2
        return getParser().Error(IDRange.Start, "directive requires gfx10+",
3646
2
                                 IDRange);
3647
4
      EnableWavefrontSize32 = Val;
3648
4
      PARSE_BITS_ENTRY(KD.kernel_code_properties,
3649
3
                       KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
3650
3
                       Val, ValRange);
3651
162
    } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") {
3652
4
      PARSE_BITS_ENTRY(
3653
4
          KD.compute_pgm_rsrc2,
3654
4
          COMPUTE_PGM_RSRC2_ENABLE_SGPR_PRIVATE_SEGMENT_WAVEFRONT_OFFSET, Val,
3655
4
          ValRange);
3656
158
    } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") {
3657
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3658
4
                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val,
3659
4
                       ValRange);
3660
154
    } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") {
3661
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3662
4
                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val,
3663
4
                       ValRange);
3664
150
    } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") {
3665
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3666
4
                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val,
3667
4
                       ValRange);
3668
146
    } else if (ID == ".amdhsa_system_sgpr_workgroup_info") {
3669
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3670
4
                       COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val,
3671
4
                       ValRange);
3672
142
    } else if (ID == ".amdhsa_system_vgpr_workitem_id") {
3673
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3674
4
                       COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val,
3675
4
                       ValRange);
3676
138
    } else if (ID == ".amdhsa_next_free_vgpr") {
3677
15
      VGPRRange = ValRange;
3678
15
      NextFreeVGPR = Val;
3679
123
    } else if (ID == ".amdhsa_next_free_sgpr") {
3680
13
      SGPRRange = ValRange;
3681
13
      NextFreeSGPR = Val;
3682
110
    } else if (ID == ".amdhsa_reserve_vcc") {
3683
8
      if (!isUInt<1>(Val))
3684
0
        return OutOfRangeError(ValRange);
3685
8
      ReserveVCC = Val;
3686
102
    } else if (ID == ".amdhsa_reserve_flat_scratch") {
3687
8
      if (IVersion.Major < 7)
3688
0
        return getParser().Error(IDRange.Start, "directive requires gfx7+",
3689
0
                                 IDRange);
3690
8
      if (!isUInt<1>(Val))
3691
0
        return OutOfRangeError(ValRange);
3692
8
      ReserveFlatScr = Val;
3693
94
    } else if (ID == ".amdhsa_reserve_xnack_mask") {
3694
8
      if (IVersion.Major < 8)
3695
0
        return getParser().Error(IDRange.Start, "directive requires gfx8+",
3696
0
                                 IDRange);
3697
8
      if (!isUInt<1>(Val))
3698
0
        return OutOfRangeError(ValRange);
3699
8
      ReserveXNACK = Val;
3700
86
    } else if (ID == ".amdhsa_float_round_mode_32") {
3701
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
3702
4
                       COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange);
3703
82
    } else if (ID == ".amdhsa_float_round_mode_16_64") {
3704
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
3705
4
                       COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange);
3706
78
    } else if (ID == ".amdhsa_float_denorm_mode_32") {
3707
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
3708
4
                       COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange);
3709
74
    } else if (ID == ".amdhsa_float_denorm_mode_16_64") {
3710
8
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
3711
8
                       COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val,
3712
8
                       ValRange);
3713
66
    } else if (ID == ".amdhsa_dx10_clamp") {
3714
8
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
3715
8
                       COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, Val, ValRange);
3716
58
    } else if (ID == ".amdhsa_ieee_mode") {
3717
8
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE,
3718
8
                       Val, ValRange);
3719
50
    } else if (ID == ".amdhsa_fp16_overflow") {
3720
4
      if (IVersion.Major < 9)
3721
0
        return getParser().Error(IDRange.Start, "directive requires gfx9+",
3722
0
                                 IDRange);
3723
8
      
PARSE_BITS_ENTRY4
(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FP16_OVFL, Val,
3724
8
                       ValRange);
3725
46
    } else if (ID == ".amdhsa_workgroup_processor_mode") {
3726
6
      if (IVersion.Major < 10)
3727
2
        return getParser().Error(IDRange.Start, "directive requires gfx10+",
3728
2
                                 IDRange);
3729
7
      
PARSE_BITS_ENTRY4
(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE, Val,
3730
7
                       ValRange);
3731
40
    } else if (ID == ".amdhsa_memory_ordered") {
3732
6
      if (IVersion.Major < 10)
3733
2
        return getParser().Error(IDRange.Start, "directive requires gfx10+",
3734
2
                                 IDRange);
3735
7
      
PARSE_BITS_ENTRY4
(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED, Val,
3736
7
                       ValRange);
3737
34
    } else if (ID == ".amdhsa_forward_progress") {
3738
6
      if (IVersion.Major < 10)
3739
2
        return getParser().Error(IDRange.Start, "directive requires gfx10+",
3740
2
                                 IDRange);
3741
7
      
PARSE_BITS_ENTRY4
(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS, Val,
3742
7
                       ValRange);
3743
28
    } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") {
3744
4
      PARSE_BITS_ENTRY(
3745
4
          KD.compute_pgm_rsrc2,
3746
4
          COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val,
3747
4
          ValRange);
3748
24
    } else if (ID == ".amdhsa_exception_fp_denorm_src") {
3749
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3750
4
                       COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
3751
4
                       Val, ValRange);
3752
20
    } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") {
3753
4
      PARSE_BITS_ENTRY(
3754
4
          KD.compute_pgm_rsrc2,
3755
4
          COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val,
3756
4
          ValRange);
3757
16
    } else if (ID == ".amdhsa_exception_fp_ieee_overflow") {
3758
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3759
4
                       COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
3760
4
                       Val, ValRange);
3761
12
    } else if (ID == ".amdhsa_exception_fp_ieee_underflow") {
3762
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3763
4
                       COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
3764
4
                       Val, ValRange);
3765
8
    } else if (ID == ".amdhsa_exception_fp_ieee_inexact") {
3766
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3767
4
                       COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
3768
4
                       Val, ValRange);
3769
4
    } else if (ID == ".amdhsa_exception_int_div_zero") {
3770
4
      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
3771
4
                       COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
3772
4
                       Val, ValRange);
3773
4
    } else {
3774
0
      return getParser().Error(IDRange.Start,
3775
0
                               "unknown .amdhsa_kernel directive", IDRange);
3776
0
    }
3777
208
3778
208
#undef PARSE_BITS_ENTRY
3779
208
  }
3780
43
3781
43
  
if (23
Seen.find(".amdhsa_next_free_vgpr") == Seen.end()23
)
3782
8
    return TokError(".amdhsa_next_free_vgpr directive is required");
3783
15
3784
15
  if (Seen.find(".amdhsa_next_free_sgpr") == Seen.end())
3785
2
    return TokError(".amdhsa_next_free_sgpr directive is required");
3786
13
3787
13
  unsigned VGPRBlocks;
3788
13
  unsigned SGPRBlocks;
3789
13
  if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
3790
13
                         ReserveXNACK, EnableWavefrontSize32, NextFreeVGPR,
3791
13
                         VGPRRange, NextFreeSGPR, SGPRRange, VGPRBlocks,
3792
13
                         SGPRBlocks))
3793
0
    return true;
3794
13
3795
13
  if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
3796
13
          VGPRBlocks))
3797
0
    return OutOfRangeError(VGPRRange);
3798
13
  AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
3799
13
                  COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks);
3800
13
3801
13
  if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
3802
13
          SGPRBlocks))
3803
0
    return OutOfRangeError(SGPRRange);
3804
13
  AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
3805
13
                  COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
3806
13
                  SGPRBlocks);
3807
13
3808
13
  if (!isUInt<COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_WIDTH>(UserSGPRCount))
3809
0
    return TokError("too many user SGPRs enabled");
3810
13
  AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT,
3811
13
                  UserSGPRCount);
3812
13
3813
13
  getTargetStreamer().EmitAmdhsaKernelDescriptor(
3814
13
      getSTI(), KernelName, KD, NextFreeVGPR, NextFreeSGPR, ReserveVCC,
3815
13
      ReserveFlatScr, ReserveXNACK);
3816
13
  return false;
3817
13
}
3818
3819
17
bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
3820
17
  uint32_t Major;
3821
17
  uint32_t Minor;
3822
17
3823
17
  if (ParseDirectiveMajorMinor(Major, Minor))
3824
0
    return true;
3825
17
3826
17
  getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
3827
17
  return false;
3828
17
}
3829
3830
26
bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
3831
26
  uint32_t Major;
3832
26
  uint32_t Minor;
3833
26
  uint32_t Stepping;
3834
26
  StringRef VendorName;
3835
26
  StringRef ArchName;
3836
26
3837
26
  // If this directive has no arguments, then use the ISA version for the
3838
26
  // targeted GPU.
3839
26
  if (getLexer().is(AsmToken::EndOfStatement)) {
3840
9
    AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
3841
9
    getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
3842
9
                                                      ISA.Stepping,
3843
9
                                                      "AMD", "AMDGPU");
3844
9
    return false;
3845
9
  }
3846
17
3847
17
  if (ParseDirectiveMajorMinor(Major, Minor))
3848
0
    return true;
3849
17
3850
17
  if (getLexer().isNot(AsmToken::Comma))
3851
0
    return TokError("stepping version number required, comma expected");
3852
17
  Lex();
3853
17
3854
17
  if (ParseAsAbsoluteExpression(Stepping))
3855
0
    return TokError("invalid stepping version");
3856
17
3857
17
  if (getLexer().isNot(AsmToken::Comma))
3858
0
    return TokError("vendor name required, comma expected");
3859
17
  Lex();
3860
17
3861
17
  if (getLexer().isNot(AsmToken::String))
3862
0
    return TokError("invalid vendor name");
3863
17
3864
17
  VendorName = getLexer().getTok().getStringContents();
3865
17
  Lex();
3866
17
3867
17
  if (getLexer().isNot(AsmToken::Comma))
3868
0
    return TokError("arch name required, comma expected");
3869
17
  Lex();
3870
17
3871
17
  if (getLexer().isNot(AsmToken::String))
3872
0
    return TokError("invalid arch name");
3873
17
3874
17
  ArchName = getLexer().getTok().getStringContents();
3875
17
  Lex();
3876
17
3877
17
  getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
3878
17
                                                    VendorName, ArchName);
3879
17
  return false;
3880
17
}
3881
3882
bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
3883
460
                                               amd_kernel_code_t &Header) {
3884
460
  // max_scratch_backing_memory_byte_size is deprecated. Ignore it while parsing
3885
460
  // assembly for backwards compatibility.
3886
460
  if (ID == "max_scratch_backing_memory_byte_size") {
3887
4
    Parser.eatToEndOfStatement();
3888
4
    return false;
3889
4
  }
3890
456
3891
456
  SmallString<40> ErrStr;
3892
456
  raw_svector_ostream Err(ErrStr);
3893
456
  if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
3894
0
    return TokError(Err.str());
3895
0
  }
3896
456
  Lex();
3897
456
3898
456
  if (ID == "enable_wavefront_size32") {
3899
14
    if (Header.code_properties & AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32) {
3900
6
      if (!isGFX10())
3901
2
        return TokError("enable_wavefront_size32=1 is only allowed on GFX10+");
3902
4
      if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize32])
3903
2
        return TokError("enable_wavefront_size32=1 requires +WavefrontSize32");
3904
8
    } else {
3905
8
      if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize64])
3906
2
        return TokError("enable_wavefront_size32=0 requires +WavefrontSize64");
3907
450
    }
3908
14
  }
3909
450
3910
450
  if (ID == "wavefront_size") {
3911
18
    if (Header.wavefront_size == 5) {
3912
6
      if (!isGFX10())
3913
2
        return TokError("wavefront_size=5 is only allowed on GFX10+");
3914
4
      if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize32])
3915
2
        return TokError("wavefront_size=5 requires +WavefrontSize32");
3916
12
    } else if (Header.wavefront_size == 6) {
3917
12
      if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize64])
3918
2
        return TokError("wavefront_size=6 requires +WavefrontSize64");
3919
444
    }
3920
18
  }
3921
444
3922
444
  if (ID == "enable_wgp_mode") {
3923
2
    if (G_00B848_WGP_MODE(Header.compute_pgm_resource_registers) && 
!isGFX10()0
)
3924
0
      return TokError("enable_wgp_mode=1 is only allowed on GFX10+");
3925
444
  }
3926
444
3927
444
  if (ID == "enable_mem_ordered") {
3928
2
    if (G_00B848_MEM_ORDERED(Header.compute_pgm_resource_registers) && 
!isGFX10()0
)
3929
0
      return TokError("enable_mem_ordered=1 is only allowed on GFX10+");
3930
444
  }
3931
444
3932
444
  if (ID == "enable_fwd_progress") {
3933
2
    if (G_00B848_FWD_PROGRESS(Header.compute_pgm_resource_registers) && 
!isGFX10()0
)
3934
0
      return TokError("enable_fwd_progress=1 is only allowed on GFX10+");
3935
444
  }
3936
444
3937
444
  return false;
3938
444
}
3939
3940
46
bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
3941
46
  amd_kernel_code_t Header;
3942
46
  AMDGPU::initDefaultAMDKernelCodeT(Header, &getSTI());
3943
46
3944
494
  while (true) {
3945
494
    // Lex EndOfStatement.  This is in a while loop, because lexing a comment
3946
494
    // will set the current token to EndOfStatement.
3947
576
    while(getLexer().is(AsmToken::EndOfStatement))
3948
82
      Lex();
3949
494
3950
494
    if (getLexer().isNot(AsmToken::Identifier))
3951
0
      return TokError("expected value identifier or .end_amd_kernel_code_t");
3952
494
3953
494
    StringRef ID = getLexer().getTok().getIdentifier();
3954
494
    Lex();
3955
494
3956
494
    if (ID == ".end_amd_kernel_code_t")
3957
34
      break;
3958
460
3959
460
    if (ParseAMDKernelCodeTValue(ID, Header))
3960
12
      return true;
3961
460
  }
3962
46
3963
46
  getTargetStreamer().EmitAMDKernelCodeT(Header);
3964
34
3965
34
  return false;
3966
46
}
3967
3968
53
bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
3969
53
  if (getLexer().isNot(AsmToken::Identifier))
3970
0
    return TokError("expected symbol name");
3971
53
3972
53
  StringRef KernelName = Parser.getTok().getString();
3973
53
3974
53
  getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
3975
53
                                           ELF::STT_AMDGPU_HSA_KERNEL);
3976
53
  Lex();
3977
53
  if (!AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI()))
3978
53
    KernelScope.initialize(getContext());
3979
53
  return false;
3980
53
}
3981
3982
24
bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
3983
24
  if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
3984
1
    return Error(getParser().getTok().getLoc(),
3985
1
                 ".amd_amdgpu_isa directive is not available on non-amdgcn "
3986
1
                 "architectures");
3987
1
  }
3988
23
3989
23
  auto ISAVersionStringFromASM = getLexer().getTok().getStringContents();
3990
23
3991
23
  std::string ISAVersionStringFromSTI;
3992
23
  raw_string_ostream ISAVersionStreamFromSTI(ISAVersionStringFromSTI);
3993
23
  IsaInfo::streamIsaVersion(&getSTI(), ISAVersionStreamFromSTI);
3994
23
3995
23
  if (ISAVersionStringFromASM != ISAVersionStreamFromSTI.str()) {
3996
15
    return Error(getParser().getTok().getLoc(),
3997
15
                 ".amd_amdgpu_isa directive does not match triple and/or mcpu "
3998
15
                 "arguments specified through the command line");
3999
15
  }
4000
8
4001
8
  getTargetStreamer().EmitISAVersion(ISAVersionStreamFromSTI.str());
4002
8
  Lex();
4003
8
4004
8
  return false;
4005
8
}
4006
4007
37
bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
4008
37
  const char *AssemblerDirectiveBegin;
4009
37
  const char *AssemblerDirectiveEnd;
4010
37
  std::tie(AssemblerDirectiveBegin, AssemblerDirectiveEnd) =
4011
37
      AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())
4012
37
          ? std::make_tuple(HSAMD::V3::AssemblerDirectiveBegin,
4013
11
                            HSAMD::V3::AssemblerDirectiveEnd)
4014
37
          : std::make_tuple(HSAMD::AssemblerDirectiveBegin,
4015
26
                            HSAMD::AssemblerDirectiveEnd);
4016
37
4017
37
  if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
4018
2
    return Error(getParser().getTok().getLoc(),
4019
2
                 (Twine(AssemblerDirectiveBegin) + Twine(" directive is "
4020
2
                 "not available on non-amdhsa OSes")).str());
4021
2
  }
4022
35
4023
35
  std::string HSAMetadataString;
4024
35
  if (ParseToEndDirective(AssemblerDirectiveBegin, AssemblerDirectiveEnd,
4025
35
                          HSAMetadataString))
4026
0
    return true;
4027
35
4028
35
  if (IsaInfo::hasCodeObjectV3(&getSTI())) {
4029
11
    if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
4030
0
      return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
4031
24
  } else {
4032
24
    if (!getTargetStreamer().EmitHSAMetadataV2(HSAMetadataString))
4033
6
      return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
4034
29
  }
4035
29
4036
29
  return false;
4037
29
}
4038
4039
/// Common code to parse out a block of text (typically YAML) between start and
4040
/// end directives.
4041
bool AMDGPUAsmParser::ParseToEndDirective(const char *AssemblerDirectiveBegin,
4042
                                          const char *AssemblerDirectiveEnd,
4043
37
                                          std::string &CollectString) {
4044
37
4045
37
  raw_string_ostream CollectStream(CollectString);
4046
37
4047
37
  getLexer().setSkipSpace(false);
4048
37
4049
37
  bool FoundEnd = false;
4050
869
  while (!getLexer().is(AsmToken::Eof)) {
4051
1.65k
    while (getLexer().is(AsmToken::Space)) {
4052
784
      CollectStream << getLexer().getTok().getString();
4053
784
      Lex();
4054
784
    }
4055
869
4056
869
    if (getLexer().is(AsmToken::Identifier)) {
4057
649
      StringRef ID = getLexer().getTok().getIdentifier();
4058
649
      if (ID == AssemblerDirectiveEnd) {
4059
37
        Lex();
4060
37
        FoundEnd = true;
4061
37
        break;
4062
37
      }
4063
832
    }
4064
832
4065
832
    CollectStream << Parser.parseStringToEndOfStatement()
4066
832
                  << getContext().getAsmInfo()->getSeparatorString();
4067
832
4068
832
    Parser.eatToEndOfStatement();
4069
832
  }
4070
37
4071
37
  getLexer().setSkipSpace(true);
4072
37
4073
37
  if (getLexer().is(AsmToken::Eof) && 
!FoundEnd0
) {
4074
0
    return TokError(Twine("expected directive ") +
4075
0
                    Twine(AssemblerDirectiveEnd) + Twine(" not found"));
4076
0
  }
4077
37
4078
37
  CollectStream.flush();
4079
37
  return false;
4080
37
}
4081
4082
/// Parse the assembler directive for new MsgPack-format PAL metadata.
4083
2
bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
4084
2
  std::string String;
4085
2
  if (ParseToEndDirective(AMDGPU::PALMD::AssemblerDirectiveBegin,
4086
2
                          AMDGPU::PALMD::AssemblerDirectiveEnd, String))
4087
0
    return true;
4088
2
4089
2
  auto PALMetadata = getTargetStreamer().getPALMetadata();
4090
2
  if (!PALMetadata->setFromString(String))
4091
0
    return Error(getParser().getTok().getLoc(), "invalid PAL metadata");
4092
2
  return false;
4093
2
}
4094
4095
/// Parse the assembler directive for old linear-format PAL metadata.
4096
4
bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
4097
4
  if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
4098
2
    return Error(getParser().getTok().getLoc(),
4099
2
                 (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
4100
2
                 "not available on non-amdpal OSes")).str());
4101
2
  }
4102
2
4103
2
  auto PALMetadata = getTargetStreamer().getPALMetadata();
4104
2
  PALMetadata->setLegacy();
4105
4
  for (;;) {
4106
4
    uint32_t Key, Value;
4107
4
    if (ParseAsAbsoluteExpression(Key)) {
4108
0
      return TokError(Twine("invalid value in ") +
4109
0
                      Twine(PALMD::AssemblerDirective));
4110
0
    }
4111
4
    if (getLexer().isNot(AsmToken::Comma)) {
4112
0
      return TokError(Twine("expected an even number of values in ") +
4113
0
                      Twine(PALMD::AssemblerDirective));
4114
0
    }
4115
4
    Lex();
4116
4
    if (ParseAsAbsoluteExpression(Value)) {
4117
0
      return TokError(Twine("invalid value in ") +
4118
0
                      Twine(PALMD::AssemblerDirective));
4119
0
    }
4120
4
    PALMetadata->setRegister(Key, Value);
4121
4
    if (getLexer().isNot(AsmToken::Comma))
4122
2
      break;
4123
2
    Lex();
4124
2
  }
4125
2
  return false;
4126
2
}
4127
4128
/// ParseDirectiveAMDGPULDS
4129
///  ::= .amdgpu_lds identifier ',' size_expression [',' align_expression]
4130
10
bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
4131
10
  if (getParser().checkForValidSection())
4132
0
    return true;
4133
10
4134
10
  StringRef Name;
4135
10
  SMLoc NameLoc = getLexer().getLoc();
4136
10
  if (getParser().parseIdentifier(Name))
4137
0
    return TokError("expected identifier in directive");
4138
10
4139
10
  MCSymbol *Symbol = getContext().getOrCreateSymbol(Name);
4140
10
  if (parseToken(AsmToken::Comma, "expected ','"))
4141
0
    return true;
4142
10
4143
10
  unsigned LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(&getSTI());
4144
10
4145
10
  int64_t Size;
4146
10
  SMLoc SizeLoc = getLexer().getLoc();
4147
10
  if (getParser().parseAbsoluteExpression(Size))
4148
0
    return true;
4149
10
  if (Size < 0)
4150
1
    return Error(SizeLoc, "size must be non-negative");
4151
9
  if (Size > LocalMemorySize)
4152
1
    return Error(SizeLoc, "size is too large");
4153
8
4154
8
  int64_t Align = 4;
4155
8
  if (getLexer().is(AsmToken::Comma)) {
4156
5
    Lex();
4157
5
    SMLoc AlignLoc = getLexer().getLoc();
4158
5
    if (getParser().parseAbsoluteExpression(Align))
4159
0
      return true;
4160
5
    if (Align < 0 || !isPowerOf2_64(Align))
4161
2
      return Error(AlignLoc, "alignment must be a power of two");
4162
3
4163
3
    // Alignment larger than the size of LDS is possible in theory, as long
4164
3
    // as the linker manages to place to symbol at address 0, but we do want
4165
3
    // to make sure the alignment fits nicely into a 32-bit integer.
4166
3
    if (Align >= 1u << 31)
4167
1
      return Error(AlignLoc, "alignment is too large");
4168
5
  }
4169
5
4170
5
  if (parseToken(AsmToken::EndOfStatement,
4171
5
                 "unexpected token in '.amdgpu_lds' directive"))
4172
0
    return true;
4173
5
4174
5
  Symbol->redefineIfPossible();
4175
5
  if (!Symbol->isUndefined())
4176
0
    return Error(NameLoc, "invalid symbol redefinition");
4177
5
4178
5
  getTargetStreamer().emitAMDGPULDS(Symbol, Size, Align);
4179
5
  return false;
4180
5
}
4181
4182
668
bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
4183
668
  StringRef IDVal = DirectiveID.getString();
4184
668
4185
668
  if (AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())) {
4186
240
    if (IDVal == ".amdgcn_target")
4187
7
      return ParseDirectiveAMDGCNTarget();
4188
233
4189
233
    if (IDVal == ".amdhsa_kernel")
4190
45
      return ParseDirectiveAMDHSAKernel();
4191
188
4192
188
    // TODO: Restructure/combine with PAL metadata directive.
4193
188
    if (IDVal == AMDGPU::HSAMD::V3::AssemblerDirectiveBegin)
4194
11
      return ParseDirectiveHSAMetadata();
4195
428
  } else {
4196
428
    if (IDVal == ".hsa_code_object_version")
4197
17
      return ParseDirectiveHSACodeObjectVersion();
4198
411
4199
411
    if (IDVal == ".hsa_code_object_isa")
4200
26
      return ParseDirectiveHSACodeObjectISA();
4201
385
4202
385
    if (IDVal == ".amd_kernel_code_t")
4203
46
      return ParseDirectiveAMDKernelCodeT();
4204
339
4205
339
    if (IDVal == ".amdgpu_hsa_kernel")
4206
53
      return ParseDirectiveAMDGPUHsaKernel();
4207
286
4208
286
    if (IDVal == ".amd_amdgpu_isa")
4209
24
      return ParseDirectiveISAVersion();
4210
262
4211
262
    if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
4212
26
      return ParseDirectiveHSAMetadata();
4213
413
  }
4214
413
4215
413
  if (IDVal == ".amdgpu_lds")
4216
10
    return ParseDirectiveAMDGPULDS();
4217
403
4218
403
  if (IDVal == PALMD::AssemblerDirectiveBegin)
4219
2
    return ParseDirectivePALMetadataBegin();
4220
401
4221
401
  if (IDVal == PALMD::AssemblerDirective)
4222
4
    return ParseDirectivePALMetadata();
4223
397
4224
397
  return true;
4225
397
}
4226
4227
bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
4228
864k
                                           unsigned RegNo) const {
4229
864k
4230
864k
  for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
4231
18.1M
       R.isValid(); 
++R17.2M
) {
4232
17.2M
    if (*R == RegNo)
4233
125
      return isGFX9() || 
isGFX10()93
;
4234
17.2M
  }
4235
864k
4236
864k
  // GFX10 has 2 more SGPRs 104 and 105.
4237
864k
  for (MCRegAliasIterator R(AMDGPU::SGPR104_SGPR105, &MRI, true);
4238
6.03M
       R.isValid(); 
++R5.17M
) {
4239
5.17M
    if (*R == RegNo)
4240
2.56k
      return hasSGPR104_SGPR105();
4241
5.17M
  }
4242
864k
4243
864k
  switch (RegNo) {
4244
861k
  case AMDGPU::SRC_SHARED_BASE:
4245
372
  case AMDGPU::SRC_SHARED_LIMIT:
4246
372
  case AMDGPU::SRC_PRIVATE_BASE:
4247
372
  case AMDGPU::SRC_PRIVATE_LIMIT:
4248
372
  case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
4249
372
    return !isCI() && 
!isSI()300
&&
!isVI()164
;
4250
10.2k
  case AMDGPU::TBA:
4251
10.2k
  case AMDGPU::TBA_LO:
4252
10.2k
  case AMDGPU::TBA_HI:
4253
10.2k
  case AMDGPU::TMA:
4254
10.2k
  case AMDGPU::TMA_LO:
4255
10.2k
  case AMDGPU::TMA_HI:
4256
10.2k
    return !isGFX9() && 
!isGFX10()10.1k
;
4257
10.2k
  case AMDGPU::XNACK_MASK:
4258
40
  case AMDGPU::XNACK_MASK_LO:
4259
40
  case AMDGPU::XNACK_MASK_HI:
4260
40
    return !isCI() && 
!isSI()34
&&
!isGFX10()22
&&
hasXNACK()22
;
4261
40
  case AMDGPU::SGPR_NULL:
4262
12
    return isGFX10();
4263
850k
  default:
4264
850k
    break;
4265
850k
  }
4266
850k
4267
850k
  if (isCI())
4268
84.1k
    return true;
4269
766k
4270
766k
  if (isSI() || 
isGFX10()749k
) {
4271
421k
    // No flat_scr on SI.
4272
421k
    // On GFX10 flat scratch is not a valid register operand and can only be
4273
421k
    // accessed with s_setreg/s_getreg.
4274
421k
    switch (RegNo) {
4275
421k
    case AMDGPU::FLAT_SCR:
4276
26
    case AMDGPU::FLAT_SCR_LO:
4277
26
    case AMDGPU::FLAT_SCR_HI:
4278
26
      return false;
4279
421k
    default:
4280
421k
      return true;
4281
345k
    }
4282
345k
  }
4283
345k
4284
345k
  // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
4285
345k
  // SI/CI have.
4286
345k
  for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
4287
5.86M
       R.isValid(); 
++R5.52M
) {
4288
5.52M
    if (*R == RegNo)
4289
26
      return hasSGPR102_SGPR103();
4290
5.52M
  }
4291
345k
4292
345k
  
return true345k
;
4293
345k
}
4294
4295
OperandMatchResultTy
4296
AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic,
4297
1.09M
                              OperandMode Mode) {
4298
1.09M
  // Try to parse with a custom parser
4299
1.09M
  OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
4300
1.09M
4301
1.09M
  // If we successfully parsed the operand or if there as an error parsing,
4302
1.09M
  // we are done.
4303
1.09M
  //
4304
1.09M
  // If we are parsing after we reach EndOfStatement then this means we
4305
1.09M
  // are appending default values to the Operands list.  This is only done
4306
1.09M
  // by custom parser, so we shouldn't continue on to the generic parsing.
4307
1.09M
  if (ResTy == MatchOperand_Success || 
ResTy == MatchOperand_ParseFail432k
||
4308
1.09M
      
getLexer().is(AsmToken::EndOfStatement)431k
)
4309
666k
    return ResTy;
4310
431k
4311
431k
  if (Mode == OperandMode_NSA && 
getLexer().is(AsmToken::LBrac)153
) {
4312
98
    unsigned Prefix = Operands.size();
4313
98
    SMLoc LBraceLoc = getTok().getLoc();
4314
98
    Parser.Lex(); // eat the '['
4315
98
4316
520
    for (;;) {
4317
520
      ResTy = parseReg(Operands);
4318
520
      if (ResTy != MatchOperand_Success)
4319
0
        return ResTy;
4320
520
4321
520
      if (getLexer().is(AsmToken::RBrac))
4322
98
        break;
4323
422
4324
422
      if (getLexer().isNot(AsmToken::Comma))
4325
0
        return MatchOperand_ParseFail;
4326
422
      Parser.Lex();
4327
422
    }
4328
98
4329
98
    if (Operands.size() - Prefix > 1) {
4330
98
      Operands.insert(Operands.begin() + Prefix,
4331
98
                      AMDGPUOperand::CreateToken(this, "[", LBraceLoc));
4332
98
      Operands.push_back(AMDGPUOperand::CreateToken(this, "]",
4333
98
                                                    getTok().getLoc()));
4334
98
    }
4335
98
4336
98
    Parser.Lex(); // eat the ']'
4337
98
    return MatchOperand_Success;
4338
431k
  }
4339
431k
4340
431k
  return parseRegOrImm(Operands);
4341
431k
}
4342
4343
333k
StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
4344
333k
  // Clear any forced encodings from the previous instruction.
4345
333k
  setForcedEncodingSize(0);
4346
333k
  setForcedDPP(false);
4347
333k
  setForcedSDWA(false);
4348
333k
4349
333k
  if (Name.endswith("_e64")) {
4350
89.0k
    setForcedEncodingSize(64);
4351
89.0k
    return Name.substr(0, Name.size() - 4);
4352
244k
  } else if (Name.endswith("_e32")) {
4353
6.48k
    setForcedEncodingSize(32);
4354
6.48k
    return Name.substr(0, Name.size() - 4);
4355
237k
  } else if (Name.endswith("_dpp")) {
4356
19.7k
    setForcedDPP(true);
4357
19.7k
    return Name.substr(0, Name.size() - 4);
4358
218k
  } else if (Name.endswith("_sdwa")) {
4359
59.2k
    setForcedSDWA(true);
4360
59.2k
    return Name.substr(0, Name.size() - 5);
4361
59.2k
  }
4362
158k
  return Name;
4363
158k
}
4364
4365
bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
4366
                                       StringRef Name,
4367
333k
                                       SMLoc NameLoc, OperandVector &Operands) {
4368
333k
  // Add the instruction mnemonic
4369
333k
  Name = parseMnemonicSuffix(Name);
4370
333k
  Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
4371
333k
4372
333k
  bool IsMIMG = Name.startswith("image_");
4373
333k
4374
1.42M
  while (!getLexer().is(AsmToken::EndOfStatement)) {
4375
1.09M
    OperandMode Mode = OperandMode_Default;
4376
1.09M
    if (IsMIMG && 
isGFX10()21.5k
&&
Operands.size() == 2691
)
4377
153
      Mode = OperandMode_NSA;
4378
1.09M
    OperandMatchResultTy Res = parseOperand(Operands, Name, Mode);
4379
1.09M
4380
1.09M
    // Eat the comma or space if there is one.
4381
1.09M
    if (getLexer().is(AsmToken::Comma))
4382
609k
      Parser.Lex();
4383
1.09M
4384
1.09M
    switch (Res) {
4385
1.09M
      
case MatchOperand_Success: break1.09M
;
4386
1.09M
      case MatchOperand_ParseFail:
4387
1.27k
        // FIXME: use real operand location rather than the current location.
4388
1.27k
        Error(getLexer().getLoc(), "failed parsing operand.");
4389
5.06k
        while (!getLexer().is(AsmToken::EndOfStatement)) {
4390
3.79k
          Parser.Lex();
4391
3.79k
        }
4392
1.27k
        return true;
4393
1.09M
      case MatchOperand_NoMatch:
4394
2.02k
        // FIXME: use real operand location rather than the current location.
4395
2.02k
        Error(getLexer().getLoc(), "not a valid operand.");
4396
20.0k
        while (!getLexer().is(AsmToken::EndOfStatement)) {
4397
17.9k
          Parser.Lex();
4398
17.9k
        }
4399
2.02k
        return true;
4400
1.09M
    }
4401
1.09M
  }
4402
333k
4403
333k
  
return false329k
;
4404
333k
}
4405
4406
//===----------------------------------------------------------------------===//
4407
// Utility functions
4408
//===----------------------------------------------------------------------===//
4409
4410
OperandMatchResultTy
4411
2.42M
AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &IntVal) {
4412
2.42M
4413
2.42M
  if (!trySkipId(Prefix, AsmToken::Colon))
4414
2.35M
    return MatchOperand_NoMatch;
4415
68.9k
4416
68.9k
  return parseExpr(IntVal) ? MatchOperand_Success : 
MatchOperand_ParseFail0
;
4417
68.9k
}
4418
4419
OperandMatchResultTy
4420
AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
4421
                                    AMDGPUOperand::ImmTy ImmTy,
4422
2.19M
                                    bool (*ConvertResult)(int64_t&)) {
4423
2.19M
  SMLoc S = getLoc();
4424
2.19M
  int64_t Value = 0;
4425
2.19M
4426
2.19M
  OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
4427
2.19M
  if (Res != MatchOperand_Success)
4428
2.12M
    return Res;
4429
68.7k
4430
68.7k
  if (ConvertResult && 
!ConvertResult(Value)3.55k
) {
4431
4
    Error(S, "invalid " + StringRef(Prefix) + " value.");
4432
4
  }
4433
68.7k
4434
68.7k
  Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
4435
68.7k
  return MatchOperand_Success;
4436
68.7k
}
4437
4438
OperandMatchResultTy
4439
AMDGPUAsmParser::parseOperandArrayWithPrefix(const char *Prefix,
4440
                                             OperandVector &Operands,
4441
                                             AMDGPUOperand::ImmTy ImmTy,
4442
94.7k
                                             bool (*ConvertResult)(int64_t&)) {
4443
94.7k
  SMLoc S = getLoc();
4444
94.7k
  if (!trySkipId(Prefix, AsmToken::Colon))
4445
92.5k
    return MatchOperand_NoMatch;
4446
2.20k
4447
2.20k
  if (!skipToken(AsmToken::LBrac, "expected a left square bracket"))
4448
33
    return MatchOperand_ParseFail;
4449
2.17k
4450
2.17k
  unsigned Val = 0;
4451
2.17k
  const unsigned MaxSize = 4;
4452
2.17k
4453
2.17k
  // FIXME: How to verify the number of elements matches the number of src
4454
2.17k
  // operands?
4455
5.89k
  for (int I = 0; ; 
++I3.72k
) {
4456
5.89k
    int64_t Op;
4457
5.89k
    SMLoc Loc = getLoc();
4458
5.89k
    if (!parseExpr(Op))
4459
69
      return MatchOperand_ParseFail;
4460
5.82k
4461
5.82k
    if (Op != 0 && 
Op != 12.49k
) {
4462
100
      Error(Loc, "invalid " + StringRef(Prefix) + " value.");
4463
100
      return MatchOperand_ParseFail;
4464
100
    }
4465
5.72k
4466
5.72k
    Val |= (Op << I);
4467
5.72k
4468
5.72k
    if (trySkipToken(AsmToken::RBrac))
4469
1.98k
      break;
4470
3.74k
4471
3.74k
    if (I + 1 == MaxSize) {
4472
20
      Error(getLoc(), "expected a closing square bracket");
4473
20
      return MatchOperand_ParseFail;
4474
20
    }
4475
3.72k
4476
3.72k
    if (!skipToken(AsmToken::Comma, "expected a comma"))
4477
1
      return MatchOperand_ParseFail;
4478
3.72k
  }
4479
2.17k
4480
2.17k
  Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
4481
1.98k
  return MatchOperand_Success;
4482
2.17k
}
4483
4484
OperandMatchResultTy
4485
AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
4486
4.30M
                               AMDGPUOperand::ImmTy ImmTy) {
4487
4.30M
  int64_t Bit = 0;
4488
4.30M
  SMLoc S = Parser.getTok().getLoc();
4489
4.30M
4490
4.30M
  // We are at the end of the statement, and this is a default argument, so
4491
4.30M
  // use a default value.
4492
4.30M
  if (getLexer().isNot(AsmToken::EndOfStatement)) {
4493
4.30M
    switch(getLexer().getKind()) {
4494
4.30M
      case AsmToken::Identifier: {
4495
4.28M
        StringRef Tok = Parser.getTok().getString();
4496
4.28M
        if (Tok == Name) {
4497
16.2k
          if (Tok == "r128" && 
isGFX9()73
)
4498
12
            Error(S, "r128 modifier is not supported on this GPU");
4499
16.2k
          if (Tok == "a16" && 
!isGFX9()264
&&
!isGFX10()220
)
4500
220
            Error(S, "a16 modifier is not supported on this GPU");
4501
16.2k
          Bit = 1;
4502
16.2k
          Parser.Lex();
4503
4.26M
        } else if (Tok.startswith("no") && 
Tok.endswith(Name)0
) {
4504
0
          Bit = 0;
4505
0
          Parser.Lex();
4506
4.26M
        } else {
4507
4.26M
          return MatchOperand_NoMatch;
4508
4.26M
        }
4509
16.2k
        break;
4510
16.2k
      }
4511
19.8k
      default:
4512
19.8k
        return MatchOperand_NoMatch;
4513
16.2k
    }
4514
16.2k
  }
4515
16.2k
4516
16.2k
  if (!isGFX10() && 
ImmTy == AMDGPUOperand::ImmTyDLC9.36k
)
4517
56
    return MatchOperand_ParseFail;
4518
16.1k
4519
16.1k
  Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
4520
16.1k
  return MatchOperand_Success;
4521
16.1k
}
4522
4523
static void addOptionalImmOperand(
4524
  MCInst& Inst, const OperandVector& Operands,
4525
  AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
4526
  AMDGPUOperand::ImmTy ImmT,
4527
450k
  int64_t Default = 0) {
4528
450k
  auto i = OptionalIdx.find(ImmT);
4529
450k
  if (i != OptionalIdx.end()) {
4530
198k
    unsigned Idx = i->second;
4531
198k
    ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
4532
252k
  } else {
4533
252k
    Inst.addOperand(MCOperand::createImm(Default));
4534
252k
  }
4535
450k
}
4536
4537
OperandMatchResultTy
4538
476k
AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
4539
476k
  if (getLexer().isNot(AsmToken::Identifier)) {
4540
3.97k
    return MatchOperand_NoMatch;
4541
3.97k
  }
4542
472k
  StringRef Tok = Parser.getTok().getString();
4543
472k
  if (Tok != Prefix) {
4544
319k
    return MatchOperand_NoMatch;
4545
319k
  }
4546
153k
4547
153k
  Parser.Lex();
4548
153k
  if (getLexer().isNot(AsmToken::Colon)) {
4549
0
    return MatchOperand_ParseFail;
4550
0
  }
4551
153k
4552
153k
  Parser.Lex();
4553
153k
  if (getLexer().isNot(AsmToken::Identifier)) {
4554
0
    return MatchOperand_ParseFail;
4555
0
  }
4556
153k
4557
153k
  Value = Parser.getTok().getString();
4558
153k
  return MatchOperand_Success;
4559
153k
}
4560
4561
// dfmt and nfmt (in a tbuffer instruction) are parsed as one to allow their
4562
// values to live in a joint format operand in the MCInst encoding.
4563
OperandMatchResultTy
4564
111k
AMDGPUAsmParser::parseDfmtNfmt(OperandVector &Operands) {
4565
111k
  SMLoc S = Parser.getTok().getLoc();
4566
111k
  int64_t Dfmt = 0, Nfmt = 0;
4567
111k
  // dfmt and nfmt can appear in either order, and each is optional.
4568
111k
  bool GotDfmt = false, GotNfmt = false;
4569
111k
  while (!GotDfmt || 
!GotNfmt150
) {
4570
111k
    if (!GotDfmt) {
4571
111k
      auto Res = parseIntWithPrefix("dfmt", Dfmt);
4572
111k
      if (Res != MatchOperand_NoMatch) {
4573
78
        if (Res != MatchOperand_Success)
4574
0
          return Res;
4575
78
        if (Dfmt >= 16) {
4576
0
          Error(Parser.getTok().getLoc(), "out of range dfmt");
4577
0
          return MatchOperand_ParseFail;
4578
0
        }
4579
78
        GotDfmt = true;
4580
78
        Parser.Lex();
4581
78
        continue;
4582
78
      }
4583
111k
    }
4584
111k
    if (!GotNfmt) {
4585
111k
      auto Res = parseIntWithPrefix("nfmt", Nfmt);
4586
111k
      if (Res != MatchOperand_NoMatch) {
4587
78
        if (Res != MatchOperand_Success)
4588
0
          return Res;
4589
78
        if (Nfmt >= 8) {
4590
0
          Error(Parser.getTok().getLoc(), "out of range nfmt");
4591
0
          return MatchOperand_ParseFail;
4592
0
        }
4593
78
        GotNfmt = true;
4594
78
        Parser.Lex();
4595
78
        continue;
4596
78
      }
4597
111k
    }
4598
111k
    break;
4599
111k
  }
4600
111k
  if (!GotDfmt && 
!GotNfmt111k
)
4601
111k
    return MatchOperand_NoMatch;
4602
81
  auto Format = Dfmt | Nfmt << 4;
4603
81
  Operands.push_back(
4604
81
      AMDGPUOperand::CreateImm(this, Format, S, AMDGPUOperand::ImmTyFORMAT));
4605
81
  return MatchOperand_Success;
4606
81
}
4607
4608
//===----------------------------------------------------------------------===//
4609
// ds
4610
//===----------------------------------------------------------------------===//
4611
4612
void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
4613
1.84k
                                    const OperandVector &Operands) {
4614
1.84k
  OptionalImmIndexMap OptionalIdx;
4615
1.84k
4616
11.0k
  for (unsigned i = 1, e = Operands.size(); i != e; 
++i9.20k
) {
4617
9.20k
    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4618
9.20k
4619
9.20k
    // Add the register arguments
4620
9.20k
    if (Op.isReg()) {
4621
5.59k
      Op.addRegOperands(Inst, 1);
4622
5.59k
      continue;
4623
5.59k
    }
4624
3.60k
4625
3.60k
    // Handle optional arguments
4626
3.60k
    OptionalIdx[Op.getImmTy()] = i;
4627
3.60k
  }
4628
1.84k
4629
1.84k
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
4630
1.84k
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
4631
1.84k
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
4632
1.84k
4633
1.84k
  Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
4634
1.84k
}
4635
4636
void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
4637
12.9k
                                bool IsGdsHardcoded) {
4638
12.9k
  OptionalImmIndexMap OptionalIdx;
4639
12.9k
4640
57.0k
  for (unsigned i = 1, e = Operands.size(); i != e; 
++i44.1k
) {
4641
44.1k
    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4642
44.1k
4643
44.1k
    // Add the register arguments
4644
44.1k
    if (Op.isReg()) {
4645
30.3k
      Op.addRegOperands(Inst, 1);
4646
30.3k
      continue;
4647
30.3k
    }
4648
13.7k
4649
13.7k
    if (Op.isToken() && 
Op.getToken() == "gds"0
) {
4650
0
      IsGdsHardcoded = true;
4651
0
      continue;
4652
0
    }
4653
13.7k
4654
13.7k
    // Handle optional arguments
4655
13.7k
    OptionalIdx[Op.getImmTy()] = i;
4656
13.7k
  }
4657
12.9k
4658
12.9k
  AMDGPUOperand::ImmTy OffsetType =
4659
12.9k
    (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_gfx10 ||
4660
12.9k
     
Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_gfx6_gfx712.9k
||
4661
12.9k
     
Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi12.6k
) ?
AMDGPUOperand::ImmTySwizzle301
:
4662
12.9k
                                                      
AMDGPUOperand::ImmTyOffset12.6k
;
4663
12.9k
4664
12.9k
  addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
4665
12.9k
4666
12.9k
  if (!IsGdsHardcoded) {
4667
12.5k
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
4668
12.5k
  }
4669
12.9k
  Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
4670
12.9k
}
4671
4672
173
void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
4673
173
  OptionalImmIndexMap OptionalIdx;
4674
173
4675
173
  unsigned OperandIdx[4];
4676
173
  unsigned EnMask = 0;
4677
173
  int SrcIdx = 0;
4678
173
4679
1.09k
  for (unsigned i = 1, e = Operands.size(); i != e; 
++i921
) {
4680
921
    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4681
921
4682
921
    // Add the register arguments
4683
921
    if (Op.isReg()) {
4684
495
      assert(SrcIdx < 4);
4685
495
      OperandIdx[SrcIdx] = Inst.size();
4686
495
      Op.addRegOperands(Inst, 1);
4687
495
      ++SrcIdx;
4688
495
      continue;
4689
495
    }
4690
426
4691
426
    if (Op.isOff()) {
4692
197
      assert(SrcIdx < 4);
4693
197
      OperandIdx[SrcIdx] = Inst.size();
4694
197
      Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
4695
197
      ++SrcIdx;
4696
197
      continue;
4697
197
    }
4698
229
4699
229
    if (Op.isImm() && 
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt191
) {
4700
173
      Op.addImmOperands(Inst, 1);
4701
173
      continue;
4702
173
    }
4703
56
4704
56
    if (Op.isToken() && 
Op.getToken() == "done"38
)
4705
38
      continue;
4706
18
4707
18
    // Handle optional arguments
4708
18
    OptionalIdx[Op.getImmTy()] = i;
4709
18
  }
4710
173
4711
173
  assert(SrcIdx == 4);
4712
173
4713
173
  bool Compr = false;
4714
173
  if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
4715
10
    Compr = true;
4716
10
    Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
4717
10
    Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
4718
10
    Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
4719
10
  }
4720
173
4721
865
  for (auto i = 0; i < SrcIdx; 
++i692
) {
4722
692
    if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
4723
479
      EnMask |= Compr? 
(0x3 << i * 2)16
:
(0x1 << i)463
;
4724
479
    }
4725
692
  }
4726
173
4727
173
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
4728
173
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
4729
173
4730
173
  Inst.addOperand(MCOperand::createImm(EnMask));
4731
173
}
4732
4733
//===----------------------------------------------------------------------===//
4734
// s_waitcnt
4735
//===----------------------------------------------------------------------===//
4736
4737
static bool
4738
encodeCnt(
4739
  const AMDGPU::IsaVersion ISA,
4740
  int64_t &IntVal,
4741
  int64_t CntVal,
4742
  bool Saturate,
4743
  unsigned (*encode)(const IsaVersion &Version, unsigned, unsigned),
4744
  unsigned (*decode)(const IsaVersion &Version, unsigned))
4745
242
{
4746
242
  bool Failed = false;
4747
242
4748
242
  IntVal = encode(ISA, IntVal, CntVal);
4749
242
  if (CntVal != decode(ISA, IntVal)) {
4750
31
    if (Saturate) {
4751
9
      IntVal = encode(ISA, IntVal, -1);
4752
22
    } else {
4753
22
      Failed = true;
4754
22
    }
4755
31
  }
4756
242
  return Failed;
4757
242
}
4758
4759
266
bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
4760
266
4761
266
  SMLoc CntLoc = getLoc();
4762
266
  StringRef CntName = getTokenStr();
4763
266
4764
266
  if (!skipToken(AsmToken::Identifier, "expected a counter name") ||
4765
266
      
!skipToken(AsmToken::LParen, "expected a left parenthesis")258
)
4766
16
    return false;
4767
250
4768
250
  int64_t CntVal;
4769
250
  SMLoc ValLoc = getLoc();
4770
250
  if (!parseExpr(CntVal))
4771
4
    return false;
4772
246
4773
246
  AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
4774
246
4775
246
  bool Failed = true;
4776
246
  bool Sat = CntName.endswith("_sat");
4777
246
4778
246
  if (CntName == "vmcnt" || 
CntName == "vmcnt_sat"157
) {
4779
98
    Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
4780
148
  } else if (CntName == "expcnt" || 
CntName == "expcnt_sat"85
) {
4781
72
    Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
4782
76
  } else if (CntName == "lgkmcnt" || 
CntName == "lgkmcnt_sat"22
) {
4783
72
    Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
4784
72
  } else {
4785
4
    Error(CntLoc, "invalid counter name " + CntName);
4786
4
    return false;
4787
4
  }
4788
242
4789
242
  if (Failed) {
4790
22
    Error(ValLoc, "too large value for " + CntName);
4791
22
    return false;
4792
22
  }
4793
220
4794
220
  if (!skipToken(AsmToken::RParen, "expected a closing parenthesis"))
4795
4
    return false;
4796
216
4797
216
  if (trySkipToken(AsmToken::Amp) || 
trySkipToken(AsmToken::Comma)168
) {
4798
72
    if (isToken(AsmToken::EndOfStatement)) {
4799
8
      Error(getLoc(), "expected a counter name");
4800
8
      return false;
4801
8
    }
4802
208
  }
4803
208
4804
208
  return true;
4805
208
}
4806
4807
OperandMatchResultTy
4808
202
AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
4809
202
  AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
4810
202
  int64_t Waitcnt = getWaitcntBitMask(ISA);
4811
202
  SMLoc S = getLoc();
4812
202
4813
202
  // If parse failed, do not return error code
4814
202
  // to avoid excessive error messages.
4815
202
  if (isToken(AsmToken::Identifier) && 
peekToken().is(AsmToken::LParen)175
) {
4816
266
    while (parseCnt(Waitcnt) && 
!isToken(AsmToken::EndOfStatement)208
)
;104
4817
162
  } else {
4818
40
    parseExpr(Waitcnt);
4819
40
  }
4820
202
4821
202
  Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
4822
202
  return MatchOperand_Success;
4823
202
}
4824
4825
bool
4826
140
AMDGPUOperand::isSWaitCnt() const {
4827
140
  return isImm();
4828
140
}
4829
4830
//===----------------------------------------------------------------------===//
4831
// hwreg
4832
//===----------------------------------------------------------------------===//
4833
4834
bool
4835
AMDGPUAsmParser::parseHwregBody(OperandInfoTy &HwReg,
4836
                                int64_t &Offset,
4837
374
                                int64_t &Width) {
4838
374
  using namespace llvm::AMDGPU::Hwreg;
4839
374
4840
374
  // The register may be specified by name or using a numeric code
4841
374
  if (isToken(AsmToken::Identifier) &&
4842
374
      
(HwReg.Id = getHwregId(getTokenStr())) >= 0102
) {
4843
86
    HwReg.IsSymbolic = true;
4844
86
    lex(); // skip message name
4845
288
  } else if (!parseExpr(HwReg.Id)) {
4846
8
    return false;
4847
8
  }
4848
366
4849
366
  if (trySkipToken(AsmToken::RParen))
4850
268
    return true;
4851
98
4852
98
  // parse optional params
4853
98
  return
4854
98
    skipToken(AsmToken::Comma, "expected a comma or a closing parenthesis") &&
4855
98
    
parseExpr(Offset)92
&&
4856
98
    
skipToken(AsmToken::Comma, "expected a comma")92
&&
4857
98
    
parseExpr(Width)86
&&
4858
98
    
skipToken(AsmToken::RParen, "expected a closing parenthesis")86
;
4859
98
}
4860
4861
bool
4862
AMDGPUAsmParser::validateHwreg(const OperandInfoTy &HwReg,
4863
                               const int64_t Offset,
4864
                               const int64_t Width,
4865
348
                               const SMLoc Loc) {
4866
348
4867
348
  using namespace llvm::AMDGPU::Hwreg;
4868
348
4869
348
  if (HwReg.IsSymbolic && 
!isValidHwreg(HwReg.Id, getSTI())86
) {
4870
43
    Error(Loc, "specified hardware register is not supported on this GPU");
4871
43
    return false;
4872
305
  } else if (!isValidHwreg(HwReg.Id)) {
4873
6
    Error(Loc, "invalid code of hardware register: only 6-bit values are legal");
4874
6
    return false;
4875
299
  } else if (!isValidHwregOffset(Offset)) {
4876
12
    Error(Loc, "invalid bit offset: only 5-bit values are legal");
4877
12
    return false;
4878
287
  } else if (!isValidHwregWidth(Width)) {
4879
12
    Error(Loc, "invalid bitfield width: only values from 1 to 32 are legal");
4880
12
    return false;
4881
12
  }
4882
275
  return true;
4883
275
}
4884
4885
OperandMatchResultTy
4886
650
AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
4887
650
  using namespace llvm::AMDGPU::Hwreg;
4888
650
4889
650
  int64_t ImmVal = 0;
4890
650
  SMLoc Loc = getLoc();
4891
650
4892
650
  // If parse failed, do not return error code
4893
650
  // to avoid excessive error messages.
4894
650
  if (trySkipId("hwreg", AsmToken::LParen)) {
4895
374
    OperandInfoTy HwReg(ID_UNKNOWN_);
4896
374
    int64_t Offset = OFFSET_DEFAULT_;
4897
374
    int64_t Width = WIDTH_DEFAULT_;
4898
374
    if (parseHwregBody(HwReg, Offset, Width) &&
4899
374
        
validateHwreg(HwReg, Offset, Width, Loc)348
) {
4900
275
      ImmVal = encodeHwreg(HwReg.Id, Offset, Width);
4901
275
    }
4902
374
  } else 
if (276
parseExpr(ImmVal)276
) {
4903
270
    if (ImmVal < 0 || !isUInt<16>(ImmVal))
4904
12
      Error(Loc, "invalid immediate: only 16-bit values are legal");
4905
270
  }
4906
650
4907
650
  Operands.push_back(AMDGPUOperand::CreateImm(this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
4908
650
  return MatchOperand_Success;
4909
650
}
4910
4911
1.14k
bool AMDGPUOperand::isHwreg() const {
4912
1.14k
  return isImmTy(ImmTyHwreg);
4913
1.14k
}
4914
4915
//===----------------------------------------------------------------------===//
4916
// sendmsg
4917
//===----------------------------------------------------------------------===//
4918
4919
bool
4920
AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg,
4921
                                  OperandInfoTy &Op,
4922
226
                                  OperandInfoTy &Stream) {
4923
226
  using namespace llvm::AMDGPU::SendMsg;
4924
226
4925
226
  if (isToken(AsmToken::Identifier) && 
(Msg.Id = getMsgId(getTokenStr())) >= 0148
) {
4926
125
    Msg.IsSymbolic = true;
4927
125
    lex(); // skip message name
4928
125
  } else 
if (101
!parseExpr(Msg.Id)101
) {
4929
12
    return false;
4930
12
  }
4931
214
4932
214
  if (trySkipToken(AsmToken::Comma)) {
4933
175
    Op.IsDefined = true;
4934
175
    if (isToken(AsmToken::Identifier) &&
4935
175
        
(Op.Id = getMsgOpId(Msg.Id, getTokenStr())) >= 068
) {
4936
51
      lex(); // skip operation name
4937
124
    } else if (!parseExpr(Op.Id)) {
4938
12
      return false;
4939
12
    }
4940
163
4941
163
    if (trySkipToken(AsmToken::Comma)) {
4942
83
      Stream.IsDefined = true;
4943
83
      if (!parseExpr(Stream.Id))
4944
4
        return false;
4945
198
    }
4946
163
  }
4947
198
4948
198
  return skipToken(AsmToken::RParen, "expected a closing parenthesis");
4949
198
}
4950
4951
bool
4952
AMDGPUAsmParser::validateSendMsg(const OperandInfoTy &Msg,
4953
                                 const OperandInfoTy &Op,
4954
                                 const OperandInfoTy &Stream,
4955
186
                                 const SMLoc S) {
4956
186
  using namespace llvm::AMDGPU::SendMsg;
4957
186
4958
186
  // Validation strictness depends on whether message is specified
4959
186
  // in a symbolc or in a numeric form. In the latter case
4960
186
  // only encoding possibility is checked.
4961
186
  bool Strict = Msg.IsSymbolic;
4962
186
4963
186
  if (!isValidMsgId(Msg.Id, getSTI(), Strict)) {
4964
14
    Error(S, "invalid message id");
4965
14
    return false;
4966
172
  } else if (Strict && 
(msgRequiresOp(Msg.Id) != Op.IsDefined)99
) {
4967
17
    Error(S, Op.IsDefined ?
4968
9
             "message does not support operations" :
4969
17
             
"missing message operation"8
);
4970
17
    return false;
4971
155
  } else if (!isValidMsgOp(Msg.Id, Op.Id, Strict)) {
4972
36
    Error(S, "invalid operation id");
4973
36
    return false;
4974
119
  } else if (Strict && 
!msgSupportsStream(Msg.Id, Op.Id)54
&&
Stream.IsDefined26
) {
4975
12
    Error(S, "message operation does not support streams");
4976
12
    return false;
4977
107
  } else if (!isValidMsgStream(Msg.Id, Op.Id, Stream.Id, Strict)) {
4978
12
    Error(S, "invalid message stream id");
4979
12
    return false;
4980
12
  }
4981
95
  return true;
4982
95
}
4983
4984
OperandMatchResultTy
4985
318
AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
4986
318
  using namespace llvm::AMDGPU::SendMsg;
4987
318
4988
318
  int64_t ImmVal = 0;
4989
318
  SMLoc Loc = getLoc();
4990
318
4991
318
  // If parse failed, do not return error code
4992
318
  // to avoid excessive error messages.
4993
318
  if (trySkipId("sendmsg", AsmToken::LParen)) {
4994
226
    OperandInfoTy Msg(ID_UNKNOWN_);
4995
226
    OperandInfoTy Op(OP_NONE_);
4996
226
    OperandInfoTy Stream(STREAM_ID_NONE_);
4997
226
    if (parseSendMsgBody(Msg, Op, Stream) &&
4998
226
        
validateSendMsg(Msg, Op, Stream, Loc)186
) {
4999
95
      ImmVal = encodeMsg(Msg.Id, Op.Id, Stream.Id);
5000
95
    }
5001
226
  } else 
if (92
parseExpr(ImmVal)92
) {
5002
92
    if (ImmVal < 0 || !isUInt<16>(ImmVal))
5003
0
      Error(Loc, "invalid immediate: only 16-bit values are legal");
5004
92
  }
5005
318
5006
318
  Operands.push_back(AMDGPUOperand::CreateImm(this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
5007
318
  return MatchOperand_Success;
5008
318
}
5009
5010
187
bool AMDGPUOperand::isSendMsg() const {
5011
187
  return isImmTy(ImmTySendMsg);
5012
187
}
5013
5014
//===----------------------------------------------------------------------===//
5015
// v_interp
5016
//===----------------------------------------------------------------------===//
5017
5018
176
OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
5019
176
  if (getLexer().getKind() != AsmToken::Identifier)
5020
0
    return MatchOperand_NoMatch;
5021
176
5022
176
  StringRef Str = Parser.getTok().getString();
5023
176
  int Slot = StringSwitch<int>(Str)
5024
176
    .Case("p10", 0)
5025
176
    .Case("p20", 1)
5026
176
    .Case("p0", 2)
5027
176
    .Default(-1);
5028
176
5029
176
  SMLoc S = Parser.getTok().getLoc();
5030
176
  if (Slot == -1)
5031
10
    return MatchOperand_ParseFail;
5032
166
5033
166
  Parser.Lex();
5034
166
  Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
5035
166
                                              AMDGPUOperand::ImmTyInterpSlot));
5036
166
  return MatchOperand_Success;
5037
166
}
5038
5039
824
OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
5040
824
  if (getLexer().getKind() != AsmToken::Identifier)
5041
0
    return MatchOperand_NoMatch;
5042
824
5043
824
  StringRef Str = Parser.getTok().getString();
5044
824
  if (!Str.startswith("attr"))
5045
3
    return MatchOperand_NoMatch;
5046
821
5047
821
  StringRef Chan = Str.take_back(2);
5048
821
  int AttrChan = StringSwitch<int>(Chan)
5049
821
    .Case(".x", 0)
5050
821
    .Case(".y", 1)
5051
821
    .Case(".z", 2)
5052
821
    .Case(".w", 3)
5053
821
    .Default(-1);
5054
821
  if (AttrChan == -1)
5055
8
    return MatchOperand_ParseFail;
5056
813
5057
813
  Str = Str.drop_back(2).drop_front(4);
5058
813
5059
813
  uint8_t Attr;
5060
813
  if (Str.getAsInteger(10, Attr))
5061
0
    return MatchOperand_ParseFail;
5062
813
5063
813
  SMLoc S = Parser.getTok().getLoc();
5064
813
  Parser.Lex();
5065
813
  if (Attr > 63) {
5066
8
    Error(S, "out of bounds attr");
5067
8
    return MatchOperand_Success;
5068
8
  }
5069
805
5070
805
  SMLoc SChan = SMLoc::getFromPointer(Chan.data());
5071
805
5072
805
  Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
5073
805
                                              AMDGPUOperand::ImmTyInterpAttr));
5074
805
  Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
5075
805
                                              AMDGPUOperand::ImmTyAttrChan));
5076
805
  return MatchOperand_Success;
5077
805
}
5078
5079
//===----------------------------------------------------------------------===//
5080
// exp
5081
//===----------------------------------------------------------------------===//
5082
5083
16
void AMDGPUAsmParser::errorExpTgt() {
5084
16
  Error(Parser.getTok().getLoc(), "invalid exp target");
5085
16
}
5086
5087
OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
5088
254
                                                      uint8_t &Val) {
5089
254
  if (Str == "null") {
5090
8
    Val = 9;
5091
8
    return MatchOperand_Success;
5092
8
  }
5093
246
5094
246
  if (Str.startswith("mrt")) {
5095
185
    Str = Str.drop_front(3);
5096
185
    if (Str == "z") { // == mrtz
5097
14
      Val = 8;
5098
14
      return MatchOperand_Success;
5099
14
    }
5100
171
5101
171
    if (Str.getAsInteger(10, Val))
5102
4
      return MatchOperand_ParseFail;
5103
167
5104
167
    if (Val > 7)
5105
2
      errorExpTgt();
5106
167
5107
167
    return MatchOperand_Success;
5108
167
  }
5109
61
5110
61
  if (Str.startswith("pos")) {
5111
21
    Str = Str.drop_front(3);
5112
21
    if (Str.getAsInteger(10, Val))
5113
4
      return MatchOperand_ParseFail;
5114
17
5115
17
    if (Val > 4 || (Val == 4 && 
!isGFX10()5
))
5116
4
      errorExpTgt();
5117
17
5118
17
    Val += 12;
5119
17
    return MatchOperand_Success;
5120
17
  }
5121
40
5122
40
  if (isGFX10() && 
Str == "prim"2
) {
5123
2
    Val = 20;
5124
2
    return MatchOperand_Success;
5125
2
  }
5126
38
5127
38
  if (Str.startswith("param")) {
5128
18
    Str = Str.drop_front(5);
5129
18
    if (Str.getAsInteger(10, Val))
5130
4
      return MatchOperand_ParseFail;
5131
14
5132
14
    if (Val >= 32)
5133
2
      errorExpTgt();
5134
14
5135
14
    Val += 32;
5136
14
    return MatchOperand_Success;
5137
14
  }
5138
20
5139
20
  if (Str.startswith("invalid_target_")) {
5140
12
    Str = Str.drop_front(15);
5141
12
    if (Str.getAsInteger(10, Val))
5142
4
      return MatchOperand_ParseFail;
5143
8
5144
8
    errorExpTgt();
5145
8
    return MatchOperand_Success;
5146
8
  }
5147
8
5148
8
  return MatchOperand_NoMatch;
5149
8
}
5150
5151
254
OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
5152
254
  uint8_t Val;
5153
254
  StringRef Str = Parser.getTok().getString();
5154
254
5155
254
  auto Res = parseExpTgtImpl(Str, Val);
5156
254
  if (Res != MatchOperand_Success)
5157
24
    return Res;
5158
230
5159
230
  SMLoc S = Parser.getTok().getLoc();
5160
230
  Parser.Lex();
5161
230
5162
230
  Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
5163
230
                                              AMDGPUOperand::ImmTyExpTgt));
5164
230
  return MatchOperand_Success;
5165
230
}
5166
5167
//===----------------------------------------------------------------------===//
5168
// parser helpers
5169
//===----------------------------------------------------------------------===//
5170
5171
bool
5172
3.16M
AMDGPUAsmParser::isId(const AsmToken &Token, const StringRef Id) const {
5173
3.16M
  return Token.is(AsmToken::Identifier) && 
Token.getString() == Id3.06M
;
5174
3.16M
}
5175
5176
bool
5177
3.15M
AMDGPUAsmParser::isId(const StringRef Id) const {
5178
3.15M
  return isId(getToken(), Id);
5179
3.15M
}
5180
5181
bool
5182
766k
AMDGPUAsmParser::isToken(const AsmToken::TokenKind Kind) const {
5183
766k
  return getTokenKind() == Kind;
5184
766k
}
5185
5186
bool
5187
640k
AMDGPUAsmParser::trySkipId(const StringRef Id) {
5188
640k
  if (isId(Id)) {
5189
2.95k
    lex();
5190
2.95k
    return true;
5191
2.95k
  }
5192
637k
  return false;