Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// This file implements the X86MCCodeEmitter class.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "MCTargetDesc/X86BaseInfo.h"
15
#include "MCTargetDesc/X86FixupKinds.h"
16
#include "MCTargetDesc/X86MCTargetDesc.h"
17
#include "llvm/ADT/SmallVector.h"
18
#include "llvm/MC/MCCodeEmitter.h"
19
#include "llvm/MC/MCContext.h"
20
#include "llvm/MC/MCExpr.h"
21
#include "llvm/MC/MCFixup.h"
22
#include "llvm/MC/MCInst.h"
23
#include "llvm/MC/MCInstrDesc.h"
24
#include "llvm/MC/MCInstrInfo.h"
25
#include "llvm/MC/MCRegisterInfo.h"
26
#include "llvm/MC/MCSubtargetInfo.h"
27
#include "llvm/MC/MCSymbol.h"
28
#include "llvm/Support/ErrorHandling.h"
29
#include "llvm/Support/raw_ostream.h"
30
#include <cassert>
31
#include <cstdint>
32
#include <cstdlib>
33
34
using namespace llvm;
35
36
#define DEBUG_TYPE "mccodeemitter"
37
38
namespace {
39
40
class X86MCCodeEmitter : public MCCodeEmitter {
41
  const MCInstrInfo &MCII;
42
  MCContext &Ctx;
43
44
public:
45
  X86MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx)
46
77.6k
    : MCII(mcii), Ctx(ctx) {
47
77.6k
  }
48
  X86MCCodeEmitter(const X86MCCodeEmitter &) = delete;
49
  X86MCCodeEmitter &operator=(const X86MCCodeEmitter &) = delete;
50
77.6k
  ~X86MCCodeEmitter() override = default;
51
52
1.09M
  bool is64BitMode(const MCSubtargetInfo &STI) const {
53
1.09M
    return STI.getFeatureBits()[X86::Mode64Bit];
54
1.09M
  }
55
56
516k
  bool is32BitMode(const MCSubtargetInfo &STI) const {
57
516k
    return STI.getFeatureBits()[X86::Mode32Bit];
58
516k
  }
59
60
1.05M
  bool is16BitMode(const MCSubtargetInfo &STI) const {
61
1.05M
    return STI.getFeatureBits()[X86::Mode16Bit];
62
1.05M
  }
63
64
  /// Is16BitMemOperand - Return true if the specified instruction has
65
  /// a 16-bit memory operand. Op specifies the operand # of the memoperand.
66
  bool Is16BitMemOperand(const MCInst &MI, unsigned Op,
67
172k
                         const MCSubtargetInfo &STI) const {
68
172k
    const MCOperand &BaseReg  = MI.getOperand(Op+X86::AddrBaseReg);
69
172k
    const MCOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg);
70
172k
    const MCOperand &Disp     = MI.getOperand(Op+X86::AddrDisp);
71
172k
72
172k
    if (
is16BitMode(STI) && 172k
BaseReg.getReg() == 088
&&
73
172k
        
Disp.isImm()30
&&
Disp.getImm() < 0x1000030
)
74
30
      return true;
75
172k
    
if (172k
(BaseReg.getReg() != 0 &&
76
167k
         X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg.getReg())) ||
77
172k
        (IndexReg.getReg() != 0 &&
78
24.8k
         X86MCRegisterClasses[X86::GR16RegClassID].contains(IndexReg.getReg())))
79
12
      return true;
80
172k
    return false;
81
172k
  }
82
83
619k
  unsigned GetX86RegNum(const MCOperand &MO) const {
84
619k
    return Ctx.getRegisterInfo()->getEncodingValue(MO.getReg()) & 0x7;
85
619k
  }
86
87
478k
  unsigned getX86RegEncoding(const MCInst &MI, unsigned OpNum) const {
88
478k
    return Ctx.getRegisterInfo()->getEncodingValue(
89
478k
                                                 MI.getOperand(OpNum).getReg());
90
478k
  }
91
92
  // Does this register require a bit to be set in REX prefix.
93
331k
  bool isREXExtendedReg(const MCInst &MI, unsigned OpNum) const {
94
331k
    return (getX86RegEncoding(MI, OpNum) >> 3) & 1;
95
331k
  }
96
97
1.66M
  void EmitByte(uint8_t C, unsigned &CurByte, raw_ostream &OS) const {
98
1.66M
    OS << (char)C;
99
1.66M
    ++CurByte;
100
1.66M
  }
101
102
  void EmitConstant(uint64_t Val, unsigned Size, unsigned &CurByte,
103
258k
                    raw_ostream &OS) const {
104
258k
    // Output the constant in little endian byte order.
105
818k
    for (unsigned i = 0; 
i != Size818k
;
++i560k
) {
106
560k
      EmitByte(Val & 255, CurByte, OS);
107
560k
      Val >>= 8;
108
560k
    }
109
258k
  }
110
111
  void EmitImmediate(const MCOperand &Disp, SMLoc Loc,
112
                     unsigned ImmSize, MCFixupKind FixupKind,
113
                     unsigned &CurByte, raw_ostream &OS,
114
                     SmallVectorImpl<MCFixup> &Fixups,
115
                     int ImmOffset = 0) const;
116
117
328k
  static uint8_t ModRMByte(unsigned Mod, unsigned RegOpcode, unsigned RM) {
118
328k
    assert(Mod < 4 && RegOpcode < 8 && RM < 8 && "ModRM Fields out of range!");
119
328k
    return RM | (RegOpcode << 3) | (Mod << 6);
120
328k
  }
121
122
  void EmitRegModRMByte(const MCOperand &ModRMReg, unsigned RegOpcodeFld,
123
162k
                        unsigned &CurByte, raw_ostream &OS) const {
124
162k
    EmitByte(ModRMByte(3, RegOpcodeFld, GetX86RegNum(ModRMReg)), CurByte, OS);
125
162k
  }
126
127
  void EmitSIBByte(unsigned SS, unsigned Index, unsigned Base,
128
34.7k
                   unsigned &CurByte, raw_ostream &OS) const {
129
34.7k
    // SIB byte is in the same format as the ModRMByte.
130
34.7k
    EmitByte(ModRMByte(SS, Index, Base), CurByte, OS);
131
34.7k
  }
132
133
  void emitMemModRMByte(const MCInst &MI, unsigned Op, unsigned RegOpcodeField,
134
                        uint64_t TSFlags, bool Rex, unsigned &CurByte,
135
                        raw_ostream &OS, SmallVectorImpl<MCFixup> &Fixups,
136
                        const MCSubtargetInfo &STI) const;
137
138
  void encodeInstruction(const MCInst &MI, raw_ostream &OS,
139
                         SmallVectorImpl<MCFixup> &Fixups,
140
                         const MCSubtargetInfo &STI) const override;
141
142
  void EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, int MemOperand,
143
                           const MCInst &MI, const MCInstrDesc &Desc,
144
                           raw_ostream &OS) const;
145
146
  void EmitSegmentOverridePrefix(unsigned &CurByte, unsigned SegOperand,
147
                                 const MCInst &MI, raw_ostream &OS) const;
148
149
  bool emitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, int MemOperand,
150
                        const MCInst &MI, const MCInstrDesc &Desc,
151
                        const MCSubtargetInfo &STI, raw_ostream &OS) const;
152
153
  uint8_t DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
154
                             int MemOperand, const MCInstrDesc &Desc) const;
155
};
156
157
} // end anonymous namespace
158
159
/// isDisp8 - Return true if this signed displacement fits in a 8-bit
160
/// sign-extended field.
161
65.4k
static bool isDisp8(int Value) {
162
65.4k
  return Value == (int8_t)Value;
163
65.4k
}
164
165
/// isCDisp8 - Return true if this signed displacement fits in a 8-bit
166
/// compressed dispacement field.
167
15.4k
static bool isCDisp8(uint64_t TSFlags, int Value, int& CValue) {
168
15.4k
  assert(((TSFlags & X86II::EncodingMask) == X86II::EVEX) &&
169
15.4k
         "Compressed 8-bit displacement is only valid for EVEX inst.");
170
15.4k
171
15.4k
  unsigned CD8_Scale =
172
15.4k
    (TSFlags & X86II::CD8_Scale_Mask) >> X86II::CD8_Scale_Shift;
173
15.4k
  if (
CD8_Scale == 015.4k
) {
174
0
    CValue = Value;
175
0
    return isDisp8(Value);
176
0
  }
177
15.4k
178
15.4k
  unsigned Mask = CD8_Scale - 1;
179
15.4k
  assert((CD8_Scale & Mask) == 0 && "Invalid memory object size.");
180
15.4k
  if (Value & Mask) // Unaligned offset
181
2.35k
    return false;
182
13.0k
  Value /= (int)CD8_Scale;
183
13.0k
  bool Ret = (Value == (int8_t)Value);
184
13.0k
185
13.0k
  if (Ret)
186
6.62k
    CValue = Value;
187
15.4k
  return Ret;
188
15.4k
}
189
190
/// getImmFixupKind - Return the appropriate fixup kind to use for an immediate
191
/// in an instruction with the specified TSFlags.
192
153k
static MCFixupKind getImmFixupKind(uint64_t TSFlags) {
193
153k
  unsigned Size = X86II::getSizeOfImm(TSFlags);
194
153k
  bool isPCRel = X86II::isImmPCRel(TSFlags);
195
153k
196
153k
  if (
X86II::isImmSigned(TSFlags)153k
) {
197
3.47k
    switch (Size) {
198
0
    
default: 0
llvm_unreachable0
("Unsupported signed fixup size!");
199
3.47k
    case 4: return MCFixupKind(X86::reloc_signed_4byte);
200
149k
    }
201
149k
  }
202
149k
  return MCFixup::getKindForSize(Size, isPCRel);
203
149k
}
204
205
/// Is32BitMemOperand - Return true if the specified instruction has
206
/// a 32-bit memory operand. Op specifies the operand # of the memoperand.
207
75.9k
static bool Is32BitMemOperand(const MCInst &MI, unsigned Op) {
208
75.9k
  const MCOperand &BaseReg  = MI.getOperand(Op+X86::AddrBaseReg);
209
75.9k
  const MCOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg);
210
75.9k
211
75.9k
  if ((BaseReg.getReg() != 0 &&
212
75.4k
       X86MCRegisterClasses[X86::GR32RegClassID].contains(BaseReg.getReg())) ||
213
75.9k
      (IndexReg.getReg() != 0 &&
214
12.4k
       X86MCRegisterClasses[X86::GR32RegClassID].contains(IndexReg.getReg())))
215
44
    return true;
216
75.9k
  
if (75.9k
BaseReg.getReg() == X86::EIP75.9k
) {
217
7
    assert(IndexReg.getReg() == 0 && "Invalid eip-based address.");
218
7
    return true;
219
7
  }
220
75.9k
  return false;
221
75.9k
}
222
223
/// Is64BitMemOperand - Return true if the specified instruction has
224
/// a 64-bit memory operand. Op specifies the operand # of the memoperand.
225
#ifndef NDEBUG
226
static bool Is64BitMemOperand(const MCInst &MI, unsigned Op) {
227
  const MCOperand &BaseReg  = MI.getOperand(Op+X86::AddrBaseReg);
228
  const MCOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg);
229
230
  if ((BaseReg.getReg() != 0 &&
231
       X86MCRegisterClasses[X86::GR64RegClassID].contains(BaseReg.getReg())) ||
232
      (IndexReg.getReg() != 0 &&
233
       X86MCRegisterClasses[X86::GR64RegClassID].contains(IndexReg.getReg())))
234
    return true;
235
  return false;
236
}
237
#endif
238
239
/// StartsWithGlobalOffsetTable - Check if this expression starts with
240
///  _GLOBAL_OFFSET_TABLE_ and if it is of the form
241
///  _GLOBAL_OFFSET_TABLE_-symbol. This is needed to support PIC on ELF
242
/// i386 as _GLOBAL_OFFSET_TABLE_ is magical. We check only simple case that
243
/// are know to be used: _GLOBAL_OFFSET_TABLE_ by itself or at the start
244
/// of a binary expression.
245
enum GlobalOffsetTableExprKind {
246
  GOT_None,
247
  GOT_Normal,
248
  GOT_SymDiff
249
};
250
static GlobalOffsetTableExprKind
251
8.29k
StartsWithGlobalOffsetTable(const MCExpr *Expr) {
252
8.29k
  const MCExpr *RHS = nullptr;
253
8.29k
  if (
Expr->getKind() == MCExpr::Binary8.29k
) {
254
6.75k
    const MCBinaryExpr *BE = static_cast<const MCBinaryExpr *>(Expr);
255
6.75k
    Expr = BE->getLHS();
256
6.75k
    RHS = BE->getRHS();
257
6.75k
  }
258
8.29k
259
8.29k
  if (Expr->getKind() != MCExpr::SymbolRef)
260
851
    return GOT_None;
261
7.43k
262
7.43k
  const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr*>(Expr);
263
7.43k
  const MCSymbol &S = Ref->getSymbol();
264
7.43k
  if (S.getName() != "_GLOBAL_OFFSET_TABLE_")
265
7.42k
    return GOT_None;
266
15
  
if (15
RHS && 15
RHS->getKind() == MCExpr::SymbolRef5
)
267
3
    return GOT_SymDiff;
268
12
  return GOT_Normal;
269
12
}
270
271
15.0k
static bool HasSecRelSymbolRef(const MCExpr *Expr) {
272
15.0k
  if (
Expr->getKind() == MCExpr::SymbolRef15.0k
) {
273
13.2k
    const MCSymbolRefExpr *Ref = static_cast<const MCSymbolRefExpr*>(Expr);
274
13.2k
    return Ref->getKind() == MCSymbolRefExpr::VK_SECREL;
275
13.2k
  }
276
1.79k
  return false;
277
1.79k
}
278
279
void X86MCCodeEmitter::
280
EmitImmediate(const MCOperand &DispOp, SMLoc Loc, unsigned Size,
281
              MCFixupKind FixupKind, unsigned &CurByte, raw_ostream &OS,
282
258k
              SmallVectorImpl<MCFixup> &Fixups, int ImmOffset) const {
283
258k
  const MCExpr *Expr = nullptr;
284
258k
  if (
DispOp.isImm()258k
) {
285
163k
    // If this is a simple integer displacement that doesn't require a
286
163k
    // relocation, emit it now.
287
163k
    if (FixupKind != FK_PCRel_1 &&
288
162k
        FixupKind != FK_PCRel_2 &&
289
163k
        
FixupKind != FK_PCRel_4162k
) {
290
162k
      EmitConstant(DispOp.getImm()+ImmOffset, Size, CurByte, OS);
291
162k
      return;
292
162k
    }
293
102
    Expr = MCConstantExpr::create(DispOp.getImm(), Ctx);
294
258k
  } else {
295
95.4k
    Expr = DispOp.getExpr();
296
95.4k
  }
297
258k
298
258k
  // If we have an immoffset, add it to the expression.
299
95.5k
  
if (95.5k
(FixupKind == FK_Data_4 ||
300
94.8k
       FixupKind == FK_Data_8 ||
301
95.5k
       
FixupKind == MCFixupKind(X86::reloc_signed_4byte)94.4k
)) {
302
8.29k
    GlobalOffsetTableExprKind Kind = StartsWithGlobalOffsetTable(Expr);
303
8.29k
    if (
Kind != GOT_None8.29k
) {
304
15
      assert(ImmOffset == 0);
305
15
306
15
      if (
Size == 815
) {
307
1
        FixupKind = MCFixupKind(X86::reloc_global_offset_table8);
308
15
      } else {
309
14
        assert(Size == 4);
310
14
        FixupKind = MCFixupKind(X86::reloc_global_offset_table);
311
14
      }
312
15
313
15
      if (Kind == GOT_Normal)
314
12
        ImmOffset = CurByte;
315
8.29k
    } else 
if (8.27k
Expr->getKind() == MCExpr::SymbolRef8.27k
) {
316
1.52k
      if (
HasSecRelSymbolRef(Expr)1.52k
) {
317
2
        FixupKind = MCFixupKind(FK_SecRel_4);
318
2
      }
319
8.27k
    } else 
if (6.74k
Expr->getKind() == MCExpr::Binary6.74k
) {
320
6.74k
      const MCBinaryExpr *Bin = static_cast<const MCBinaryExpr*>(Expr);
321
6.74k
      if (HasSecRelSymbolRef(Bin->getLHS())
322
6.74k
          || 
HasSecRelSymbolRef(Bin->getRHS())6.74k
) {
323
1
        FixupKind = MCFixupKind(FK_SecRel_4);
324
1
      }
325
8.27k
    }
326
8.29k
  }
327
95.5k
328
95.5k
  // If the fixup is pc-relative, we need to bias the value to be relative to
329
95.5k
  // the start of the field, not the end of the field.
330
95.5k
  if (FixupKind == FK_PCRel_4 ||
331
62.8k
      FixupKind == MCFixupKind(X86::reloc_riprel_4byte) ||
332
51.7k
      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_movq_load) ||
333
49.7k
      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax) ||
334
49.7k
      FixupKind == MCFixupKind(X86::reloc_riprel_4byte_relax_rex))
335
46.0k
    ImmOffset -= 4;
336
95.5k
  if (FixupKind == FK_PCRel_2)
337
5
    ImmOffset -= 2;
338
95.5k
  if (FixupKind == FK_PCRel_1)
339
40.0k
    ImmOffset -= 1;
340
95.5k
341
95.5k
  if (ImmOffset)
342
86.0k
    Expr = MCBinaryExpr::createAdd(Expr, MCConstantExpr::create(ImmOffset, Ctx),
343
86.0k
                                   Ctx);
344
258k
345
258k
  // Emit a symbolic constant as a fixup and 4 zeros.
346
258k
  Fixups.push_back(MCFixup::create(CurByte, Expr, FixupKind, Loc));
347
258k
  EmitConstant(0, Size, CurByte, OS);
348
258k
}
349
350
void X86MCCodeEmitter::emitMemModRMByte(const MCInst &MI, unsigned Op,
351
                                        unsigned RegOpcodeField,
352
                                        uint64_t TSFlags, bool Rex,
353
                                        unsigned &CurByte, raw_ostream &OS,
354
                                        SmallVectorImpl<MCFixup> &Fixups,
355
130k
                                        const MCSubtargetInfo &STI) const {
356
130k
  const MCOperand &Disp     = MI.getOperand(Op+X86::AddrDisp);
357
130k
  const MCOperand &Base     = MI.getOperand(Op+X86::AddrBaseReg);
358
130k
  const MCOperand &Scale    = MI.getOperand(Op+X86::AddrScaleAmt);
359
130k
  const MCOperand &IndexReg = MI.getOperand(Op+X86::AddrIndexReg);
360
130k
  unsigned BaseReg = Base.getReg();
361
130k
  bool HasEVEX = (TSFlags & X86II::EncodingMask) == X86II::EVEX;
362
130k
363
130k
  // Handle %rip relative addressing.
364
130k
  if (BaseReg == X86::RIP ||
365
130k
      
BaseReg == X86::EIP117k
) { // [disp32+rIP] in X86-64 mode
366
13.3k
    assert(is64BitMode(STI) && "Rip-relative addressing requires 64-bit mode");
367
13.3k
    assert(IndexReg.getReg() == 0 && "Invalid rip-relative address");
368
13.3k
    EmitByte(ModRMByte(0, RegOpcodeField, 5), CurByte, OS);
369
13.3k
370
13.3k
    unsigned Opcode = MI.getOpcode();
371
13.3k
    // movq loads are handled with a special relocation form which allows the
372
13.3k
    // linker to eliminate some loads for GOT references which end up in the
373
13.3k
    // same linkage unit.
374
13.3k
    unsigned FixupKind = [=]() {
375
13.3k
      switch (Opcode) {
376
11.0k
      default:
377
11.0k
        return X86::reloc_riprel_4byte;
378
2.00k
      case X86::MOV64rm:
379
2.00k
        assert(Rex);
380
2.00k
        return X86::reloc_riprel_4byte_movq_load;
381
215
      case X86::CALL64m:
382
215
      case X86::JMP64m:
383
215
      case X86::TEST64rm:
384
215
      case X86::ADC64rm:
385
215
      case X86::ADD64rm:
386
215
      case X86::AND64rm:
387
215
      case X86::CMP64rm:
388
215
      case X86::OR64rm:
389
215
      case X86::SBB64rm:
390
215
      case X86::SUB64rm:
391
215
      case X86::XOR64rm:
392
152
        return Rex ? X86::reloc_riprel_4byte_relax_rex
393
63
                   : X86::reloc_riprel_4byte_relax;
394
0
      }
395
0
    }();
396
13.3k
397
13.3k
    // rip-relative addressing is actually relative to the *next* instruction.
398
13.3k
    // Since an immediate can follow the mod/rm byte for an instruction, this
399
13.3k
    // means that we need to bias the immediate field of the instruction with
400
13.3k
    // the size of the immediate field.  If we have this case, add it into the
401
13.3k
    // expression to emit.
402
13.3k
    int ImmSize = X86II::hasImm(TSFlags) ? 
X86II::getSizeOfImm(TSFlags)828
:
012.4k
;
403
13.3k
404
13.3k
    EmitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind),
405
13.3k
                  CurByte, OS, Fixups, -ImmSize);
406
13.3k
    return;
407
13.3k
  }
408
117k
409
117k
  
unsigned BaseRegNo = BaseReg ? 117k
GetX86RegNum(Base)114k
:
-1U2.76k
;
410
117k
411
117k
  // 16-bit addressing forms of the ModR/M byte have a different encoding for
412
117k
  // the R/M field and are far more limited in which registers can be used.
413
117k
  if (
Is16BitMemOperand(MI, Op, STI)117k
) {
414
21
    if (
BaseReg21
) {
415
6
      // For 32-bit addressing, the row and column values in Table 2-2 are
416
6
      // basically the same. It's AX/CX/DX/BX/SP/BP/SI/DI in that order, with
417
6
      // some special cases. And GetX86RegNum reflects that numbering.
418
6
      // For 16-bit addressing it's more fun, as shown in the SDM Vol 2A,
419
6
      // Table 2-1 "16-Bit Addressing Forms with the ModR/M byte". We can only
420
6
      // use SI/DI/BP/BX, which have "row" values 4-7 in no particular order,
421
6
      // while values 0-3 indicate the allowed combinations (base+index) of
422
6
      // those: 0 for BX+SI, 1 for BX+DI, 2 for BP+SI, 3 for BP+DI.
423
6
      //
424
6
      // R16Table[] is a lookup from the normal RegNo, to the row values from
425
6
      // Table 2-1 for 16-bit addressing modes. Where zero means disallowed.
426
6
      static const unsigned R16Table[] = { 0, 0, 0, 7, 0, 6, 4, 5 };
427
6
      unsigned RMfield = R16Table[BaseRegNo];
428
6
429
6
      assert(RMfield && "invalid 16-bit base register");
430
6
431
6
      if (
IndexReg.getReg()6
) {
432
2
        unsigned IndexReg16 = R16Table[GetX86RegNum(IndexReg)];
433
2
434
2
        assert(IndexReg16 && "invalid 16-bit index register");
435
2
        // We must have one of SI/DI (4,5), and one of BP/BX (6,7).
436
2
        assert(((IndexReg16 ^ RMfield) & 2) &&
437
2
               "invalid 16-bit base/index register combination");
438
2
        assert(Scale.getImm() == 1 &&
439
2
               "invalid scale for 16-bit memory reference");
440
2
441
2
        // Allow base/index to appear in either order (although GAS doesn't).
442
2
        if (IndexReg16 & 2)
443
1
          RMfield = (RMfield & 1) | ((7 - IndexReg16) << 1);
444
2
        else
445
1
          RMfield = (IndexReg16 & 1) | ((7 - RMfield) << 1);
446
2
      }
447
6
448
6
      if (
Disp.isImm() && 6
isDisp8(Disp.getImm())6
) {
449
5
        if (
Disp.getImm() == 0 && 5
BaseRegNo != N86::EBP5
) {
450
5
          // There is no displacement; just the register.
451
5
          EmitByte(ModRMByte(0, RegOpcodeField, RMfield), CurByte, OS);
452
5
          return;
453
5
        }
454
0
        // Use the [REG]+disp8 form, including for [BP] which cannot be encoded.
455
0
        EmitByte(ModRMByte(1, RegOpcodeField, RMfield), CurByte, OS);
456
0
        EmitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups);
457
0
        return;
458
0
      }
459
1
      // This is the [REG]+disp16 case.
460
1
      EmitByte(ModRMByte(2, RegOpcodeField, RMfield), CurByte, OS);
461
21
    } else {
462
15
      // There is no BaseReg; this is the plain [disp16] case.
463
15
      EmitByte(ModRMByte(0, RegOpcodeField, 6), CurByte, OS);
464
15
    }
465
21
466
21
    // Emit 16-bit displacement for plain disp16 or [REG]+disp16 cases.
467
16
    EmitImmediate(Disp, MI.getLoc(), 2, FK_Data_2, CurByte, OS, Fixups);
468
16
    return;
469
117k
  }
470
117k
471
117k
  // Determine whether a SIB byte is needed.
472
117k
  // If no BaseReg, issue a RIP relative instruction only if the MCE can
473
117k
  // resolve addresses on-the-fly, otherwise use SIB (Intel Manual 2A, table
474
117k
  // 2-7) and absolute references.
475
117k
476
117k
  if (// The SIB byte must be used if there is an index register.
477
117k
      IndexReg.getReg() == 0 &&
478
117k
      // The SIB byte must be used if the base is ESP/RSP/R12, all of which
479
117k
      // encode to an R/M value of 4, which indicates that a SIB byte is
480
117k
      // present.
481
98.6k
      BaseRegNo != N86::ESP &&
482
117k
      // If there is no base register and we're in 64-bit mode, we need a SIB
483
117k
      // byte to emit an addr that is just 'disp32' (the non-RIP relative form).
484
117k
      
(!is64BitMode(STI) || 82.8k
BaseReg != 044.7k
)) {
485
82.5k
486
82.5k
    if (
BaseReg == 082.5k
) { // [disp32] in X86-32 mode
487
2.10k
      EmitByte(ModRMByte(0, RegOpcodeField, 5), CurByte, OS);
488
2.10k
      EmitImmediate(Disp, MI.getLoc(), 4, FK_Data_4, CurByte, OS, Fixups);
489
2.10k
      return;
490
2.10k
    }
491
80.4k
492
80.4k
    // If the base is not EBP/ESP and there is no displacement, use simple
493
80.4k
    // indirect register encoding, this handles addresses like [EAX].  The
494
80.4k
    // encoding for [EBP] with no displacement means [disp32] so we handle it
495
80.4k
    // by emitting a displacement of 0 below.
496
80.4k
    
if (80.4k
Disp.isImm() && 80.4k
Disp.getImm() == 072.7k
&&
BaseRegNo != N86::EBP17.9k
) {
497
17.4k
      EmitByte(ModRMByte(0, RegOpcodeField, BaseRegNo), CurByte, OS);
498
17.4k
      return;
499
17.4k
    }
500
63.0k
501
63.0k
    // Otherwise, if the displacement fits in a byte, encode as [REG+disp8].
502
63.0k
    
if (63.0k
Disp.isImm()63.0k
) {
503
55.3k
      if (
!HasEVEX && 55.3k
isDisp8(Disp.getImm())42.6k
) {
504
38.8k
        EmitByte(ModRMByte(1, RegOpcodeField, BaseRegNo), CurByte, OS);
505
38.8k
        EmitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups);
506
38.8k
        return;
507
38.8k
      }
508
16.4k
      // Try EVEX compressed 8-bit displacement first; if failed, fall back to
509
16.4k
      // 32-bit displacement.
510
16.4k
      int CDisp8 = 0;
511
16.4k
      if (
HasEVEX && 16.4k
isCDisp8(TSFlags, Disp.getImm(), CDisp8)12.6k
) {
512
6.39k
        EmitByte(ModRMByte(1, RegOpcodeField, BaseRegNo), CurByte, OS);
513
6.39k
        EmitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups,
514
6.39k
                      CDisp8 - Disp.getImm());
515
6.39k
        return;
516
6.39k
      }
517
17.8k
    }
518
17.8k
519
17.8k
    // Otherwise, emit the most general non-SIB encoding: [REG+disp32]
520
17.8k
    EmitByte(ModRMByte(2, RegOpcodeField, BaseRegNo), CurByte, OS);
521
17.8k
    unsigned Opcode = MI.getOpcode();
522
1.34k
    unsigned FixupKind = Opcode == X86::MOV32rm ? X86::reloc_signed_4byte_relax
523
16.4k
                                                : X86::reloc_signed_4byte;
524
82.5k
    EmitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(FixupKind), CurByte, OS,
525
82.5k
                  Fixups);
526
82.5k
    return;
527
82.5k
  }
528
34.7k
529
34.7k
  // We need a SIB byte, so start by outputting the ModR/M byte first
530
117k
  assert(IndexReg.getReg() != X86::ESP &&
531
34.7k
         IndexReg.getReg() != X86::RSP && "Cannot use ESP as index reg!");
532
34.7k
533
34.7k
  bool ForceDisp32 = false;
534
34.7k
  bool ForceDisp8  = false;
535
34.7k
  int CDisp8 = 0;
536
34.7k
  int ImmOffset = 0;
537
34.7k
  if (
BaseReg == 034.7k
) {
538
638
    // If there is no base register, we emit the special case SIB byte with
539
638
    // MOD=0, BASE=5, to JUST get the index, scale, and displacement.
540
638
    EmitByte(ModRMByte(0, RegOpcodeField, 4), CurByte, OS);
541
638
    ForceDisp32 = true;
542
34.7k
  } else 
if (34.1k
!Disp.isImm()34.1k
) {
543
261
    // Emit the normal disp32 encoding.
544
261
    EmitByte(ModRMByte(2, RegOpcodeField, 4), CurByte, OS);
545
261
    ForceDisp32 = true;
546
34.1k
  } else 
if (33.8k
Disp.getImm() == 0 &&
547
33.8k
             // Base reg can't be anything that ends up with '5' as the base
548
33.8k
             // reg, it is the magic [*] nomenclature that indicates no base.
549
33.8k
             
BaseRegNo != N86::EBP8.39k
) {
550
8.28k
    // Emit no displacement ModR/M byte
551
8.28k
    EmitByte(ModRMByte(0, RegOpcodeField, 4), CurByte, OS);
552
33.8k
  } else 
if (25.5k
!HasEVEX && 25.5k
isDisp8(Disp.getImm())22.8k
) {
553
18.2k
    // Emit the disp8 encoding.
554
18.2k
    EmitByte(ModRMByte(1, RegOpcodeField, 4), CurByte, OS);
555
18.2k
    ForceDisp8 = true;           // Make sure to force 8 bit disp if Base=EBP
556
25.5k
  } else 
if (7.36k
HasEVEX && 7.36k
isCDisp8(TSFlags, Disp.getImm(), CDisp8)2.75k
) {
557
231
    // Emit the disp8 encoding.
558
231
    EmitByte(ModRMByte(1, RegOpcodeField, 4), CurByte, OS);
559
231
    ForceDisp8 = true;           // Make sure to force 8 bit disp if Base=EBP
560
231
    ImmOffset = CDisp8 - Disp.getImm();
561
7.36k
  } else {
562
7.12k
    // Emit the normal disp32 encoding.
563
7.12k
    EmitByte(ModRMByte(2, RegOpcodeField, 4), CurByte, OS);
564
7.12k
  }
565
34.7k
566
34.7k
  // Calculate what the SS field value should be...
567
34.7k
  static const unsigned SSTable[] = { ~0U, 0, 1, ~0U, 2, ~0U, ~0U, ~0U, 3 };
568
34.7k
  unsigned SS = SSTable[Scale.getImm()];
569
34.7k
570
34.7k
  if (
BaseReg == 034.7k
) {
571
638
    // Handle the SIB byte for the case where there is no base, see Intel
572
638
    // Manual 2A, table 2-7. The displacement has already been output.
573
638
    unsigned IndexRegNo;
574
638
    if (IndexReg.getReg())
575
327
      IndexRegNo = GetX86RegNum(IndexReg);
576
638
    else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
577
311
      IndexRegNo = 4;
578
638
    EmitSIBByte(SS, IndexRegNo, 5, CurByte, OS);
579
34.7k
  } else {
580
34.1k
    unsigned IndexRegNo;
581
34.1k
    if (IndexReg.getReg())
582
18.3k
      IndexRegNo = GetX86RegNum(IndexReg);
583
34.1k
    else
584
15.7k
      IndexRegNo = 4;   // For example [ESP+1*<noreg>+4]
585
34.1k
    EmitSIBByte(SS, IndexRegNo, GetX86RegNum(Base), CurByte, OS);
586
34.1k
  }
587
34.7k
588
34.7k
  // Do we need to output a displacement?
589
34.7k
  if (ForceDisp8)
590
18.4k
    EmitImmediate(Disp, MI.getLoc(), 1, FK_Data_1, CurByte, OS, Fixups, ImmOffset);
591
16.3k
  else 
if (16.3k
ForceDisp32 || 16.3k
Disp.getImm() != 015.4k
)
592
8.02k
    EmitImmediate(Disp, MI.getLoc(), 4, MCFixupKind(X86::reloc_signed_4byte),
593
8.02k
                  CurByte, OS, Fixups);
594
130k
}
595
596
/// EmitVEXOpcodePrefix - AVX instructions are encoded using a opcode prefix
597
/// called VEX.
598
void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
599
                                           int MemOperand, const MCInst &MI,
600
                                           const MCInstrDesc &Desc,
601
44.9k
                                           raw_ostream &OS) const {
602
44.9k
  assert(!(TSFlags & X86II::LOCK) && "Can't have LOCK VEX.");
603
44.9k
604
44.9k
  uint64_t Encoding = TSFlags & X86II::EncodingMask;
605
44.9k
  bool HasEVEX_K = TSFlags & X86II::EVEX_K;
606
44.9k
  bool HasVEX_4V = TSFlags & X86II::VEX_4V;
607
44.9k
  bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
608
44.9k
609
44.9k
  // VEX_R: opcode externsion equivalent to REX.R in
610
44.9k
  // 1's complement (inverted) form
611
44.9k
  //
612
44.9k
  //  1: Same as REX_R=0 (must be 1 in 32-bit mode)
613
44.9k
  //  0: Same as REX_R=1 (64 bit mode only)
614
44.9k
  //
615
44.9k
  uint8_t VEX_R = 0x1;
616
44.9k
  uint8_t EVEX_R2 = 0x1;
617
44.9k
618
44.9k
  // VEX_X: equivalent to REX.X, only used when a
619
44.9k
  // register is used for index in SIB Byte.
620
44.9k
  //
621
44.9k
  //  1: Same as REX.X=0 (must be 1 in 32-bit mode)
622
44.9k
  //  0: Same as REX.X=1 (64-bit mode only)
623
44.9k
  uint8_t VEX_X = 0x1;
624
44.9k
625
44.9k
  // VEX_B:
626
44.9k
  //
627
44.9k
  //  1: Same as REX_B=0 (ignored in 32-bit mode)
628
44.9k
  //  0: Same as REX_B=1 (64 bit mode only)
629
44.9k
  //
630
44.9k
  uint8_t VEX_B = 0x1;
631
44.9k
632
44.9k
  // VEX_W: opcode specific (use like REX.W, or used for
633
44.9k
  // opcode extension, or ignored, depending on the opcode byte)
634
44.9k
  uint8_t VEX_W = (TSFlags & X86II::VEX_W) ? 
113.8k
:
031.1k
;
635
44.9k
636
44.9k
  // VEX_5M (VEX m-mmmmm field):
637
44.9k
  //
638
44.9k
  //  0b00000: Reserved for future use
639
44.9k
  //  0b00001: implied 0F leading opcode
640
44.9k
  //  0b00010: implied 0F 38 leading opcode bytes
641
44.9k
  //  0b00011: implied 0F 3A leading opcode bytes
642
44.9k
  //  0b00100-0b11111: Reserved for future use
643
44.9k
  //  0b01000: XOP map select - 08h instructions with imm byte
644
44.9k
  //  0b01001: XOP map select - 09h instructions with no imm byte
645
44.9k
  //  0b01010: XOP map select - 0Ah instructions with imm dword
646
44.9k
  uint8_t VEX_5M;
647
44.9k
  switch (TSFlags & X86II::OpMapMask) {
648
0
  
default: 0
llvm_unreachable0
("Invalid prefix!");
649
24.0k
  case X86II::TB:   VEX_5M = 0x1; break; // 0F
650
15.2k
  case X86II::T8:   VEX_5M = 0x2; break; // 0F 38
651
5.45k
  case X86II::TA:   VEX_5M = 0x3; break; // 0F 3A
652
91
  case X86II::XOP8: VEX_5M = 0x8; break;
653
128
  case X86II::XOP9: VEX_5M = 0x9; break;
654
29
  case X86II::XOPA: VEX_5M = 0xA; break;
655
44.9k
  }
656
44.9k
657
44.9k
  // VEX_4V (VEX vvvv field): a register specifier
658
44.9k
  // (in 1's complement form) or 1111 if unused.
659
44.9k
  uint8_t VEX_4V = 0xf;
660
44.9k
  uint8_t EVEX_V2 = 0x1;
661
44.9k
662
44.9k
  // EVEX_L2/VEX_L (Vector Length):
663
44.9k
  //
664
44.9k
  // L2 L
665
44.9k
  //  0 0: scalar or 128-bit vector
666
44.9k
  //  0 1: 256-bit vector
667
44.9k
  //  1 0: 512-bit vector
668
44.9k
  //
669
44.9k
  uint8_t VEX_L = (TSFlags & X86II::VEX_L) ? 
111.3k
:
033.6k
;
670
44.9k
  uint8_t EVEX_L2 = (TSFlags & X86II::EVEX_L2) ? 
113.5k
:
031.4k
;
671
44.9k
672
44.9k
  // VEX_PP: opcode extension providing equivalent
673
44.9k
  // functionality of a SIMD prefix
674
44.9k
  //
675
44.9k
  //  0b00: None
676
44.9k
  //  0b01: 66
677
44.9k
  //  0b10: F3
678
44.9k
  //  0b11: F2
679
44.9k
  //
680
44.9k
  uint8_t VEX_PP;
681
44.9k
  switch (TSFlags & X86II::OpPrefixMask) {
682
0
  
default: 0
llvm_unreachable0
("Invalid op prefix!");
683
7.53k
  case X86II::PS: VEX_PP = 0x0; break; // none
684
31.0k
  case X86II::PD: VEX_PP = 0x1; break; // 66
685
3.77k
  case X86II::XS: VEX_PP = 0x2; break; // F3
686
2.62k
  case X86II::XD: VEX_PP = 0x3; break; // F2
687
44.9k
  }
688
44.9k
689
44.9k
  // EVEX_U
690
44.9k
  uint8_t EVEX_U = 1; // Always '1' so far
691
44.9k
692
44.9k
  // EVEX_z
693
44.9k
  uint8_t EVEX_z = (HasEVEX_K && 
(TSFlags & X86II::EVEX_Z)6.44k
) ?
12.57k
:
042.4k
;
694
44.9k
695
44.9k
  // EVEX_b
696
44.9k
  uint8_t EVEX_b = (TSFlags & X86II::EVEX_B) ? 
17.14k
:
037.8k
;
697
44.9k
698
44.9k
  // EVEX_rc
699
44.9k
  uint8_t EVEX_rc = 0;
700
44.9k
701
44.9k
  // EVEX_aaa
702
44.9k
  uint8_t EVEX_aaa = 0;
703
44.9k
704
44.9k
  bool EncodeRC = false;
705
44.9k
706
44.9k
  // Classify VEX_B, VEX_4V, VEX_R, VEX_X
707
44.9k
  unsigned NumOps = Desc.getNumOperands();
708
44.9k
  unsigned CurOp = X86II::getOperandBias(Desc);
709
44.9k
710
44.9k
  switch (TSFlags & X86II::FormMask) {
711
0
  
default: 0
llvm_unreachable0
("Unexpected form in EmitVEXOpcodePrefix!");
712
200
  case X86II::RawFrm:
713
200
    break;
714
3.60k
  case X86II::MRMDestMem: {
715
3.60k
    // MRMDestMem instructions forms:
716
3.60k
    //  MemAddr, src1(ModR/M)
717
3.60k
    //  MemAddr, src1(VEX_4V), src2(ModR/M)
718
3.60k
    //  MemAddr, src1(ModR/M), imm8
719
3.60k
    //
720
3.60k
    unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
721
3.60k
    VEX_B = ~(BaseRegEnc >> 3) & 1;
722
3.60k
    unsigned IndexRegEnc = getX86RegEncoding(MI, MemOperand+X86::AddrIndexReg);
723
3.60k
    VEX_X = ~(IndexRegEnc >> 3) & 1;
724
3.60k
    if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
725
3.58k
      EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
726
3.60k
727
3.60k
    CurOp += X86::AddrNumOperands;
728
3.60k
729
3.60k
    if (HasEVEX_K)
730
567
      EVEX_aaa = getX86RegEncoding(MI, CurOp++);
731
3.60k
732
3.60k
    if (
HasVEX_4V3.60k
) {
733
24
      unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
734
24
      VEX_4V = ~VRegEnc & 0xf;
735
24
      EVEX_V2 = ~(VRegEnc >> 4) & 1;
736
24
    }
737
3.60k
738
3.60k
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
739
3.60k
    VEX_R = ~(RegEnc >> 3) & 1;
740
3.60k
    EVEX_R2 = ~(RegEnc >> 4) & 1;
741
3.60k
    break;
742
44.9k
  }
743
20.6k
  case X86II::MRMSrcMem: {
744
20.6k
    // MRMSrcMem instructions forms:
745
20.6k
    //  src1(ModR/M), MemAddr
746
20.6k
    //  src1(ModR/M), src2(VEX_4V), MemAddr
747
20.6k
    //  src1(ModR/M), MemAddr, imm8
748
20.6k
    //  src1(ModR/M), MemAddr, src2(Imm[7:4])
749
20.6k
    //
750
20.6k
    //  FMA4:
751
20.6k
    //  dst(ModR/M.reg), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
752
20.6k
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
753
20.6k
    VEX_R = ~(RegEnc >> 3) & 1;
754
20.6k
    EVEX_R2 = ~(RegEnc >> 4) & 1;
755
20.6k
756
20.6k
    if (HasEVEX_K)
757
769
      EVEX_aaa = getX86RegEncoding(MI, CurOp++);
758
20.6k
759
20.6k
    if (
HasVEX_4V20.6k
) {
760
13.5k
      unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
761
13.5k
      VEX_4V = ~VRegEnc & 0xf;
762
13.5k
      EVEX_V2 = ~(VRegEnc >> 4) & 1;
763
13.5k
    }
764
20.6k
765
20.6k
    unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
766
20.6k
    VEX_B = ~(BaseRegEnc >> 3) & 1;
767
20.6k
    unsigned IndexRegEnc = getX86RegEncoding(MI, MemOperand+X86::AddrIndexReg);
768
20.6k
    VEX_X = ~(IndexRegEnc >> 3) & 1;
769
20.6k
    if (!HasVEX_4V) // Only needed with VSIB which don't use VVVV.
770
7.10k
      EVEX_V2 = ~(IndexRegEnc >> 4) & 1;
771
20.6k
772
20.6k
    break;
773
44.9k
  }
774
95
  case X86II::MRMSrcMem4VOp3: {
775
95
    // Instruction format for 4VOp3:
776
95
    //   src1(ModR/M), MemAddr, src3(VEX_4V)
777
95
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
778
95
    VEX_R = ~(RegEnc >> 3) & 1;
779
95
780
95
    unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
781
95
    VEX_B = ~(BaseRegEnc >> 3) & 1;
782
95
    unsigned IndexRegEnc = getX86RegEncoding(MI, MemOperand+X86::AddrIndexReg);
783
95
    VEX_X = ~(IndexRegEnc >> 3) & 1;
784
95
785
95
    VEX_4V = ~getX86RegEncoding(MI, CurOp + X86::AddrNumOperands) & 0xf;
786
95
    break;
787
44.9k
  }
788
49
  case X86II::MRMSrcMemOp4: {
789
49
    //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
790
49
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
791
49
    VEX_R = ~(RegEnc >> 3) & 1;
792
49
793
49
    unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
794
49
    VEX_4V = ~VRegEnc & 0xf;
795
49
796
49
    unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
797
49
    VEX_B = ~(BaseRegEnc >> 3) & 1;
798
49
    unsigned IndexRegEnc = getX86RegEncoding(MI, MemOperand+X86::AddrIndexReg);
799
49
    VEX_X = ~(IndexRegEnc >> 3) & 1;
800
49
    break;
801
44.9k
  }
802
558
  
case X86II::MRM0m: 558
case X86II::MRM1m:
803
558
  
case X86II::MRM2m: 558
case X86II::MRM3m:
804
558
  
case X86II::MRM4m: 558
case X86II::MRM5m:
805
558
  
case X86II::MRM6m: 558
case X86II::MRM7m: {
806
558
    // MRM[0-9]m instructions forms:
807
558
    //  MemAddr
808
558
    //  src1(VEX_4V), MemAddr
809
558
    if (
HasVEX_4V558
) {
810
548
      unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
811
548
      VEX_4V = ~VRegEnc & 0xf;
812
548
      EVEX_V2 = ~(VRegEnc >> 4) & 1;
813
548
    }
814
558
815
558
    if (HasEVEX_K)
816
0
      EVEX_aaa = getX86RegEncoding(MI, CurOp++);
817
558
818
558
    unsigned BaseRegEnc = getX86RegEncoding(MI, MemOperand + X86::AddrBaseReg);
819
558
    VEX_B = ~(BaseRegEnc >> 3) & 1;
820
558
    unsigned IndexRegEnc = getX86RegEncoding(MI, MemOperand+X86::AddrIndexReg);
821
558
    VEX_X = ~(IndexRegEnc >> 3) & 1;
822
558
    break;
823
558
  }
824
17.2k
  case X86II::MRMSrcReg: {
825
17.2k
    // MRMSrcReg instructions forms:
826
17.2k
    //  dst(ModR/M), src1(VEX_4V), src2(ModR/M), src3(Imm[7:4])
827
17.2k
    //  dst(ModR/M), src1(ModR/M)
828
17.2k
    //  dst(ModR/M), src1(ModR/M), imm8
829
17.2k
    //
830
17.2k
    //  FMA4:
831
17.2k
    //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
832
17.2k
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
833
17.2k
    VEX_R = ~(RegEnc >> 3) & 1;
834
17.2k
    EVEX_R2 = ~(RegEnc >> 4) & 1;
835
17.2k
836
17.2k
    if (HasEVEX_K)
837
4.36k
      EVEX_aaa = getX86RegEncoding(MI, CurOp++);
838
17.2k
839
17.2k
    if (
HasVEX_4V17.2k
) {
840
11.4k
      unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
841
11.4k
      VEX_4V = ~VRegEnc & 0xf;
842
11.4k
      EVEX_V2 = ~(VRegEnc >> 4) & 1;
843
11.4k
    }
844
17.2k
845
17.2k
    RegEnc = getX86RegEncoding(MI, CurOp++);
846
17.2k
    VEX_B = ~(RegEnc >> 3) & 1;
847
17.2k
    VEX_X = ~(RegEnc >> 4) & 1;
848
17.2k
849
17.2k
    if (
EVEX_b17.2k
) {
850
1.79k
      if (
HasEVEX_RC1.79k
) {
851
1.53k
        unsigned RcOperand = NumOps-1;
852
1.53k
        assert(RcOperand >= CurOp);
853
1.53k
        EVEX_rc = MI.getOperand(RcOperand).getImm() & 0x3;
854
1.53k
      }
855
1.79k
      EncodeRC = true;
856
1.79k
    }
857
17.2k
    break;
858
558
  }
859
549
  case X86II::MRMSrcReg4VOp3: {
860
549
    // Instruction format for 4VOp3:
861
549
    //   src1(ModR/M), src2(ModR/M), src3(VEX_4V)
862
549
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
863
549
    VEX_R = ~(RegEnc >> 3) & 1;
864
549
865
549
    RegEnc = getX86RegEncoding(MI, CurOp++);
866
549
    VEX_B = ~(RegEnc >> 3) & 1;
867
549
868
549
    VEX_4V = ~getX86RegEncoding(MI, CurOp++) & 0xf;
869
549
    break;
870
558
  }
871
45
  case X86II::MRMSrcRegOp4: {
872
45
    //  dst(ModR/M.reg), src1(VEX_4V), src2(Imm[7:4]), src3(ModR/M),
873
45
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
874
45
    VEX_R = ~(RegEnc >> 3) & 1;
875
45
876
45
    unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
877
45
    VEX_4V = ~VRegEnc & 0xf;
878
45
879
45
    // Skip second register source (encoded in Imm[7:4])
880
45
    ++CurOp;
881
45
882
45
    RegEnc = getX86RegEncoding(MI, CurOp++);
883
45
    VEX_B = ~(RegEnc >> 3) & 1;
884
45
    VEX_X = ~(RegEnc >> 4) & 1;
885
45
    break;
886
558
  }
887
1.43k
  case X86II::MRMDestReg: {
888
1.43k
    // MRMDestReg instructions forms:
889
1.43k
    //  dst(ModR/M), src(ModR/M)
890
1.43k
    //  dst(ModR/M), src(ModR/M), imm8
891
1.43k
    //  dst(ModR/M), src1(VEX_4V), src2(ModR/M)
892
1.43k
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
893
1.43k
    VEX_B = ~(RegEnc >> 3) & 1;
894
1.43k
    VEX_X = ~(RegEnc >> 4) & 1;
895
1.43k
896
1.43k
    if (HasEVEX_K)
897
613
      EVEX_aaa = getX86RegEncoding(MI, CurOp++);
898
1.43k
899
1.43k
    if (
HasVEX_4V1.43k
) {
900
26
      unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
901
26
      VEX_4V = ~VRegEnc & 0xf;
902
26
      EVEX_V2 = ~(VRegEnc >> 4) & 1;
903
26
    }
904
1.43k
905
1.43k
    RegEnc = getX86RegEncoding(MI, CurOp++);
906
1.43k
    VEX_R = ~(RegEnc >> 3) & 1;
907
1.43k
    EVEX_R2 = ~(RegEnc >> 4) & 1;
908
1.43k
    if (EVEX_b)
909
6
      EncodeRC = true;
910
1.43k
    break;
911
558
  }
912
605
  
case X86II::MRM0r: 605
case X86II::MRM1r:
913
605
  
case X86II::MRM2r: 605
case X86II::MRM3r:
914
605
  
case X86II::MRM4r: 605
case X86II::MRM5r:
915
605
  
case X86II::MRM6r: 605
case X86II::MRM7r: {
916
605
    // MRM0r-MRM7r instructions forms:
917
605
    //  dst(VEX_4V), src(ModR/M), imm8
918
605
    if (
HasVEX_4V605
) {
919
594
      unsigned VRegEnc = getX86RegEncoding(MI, CurOp++);
920
594
      VEX_4V = ~VRegEnc & 0xf;
921
594
      EVEX_V2 = ~(VRegEnc >> 4) & 1;
922
594
    }
923
605
    if (HasEVEX_K)
924
136
      EVEX_aaa = getX86RegEncoding(MI, CurOp++);
925
605
926
605
    unsigned RegEnc = getX86RegEncoding(MI, CurOp++);
927
605
    VEX_B = ~(RegEnc >> 3) & 1;
928
605
    VEX_X = ~(RegEnc >> 4) & 1;
929
605
    break;
930
44.9k
  }
931
44.9k
  }
932
44.9k
933
44.9k
  
if (44.9k
Encoding == X86II::VEX || 44.9k
Encoding == X86II::XOP30.2k
) {
934
14.9k
    // VEX opcode prefix can have 2 or 3 bytes
935
14.9k
    //
936
14.9k
    //  3 bytes:
937
14.9k
    //    +-----+ +--------------+ +-------------------+
938
14.9k
    //    | C4h | | RXB | m-mmmm | | W | vvvv | L | pp |
939
14.9k
    //    +-----+ +--------------+ +-------------------+
940
14.9k
    //  2 bytes:
941
14.9k
    //    +-----+ +-------------------+
942
14.9k
    //    | C5h | | R | vvvv | L | pp |
943
14.9k
    //    +-----+ +-------------------+
944
14.9k
    //
945
14.9k
    //  XOP uses a similar prefix:
946
14.9k
    //    +-----+ +--------------+ +-------------------+
947
14.9k
    //    | 8Fh | | RXB | m-mmmm | | W | vvvv | L | pp |
948
14.9k
    //    +-----+ +--------------+ +-------------------+
949
14.9k
    uint8_t LastByte = VEX_PP | (VEX_L << 2) | (VEX_4V << 3);
950
14.9k
951
14.9k
    // Can we use the 2 byte VEX prefix?
952
14.9k
    if (
Encoding == X86II::VEX && 14.9k
VEX_B14.7k
&&
VEX_X13.5k
&&
!VEX_W13.4k
&&
(VEX_5M == 1)12.3k
) {
953
9.80k
      EmitByte(0xC5, CurByte, OS);
954
9.80k
      EmitByte(LastByte | (VEX_R << 7), CurByte, OS);
955
9.80k
      return;
956
9.80k
    }
957
5.18k
958
5.18k
    // 3 byte VEX prefix
959
5.18k
    
EmitByte(Encoding == X86II::XOP ? 5.18k
0x8F248
:
0xC44.94k
, CurByte, OS);
960
14.9k
    EmitByte(VEX_R << 7 | VEX_X << 6 | VEX_B << 5 | VEX_5M, CurByte, OS);
961
14.9k
    EmitByte(LastByte | (VEX_W << 7), CurByte, OS);
962
44.9k
  } else {
963
30.0k
    assert(Encoding == X86II::EVEX && "unknown encoding!");
964
30.0k
    // EVEX opcode prefix can have 4 bytes
965
30.0k
    //
966
30.0k
    // +-----+ +--------------+ +-------------------+ +------------------------+
967
30.0k
    // | 62h | | RXBR' | 00mm | | W | vvvv | U | pp | | z | L'L | b | v' | aaa |
968
30.0k
    // +-----+ +--------------+ +-------------------+ +------------------------+
969
30.0k
    assert((VEX_5M & 0x3) == VEX_5M
970
30.0k
           && "More than 2 significant bits in VEX.m-mmmm fields for EVEX!");
971
30.0k
972
30.0k
    EmitByte(0x62, CurByte, OS);
973
30.0k
    EmitByte((VEX_R   << 7) |
974
30.0k
             (VEX_X   << 6) |
975
30.0k
             (VEX_B   << 5) |
976
30.0k
             (EVEX_R2 << 4) |
977
30.0k
             VEX_5M, CurByte, OS);
978
30.0k
    EmitByte((VEX_W   << 7) |
979
30.0k
             (VEX_4V  << 3) |
980
30.0k
             (EVEX_U  << 2) |
981
30.0k
             VEX_PP, CurByte, OS);
982
30.0k
    if (EncodeRC)
983
1.80k
      EmitByte((EVEX_z  << 7) |
984
1.80k
               (EVEX_rc << 5) |
985
1.80k
               (EVEX_b  << 4) |
986
1.80k
               (EVEX_V2 << 3) |
987
1.80k
               EVEX_aaa, CurByte, OS);
988
30.0k
    else
989
28.2k
      EmitByte((EVEX_z  << 7) |
990
28.2k
               (EVEX_L2 << 6) |
991
28.2k
               (VEX_L   << 5) |
992
28.2k
               (EVEX_b  << 4) |
993
28.2k
               (EVEX_V2 << 3) |
994
28.2k
               EVEX_aaa, CurByte, OS);
995
30.0k
  }
996
44.9k
}
997
998
/// DetermineREXPrefix - Determine if the MCInst has to be encoded with a X86-64
999
/// REX prefix which specifies 1) 64-bit instructions, 2) non-default operand
1000
/// size, and 3) use of X86-64 extended registers.
1001
uint8_t X86MCCodeEmitter::DetermineREXPrefix(const MCInst &MI, uint64_t TSFlags,
1002
                                             int MemOperand,
1003
229k
                                             const MCInstrDesc &Desc) const {
1004
229k
  uint8_t REX = 0;
1005
229k
  bool UsesHighByteReg = false;
1006
229k
1007
229k
  if (TSFlags & X86II::REX_W)
1008
85.9k
    REX |= 1 << 3; // set REX.W
1009
229k
1010
229k
  if (
MI.getNumOperands() == 0229k
)
return REX4.90k
;
1011
224k
1012
224k
  unsigned NumOps = MI.getNumOperands();
1013
224k
  unsigned CurOp = X86II::getOperandBias(Desc);
1014
224k
1015
224k
  // If it accesses SPL, BPL, SIL, or DIL, then it requires a 0x40 REX prefix.
1016
798k
  for (unsigned i = CurOp; 
i != NumOps798k
;
++i574k
) {
1017
574k
    const MCOperand &MO = MI.getOperand(i);
1018
574k
    if (
!MO.isReg()574k
)
continue183k
;
1019
390k
    unsigned Reg = MO.getReg();
1020
390k
    if (
Reg == X86::AH || 390k
Reg == X86::BH390k
||
Reg == X86::CH390k
||
Reg == X86::DH390k
)
1021
9
      UsesHighByteReg = true;
1022
390k
    if (X86II::isX86_64NonExtLowByteReg(Reg))
1023
390k
      // FIXME: The caller of DetermineREXPrefix slaps this prefix onto anything
1024
390k
      // that returns non-zero.
1025
897
      REX |= 0x40; // REX fixed encoding prefix
1026
574k
  }
1027
224k
1028
224k
  switch (TSFlags & X86II::FormMask) {
1029
33.4k
  case X86II::AddRegFrm:
1030
33.4k
    REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1031
33.4k
    break;
1032
7.39k
  case X86II::MRMSrcReg:
1033
7.39k
    REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1034
7.39k
    REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1035
7.39k
    break;
1036
33.7k
  case X86II::MRMSrcMem: {
1037
33.7k
    REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1038
33.7k
    REX |= isREXExtendedReg(MI, MemOperand+X86::AddrBaseReg) << 0; // REX.B
1039
33.7k
    REX |= isREXExtendedReg(MI, MemOperand+X86::AddrIndexReg) << 1; // REX.X
1040
33.7k
    CurOp += X86::AddrNumOperands;
1041
33.7k
    break;
1042
224k
  }
1043
53.6k
  case X86II::MRMDestReg:
1044
53.6k
    REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1045
53.6k
    REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1046
53.6k
    break;
1047
11.1k
  case X86II::MRMDestMem:
1048
11.1k
    REX |= isREXExtendedReg(MI, MemOperand+X86::AddrBaseReg) << 0; // REX.B
1049
11.1k
    REX |= isREXExtendedReg(MI, MemOperand+X86::AddrIndexReg) << 1; // REX.X
1050
11.1k
    CurOp += X86::AddrNumOperands;
1051
11.1k
    REX |= isREXExtendedReg(MI, CurOp++) << 2; // REX.R
1052
11.1k
    break;
1053
7.42k
  case X86II::MRMXm:
1054
7.42k
  
case X86II::MRM0m: 7.42k
case X86II::MRM1m:
1055
7.42k
  
case X86II::MRM2m: 7.42k
case X86II::MRM3m:
1056
7.42k
  
case X86II::MRM4m: 7.42k
case X86II::MRM5m:
1057
7.42k
  
case X86II::MRM6m: 7.42k
case X86II::MRM7m:
1058
7.42k
    REX |= isREXExtendedReg(MI, MemOperand+X86::AddrBaseReg) << 0; // REX.B
1059
7.42k
    REX |= isREXExtendedReg(MI, MemOperand+X86::AddrIndexReg) << 1; // REX.X
1060
7.42k
    break;
1061
26.2k
  case X86II::MRMXr:
1062
26.2k
  
case X86II::MRM0r: 26.2k
case X86II::MRM1r:
1063
26.2k
  
case X86II::MRM2r: 26.2k
case X86II::MRM3r:
1064
26.2k
  
case X86II::MRM4r: 26.2k
case X86II::MRM5r:
1065
26.2k
  
case X86II::MRM6r: 26.2k
case X86II::MRM7r:
1066
26.2k
    REX |= isREXExtendedReg(MI, CurOp++) << 0; // REX.B
1067
26.2k
    break;
1068
224k
  }
1069
224k
  
if (224k
REX && 224k
UsesHighByteReg106k
)
1070
1
    report_fatal_error("Cannot encode high byte register in REX-prefixed instruction");
1071
224k
1072
224k
  return REX;
1073
224k
}
1074
1075
/// EmitSegmentOverridePrefix - Emit segment override opcode prefix as needed
1076
void X86MCCodeEmitter::EmitSegmentOverridePrefix(unsigned &CurByte,
1077
                                                 unsigned SegOperand,
1078
                                                 const MCInst &MI,
1079
130k
                                                 raw_ostream &OS) const {
1080
130k
  // Check for explicit segment override on memory operand.
1081
130k
  switch (MI.getOperand(SegOperand).getReg()) {
1082
0
  
default: 0
llvm_unreachable0
("Unknown segment register!");
1083
130k
  case 0: break;
1084
14
  case X86::CS: EmitByte(0x2E, CurByte, OS); break;
1085
4
  case X86::SS: EmitByte(0x36, CurByte, OS); break;
1086
0
  case X86::DS: EmitByte(0x3E, CurByte, OS); break;
1087
6
  case X86::ES: EmitByte(0x26, CurByte, OS); break;
1088
91
  case X86::FS: EmitByte(0x64, CurByte, OS); break;
1089
105
  case X86::GS: EmitByte(0x65, CurByte, OS); break;
1090
130k
  }
1091
130k
}
1092
1093
/// Emit all instruction prefixes prior to the opcode.
1094
///
1095
/// MemOperand is the operand # of the start of a memory operand if present.  If
1096
/// Not present, it is -1.
1097
///
1098
/// Returns true if a REX prefix was used.
1099
bool X86MCCodeEmitter::emitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
1100
                                        int MemOperand, const MCInst &MI,
1101
                                        const MCInstrDesc &Desc,
1102
                                        const MCSubtargetInfo &STI,
1103
416k
                                        raw_ostream &OS) const {
1104
416k
  bool Ret = false;
1105
416k
  // Emit the operand size opcode prefix as needed.
1106
416k
  if (
(TSFlags & X86II::OpSizeMask) == (is16BitMode(STI) ? 416k
X86II::OpSize32478
1107
416k
                                                         : X86II::OpSize16))
1108
1.33k
    EmitByte(0x66, CurByte, OS);
1109
416k
1110
416k
  // Emit the LOCK opcode prefix.
1111
416k
  if (TSFlags & X86II::LOCK)
1112
1.05k
    EmitByte(0xF0, CurByte, OS);
1113
416k
1114
416k
  switch (TSFlags & X86II::OpPrefixMask) {
1115
6.30k
  case X86II::PD:   // 66
1116
6.30k
    EmitByte(0x66, CurByte, OS);
1117
6.30k
    break;
1118
2.50k
  case X86II::XS:   // F3
1119
2.50k
    EmitByte(0xF3, CurByte, OS);
1120
2.50k
    break;
1121
2.11k
  case X86II::XD:   // F2
1122
2.11k
    EmitByte(0xF2, CurByte, OS);
1123
2.11k
    break;
1124
416k
  }
1125
416k
1126
416k
  // Handle REX prefix.
1127
416k
  // FIXME: Can this come before F2 etc to simplify emission?
1128
416k
  
if (416k
is64BitMode(STI)416k
) {
1129
229k
    if (uint8_t 
REX229k
= DetermineREXPrefix(MI, TSFlags, MemOperand, Desc)) {
1130
107k
      EmitByte(0x40 | REX, CurByte, OS);
1131
107k
      Ret = true;
1132
107k
    }
1133
229k
  }
1134
416k
1135
416k
  // 0x0F escape code must be emitted just before the opcode.
1136
416k
  switch (TSFlags & X86II::OpMapMask) {
1137
35.5k
  case X86II::TB:  // Two-byte opcode map
1138
35.5k
  case X86II::T8:  // 0F 38
1139
35.5k
  case X86II::TA:  // 0F 3A
1140
35.5k
    EmitByte(0x0F, CurByte, OS);
1141
35.5k
    break;
1142
416k
  }
1143
416k
1144
416k
  switch (TSFlags & X86II::OpMapMask) {
1145
555
  case X86II::T8:    // 0F 38
1146
555
    EmitByte(0x38, CurByte, OS);
1147
555
    break;
1148
91
  case X86II::TA:    // 0F 3A
1149
91
    EmitByte(0x3A, CurByte, OS);
1150
91
    break;
1151
416k
  }
1152
416k
  return Ret;
1153
416k
}
1154
1155
void X86MCCodeEmitter::
1156
encodeInstruction(const MCInst &MI, raw_ostream &OS,
1157
                  SmallVectorImpl<MCFixup> &Fixups,
1158
461k
                  const MCSubtargetInfo &STI) const {
1159
461k
  unsigned Opcode = MI.getOpcode();
1160
461k
  const MCInstrDesc &Desc = MCII.get(Opcode);
1161
461k
  uint64_t TSFlags = Desc.TSFlags;
1162
461k
1163
461k
  // Pseudo instructions don't get encoded.
1164
461k
  if ((TSFlags & X86II::FormMask) == X86II::Pseudo)
1165
0
    return;
1166
461k
1167
461k
  unsigned NumOps = Desc.getNumOperands();
1168
461k
  unsigned CurOp = X86II::getOperandBias(Desc);
1169
461k
1170
461k
  // Keep track of the current byte being emitted.
1171
461k
  unsigned CurByte = 0;
1172
461k
1173
461k
  // Encoding type for this instruction.
1174
461k
  uint64_t Encoding = TSFlags & X86II::EncodingMask;
1175
461k
1176
461k
  // It uses the VEX.VVVV field?
1177
461k
  bool HasVEX_4V = TSFlags & X86II::VEX_4V;
1178
461k
  bool HasVEX_I8Reg = (TSFlags & X86II::ImmMask) == X86II::Imm8Reg;
1179
461k
1180
461k
  // It uses the EVEX.aaa field?
1181
461k
  bool HasEVEX_K = TSFlags & X86II::EVEX_K;
1182
461k
  bool HasEVEX_RC = TSFlags & X86II::EVEX_RC;
1183
461k
1184
461k
  // Used if a register is encoded in 7:4 of immediate.
1185
461k
  unsigned I8RegNum = 0;
1186
461k
1187
461k
  // Determine where the memory operand starts, if present.
1188
461k
  int MemoryOperand = X86II::getMemoryOperandNo(TSFlags);
1189
461k
  if (
MemoryOperand != -1461k
)
MemoryOperand += CurOp130k
;
1190
461k
1191
461k
  // Emit segment override opcode prefix as needed.
1192
461k
  if (MemoryOperand >= 0)
1193
130k
    EmitSegmentOverridePrefix(CurByte, MemoryOperand+X86::AddrSegmentReg,
1194
130k
                              MI, OS);
1195
461k
1196
461k
  // Emit the repeat opcode prefix as needed.
1197
461k
  if (TSFlags & X86II::REP)
1198
36
    EmitByte(0xF3, CurByte, OS);
1199
461k
1200
461k
  // Emit the address size opcode prefix as needed.
1201
461k
  bool need_address_override;
1202
461k
  uint64_t AdSize = TSFlags & X86II::AdSizeMask;
1203
461k
  if (
(is16BitMode(STI) && 461k
AdSize == X86II::AdSize32478
) ||
1204
461k
      
(is32BitMode(STI) && 461k
AdSize == X86II::AdSize16189k
) ||
1205
461k
      
(is64BitMode(STI) && 461k
AdSize == X86II::AdSize32271k
)) {
1206
33
    need_address_override = true;
1207
461k
  } else 
if (461k
MemoryOperand < 0461k
) {
1208
331k
    need_address_override = false;
1209
461k
  } else 
if (130k
is64BitMode(STI)130k
) {
1210
75.9k
    assert(!Is16BitMemOperand(MI, MemoryOperand, STI));
1211
75.9k
    need_address_override = Is32BitMemOperand(MI, MemoryOperand);
1212
130k
  } else 
if (54.7k
is32BitMode(STI)54.7k
) {
1213
54.6k
    assert(!Is64BitMemOperand(MI, MemoryOperand));
1214
54.6k
    need_address_override = Is16BitMemOperand(MI, MemoryOperand, STI);
1215
54.7k
  } else {
1216
54
    assert(is16BitMode(STI));
1217
54
    assert(!Is64BitMemOperand(MI, MemoryOperand));
1218
54
    need_address_override = !Is16BitMemOperand(MI, MemoryOperand, STI);
1219
54
  }
1220
461k
1221
461k
  if (need_address_override)
1222
115
    EmitByte(0x67, CurByte, OS);
1223
461k
1224
461k
  bool Rex = false;
1225
461k
  if (Encoding == 0)
1226
416k
    Rex = emitOpcodePrefix(TSFlags, CurByte, MemoryOperand, MI, Desc, STI, OS);
1227
461k
  else
1228
45.0k
    EmitVEXOpcodePrefix(TSFlags, CurByte, MemoryOperand, MI, Desc, OS);
1229
461k
1230
461k
  uint8_t BaseOpcode = X86II::getBaseOpcodeFor(TSFlags);
1231
461k
1232
461k
  if (TSFlags & X86II::Has3DNow0F0FOpcode)
1233
26
    BaseOpcode = 0x0F;   // Weird 3DNow! encoding.
1234
461k
1235
461k
  uint64_t Form = TSFlags & X86II::FormMask;
1236
461k
  switch (Form) {
1237
0
  default: errs() << "FORM: " << Form << "\n";
1238
0
    llvm_unreachable("Unknown FormMask value in X86MCCodeEmitter!");
1239
0
  case X86II::Pseudo:
1240
0
    llvm_unreachable("Pseudo instruction shouldn't be emitted");
1241
70
  case X86II::RawFrmDstSrc: {
1242
70
    unsigned siReg = MI.getOperand(1).getReg();
1243
70
    assert(((siReg == X86::SI && MI.getOperand(0).getReg() == X86::DI) ||
1244
70
            (siReg == X86::ESI && MI.getOperand(0).getReg() == X86::EDI) ||
1245
70
            (siReg == X86::RSI && MI.getOperand(0).getReg() == X86::RDI)) &&
1246
70
           "SI and DI register sizes do not match");
1247
70
    // Emit segment override opcode prefix as needed (not for %ds).
1248
70
    if (MI.getOperand(2).getReg() != X86::DS)
1249
60
      EmitSegmentOverridePrefix(CurByte, 2, MI, OS);
1250
70
    // Emit AdSize prefix as needed.
1251
70
    if (
(!is32BitMode(STI) && 70
siReg == X86::ESI53
) ||
1252
57
        
(is32BitMode(STI) && 57
siReg == X86::SI17
))
1253
13
      EmitByte(0x67, CurByte, OS);
1254
70
    CurOp += 3; // Consume operands.
1255
70
    EmitByte(BaseOpcode, CurByte, OS);
1256
70
    break;
1257
461k
  }
1258
115
  case X86II::RawFrmSrc: {
1259
115
    unsigned siReg = MI.getOperand(0).getReg();
1260
115
    // Emit segment override opcode prefix as needed (not for %ds).
1261
115
    if (MI.getOperand(1).getReg() != X86::DS)
1262
86
      EmitSegmentOverridePrefix(CurByte, 1, MI, OS);
1263
115
    // Emit AdSize prefix as needed.
1264
115
    if (
(!is32BitMode(STI) && 115
siReg == X86::ESI80
) ||
1265
101
        
(is32BitMode(STI) && 101
siReg == X86::SI35
))
1266
15
      EmitByte(0x67, CurByte, OS);
1267
115
    CurOp += 2; // Consume operands.
1268
115
    EmitByte(BaseOpcode, CurByte, OS);
1269
115
    break;
1270
461k
  }
1271
99
  case X86II::RawFrmDst: {
1272
99
    unsigned siReg = MI.getOperand(0).getReg();
1273
99
    // Emit AdSize prefix as needed.
1274
99
    if (
(!is32BitMode(STI) && 99
siReg == X86::EDI70
) ||
1275
83
        
(is32BitMode(STI) && 83
siReg == X86::DI29
))
1276
18
      EmitByte(0x67, CurByte, OS);
1277
99
    ++CurOp; // Consume operand.
1278
99
    EmitByte(BaseOpcode, CurByte, OS);
1279
99
    break;
1280
461k
  }
1281
97.0k
  case X86II::RawFrm:
1282
97.0k
    EmitByte(BaseOpcode, CurByte, OS);
1283
97.0k
    break;
1284
169
  case X86II::RawFrmMemOffs:
1285
169
    // Emit segment override opcode prefix as needed.
1286
169
    EmitSegmentOverridePrefix(CurByte, 1, MI, OS);
1287
169
    EmitByte(BaseOpcode, CurByte, OS);
1288
169
    EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1289
169
                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1290
169
                  CurByte, OS, Fixups);
1291
169
    ++CurOp; // skip segment operand
1292
169
    break;
1293
5
  case X86II::RawFrmImm8:
1294
5
    EmitByte(BaseOpcode, CurByte, OS);
1295
5
    EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1296
5
                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1297
5
                  CurByte, OS, Fixups);
1298
5
    EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 1, FK_Data_1, CurByte,
1299
5
                  OS, Fixups);
1300
5
    break;
1301
27
  case X86II::RawFrmImm16:
1302
27
    EmitByte(BaseOpcode, CurByte, OS);
1303
27
    EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1304
27
                  X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1305
27
                  CurByte, OS, Fixups);
1306
27
    EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(), 2, FK_Data_2, CurByte,
1307
27
                  OS, Fixups);
1308
27
    break;
1309
461k
1310
69.1k
  case X86II::AddRegFrm:
1311
69.1k
    EmitByte(BaseOpcode + GetX86RegNum(MI.getOperand(CurOp++)), CurByte, OS);
1312
69.1k
    break;
1313
461k
1314
78.0k
  case X86II::MRMDestReg: {
1315
78.0k
    EmitByte(BaseOpcode, CurByte, OS);
1316
78.0k
    unsigned SrcRegNum = CurOp + 1;
1317
78.0k
1318
78.0k
    if (HasEVEX_K) // Skip writemask
1319
613
      ++SrcRegNum;
1320
78.0k
1321
78.0k
    if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1322
26
      ++SrcRegNum;
1323
78.0k
1324
78.0k
    EmitRegModRMByte(MI.getOperand(CurOp),
1325
78.0k
                     GetX86RegNum(MI.getOperand(SrcRegNum)), CurByte, OS);
1326
78.0k
    CurOp = SrcRegNum + 1;
1327
78.0k
    break;
1328
461k
  }
1329
25.6k
  case X86II::MRMDestMem: {
1330
25.6k
    EmitByte(BaseOpcode, CurByte, OS);
1331
25.6k
    unsigned SrcRegNum = CurOp + X86::AddrNumOperands;
1332
25.6k
1333
25.6k
    if (HasEVEX_K) // Skip writemask
1334
567
      ++SrcRegNum;
1335
25.6k
1336
25.6k
    if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1337
24
      ++SrcRegNum;
1338
25.6k
1339
25.6k
    emitMemModRMByte(MI, CurOp, GetX86RegNum(MI.getOperand(SrcRegNum)), TSFlags,
1340
25.6k
                     Rex, CurByte, OS, Fixups, STI);
1341
25.6k
    CurOp = SrcRegNum + 1;
1342
25.6k
    break;
1343
461k
  }
1344
29.2k
  case X86II::MRMSrcReg: {
1345
29.2k
    EmitByte(BaseOpcode, CurByte, OS);
1346
29.2k
    unsigned SrcRegNum = CurOp + 1;
1347
29.2k
1348
29.2k
    if (HasEVEX_K) // Skip writemask
1349
4.36k
      ++SrcRegNum;
1350
29.2k
1351
29.2k
    if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
1352
11.4k
      ++SrcRegNum;
1353
29.2k
1354
29.2k
    EmitRegModRMByte(MI.getOperand(SrcRegNum),
1355
29.2k
                     GetX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1356
29.2k
    CurOp = SrcRegNum + 1;
1357
29.2k
    if (HasVEX_I8Reg)
1358
114
      I8RegNum = getX86RegEncoding(MI, CurOp++);
1359
29.2k
    // do not count the rounding control operand
1360
29.2k
    if (HasEVEX_RC)
1361
1.53k
      --NumOps;
1362
29.2k
    break;
1363
461k
  }
1364
549
  case X86II::MRMSrcReg4VOp3: {
1365
549
    EmitByte(BaseOpcode, CurByte, OS);
1366
549
    unsigned SrcRegNum = CurOp + 1;
1367
549
1368
549
    EmitRegModRMByte(MI.getOperand(SrcRegNum),
1369
549
                     GetX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1370
549
    CurOp = SrcRegNum + 1;
1371
549
    ++CurOp; // Encoded in VEX.VVVV
1372
549
    break;
1373
461k
  }
1374
45
  case X86II::MRMSrcRegOp4: {
1375
45
    EmitByte(BaseOpcode, CurByte, OS);
1376
45
    unsigned SrcRegNum = CurOp + 1;
1377
45
1378
45
    // Skip 1st src (which is encoded in VEX_VVVV)
1379
45
    ++SrcRegNum;
1380
45
1381
45
    // Capture 2nd src (which is encoded in Imm[7:4])
1382
45
    assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1383
45
    I8RegNum = getX86RegEncoding(MI, SrcRegNum++);
1384
45
1385
45
    EmitRegModRMByte(MI.getOperand(SrcRegNum),
1386
45
                     GetX86RegNum(MI.getOperand(CurOp)), CurByte, OS);
1387
45
    CurOp = SrcRegNum + 1;
1388
45
    break;
1389
461k
  }
1390
86.6k
  case X86II::MRMSrcMem: {
1391
86.6k
    unsigned FirstMemOp = CurOp+1;
1392
86.6k
1393
86.6k
    if (HasEVEX_K) // Skip writemask
1394
769
      ++FirstMemOp;
1395
86.6k
1396
86.6k
    if (HasVEX_4V)
1397
13.5k
      ++FirstMemOp;  // Skip the register source (which is encoded in VEX_VVVV).
1398
86.6k
1399
86.6k
    EmitByte(BaseOpcode, CurByte, OS);
1400
86.6k
1401
86.6k
    emitMemModRMByte(MI, FirstMemOp, GetX86RegNum(MI.getOperand(CurOp)),
1402
86.6k
                     TSFlags, Rex, CurByte, OS, Fixups, STI);
1403
86.6k
    CurOp = FirstMemOp + X86::AddrNumOperands;
1404
86.6k
    if (HasVEX_I8Reg)
1405
66
      I8RegNum = getX86RegEncoding(MI, CurOp++);
1406
86.6k
    break;
1407
461k
  }
1408
95
  case X86II::MRMSrcMem4VOp3: {
1409
95
    unsigned FirstMemOp = CurOp+1;
1410
95
1411
95
    EmitByte(BaseOpcode, CurByte, OS);
1412
95
1413
95
    emitMemModRMByte(MI, FirstMemOp, GetX86RegNum(MI.getOperand(CurOp)),
1414
95
                     TSFlags, Rex, CurByte, OS, Fixups, STI);
1415
95
    CurOp = FirstMemOp + X86::AddrNumOperands;
1416
95
    ++CurOp; // Encoded in VEX.VVVV.
1417
95
    break;
1418
461k
  }
1419
49
  case X86II::MRMSrcMemOp4: {
1420
49
    unsigned FirstMemOp = CurOp+1;
1421
49
1422
49
    ++FirstMemOp;  // Skip the register source (which is encoded in VEX_VVVV).
1423
49
1424
49
    // Capture second register source (encoded in Imm[7:4])
1425
49
    assert(HasVEX_I8Reg && "MRMSrcRegOp4 should imply VEX_I8Reg");
1426
49
    I8RegNum = getX86RegEncoding(MI, FirstMemOp++);
1427
49
1428
49
    EmitByte(BaseOpcode, CurByte, OS);
1429
49
1430
49
    emitMemModRMByte(MI, FirstMemOp, GetX86RegNum(MI.getOperand(CurOp)),
1431
49
                     TSFlags, Rex, CurByte, OS, Fixups, STI);
1432
49
    CurOp = FirstMemOp + X86::AddrNumOperands;
1433
49
    break;
1434
461k
  }
1435
461k
1436
54.8k
  case X86II::MRMXr:
1437
54.8k
  
case X86II::MRM0r: 54.8k
case X86II::MRM1r:
1438
54.8k
  
case X86II::MRM2r: 54.8k
case X86II::MRM3r:
1439
54.8k
  
case X86II::MRM4r: 54.8k
case X86II::MRM5r:
1440
54.8k
  
case X86II::MRM6r: 54.8k
case X86II::MRM7r:
1441
54.8k
    if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1442
594
      ++CurOp;
1443
54.8k
    if (HasEVEX_K) // Skip writemask
1444
136
      ++CurOp;
1445
54.8k
    EmitByte(BaseOpcode, CurByte, OS);
1446
54.8k
    EmitRegModRMByte(MI.getOperand(CurOp++),
1447
54.8k
                     (Form == X86II::MRMXr) ? 
01.46k
:
Form-X86II::MRM0r53.4k
,
1448
54.8k
                     CurByte, OS);
1449
54.8k
    break;
1450
54.8k
1451
18.1k
  case X86II::MRMXm:
1452
18.1k
  
case X86II::MRM0m: 18.1k
case X86II::MRM1m:
1453
18.1k
  
case X86II::MRM2m: 18.1k
case X86II::MRM3m:
1454
18.1k
  
case X86II::MRM4m: 18.1k
case X86II::MRM5m:
1455
18.1k
  
case X86II::MRM6m: 18.1k
case X86II::MRM7m:
1456
18.1k
    if (HasVEX_4V) // Skip the register dst (which is encoded in VEX_VVVV).
1457
548
      ++CurOp;
1458
18.1k
    if (HasEVEX_K) // Skip writemask
1459
0
      ++CurOp;
1460
18.1k
    EmitByte(BaseOpcode, CurByte, OS);
1461
18.1k
    emitMemModRMByte(MI, CurOp,
1462
18.1k
                     (Form == X86II::MRMXm) ? 
0119
:
Form - X86II::MRM0m18.0k
, TSFlags,
1463
18.1k
                     Rex, CurByte, OS, Fixups, STI);
1464
18.1k
    CurOp += X86::AddrNumOperands;
1465
18.1k
    break;
1466
18.1k
1467
1.58k
  
case X86II::MRM_C0: 1.58k
case X86II::MRM_C1: 1.58k
case X86II::MRM_C2:
1468
1.58k
  
case X86II::MRM_C3: 1.58k
case X86II::MRM_C4: 1.58k
case X86II::MRM_C5:
1469
1.58k
  
case X86II::MRM_C6: 1.58k
case X86II::MRM_C7: 1.58k
case X86II::MRM_C8:
1470
1.58k
  
case X86II::MRM_C9: 1.58k
case X86II::MRM_CA: 1.58k
case X86II::MRM_CB:
1471
1.58k
  
case X86II::MRM_CC: 1.58k
case X86II::MRM_CD: 1.58k
case X86II::MRM_CE:
1472
1.58k
  
case X86II::MRM_CF: 1.58k
case X86II::MRM_D0: 1.58k
case X86II::MRM_D1:
1473
1.58k
  
case X86II::MRM_D2: 1.58k
case X86II::MRM_D3: 1.58k
case X86II::MRM_D4:
1474
1.58k
  
case X86II::MRM_D5: 1.58k
case X86II::MRM_D6: 1.58k
case X86II::MRM_D7:
1475
1.58k
  
case X86II::MRM_D8: 1.58k
case X86II::MRM_D9: 1.58k
case X86II::MRM_DA:
1476
1.58k
  
case X86II::MRM_DB: 1.58k
case X86II::MRM_DC: 1.58k
case X86II::MRM_DD:
1477
1.58k
  
case X86II::MRM_DE: 1.58k
case X86II::MRM_DF: 1.58k
case X86II::MRM_E0:
1478
1.58k
  
case X86II::MRM_E1: 1.58k
case X86II::MRM_E2: 1.58k
case X86II::MRM_E3:
1479
1.58k
  
case X86II::MRM_E4: 1.58k
case X86II::MRM_E5: 1.58k
case X86II::MRM_E6:
1480
1.58k
  
case X86II::MRM_E7: 1.58k
case X86II::MRM_E8: 1.58k
case X86II::MRM_E9:
1481
1.58k
  
case X86II::MRM_EA: 1.58k
case X86II::MRM_EB: 1.58k
case X86II::MRM_EC:
1482
1.58k
  
case X86II::MRM_ED: 1.58k
case X86II::MRM_EE: 1.58k
case X86II::MRM_EF:
1483
1.58k
  
case X86II::MRM_F0: 1.58k
case X86II::MRM_F1: 1.58k
case X86II::MRM_F2:
1484
1.58k
  
case X86II::MRM_F3: 1.58k
case X86II::MRM_F4: 1.58k
case X86II::MRM_F5:
1485
1.58k
  
case X86II::MRM_F6: 1.58k
case X86II::MRM_F7: 1.58k
case X86II::MRM_F8:
1486
1.58k
  
case X86II::MRM_F9: 1.58k
case X86II::MRM_FA: 1.58k
case X86II::MRM_FB:
1487
1.58k
  
case X86II::MRM_FC: 1.58k
case X86II::MRM_FD: 1.58k
case X86II::MRM_FE:
1488
1.58k
  case X86II::MRM_FF:
1489
1.58k
    EmitByte(BaseOpcode, CurByte, OS);
1490
1.58k
    EmitByte(0xC0 + Form - X86II::MRM_C0, CurByte, OS);
1491
1.58k
    break;
1492
461k
  }
1493
461k
1494
461k
  
if (461k
HasVEX_I8Reg461k
) {
1495
274
    // The last source register of a 4 operand instruction in AVX is encoded
1496
274
    // in bits[7:4] of a immediate byte.
1497
274
    assert(I8RegNum < 16 && "Register encoding out of range");
1498
274
    I8RegNum <<= 4;
1499
274
    if (
CurOp != NumOps274
) {
1500
14
      unsigned Val = MI.getOperand(CurOp++).getImm();
1501
14
      assert(Val < 16 && "Immediate operand value out of range");
1502
14
      I8RegNum |= Val;
1503
14
    }
1504
274
    EmitImmediate(MCOperand::createImm(I8RegNum), MI.getLoc(), 1, FK_Data_1,
1505
274
                  CurByte, OS, Fixups);
1506
461k
  } else {
1507
461k
    // If there is a remaining operand, it must be a trailing immediate. Emit it
1508
461k
    // according to the right size for the instruction. Some instructions
1509
461k
    // (SSE4a extrq and insertq) have two trailing immediates.
1510
614k
    while (
CurOp != NumOps && 614k
NumOps - CurOp <= 2153k
) {
1511
153k
      EmitImmediate(MI.getOperand(CurOp++), MI.getLoc(),
1512
153k
                    X86II::getSizeOfImm(TSFlags), getImmFixupKind(TSFlags),
1513
153k
                    CurByte, OS, Fixups);
1514
153k
    }
1515
461k
  }
1516
461k
1517
461k
  if (TSFlags & X86II::Has3DNow0F0FOpcode)
1518
26
    EmitByte(X86II::getBaseOpcodeFor(TSFlags), CurByte, OS);
1519
461k
1520
#ifndef NDEBUG
1521
  // FIXME: Verify.
1522
  if (/*!Desc.isVariadic() &&*/ CurOp != NumOps) {
1523
    errs() << "Cannot encode all operands of: ";
1524
    MI.dump();
1525
    errs() << '\n';
1526
    abort();
1527
  }
1528
#endif
1529
}
1530
1531
MCCodeEmitter *llvm::createX86MCCodeEmitter(const MCInstrInfo &MCII,
1532
                                            const MCRegisterInfo &MRI,
1533
77.6k
                                            MCContext &Ctx) {
1534
77.6k
  return new X86MCCodeEmitter(MCII, Ctx);
1535
77.6k
}