Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86InstructionSelector.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- X86InstructionSelector.cpp -----------------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
/// \file
9
/// This file implements the targeting of the InstructionSelector class for
10
/// X86.
11
/// \todo This should be generated by TableGen.
12
//===----------------------------------------------------------------------===//
13
14
#include "MCTargetDesc/X86BaseInfo.h"
15
#include "X86InstrBuilder.h"
16
#include "X86InstrInfo.h"
17
#include "X86RegisterBankInfo.h"
18
#include "X86RegisterInfo.h"
19
#include "X86Subtarget.h"
20
#include "X86TargetMachine.h"
21
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
22
#include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
23
#include "llvm/CodeGen/GlobalISel/RegisterBank.h"
24
#include "llvm/CodeGen/GlobalISel/Utils.h"
25
#include "llvm/CodeGen/MachineBasicBlock.h"
26
#include "llvm/CodeGen/MachineConstantPool.h"
27
#include "llvm/CodeGen/MachineFunction.h"
28
#include "llvm/CodeGen/MachineInstr.h"
29
#include "llvm/CodeGen/MachineInstrBuilder.h"
30
#include "llvm/CodeGen/MachineMemOperand.h"
31
#include "llvm/CodeGen/MachineOperand.h"
32
#include "llvm/CodeGen/MachineRegisterInfo.h"
33
#include "llvm/CodeGen/TargetOpcodes.h"
34
#include "llvm/CodeGen/TargetRegisterInfo.h"
35
#include "llvm/IR/DataLayout.h"
36
#include "llvm/IR/InstrTypes.h"
37
#include "llvm/Support/AtomicOrdering.h"
38
#include "llvm/Support/CodeGen.h"
39
#include "llvm/Support/Debug.h"
40
#include "llvm/Support/ErrorHandling.h"
41
#include "llvm/Support/LowLevelTypeImpl.h"
42
#include "llvm/Support/MathExtras.h"
43
#include "llvm/Support/raw_ostream.h"
44
#include <cassert>
45
#include <cstdint>
46
#include <tuple>
47
48
#define DEBUG_TYPE "X86-isel"
49
50
using namespace llvm;
51
52
namespace {
53
54
#define GET_GLOBALISEL_PREDICATE_BITSET
55
#include "X86GenGlobalISel.inc"
56
#undef GET_GLOBALISEL_PREDICATE_BITSET
57
58
class X86InstructionSelector : public InstructionSelector {
59
public:
60
  X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
61
                         const X86RegisterBankInfo &RBI);
62
63
  bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override;
64
0
  static const char *getName() { return DEBUG_TYPE; }
65
66
private:
67
  /// tblgen-erated 'select' implementation, used as the initial selector for
68
  /// the patterns that don't require complex C++.
69
  bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
70
71
  // TODO: remove after supported by Tablegen-erated instruction selection.
72
  unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
73
                          uint64_t Alignment) const;
74
75
  bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
76
                         MachineFunction &MF) const;
77
  bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
78
                             MachineFunction &MF) const;
79
  bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
80
                         MachineFunction &MF) const;
81
  bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
82
                      MachineFunction &MF) const;
83
  bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
84
                             MachineFunction &MF) const;
85
  bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
86
                  MachineFunction &MF) const;
87
  bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
88
                    MachineFunction &MF) const;
89
  bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
90
                 MachineFunction &MF) const;
91
  bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
92
                  MachineFunction &MF) const;
93
  bool selectUadde(MachineInstr &I, MachineRegisterInfo &MRI,
94
                   MachineFunction &MF) const;
95
  bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const;
96
  bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
97
                           MachineFunction &MF,
98
                           CodeGenCoverage &CoverageInfo) const;
99
  bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI,
100
                         MachineFunction &MF,
101
                         CodeGenCoverage &CoverageInfo) const;
102
  bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
103
                    MachineFunction &MF) const;
104
  bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
105
                     MachineFunction &MF) const;
106
  bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
107
                        MachineFunction &MF) const;
108
  bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
109
                          const unsigned DstReg,
110
                          const TargetRegisterClass *DstRC,
111
                          const unsigned SrcReg,
112
                          const TargetRegisterClass *SrcRC) const;
113
  bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
114
                     MachineFunction &MF) const;
115
  bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
116
  bool selectShift(MachineInstr &I, MachineRegisterInfo &MRI,
117
                   MachineFunction &MF) const;
118
  bool selectDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
119
                    MachineFunction &MF) const;
120
  bool selectIntrinsicWSideEffects(MachineInstr &I, MachineRegisterInfo &MRI,
121
                                   MachineFunction &MF) const;
122
123
  // emit insert subreg instruction and insert it before MachineInstr &I
124
  bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
125
                        MachineRegisterInfo &MRI, MachineFunction &MF) const;
126
  // emit extract subreg instruction and insert it before MachineInstr &I
127
  bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
128
                         MachineRegisterInfo &MRI, MachineFunction &MF) const;
129
130
  const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
131
  const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
132
                                         MachineRegisterInfo &MRI) const;
133
134
  const X86TargetMachine &TM;
135
  const X86Subtarget &STI;
136
  const X86InstrInfo &TII;
137
  const X86RegisterInfo &TRI;
138
  const X86RegisterBankInfo &RBI;
139
140
#define GET_GLOBALISEL_PREDICATES_DECL
141
#include "X86GenGlobalISel.inc"
142
#undef GET_GLOBALISEL_PREDICATES_DECL
143
144
#define GET_GLOBALISEL_TEMPORARIES_DECL
145
#include "X86GenGlobalISel.inc"
146
#undef GET_GLOBALISEL_TEMPORARIES_DECL
147
};
148
149
} // end anonymous namespace
150
151
#define GET_GLOBALISEL_IMPL
152
#include "X86GenGlobalISel.inc"
153
#undef GET_GLOBALISEL_IMPL
154
155
X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
156
                                               const X86Subtarget &STI,
157
                                               const X86RegisterBankInfo &RBI)
158
    : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()),
159
      TRI(*STI.getRegisterInfo()), RBI(RBI),
160
#define GET_GLOBALISEL_PREDICATES_INIT
161
#include "X86GenGlobalISel.inc"
162
#undef GET_GLOBALISEL_PREDICATES_INIT
163
#define GET_GLOBALISEL_TEMPORARIES_INIT
164
#include "X86GenGlobalISel.inc"
165
#undef GET_GLOBALISEL_TEMPORARIES_INIT
166
15.2k
{
167
15.2k
}
168
169
// FIXME: This should be target-independent, inferred from the types declared
170
// for each class in the bank.
171
const TargetRegisterClass *
172
2.81k
X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
173
2.81k
  if (RB.getID() == X86::GPRRegBankID) {
174
1.27k
    if (Ty.getSizeInBits() <= 8)
175
197
      return &X86::GR8RegClass;
176
1.08k
    if (Ty.getSizeInBits() == 16)
177
54
      return &X86::GR16RegClass;
178
1.02k
    if (Ty.getSizeInBits() == 32)
179
528
      return &X86::GR32RegClass;
180
498
    if (Ty.getSizeInBits() == 64)
181
498
      return &X86::GR64RegClass;
182
1.53k
  }
183
1.53k
  if (RB.getID() == X86::VECRRegBankID) {
184
1.53k
    if (Ty.getSizeInBits() == 32)
185
206
      return STI.hasAVX512() ? 
&X86::FR32XRegClass66
:
&X86::FR32RegClass140
;
186
1.32k
    if (Ty.getSizeInBits() == 64)
187
188
      return STI.hasAVX512() ? 
&X86::FR64XRegClass60
:
&X86::FR64RegClass128
;
188
1.13k
    if (Ty.getSizeInBits() == 128)
189
852
      return STI.hasAVX512() ? 
&X86::VR128XRegClass291
:
&X86::VR128RegClass561
;
190
287
    if (Ty.getSizeInBits() == 256)
191
213
      return STI.hasAVX512() ? 
&X86::VR256XRegClass86
:
&X86::VR256RegClass127
;
192
74
    if (Ty.getSizeInBits() == 512)
193
74
      return &X86::VR512RegClass;
194
0
  }
195
0
196
0
  llvm_unreachable("Unknown RegBank!");
197
0
}
198
199
const TargetRegisterClass *
200
X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg,
201
152
                                    MachineRegisterInfo &MRI) const {
202
152
  const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
203
152
  return getRegClass(Ty, RegBank);
204
152
}
205
206
23
static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
207
23
  unsigned SubIdx = X86::NoSubRegister;
208
23
  if (RC == &X86::GR32RegClass) {
209
9
    SubIdx = X86::sub_32bit;
210
14
  } else if (RC == &X86::GR16RegClass) {
211
5
    SubIdx = X86::sub_16bit;
212
9
  } else if (RC == &X86::GR8RegClass) {
213
9
    SubIdx = X86::sub_8bit;
214
9
  }
215
23
216
23
  return SubIdx;
217
23
}
218
219
0
static const TargetRegisterClass *getRegClassFromGRPhysReg(unsigned Reg) {
220
0
  assert(TargetRegisterInfo::isPhysicalRegister(Reg));
221
0
  if (X86::GR64RegClass.contains(Reg))
222
0
    return &X86::GR64RegClass;
223
0
  if (X86::GR32RegClass.contains(Reg))
224
0
    return &X86::GR32RegClass;
225
0
  if (X86::GR16RegClass.contains(Reg))
226
0
    return &X86::GR16RegClass;
227
0
  if (X86::GR8RegClass.contains(Reg))
228
0
    return &X86::GR8RegClass;
229
0
230
0
  llvm_unreachable("Unknown RegClass for PhysReg!");
231
0
}
232
233
// Set X86 Opcode and constrain DestReg.
234
bool X86InstructionSelector::selectCopy(MachineInstr &I,
235
2.71k
                                        MachineRegisterInfo &MRI) const {
236
2.71k
  unsigned DstReg = I.getOperand(0).getReg();
237
2.71k
  const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
238
2.71k
  const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
239
2.71k
240
2.71k
  unsigned SrcReg = I.getOperand(1).getReg();
241
2.71k
  const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
242
2.71k
  const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
243
2.71k
244
2.71k
  if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
245
1.02k
    assert(I.isCopy() && "Generic operators do not allow physical registers");
246
1.02k
247
1.02k
    if (DstSize > SrcSize && 
SrcRegBank.getID() == X86::GPRRegBankID0
&&
248
1.02k
        
DstRegBank.getID() == X86::GPRRegBankID0
) {
249
0
250
0
      const TargetRegisterClass *SrcRC =
251
0
          getRegClass(MRI.getType(SrcReg), SrcRegBank);
252
0
      const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
253
0
254
0
      if (SrcRC != DstRC) {
255
0
        // This case can be generated by ABI lowering, performe anyext
256
0
        unsigned ExtSrc = MRI.createVirtualRegister(DstRC);
257
0
        BuildMI(*I.getParent(), I, I.getDebugLoc(),
258
0
                TII.get(TargetOpcode::SUBREG_TO_REG))
259
0
            .addDef(ExtSrc)
260
0
            .addImm(0)
261
0
            .addReg(SrcReg)
262
0
            .addImm(getSubRegIndex(SrcRC));
263
0
264
0
        I.getOperand(1).setReg(ExtSrc);
265
0
      }
266
0
    }
267
1.02k
268
1.02k
    return true;
269
1.02k
  }
270
1.69k
271
1.69k
  assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
272
1.69k
         "No phys reg on generic operators");
273
1.69k
  assert((DstSize == SrcSize ||
274
1.69k
          // Copies are a mean to setup initial types, the number of
275
1.69k
          // bits may not exactly match.
276
1.69k
          (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
277
1.69k
           DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
278
1.69k
         "Copy with different width?!");
279
1.69k
280
1.69k
  const TargetRegisterClass *DstRC =
281
1.69k
      getRegClass(MRI.getType(DstReg), DstRegBank);
282
1.69k
283
1.69k
  if (SrcRegBank.getID() == X86::GPRRegBankID &&
284
1.69k
      
DstRegBank.getID() == X86::GPRRegBankID980
&&
SrcSize > DstSize936
&&
285
1.69k
      
TargetRegisterInfo::isPhysicalRegister(SrcReg)0
) {
286
0
    // Change the physical register to performe truncate.
287
0
288
0
    const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
289
0
290
0
    if (DstRC != SrcRC) {
291
0
      I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
292
0
      I.getOperand(1).substPhysReg(SrcReg, TRI);
293
0
    }
294
0
  }
295
1.69k
296
1.69k
  // No need to constrain SrcReg. It will get constrained when
297
1.69k
  // we hit another of its use or its defs.
298
1.69k
  // Copies do not have constraints.
299
1.69k
  const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
300
1.69k
  if (!OldRC || 
!DstRC->hasSubClassEq(OldRC)1.59k
) {
301
107
    if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
302
0
      LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
303
0
                        << " operand\n");
304
0
      return false;
305
0
    }
306
1.69k
  }
307
1.69k
  I.setDesc(TII.get(X86::COPY));
308
1.69k
  return true;
309
1.69k
}
310
311
bool X86InstructionSelector::select(MachineInstr &I,
312
6.10k
                                    CodeGenCoverage &CoverageInfo) const {
313
6.10k
  assert(I.getParent() && "Instruction should be in a basic block!");
314
6.10k
  assert(I.getParent()->getParent() && "Instruction should be in a function!");
315
6.10k
316
6.10k
  MachineBasicBlock &MBB = *I.getParent();
317
6.10k
  MachineFunction &MF = *MBB.getParent();
318
6.10k
  MachineRegisterInfo &MRI = MF.getRegInfo();
319
6.10k
320
6.10k
  unsigned Opcode = I.getOpcode();
321
6.10k
  if (!isPreISelGenericOpcode(Opcode)) {
322
3.83k
    // Certain non-generic instructions also need some special handling.
323
3.83k
324
3.83k
    if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
325
0
      return false;
326
3.83k
327
3.83k
    if (I.isCopy())
328
2.71k
      return selectCopy(I, MRI);
329
1.11k
330
1.11k
    return true;
331
1.11k
  }
332
2.27k
333
2.27k
  assert(I.getNumOperands() == I.getNumExplicitOperands() &&
334
2.27k
         "Generic instruction has unexpected implicit operands\n");
335
2.27k
336
2.27k
  if (selectImpl(I, CoverageInfo))
337
803
    return true;
338
1.46k
339
1.46k
  LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
340
1.46k
341
1.46k
  // TODO: This should be implemented by tblgen.
342
1.46k
  switch (I.getOpcode()) {
343
1.46k
  default:
344
0
    return false;
345
1.46k
  case TargetOpcode::G_STORE:
346
508
  case TargetOpcode::G_LOAD:
347
508
    return selectLoadStoreOp(I, MRI, MF);
348
508
  case TargetOpcode::G_GEP:
349
46
  case TargetOpcode::G_FRAME_INDEX:
350
46
    return selectFrameIndexOrGep(I, MRI, MF);
351
46
  case TargetOpcode::G_GLOBAL_VALUE:
352
25
    return selectGlobalValue(I, MRI, MF);
353
46
  case TargetOpcode::G_CONSTANT:
354
35
    return selectConstant(I, MRI, MF);
355
46
  case TargetOpcode::G_FCONSTANT:
356
15
    return materializeFP(I, MRI, MF);
357
295
  case TargetOpcode::G_PTRTOINT:
358
295
  case TargetOpcode::G_TRUNC:
359
295
    return selectTruncOrPtrToInt(I, MRI, MF);
360
295
  case TargetOpcode::G_INTTOPTR:
361
4
    return selectCopy(I, MRI);
362
295
  case TargetOpcode::G_ZEXT:
363
37
    return selectZext(I, MRI, MF);
364
295
  case TargetOpcode::G_ANYEXT:
365
153
    return selectAnyext(I, MRI, MF);
366
295
  case TargetOpcode::G_ICMP:
367
45
    return selectCmp(I, MRI, MF);
368
295
  case TargetOpcode::G_FCMP:
369
28
    return selectFCmp(I, MRI, MF);
370
295
  case TargetOpcode::G_UADDE:
371
4
    return selectUadde(I, MRI, MF);
372
295
  case TargetOpcode::G_UNMERGE_VALUES:
373
28
    return selectUnmergeValues(I, MRI, MF, CoverageInfo);
374
295
  case TargetOpcode::G_MERGE_VALUES:
375
16
  case TargetOpcode::G_CONCAT_VECTORS:
376
16
    return selectMergeValues(I, MRI, MF, CoverageInfo);
377
70
  case TargetOpcode::G_EXTRACT:
378
70
    return selectExtract(I, MRI, MF);
379
34
  case TargetOpcode::G_INSERT:
380
34
    return selectInsert(I, MRI, MF);
381
19
  case TargetOpcode::G_BRCOND:
382
19
    return selectCondBranch(I, MRI, MF);
383
22
  case TargetOpcode::G_IMPLICIT_DEF:
384
22
  case TargetOpcode::G_PHI:
385
22
    return selectImplicitDefOrPHI(I, MRI);
386
53
  case TargetOpcode::G_SHL:
387
53
  case TargetOpcode::G_ASHR:
388
53
  case TargetOpcode::G_LSHR:
389
53
    return selectShift(I, MRI, MF);
390
53
  case TargetOpcode::G_SDIV:
391
28
  case TargetOpcode::G_UDIV:
392
28
  case TargetOpcode::G_SREM:
393
28
  case TargetOpcode::G_UREM:
394
28
    return selectDivRem(I, MRI, MF);
395
28
  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
396
2
    return selectIntrinsicWSideEffects(I, MRI, MF);
397
0
  }
398
0
399
0
  return false;
400
0
}
401
402
unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
403
                                                const RegisterBank &RB,
404
                                                unsigned Opc,
405
523
                                                uint64_t Alignment) const {
406
523
  bool Isload = (Opc == TargetOpcode::G_LOAD);
407
523
  bool HasAVX = STI.hasAVX();
408
523
  bool HasAVX512 = STI.hasAVX512();
409
523
  bool HasVLX = STI.hasVLX();
410
523
411
523
  if (Ty == LLT::scalar(8)) {
412
51
    if (X86::GPRRegBankID == RB.getID())
413
51
      return Isload ? 
X86::MOV8rm44
:
X86::MOV8mr7
;
414
472
  } else if (Ty == LLT::scalar(16)) {
415
30
    if (X86::GPRRegBankID == RB.getID())
416
30
      return Isload ? 
X86::MOV16rm27
:
X86::MOV16mr3
;
417
442
  } else if (Ty == LLT::scalar(32) || 
Ty == LLT::pointer(0, 32)211
) {
418
286
    if (X86::GPRRegBankID == RB.getID())
419
275
      return Isload ? 
X86::MOV32rm173
:
X86::MOV32mr102
;
420
11
    if (X86::VECRRegBankID == RB.getID())
421
11
      return Isload ? 
(HasAVX512 10
?
X86::VMOVSSZrm_alt0
:
422
10
                       HasAVX    ? 
X86::VMOVSSrm_alt0
:
423
10
                                   X86::MOVSSrm_alt)
424
11
                    : 
(HasAVX512 1
?
X86::VMOVSSZmr0
:
425
1
                       HasAVX    ? 
X86::VMOVSSmr0
:
426
1
                                   X86::MOVSSmr);
427
156
  } else if (Ty == LLT::scalar(64) || 
Ty == LLT::pointer(0, 64)93
) {
428
98
    if (X86::GPRRegBankID == RB.getID())
429
92
      return Isload ? 
X86::MOV64rm49
:
X86::MOV64mr43
;
430
6
    if (X86::VECRRegBankID == RB.getID())
431
6
      return Isload ? 
(HasAVX512 5
?
X86::VMOVSDZrm_alt0
:
432
5
                       HasAVX    ? 
X86::VMOVSDrm_alt0
:
433
5
                                   X86::MOVSDrm_alt)
434
6
                    : 
(HasAVX512 1
?
X86::VMOVSDZmr0
:
435
1
                       HasAVX    ? 
X86::VMOVSDmr0
:
436
1
                                   X86::MOVSDmr);
437
58
  } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
438
26
    if (Alignment >= 16)
439
12
      return Isload ? 
(HasVLX 6
?
X86::VMOVAPSZ128rm3
440
6
                              : HasAVX512
441
3
                                    ? 
X86::VMOVAPSZ128rm_NOVLX1
442
3
                                    : 
HasAVX 2
?
X86::VMOVAPSrm1
:
X86::MOVAPSrm1
)
443
12
                    : 
(HasVLX 6
?
X86::VMOVAPSZ128mr3
444
6
                              : HasAVX512
445
3
                                    ? 
X86::VMOVAPSZ128mr_NOVLX1
446
3
                                    : 
HasAVX 2
?
X86::VMOVAPSmr1
:
X86::MOVAPSmr1
);
447
14
    else
448
14
      return Isload ? 
(HasVLX 8
?
X86::VMOVUPSZ128rm3
449
8
                              : HasAVX512
450
5
                                    ? 
X86::VMOVUPSZ128rm_NOVLX1
451
5
                                    : 
HasAVX 4
?
X86::VMOVUPSrm1
:
X86::MOVUPSrm3
)
452
14
                    : 
(HasVLX 6
?
X86::VMOVUPSZ128mr3
453
6
                              : HasAVX512
454
3
                                    ? 
X86::VMOVUPSZ128mr_NOVLX1
455
3
                                    : 
HasAVX 2
?
X86::VMOVUPSmr1
:
X86::MOVUPSmr1
);
456
32
  } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
457
20
    if (Alignment >= 32)
458
10
      return Isload ? 
(HasVLX 5
?
X86::VMOVAPSZ256rm3
459
5
                              : 
HasAVX512 2
?
X86::VMOVAPSZ256rm_NOVLX1
460
2
                                          : 
X86::VMOVAPSYrm1
)
461
10
                    : 
(HasVLX 5
?
X86::VMOVAPSZ256mr3
462
5
                              : 
HasAVX512 2
?
X86::VMOVAPSZ256mr_NOVLX1
463
2
                                          : 
X86::VMOVAPSYmr1
);
464
10
    else
465
10
      return Isload ? 
(HasVLX 5
?
X86::VMOVUPSZ256rm3
466
5
                              : 
HasAVX512 2
?
X86::VMOVUPSZ256rm_NOVLX1
467
2
                                          : 
X86::VMOVUPSYrm1
)
468
10
                    : 
(HasVLX 5
?
X86::VMOVUPSZ256mr3
469
5
                              : 
HasAVX512 2
?
X86::VMOVUPSZ256mr_NOVLX1
470
2
                                          : 
X86::VMOVUPSYmr1
);
471
12
  } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
472
12
    if (Alignment >= 64)
473
2
      return Isload ? 
X86::VMOVAPSZrm0
: X86::VMOVAPSZmr;
474
10
    else
475
10
      return Isload ? 
X86::VMOVUPSZrm6
:
X86::VMOVUPSZmr4
;
476
0
  }
477
0
  return Opc;
478
0
}
479
480
// Fill in an address from the given instruction.
481
static void X86SelectAddress(const MachineInstr &I,
482
                             const MachineRegisterInfo &MRI,
483
508
                             X86AddressMode &AM) {
484
508
  assert(I.getOperand(0).isReg() && "unsupported opperand.");
485
508
  assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
486
508
         "unsupported type.");
487
508
488
508
  if (I.getOpcode() == TargetOpcode::G_GEP) {
489
62
    if (auto COff = getConstantVRegVal(I.getOperand(2).getReg(), MRI)) {
490
62
      int64_t Imm = *COff;
491
62
      if (isInt<32>(Imm)) { // Check for displacement overflow.
492
42
        AM.Disp = static_cast<int32_t>(Imm);
493
42
        AM.Base.Reg = I.getOperand(1).getReg();
494
42
        return;
495
42
      }
496
446
    }
497
446
  } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
498
155
    AM.Base.FrameIndex = I.getOperand(1).getIndex();
499
155
    AM.BaseType = X86AddressMode::FrameIndexBase;
500
155
    return;
501
155
  }
502
311
503
311
  // Default behavior.
504
311
  AM.Base.Reg = I.getOperand(0).getReg();
505
311
}
506
507
bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
508
                                               MachineRegisterInfo &MRI,
509
508
                                               MachineFunction &MF) const {
510
508
  unsigned Opc = I.getOpcode();
511
508
512
508
  assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
513
508
         "unexpected instruction");
514
508
515
508
  const unsigned DefReg = I.getOperand(0).getReg();
516
508
  LLT Ty = MRI.getType(DefReg);
517
508
  const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
518
508
519
508
  assert(I.hasOneMemOperand());
520
508
  auto &MemOp = **I.memoperands_begin();
521
508
  if (MemOp.isAtomic()) {
522
80
    // Note: for unordered operations, we rely on the fact the appropriate MMO
523
80
    // is already on the instruction we're mutating, and thus we don't need to
524
80
    // make any changes.  So long as we select an opcode which is capable of
525
80
    // loading or storing the appropriate size atomically, the rest of the
526
80
    // backend is required to respect the MMO state. 
527
80
    if (!MemOp.isUnordered()) {
528
0
      LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
529
0
      return false;
530
0
    }
531
80
    if (MemOp.getAlignment() < Ty.getSizeInBits()/8) {
532
0
      LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
533
0
      return false;
534
0
    }
535
508
  }
536
508
537
508
  unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlignment());
538
508
  if (NewOpc == Opc)
539
0
    return false;
540
508
541
508
  X86AddressMode AM;
542
508
  X86SelectAddress(*MRI.getVRegDef(I.getOperand(1).getReg()), MRI, AM);
543
508
544
508
  I.setDesc(TII.get(NewOpc));
545
508
  MachineInstrBuilder MIB(MF, I);
546
508
  if (Opc == TargetOpcode::G_LOAD) {
547
323
    I.RemoveOperand(1);
548
323
    addFullAddress(MIB, AM);
549
323
  } else {
550
185
    // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
551
185
    I.RemoveOperand(1);
552
185
    I.RemoveOperand(0);
553
185
    addFullAddress(MIB, AM).addUse(DefReg);
554
185
  }
555
508
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
556
508
}
557
558
71
static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
559
71
  if (Ty == LLT::pointer(0, 64))
560
58
    return X86::LEA64r;
561
13
  else if (Ty == LLT::pointer(0, 32))
562
13
    return STI.isTarget64BitILP32() ? 
X86::LEA64_32r6
:
X86::LEA32r7
;
563
13
  else
564
13
    llvm_unreachable("Can't get LEA opcode. Unsupported type.");
565
71
}
566
567
bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
568
                                                   MachineRegisterInfo &MRI,
569
46
                                                   MachineFunction &MF) const {
570
46
  unsigned Opc = I.getOpcode();
571
46
572
46
  assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_GEP) &&
573
46
         "unexpected instruction");
574
46
575
46
  const unsigned DefReg = I.getOperand(0).getReg();
576
46
  LLT Ty = MRI.getType(DefReg);
577
46
578
46
  // Use LEA to calculate frame index and GEP
579
46
  unsigned NewOpc = getLeaOP(Ty, STI);
580
46
  I.setDesc(TII.get(NewOpc));
581
46
  MachineInstrBuilder MIB(MF, I);
582
46
583
46
  if (Opc == TargetOpcode::G_FRAME_INDEX) {
584
6
    addOffset(MIB, 0);
585
40
  } else {
586
40
    MachineOperand &InxOp = I.getOperand(2);
587
40
    I.addOperand(InxOp);        // set IndexReg
588
40
    InxOp.ChangeToImmediate(1); // set Scale
589
40
    MIB.addImm(0).addReg(0);
590
40
  }
591
46
592
46
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
593
46
}
594
595
bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
596
                                               MachineRegisterInfo &MRI,
597
25
                                               MachineFunction &MF) const {
598
25
  assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
599
25
         "unexpected instruction");
600
25
601
25
  auto GV = I.getOperand(1).getGlobal();
602
25
  if (GV->isThreadLocal()) {
603
0
    return false; // TODO: we don't support TLS yet.
604
0
  }
605
25
606
25
  // Can't handle alternate code models yet.
607
25
  if (TM.getCodeModel() != CodeModel::Small)
608
0
    return false;
609
25
610
25
  X86AddressMode AM;
611
25
  AM.GV = GV;
612
25
  AM.GVOpFlags = STI.classifyGlobalReference(GV);
613
25
614
25
  // TODO: The ABI requires an extra load. not supported yet.
615
25
  if (isGlobalStubReference(AM.GVOpFlags))
616
0
    return false;
617
25
618
25
  // TODO: This reference is relative to the pic base. not supported yet.
619
25
  if (isGlobalRelativeToPICBase(AM.GVOpFlags))
620
0
    return false;
621
25
622
25
  if (STI.isPICStyleRIPRel()) {
623
13
    // Use rip-relative addressing.
624
13
    assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
625
13
    AM.Base.Reg = X86::RIP;
626
13
  }
627
25
628
25
  const unsigned DefReg = I.getOperand(0).getReg();
629
25
  LLT Ty = MRI.getType(DefReg);
630
25
  unsigned NewOpc = getLeaOP(Ty, STI);
631
25
632
25
  I.setDesc(TII.get(NewOpc));
633
25
  MachineInstrBuilder MIB(MF, I);
634
25
635
25
  I.RemoveOperand(1);
636
25
  addFullAddress(MIB, AM);
637
25
638
25
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
639
25
}
640
641
bool X86InstructionSelector::selectConstant(MachineInstr &I,
642
                                            MachineRegisterInfo &MRI,
643
35
                                            MachineFunction &MF) const {
644
35
  assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
645
35
         "unexpected instruction");
646
35
647
35
  const unsigned DefReg = I.getOperand(0).getReg();
648
35
  LLT Ty = MRI.getType(DefReg);
649
35
650
35
  if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
651
0
    return false;
652
35
653
35
  uint64_t Val = 0;
654
35
  if (I.getOperand(1).isCImm()) {
655
35
    Val = I.getOperand(1).getCImm()->getZExtValue();
656
35
    I.getOperand(1).ChangeToImmediate(Val);
657
35
  } else 
if (0
I.getOperand(1).isImm()0
) {
658
0
    Val = I.getOperand(1).getImm();
659
0
  } else
660
0
    llvm_unreachable("Unsupported operand type.");
661
35
662
35
  unsigned NewOpc;
663
35
  switch (Ty.getSizeInBits()) {
664
35
  case 8:
665
0
    NewOpc = X86::MOV8ri;
666
0
    break;
667
35
  case 16:
668
0
    NewOpc = X86::MOV16ri;
669
0
    break;
670
35
  case 32:
671
21
    NewOpc = X86::MOV32ri;
672
21
    break;
673
35
  case 64:
674
14
    // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
675
14
    if (isInt<32>(Val))
676
1
      NewOpc = X86::MOV64ri32;
677
13
    else
678
13
      NewOpc = X86::MOV64ri;
679
14
    break;
680
35
  default:
681
0
    llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
682
35
  }
683
35
684
35
  I.setDesc(TII.get(NewOpc));
685
35
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
686
35
}
687
688
// Helper function for selectTruncOrPtrToInt and selectAnyext.
689
// Returns true if DstRC lives on a floating register class and
690
// SrcRC lives on a 128-bit vector class.
691
static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
692
448
                            const TargetRegisterClass *SrcRC) {
693
448
  return (DstRC == &X86::FR32RegClass || 
DstRC == &X86::FR32XRegClass323
||
694
448
          
DstRC == &X86::FR64RegClass267
||
DstRC == &X86::FR64XRegClass150
) &&
695
448
         
(350
SrcRC == &X86::VR128RegClass350
||
SrcRC == &X86::VR128XRegClass108
);
696
448
}
697
698
bool X86InstructionSelector::selectTurnIntoCOPY(
699
    MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg,
700
    const TargetRegisterClass *DstRC, const unsigned SrcReg,
701
350
    const TargetRegisterClass *SrcRC) const {
702
350
703
350
  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
704
350
      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
705
0
    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
706
0
                      << " operand\n");
707
0
    return false;
708
0
  }
709
350
  I.setDesc(TII.get(X86::COPY));
710
350
  return true;
711
350
}
712
713
bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
714
                                                   MachineRegisterInfo &MRI,
715
295
                                                   MachineFunction &MF) const {
716
295
  assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
717
295
          I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
718
295
         "unexpected instruction");
719
295
720
295
  const unsigned DstReg = I.getOperand(0).getReg();
721
295
  const unsigned SrcReg = I.getOperand(1).getReg();
722
295
723
295
  const LLT DstTy = MRI.getType(DstReg);
724
295
  const LLT SrcTy = MRI.getType(SrcReg);
725
295
726
295
  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
727
295
  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
728
295
729
295
  if (DstRB.getID() != SrcRB.getID()) {
730
0
    LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
731
0
                      << " input/output on different banks\n");
732
0
    return false;
733
0
  }
734
295
735
295
  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
736
295
  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
737
295
738
295
  if (!DstRC || !SrcRC)
739
0
    return false;
740
295
741
295
  // If that's truncation of the value that lives on the vector class and goes
742
295
  // into the floating class, just replace it with copy, as we are able to
743
295
  // select it as a regular move.
744
295
  if (canTurnIntoCOPY(DstRC, SrcRC))
745
231
    return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
746
64
747
64
  if (DstRB.getID() != X86::GPRRegBankID)
748
0
    return false;
749
64
750
64
  unsigned SubIdx;
751
64
  if (DstRC == SrcRC) {
752
5
    // Nothing to be done
753
5
    SubIdx = X86::NoSubRegister;
754
59
  } else if (DstRC == &X86::GR32RegClass) {
755
5
    SubIdx = X86::sub_32bit;
756
54
  } else if (DstRC == &X86::GR16RegClass) {
757
6
    SubIdx = X86::sub_16bit;
758
48
  } else if (DstRC == &X86::GR8RegClass) {
759
48
    SubIdx = X86::sub_8bit;
760
48
  } else {
761
0
    return false;
762
0
  }
763
64
764
64
  SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
765
64
766
64
  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
767
64
      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
768
0
    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
769
0
                      << "\n");
770
0
    return false;
771
0
  }
772
64
773
64
  I.getOperand(1).setSubReg(SubIdx);
774
64
775
64
  I.setDesc(TII.get(X86::COPY));
776
64
  return true;
777
64
}
778
779
bool X86InstructionSelector::selectZext(MachineInstr &I,
780
                                        MachineRegisterInfo &MRI,
781
37
                                        MachineFunction &MF) const {
782
37
  assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
783
37
784
37
  const unsigned DstReg = I.getOperand(0).getReg();
785
37
  const unsigned SrcReg = I.getOperand(1).getReg();
786
37
787
37
  const LLT DstTy = MRI.getType(DstReg);
788
37
  const LLT SrcTy = MRI.getType(SrcReg);
789
37
790
37
  assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
791
37
         "8=>32 Zext is handled by tablegen");
792
37
  assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
793
37
         "16=>32 Zext is handled by tablegen");
794
37
795
37
  const static struct ZextEntry {
796
37
    LLT SrcTy;
797
37
    LLT DstTy;
798
37
    unsigned MovOp;
799
37
    bool NeedSubregToReg;
800
37
  } OpTable[] = {
801
37
      {LLT::scalar(8), LLT::scalar(16), X86::MOVZX16rr8, false},  // i8  => i16
802
37
      {LLT::scalar(8), LLT::scalar(64), X86::MOVZX32rr8, true},   // i8  => i64
803
37
      {LLT::scalar(16), LLT::scalar(64), X86::MOVZX32rr16, true}, // i16 => i64
804
37
      {LLT::scalar(32), LLT::scalar(64), 0, true}                 // i32 => i64
805
37
  };
806
37
807
37
  auto ZextEntryIt =
808
37
      std::find_if(std::begin(OpTable), std::end(OpTable),
809
148
                   [SrcTy, DstTy](const ZextEntry &El) {
810
148
                     return El.DstTy == DstTy && 
El.SrcTy == SrcTy17
;
811
148
                   });
812
37
813
37
  // Here we try to select Zext into a MOVZ and/or SUBREG_TO_REG instruction.
814
37
  if (ZextEntryIt != std::end(OpTable)) {
815
4
    const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
816
4
    const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
817
4
    const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
818
4
    const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
819
4
820
4
    if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
821
4
        !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
822
0
      LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
823
0
                        << " operand\n");
824
0
      return false;
825
0
    }
826
4
827
4
    unsigned TransitRegTo = DstReg;
828
4
    unsigned TransitRegFrom = SrcReg;
829
4
    if (ZextEntryIt->MovOp) {
830
0
      // If we select Zext into MOVZ + SUBREG_TO_REG, we need to have
831
0
      // a transit register in between: create it here.
832
0
      if (ZextEntryIt->NeedSubregToReg) {
833
0
        TransitRegFrom = MRI.createVirtualRegister(
834
0
            getRegClass(LLT::scalar(32), DstReg, MRI));
835
0
        TransitRegTo = TransitRegFrom;
836
0
      }
837
0
838
0
      BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(ZextEntryIt->MovOp))
839
0
          .addDef(TransitRegTo)
840
0
          .addReg(SrcReg);
841
0
    }
842
4
    if (ZextEntryIt->NeedSubregToReg) {
843
4
      BuildMI(*I.getParent(), I, I.getDebugLoc(),
844
4
              TII.get(TargetOpcode::SUBREG_TO_REG))
845
4
          .addDef(DstReg)
846
4
          .addImm(0)
847
4
          .addReg(TransitRegFrom)
848
4
          .addImm(X86::sub_32bit);
849
4
    }
850
4
    I.eraseFromParent();
851
4
    return true;
852
4
  }
853
33
854
33
  if (SrcTy != LLT::scalar(1))
855
0
    return false;
856
33
857
33
  unsigned AndOpc;
858
33
  if (DstTy == LLT::scalar(8))
859
2
    AndOpc = X86::AND8ri;
860
31
  else if (DstTy == LLT::scalar(16))
861
2
    AndOpc = X86::AND16ri8;
862
29
  else if (DstTy == LLT::scalar(32))
863
28
    AndOpc = X86::AND32ri8;
864
1
  else if (DstTy == LLT::scalar(64))
865
1
    AndOpc = X86::AND64ri8;
866
0
  else
867
0
    return false;
868
33
869
33
  unsigned DefReg = SrcReg;
870
33
  if (DstTy != LLT::scalar(8)) {
871
31
    DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
872
31
    BuildMI(*I.getParent(), I, I.getDebugLoc(),
873
31
            TII.get(TargetOpcode::SUBREG_TO_REG), DefReg)
874
31
        .addImm(0)
875
31
        .addReg(SrcReg)
876
31
        .addImm(X86::sub_8bit);
877
31
  }
878
33
879
33
  MachineInstr &AndInst =
880
33
      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
881
33
           .addReg(DefReg)
882
33
           .addImm(1);
883
33
884
33
  constrainSelectedInstRegOperands(AndInst, TII, TRI, RBI);
885
33
886
33
  I.eraseFromParent();
887
33
  return true;
888
33
}
889
890
bool X86InstructionSelector::selectAnyext(MachineInstr &I,
891
                                          MachineRegisterInfo &MRI,
892
153
                                          MachineFunction &MF) const {
893
153
  assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
894
153
895
153
  const unsigned DstReg = I.getOperand(0).getReg();
896
153
  const unsigned SrcReg = I.getOperand(1).getReg();
897
153
898
153
  const LLT DstTy = MRI.getType(DstReg);
899
153
  const LLT SrcTy = MRI.getType(SrcReg);
900
153
901
153
  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
902
153
  const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
903
153
904
153
  assert(DstRB.getID() == SrcRB.getID() &&
905
153
         "G_ANYEXT input/output on different banks\n");
906
153
907
153
  assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
908
153
         "G_ANYEXT incorrect operand size");
909
153
910
153
  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
911
153
  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
912
153
913
153
  // If that's ANY_EXT of the value that lives on the floating class and goes
914
153
  // into the vector class, just replace it with copy, as we are able to select
915
153
  // it as a regular move.
916
153
  if (canTurnIntoCOPY(SrcRC, DstRC))
917
119
    return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC);
918
34
919
34
  if (DstRB.getID() != X86::GPRRegBankID)
920
0
    return false;
921
34
922
34
  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
923
34
      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
924
0
    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
925
0
                      << " operand\n");
926
0
    return false;
927
0
  }
928
34
929
34
  if (SrcRC == DstRC) {
930
11
    I.setDesc(TII.get(X86::COPY));
931
11
    return true;
932
11
  }
933
23
934
23
  BuildMI(*I.getParent(), I, I.getDebugLoc(),
935
23
          TII.get(TargetOpcode::SUBREG_TO_REG))
936
23
      .addDef(DstReg)
937
23
      .addImm(0)
938
23
      .addReg(SrcReg)
939
23
      .addImm(getSubRegIndex(SrcRC));
940
23
941
23
  I.eraseFromParent();
942
23
  return true;
943
23
}
944
945
bool X86InstructionSelector::selectCmp(MachineInstr &I,
946
                                       MachineRegisterInfo &MRI,
947
45
                                       MachineFunction &MF) const {
948
45
  assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
949
45
950
45
  X86::CondCode CC;
951
45
  bool SwapArgs;
952
45
  std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
953
45
      (CmpInst::Predicate)I.getOperand(1).getPredicate());
954
45
955
45
  unsigned LHS = I.getOperand(2).getReg();
956
45
  unsigned RHS = I.getOperand(3).getReg();
957
45
958
45
  if (SwapArgs)
959
0
    std::swap(LHS, RHS);
960
45
961
45
  unsigned OpCmp;
962
45
  LLT Ty = MRI.getType(LHS);
963
45
964
45
  switch (Ty.getSizeInBits()) {
965
45
  default:
966
0
    return false;
967
45
  case 8:
968
2
    OpCmp = X86::CMP8rr;
969
2
    break;
970
45
  case 16:
971
2
    OpCmp = X86::CMP16rr;
972
2
    break;
973
45
  case 32:
974
39
    OpCmp = X86::CMP32rr;
975
39
    break;
976
45
  case 64:
977
2
    OpCmp = X86::CMP64rr;
978
2
    break;
979
45
  }
980
45
981
45
  MachineInstr &CmpInst =
982
45
      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
983
45
           .addReg(LHS)
984
45
           .addReg(RHS);
985
45
986
45
  MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
987
45
                                   TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC);
988
45
989
45
  constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
990
45
  constrainSelectedInstRegOperands(SetInst, TII, TRI, RBI);
991
45
992
45
  I.eraseFromParent();
993
45
  return true;
994
45
}
995
996
bool X86InstructionSelector::selectFCmp(MachineInstr &I,
997
                                        MachineRegisterInfo &MRI,
998
28
                                        MachineFunction &MF) const {
999
28
  assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
1000
28
1001
28
  unsigned LhsReg = I.getOperand(2).getReg();
1002
28
  unsigned RhsReg = I.getOperand(3).getReg();
1003
28
  CmpInst::Predicate Predicate =
1004
28
      (CmpInst::Predicate)I.getOperand(1).getPredicate();
1005
28
1006
28
  // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1007
28
  static const uint16_t SETFOpcTable[2][3] = {
1008
28
      {X86::COND_E, X86::COND_NP, X86::AND8rr},
1009
28
      {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1010
28
  const uint16_t *SETFOpc = nullptr;
1011
28
  switch (Predicate) {
1012
28
  default:
1013
24
    break;
1014
28
  case CmpInst::FCMP_OEQ:
1015
2
    SETFOpc = &SETFOpcTable[0][0];
1016
2
    break;
1017
28
  case CmpInst::FCMP_UNE:
1018
2
    SETFOpc = &SETFOpcTable[1][0];
1019
2
    break;
1020
28
  }
1021
28
1022
28
  // Compute the opcode for the CMP instruction.
1023
28
  unsigned OpCmp;
1024
28
  LLT Ty = MRI.getType(LhsReg);
1025
28
  switch (Ty.getSizeInBits()) {
1026
28
  default:
1027
0
    return false;
1028
28
  case 32:
1029
14
    OpCmp = X86::UCOMISSrr;
1030
14
    break;
1031
28
  case 64:
1032
14
    OpCmp = X86::UCOMISDrr;
1033
14
    break;
1034
28
  }
1035
28
1036
28
  unsigned ResultReg = I.getOperand(0).getReg();
1037
28
  RBI.constrainGenericRegister(
1038
28
      ResultReg,
1039
28
      *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI);
1040
28
  if (SETFOpc) {
1041
4
    MachineInstr &CmpInst =
1042
4
        *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1043
4
             .addReg(LhsReg)
1044
4
             .addReg(RhsReg);
1045
4
1046
4
    unsigned FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass);
1047
4
    unsigned FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass);
1048
4
    MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1049
4
                                  TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]);
1050
4
    MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1051
4
                                  TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]);
1052
4
    MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1053
4
                                  TII.get(SETFOpc[2]), ResultReg)
1054
4
                              .addReg(FlagReg1)
1055
4
                              .addReg(FlagReg2);
1056
4
    constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1057
4
    constrainSelectedInstRegOperands(Set1, TII, TRI, RBI);
1058
4
    constrainSelectedInstRegOperands(Set2, TII, TRI, RBI);
1059
4
    constrainSelectedInstRegOperands(Set3, TII, TRI, RBI);
1060
4
1061
4
    I.eraseFromParent();
1062
4
    return true;
1063
4
  }
1064
24
1065
24
  X86::CondCode CC;
1066
24
  bool SwapArgs;
1067
24
  std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1068
24
  assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1069
24
1070
24
  if (SwapArgs)
1071
8
    std::swap(LhsReg, RhsReg);
1072
24
1073
24
  // Emit a compare of LHS/RHS.
1074
24
  MachineInstr &CmpInst =
1075
24
      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1076
24
           .addReg(LhsReg)
1077
24
           .addReg(RhsReg);
1078
24
1079
24
  MachineInstr &Set =
1080
24
      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC);
1081
24
  constrainSelectedInstRegOperands(CmpInst, TII, TRI, RBI);
1082
24
  constrainSelectedInstRegOperands(Set, TII, TRI, RBI);
1083
24
  I.eraseFromParent();
1084
24
  return true;
1085
24
}
1086
1087
bool X86InstructionSelector::selectUadde(MachineInstr &I,
1088
                                         MachineRegisterInfo &MRI,
1089
4
                                         MachineFunction &MF) const {
1090
4
  assert((I.getOpcode() == TargetOpcode::G_UADDE) && "unexpected instruction");
1091
4
1092
4
  const unsigned DstReg = I.getOperand(0).getReg();
1093
4
  const unsigned CarryOutReg = I.getOperand(1).getReg();
1094
4
  const unsigned Op0Reg = I.getOperand(2).getReg();
1095
4
  const unsigned Op1Reg = I.getOperand(3).getReg();
1096
4
  unsigned CarryInReg = I.getOperand(4).getReg();
1097
4
1098
4
  const LLT DstTy = MRI.getType(DstReg);
1099
4
1100
4
  if (DstTy != LLT::scalar(32))
1101
0
    return false;
1102
4
1103
4
  // find CarryIn def instruction.
1104
4
  MachineInstr *Def = MRI.getVRegDef(CarryInReg);
1105
6
  while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1106
2
    CarryInReg = Def->getOperand(1).getReg();
1107
2
    Def = MRI.getVRegDef(CarryInReg);
1108
2
  }
1109
4
1110
4
  unsigned Opcode;
1111
4
  if (Def->getOpcode() == TargetOpcode::G_UADDE) {
1112
2
    // carry set by prev ADD.
1113
2
1114
2
    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), X86::EFLAGS)
1115
2
        .addReg(CarryInReg);
1116
2
1117
2
    if (!RBI.constrainGenericRegister(CarryInReg, X86::GR32RegClass, MRI))
1118
0
      return false;
1119
2
1120
2
    Opcode = X86::ADC32rr;
1121
2
  } else if (auto val = getConstantVRegVal(CarryInReg, MRI)) {
1122
2
    // carry is constant, support only 0.
1123
2
    if (*val != 0)
1124
0
      return false;
1125
2
1126
2
    Opcode = X86::ADD32rr;
1127
2
  } else
1128
0
    return false;
1129
4
1130
4
  MachineInstr &AddInst =
1131
4
      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1132
4
           .addReg(Op0Reg)
1133
4
           .addReg(Op1Reg);
1134
4
1135
4
  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
1136
4
      .addReg(X86::EFLAGS);
1137
4
1138
4
  if (!constrainSelectedInstRegOperands(AddInst, TII, TRI, RBI) ||
1139
4
      !RBI.constrainGenericRegister(CarryOutReg, X86::GR32RegClass, MRI))
1140
0
    return false;
1141
4
1142
4
  I.eraseFromParent();
1143
4
  return true;
1144
4
}
1145
1146
bool X86InstructionSelector::selectExtract(MachineInstr &I,
1147
                                           MachineRegisterInfo &MRI,
1148
70
                                           MachineFunction &MF) const {
1149
70
  assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1150
70
         "unexpected instruction");
1151
70
1152
70
  const unsigned DstReg = I.getOperand(0).getReg();
1153
70
  const unsigned SrcReg = I.getOperand(1).getReg();
1154
70
  int64_t Index = I.getOperand(2).getImm();
1155
70
1156
70
  const LLT DstTy = MRI.getType(DstReg);
1157
70
  const LLT SrcTy = MRI.getType(SrcReg);
1158
70
1159
70
  // Meanwile handle vector type only.
1160
70
  if (!DstTy.isVector())
1161
0
    return false;
1162
70
1163
70
  if (Index % DstTy.getSizeInBits() != 0)
1164
0
    return false; // Not extract subvector.
1165
70
1166
70
  if (Index == 0) {
1167
34
    // Replace by extract subreg copy.
1168
34
    if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1169
0
      return false;
1170
34
1171
34
    I.eraseFromParent();
1172
34
    return true;
1173
34
  }
1174
36
1175
36
  bool HasAVX = STI.hasAVX();
1176
36
  bool HasAVX512 = STI.hasAVX512();
1177
36
  bool HasVLX = STI.hasVLX();
1178
36
1179
36
  if (SrcTy.getSizeInBits() == 256 && 
DstTy.getSizeInBits() == 12828
) {
1180
28
    if (HasVLX)
1181
2
      I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
1182
26
    else if (HasAVX)
1183
26
      I.setDesc(TII.get(X86::VEXTRACTF128rr));
1184
0
    else
1185
0
      return false;
1186
8
  } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1187
8
    if (DstTy.getSizeInBits() == 128)
1188
5
      I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
1189
3
    else if (DstTy.getSizeInBits() == 256)
1190
3
      I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
1191
0
    else
1192
0
      return false;
1193
0
  } else
1194
0
    return false;
1195
36
1196
36
  // Convert to X86 VEXTRACT immediate.
1197
36
  Index = Index / DstTy.getSizeInBits();
1198
36
  I.getOperand(2).setImm(Index);
1199
36
1200
36
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1201
36
}
1202
1203
bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
1204
                                               MachineInstr &I,
1205
                                               MachineRegisterInfo &MRI,
1206
34
                                               MachineFunction &MF) const {
1207
34
  const LLT DstTy = MRI.getType(DstReg);
1208
34
  const LLT SrcTy = MRI.getType(SrcReg);
1209
34
  unsigned SubIdx = X86::NoSubRegister;
1210
34
1211
34
  if (!DstTy.isVector() || !SrcTy.isVector())
1212
0
    return false;
1213
34
1214
34
  assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1215
34
         "Incorrect Src/Dst register size");
1216
34
1217
34
  if (DstTy.getSizeInBits() == 128)
1218
31
    SubIdx = X86::sub_xmm;
1219
3
  else if (DstTy.getSizeInBits() == 256)
1220
3
    SubIdx = X86::sub_ymm;
1221
0
  else
1222
0
    return false;
1223
34
1224
34
  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1225
34
  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1226
34
1227
34
  SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
1228
34
1229
34
  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1230
34
      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1231
0
    LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
1232
0
    return false;
1233
0
  }
1234
34
1235
34
  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
1236
34
      .addReg(SrcReg, 0, SubIdx);
1237
34
1238
34
  return true;
1239
34
}
1240
1241
bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1242
                                              MachineInstr &I,
1243
                                              MachineRegisterInfo &MRI,
1244
20
                                              MachineFunction &MF) const {
1245
20
  const LLT DstTy = MRI.getType(DstReg);
1246
20
  const LLT SrcTy = MRI.getType(SrcReg);
1247
20
  unsigned SubIdx = X86::NoSubRegister;
1248
20
1249
20
  // TODO: support scalar types
1250
20
  if (!DstTy.isVector() || !SrcTy.isVector())
1251
0
    return false;
1252
20
1253
20
  assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1254
20
         "Incorrect Src/Dst register size");
1255
20
1256
20
  if (SrcTy.getSizeInBits() == 128)
1257
18
    SubIdx = X86::sub_xmm;
1258
2
  else if (SrcTy.getSizeInBits() == 256)
1259
2
    SubIdx = X86::sub_ymm;
1260
0
  else
1261
0
    return false;
1262
20
1263
20
  const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1264
20
  const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1265
20
1266
20
  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1267
20
      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1268
0
    LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1269
0
    return false;
1270
0
  }
1271
20
1272
20
  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1273
20
      .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1274
20
      .addReg(SrcReg);
1275
20
1276
20
  return true;
1277
20
}
1278
1279
bool X86InstructionSelector::selectInsert(MachineInstr &I,
1280
                                          MachineRegisterInfo &MRI,
1281
34
                                          MachineFunction &MF) const {
1282
34
  assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1283
34
1284
34
  const unsigned DstReg = I.getOperand(0).getReg();
1285
34
  const unsigned SrcReg = I.getOperand(1).getReg();
1286
34
  const unsigned InsertReg = I.getOperand(2).getReg();
1287
34
  int64_t Index = I.getOperand(3).getImm();
1288
34
1289
34
  const LLT DstTy = MRI.getType(DstReg);
1290
34
  const LLT InsertRegTy = MRI.getType(InsertReg);
1291
34
1292
34
  // Meanwile handle vector type only.
1293
34
  if (!DstTy.isVector())
1294
0
    return false;
1295
34
1296
34
  if (Index % InsertRegTy.getSizeInBits() != 0)
1297
0
    return false; // Not insert subvector.
1298
34
1299
34
  if (Index == 0 && 
MRI.getVRegDef(SrcReg)->isImplicitDef()8
) {
1300
4
    // Replace by subreg copy.
1301
4
    if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1302
0
      return false;
1303
4
1304
4
    I.eraseFromParent();
1305
4
    return true;
1306
4
  }
1307
30
1308
30
  bool HasAVX = STI.hasAVX();
1309
30
  bool HasAVX512 = STI.hasAVX512();
1310
30
  bool HasVLX = STI.hasVLX();
1311
30
1312
30
  if (DstTy.getSizeInBits() == 256 && 
InsertRegTy.getSizeInBits() == 12820
) {
1313
20
    if (HasVLX)
1314
4
      I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1315
16
    else if (HasAVX)
1316
16
      I.setDesc(TII.get(X86::VINSERTF128rr));
1317
0
    else
1318
0
      return false;
1319
10
  } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1320
10
    if (InsertRegTy.getSizeInBits() == 128)
1321
6
      I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1322
4
    else if (InsertRegTy.getSizeInBits() == 256)
1323
4
      I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1324
0
    else
1325
0
      return false;
1326
0
  } else
1327
0
    return false;
1328
30
1329
30
  // Convert to X86 VINSERT immediate.
1330
30
  Index = Index / InsertRegTy.getSizeInBits();
1331
30
1332
30
  I.getOperand(3).setImm(Index);
1333
30
1334
30
  return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
1335
30
}
1336
1337
bool X86InstructionSelector::selectUnmergeValues(
1338
    MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF,
1339
28
    CodeGenCoverage &CoverageInfo) const {
1340
28
  assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1341
28
         "unexpected instruction");
1342
28
1343
28
  // Split to extracts.
1344
28
  unsigned NumDefs = I.getNumOperands() - 1;
1345
28
  unsigned SrcReg = I.getOperand(NumDefs).getReg();
1346
28
  unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1347
28
1348
86
  for (unsigned Idx = 0; Idx < NumDefs; 
++Idx58
) {
1349
58
    MachineInstr &ExtrInst =
1350
58
        *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1351
58
                 TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1352
58
             .addReg(SrcReg)
1353
58
             .addImm(Idx * DefSize);
1354
58
1355
58
    if (!select(ExtrInst, CoverageInfo))
1356
0
      return false;
1357
58
  }
1358
28
1359
28
  I.eraseFromParent();
1360
28
  return true;
1361
28
}
1362
1363
bool X86InstructionSelector::selectMergeValues(
1364
    MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF,
1365
16
    CodeGenCoverage &CoverageInfo) const {
1366
16
  assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1367
16
          I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1368
16
         "unexpected instruction");
1369
16
1370
16
  // Split to inserts.
1371
16
  unsigned DstReg = I.getOperand(0).getReg();
1372
16
  unsigned SrcReg0 = I.getOperand(1).getReg();
1373
16
1374
16
  const LLT DstTy = MRI.getType(DstReg);
1375
16
  const LLT SrcTy = MRI.getType(SrcReg0);
1376
16
  unsigned SrcSize = SrcTy.getSizeInBits();
1377
16
1378
16
  const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1379
16
1380
16
  // For the first src use insertSubReg.
1381
16
  unsigned DefReg = MRI.createGenericVirtualRegister(DstTy);
1382
16
  MRI.setRegBank(DefReg, RegBank);
1383
16
  if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1384
0
    return false;
1385
16
1386
34
  
for (unsigned Idx = 2; 16
Idx < I.getNumOperands();
++Idx18
) {
1387
18
    unsigned Tmp = MRI.createGenericVirtualRegister(DstTy);
1388
18
    MRI.setRegBank(Tmp, RegBank);
1389
18
1390
18
    MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1391
18
                                        TII.get(TargetOpcode::G_INSERT), Tmp)
1392
18
                                    .addReg(DefReg)
1393
18
                                    .addReg(I.getOperand(Idx).getReg())
1394
18
                                    .addImm((Idx - 1) * SrcSize);
1395
18
1396
18
    DefReg = Tmp;
1397
18
1398
18
    if (!select(InsertInst, CoverageInfo))
1399
0
      return false;
1400
18
  }
1401
16
1402
16
  MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1403
16
                                    TII.get(TargetOpcode::COPY), DstReg)
1404
16
                                .addReg(DefReg);
1405
16
1406
16
  if (!select(CopyInst, CoverageInfo))
1407
0
    return false;
1408
16
1409
16
  I.eraseFromParent();
1410
16
  return true;
1411
16
}
1412
1413
bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1414
                                              MachineRegisterInfo &MRI,
1415
19
                                              MachineFunction &MF) const {
1416
19
  assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1417
19
1418
19
  const unsigned CondReg = I.getOperand(0).getReg();
1419
19
  MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1420
19
1421
19
  MachineInstr &TestInst =
1422
19
      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1423
19
           .addReg(CondReg)
1424
19
           .addImm(1);
1425
19
  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1))
1426
19
      .addMBB(DestMBB).addImm(X86::COND_NE);
1427
19
1428
19
  constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1429
19
1430
19
  I.eraseFromParent();
1431
19
  return true;
1432
19
}
1433
1434
bool X86InstructionSelector::materializeFP(MachineInstr &I,
1435
                                           MachineRegisterInfo &MRI,
1436
15
                                           MachineFunction &MF) const {
1437
15
  assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1438
15
         "unexpected instruction");
1439
15
1440
15
  // Can't handle alternate code models yet.
1441
15
  CodeModel::Model CM = TM.getCodeModel();
1442
15
  if (CM != CodeModel::Small && 
CM != CodeModel::Large6
)
1443
0
    return false;
1444
15
1445
15
  const unsigned DstReg = I.getOperand(0).getReg();
1446
15
  const LLT DstTy = MRI.getType(DstReg);
1447
15
  const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1448
15
  unsigned Align = DstTy.getSizeInBits();
1449
15
  const DebugLoc &DbgLoc = I.getDebugLoc();
1450
15
1451
15
  unsigned Opc = getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Align);
1452
15
1453
15
  // Create the load from the constant pool.
1454
15
  const ConstantFP *CFP = I.getOperand(1).getFPImm();
1455
15
  unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Align);
1456
15
  MachineInstr *LoadInst = nullptr;
1457
15
  unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1458
15
1459
15
  if (CM == CodeModel::Large && 
STI.is64Bit()6
) {
1460
3
    // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1461
3
    // they cannot be folded into immediate fields.
1462
3
1463
3
    unsigned AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1464
3
    BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1465
3
        .addConstantPoolIndex(CPI, 0, OpFlag);
1466
3
1467
3
    MachineMemOperand *MMO = MF.getMachineMemOperand(
1468
3
        MachinePointerInfo::getConstantPool(MF), MachineMemOperand::MOLoad,
1469
3
        MF.getDataLayout().getPointerSize(), Align);
1470
3
1471
3
    LoadInst =
1472
3
        addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1473
3
                     AddrReg)
1474
3
            .addMemOperand(MMO);
1475
3
1476
12
  } else if (CM == CodeModel::Small || 
!STI.is64Bit()3
) {
1477
12
    // Handle the case when globals fit in our immediate field.
1478
12
    // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1479
12
1480
12
    // x86-32 PIC requires a PIC base register for constant pools.
1481
12
    unsigned PICBase = 0;
1482
12
    if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1483
0
      // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1484
0
      // In DAGISEL the code that initialize it generated by the CGBR pass.
1485
0
      return false; // TODO support the mode.
1486
12
    } else if (STI.is64Bit() && 
TM.getCodeModel() == CodeModel::Small6
)
1487
6
      PICBase = X86::RIP;
1488
12
1489
12
    LoadInst = addConstantPoolReference(
1490
12
        BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1491
12
        OpFlag);
1492
12
  } else
1493
0
    return false;
1494
15
1495
15
  constrainSelectedInstRegOperands(*LoadInst, TII, TRI, RBI);
1496
15
  I.eraseFromParent();
1497
15
  return true;
1498
15
}
1499
1500
bool X86InstructionSelector::selectImplicitDefOrPHI(
1501
22
    MachineInstr &I, MachineRegisterInfo &MRI) const {
1502
22
  assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1503
22
          I.getOpcode() == TargetOpcode::G_PHI) &&
1504
22
         "unexpected instruction");
1505
22
1506
22
  unsigned DstReg = I.getOperand(0).getReg();
1507
22
1508
22
  if (!MRI.getRegClassOrNull(DstReg)) {
1509
13
    const LLT DstTy = MRI.getType(DstReg);
1510
13
    const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1511
13
1512
13
    if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1513
0
      LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1514
0
                        << " operand\n");
1515
0
      return false;
1516
0
    }
1517
22
  }
1518
22
1519
22
  if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1520
10
    I.setDesc(TII.get(X86::IMPLICIT_DEF));
1521
12
  else
1522
12
    I.setDesc(TII.get(X86::PHI));
1523
22
1524
22
  return true;
1525
22
}
1526
1527
// Currently GlobalIsel TableGen generates patterns for shift imm and shift 1,
1528
// but with shiftCount i8. In G_LSHR/G_ASHR/G_SHL like LLVM-IR both arguments
1529
// has the same type, so for now only shift i8 can use auto generated
1530
// TableGen patterns.
1531
bool X86InstructionSelector::selectShift(MachineInstr &I,
1532
                                         MachineRegisterInfo &MRI,
1533
53
                                         MachineFunction &MF) const {
1534
53
1535
53
  assert((I.getOpcode() == TargetOpcode::G_SHL ||
1536
53
          I.getOpcode() == TargetOpcode::G_ASHR ||
1537
53
          I.getOpcode() == TargetOpcode::G_LSHR) &&
1538
53
         "unexpected instruction");
1539
53
1540
53
  unsigned DstReg = I.getOperand(0).getReg();
1541
53
  const LLT DstTy = MRI.getType(DstReg);
1542
53
  const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1543
53
1544
53
  const static struct ShiftEntry {
1545
53
    unsigned SizeInBits;
1546
53
    unsigned OpLSHR;
1547
53
    unsigned OpASHR;
1548
53
    unsigned OpSHL;
1549
53
  } OpTable[] = {
1550
53
      {8, X86::SHR8rCL, X86::SAR8rCL, X86::SHL8rCL},     // i8
1551
53
      {16, X86::SHR16rCL, X86::SAR16rCL, X86::SHL16rCL}, // i16
1552
53
      {32, X86::SHR32rCL, X86::SAR32rCL, X86::SHL32rCL}, // i32
1553
53
      {64, X86::SHR64rCL, X86::SAR64rCL, X86::SHL64rCL}  // i64
1554
53
  };
1555
53
1556
53
  if (DstRB.getID() != X86::GPRRegBankID)
1557
0
    return false;
1558
53
1559
53
  auto ShiftEntryIt = std::find_if(
1560
142
      std::begin(OpTable), std::end(OpTable), [DstTy](const ShiftEntry &El) {
1561
142
        return El.SizeInBits == DstTy.getSizeInBits();
1562
142
      });
1563
53
  if (ShiftEntryIt == std::end(OpTable))
1564
0
    return false;
1565
53
1566
53
  unsigned Opcode = 0;
1567
53
  switch (I.getOpcode()) {
1568
53
  case TargetOpcode::G_SHL:
1569
21
    Opcode = ShiftEntryIt->OpSHL;
1570
21
    break;
1571
53
  case TargetOpcode::G_ASHR:
1572
19
    Opcode = ShiftEntryIt->OpASHR;
1573
19
    break;
1574
53
  case TargetOpcode::G_LSHR:
1575
13
    Opcode = ShiftEntryIt->OpLSHR;
1576
13
    break;
1577
53
  default:
1578
0
    return false;
1579
53
  }
1580
53
1581
53
  unsigned Op0Reg = I.getOperand(1).getReg();
1582
53
  unsigned Op1Reg = I.getOperand(2).getReg();
1583
53
1584
53
  assert(MRI.getType(Op1Reg).getSizeInBits() == 8);
1585
53
1586
53
  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1587
53
          X86::CL)
1588
53
    .addReg(Op1Reg);
1589
53
1590
53
  MachineInstr &ShiftInst =
1591
53
      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1592
53
           .addReg(Op0Reg);
1593
53
1594
53
  constrainSelectedInstRegOperands(ShiftInst, TII, TRI, RBI);
1595
53
  I.eraseFromParent();
1596
53
  return true;
1597
53
}
1598
1599
bool X86InstructionSelector::selectDivRem(MachineInstr &I,
1600
                                          MachineRegisterInfo &MRI,
1601
28
                                          MachineFunction &MF) const {
1602
28
  // The implementation of this function is taken from X86FastISel.
1603
28
  assert((I.getOpcode() == TargetOpcode::G_SDIV ||
1604
28
          I.getOpcode() == TargetOpcode::G_SREM ||
1605
28
          I.getOpcode() == TargetOpcode::G_UDIV ||
1606
28
          I.getOpcode() == TargetOpcode::G_UREM) &&
1607
28
         "unexpected instruction");
1608
28
1609
28
  const unsigned DstReg = I.getOperand(0).getReg();
1610
28
  const unsigned Op1Reg = I.getOperand(1).getReg();
1611
28
  const unsigned Op2Reg = I.getOperand(2).getReg();
1612
28
1613
28
  const LLT RegTy = MRI.getType(DstReg);
1614
28
  assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1615
28
         "Arguments and return value types must match");
1616
28
1617
28
  const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI);
1618
28
  if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1619
0
    return false;
1620
28
1621
28
  const static unsigned NumTypes = 4; // i8, i16, i32, i64
1622
28
  const static unsigned NumOps = 4;   // SDiv, SRem, UDiv, URem
1623
28
  const static bool S = true;         // IsSigned
1624
28
  const static bool U = false;        // !IsSigned
1625
28
  const static unsigned Copy = TargetOpcode::COPY;
1626
28
  // For the X86 IDIV instruction, in most cases the dividend
1627
28
  // (numerator) must be in a specific register pair highreg:lowreg,
1628
28
  // producing the quotient in lowreg and the remainder in highreg.
1629
28
  // For most data types, to set up the instruction, the dividend is
1630
28
  // copied into lowreg, and lowreg is sign-extended into highreg.  The
1631
28
  // exception is i8, where the dividend is defined as a single register rather
1632
28
  // than a register pair, and we therefore directly sign-extend the dividend
1633
28
  // into lowreg, instead of copying, and ignore the highreg.
1634
28
  const static struct DivRemEntry {
1635
28
    // The following portion depends only on the data type.
1636
28
    unsigned SizeInBits;
1637
28
    unsigned LowInReg;  // low part of the register pair
1638
28
    unsigned HighInReg; // high part of the register pair
1639
28
    // The following portion depends on both the data type and the operation.
1640
28
    struct DivRemResult {
1641
28
      unsigned OpDivRem;        // The specific DIV/IDIV opcode to use.
1642
28
      unsigned OpSignExtend;    // Opcode for sign-extending lowreg into
1643
28
                                // highreg, or copying a zero into highreg.
1644
28
      unsigned OpCopy;          // Opcode for copying dividend into lowreg, or
1645
28
                                // zero/sign-extending into lowreg for i8.
1646
28
      unsigned DivRemResultReg; // Register containing the desired result.
1647
28
      bool IsOpSigned;          // Whether to use signed or unsigned form.
1648
28
    } ResultTable[NumOps];
1649
28
  } OpTable[NumTypes] = {
1650
28
      {8,
1651
28
       X86::AX,
1652
28
       0,
1653
28
       {
1654
28
           {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv
1655
28
           {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem
1656
28
           {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U},  // UDiv
1657
28
           {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U},  // URem
1658
28
       }},                                                // i8
1659
28
      {16,
1660
28
       X86::AX,
1661
28
       X86::DX,
1662
28
       {
1663
28
           {X86::IDIV16r, X86::CWD, Copy, X86::AX, S},    // SDiv
1664
28
           {X86::IDIV16r, X86::CWD, Copy, X86::DX, S},    // SRem
1665
28
           {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv
1666
28
           {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem
1667
28
       }},                                                // i16
1668
28
      {32,
1669
28
       X86::EAX,
1670
28
       X86::EDX,
1671
28
       {
1672
28
           {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S},    // SDiv
1673
28
           {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S},    // SRem
1674
28
           {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv
1675
28
           {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem
1676
28
       }},                                                 // i32
1677
28
      {64,
1678
28
       X86::RAX,
1679
28
       X86::RDX,
1680
28
       {
1681
28
           {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S},    // SDiv
1682
28
           {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S},    // SRem
1683
28
           {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv
1684
28
           {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem
1685
28
       }},                                                 // i64
1686
28
  };
1687
28
1688
28
  auto OpEntryIt = std::find_if(std::begin(OpTable), std::end(OpTable),
1689
64
                                [RegTy](const DivRemEntry &El) {
1690
64
                                  return El.SizeInBits == RegTy.getSizeInBits();
1691
64
                                });
1692
28
  if (OpEntryIt == std::end(OpTable))
1693
0
    return false;
1694
28
1695
28
  unsigned OpIndex;
1696
28
  switch (I.getOpcode()) {
1697
28
  default:
1698
0
    llvm_unreachable("Unexpected div/rem opcode");
1699
28
  case TargetOpcode::G_SDIV:
1700
7
    OpIndex = 0;
1701
7
    break;
1702
28
  case TargetOpcode::G_SREM:
1703
7
    OpIndex = 1;
1704
7
    break;
1705
28
  case TargetOpcode::G_UDIV:
1706
7
    OpIndex = 2;
1707
7
    break;
1708
28
  case TargetOpcode::G_UREM:
1709
7
    OpIndex = 3;
1710
7
    break;
1711
28
  }
1712
28
1713
28
  const DivRemEntry &TypeEntry = *OpEntryIt;
1714
28
  const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1715
28
1716
28
  const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB);
1717
28
  if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) ||
1718
28
      !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) ||
1719
28
      !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) {
1720
0
    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1721
0
                      << " operand\n");
1722
0
    return false;
1723
0
  }
1724
28
1725
28
  // Move op1 into low-order input register.
1726
28
  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy),
1727
28
          TypeEntry.LowInReg)
1728
28
      .addReg(Op1Reg);
1729
28
  // Zero-extend or sign-extend into high-order input register.
1730
28
  if (OpEntry.OpSignExtend) {
1731
20
    if (OpEntry.IsOpSigned)
1732
10
      BuildMI(*I.getParent(), I, I.getDebugLoc(),
1733
10
              TII.get(OpEntry.OpSignExtend));
1734
10
    else {
1735
10
      unsigned Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass);
1736
10
      BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0),
1737
10
              Zero32);
1738
10
1739
10
      // Copy the zero into the appropriate sub/super/identical physical
1740
10
      // register. Unfortunately the operations needed are not uniform enough
1741
10
      // to fit neatly into the table above.
1742
10
      if (RegTy.getSizeInBits() == 16) {
1743
4
        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1744
4
                TypeEntry.HighInReg)
1745
4
            .addReg(Zero32, 0, X86::sub_16bit);
1746
6
      } else if (RegTy.getSizeInBits() == 32) {
1747
4
        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1748
4
                TypeEntry.HighInReg)
1749
4
            .addReg(Zero32);
1750
4
      } else 
if (2
RegTy.getSizeInBits() == 642
) {
1751
2
        BuildMI(*I.getParent(), I, I.getDebugLoc(),
1752
2
                TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1753
2
            .addImm(0)
1754
2
            .addReg(Zero32)
1755
2
            .addImm(X86::sub_32bit);
1756
2
      }
1757
10
    }
1758
20
  }
1759
28
  // Generate the DIV/IDIV instruction.
1760
28
  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpDivRem))
1761
28
      .addReg(Op2Reg);
1762
28
  // For i8 remainder, we can't reference ah directly, as we'll end
1763
28
  // up with bogus copies like %r9b = COPY %ah. Reference ax
1764
28
  // instead to prevent ah references in a rex instruction.
1765
28
  //
1766
28
  // The current assumption of the fast register allocator is that isel
1767
28
  // won't generate explicit references to the GR8_NOREX registers. If
1768
28
  // the allocator and/or the backend get enhanced to be more robust in
1769
28
  // that regard, this can be, and should be, removed.
1770
28
  if ((I.getOpcode() == Instruction::SRem ||
1771
28
       I.getOpcode() == Instruction::URem) &&
1772
28
      
OpEntry.DivRemResultReg == X86::AH0
&&
STI.is64Bit()0
) {
1773
0
    unsigned SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1774
0
    unsigned ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1775
0
    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg)
1776
0
        .addReg(X86::AX);
1777
0
1778
0
    // Shift AX right by 8 bits instead of using AH.
1779
0
    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri),
1780
0
            ResultSuperReg)
1781
0
        .addReg(SourceSuperReg)
1782
0
        .addImm(8);
1783
0
1784
0
    // Now reference the 8-bit subreg of the result.
1785
0
    BuildMI(*I.getParent(), I, I.getDebugLoc(),
1786
0
            TII.get(TargetOpcode::SUBREG_TO_REG))
1787
0
        .addDef(DstReg)
1788
0
        .addImm(0)
1789
0
        .addReg(ResultSuperReg)
1790
0
        .addImm(X86::sub_8bit);
1791
28
  } else {
1792
28
    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1793
28
            DstReg)
1794
28
        .addReg(OpEntry.DivRemResultReg);
1795
28
  }
1796
28
  I.eraseFromParent();
1797
28
  return true;
1798
28
}
1799
1800
bool X86InstructionSelector::selectIntrinsicWSideEffects(
1801
2
    MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const {
1802
2
1803
2
  assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
1804
2
         "unexpected instruction");
1805
2
1806
2
  if (I.getOperand(0).getIntrinsicID() != Intrinsic::trap)
1807
1
    return false;
1808
1
1809
1
  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TRAP));
1810
1
1811
1
  I.eraseFromParent();
1812
1
  return true;
1813
1
}
1814
1815
InstructionSelector *
1816
llvm::createX86InstructionSelector(const X86TargetMachine &TM,
1817
                                   X86Subtarget &Subtarget,
1818
15.2k
                                   X86RegisterBankInfo &RBI) {
1819
15.2k
  return new X86InstructionSelector(TM, Subtarget, RBI);
1820
15.2k
}