Coverage Report

Created: 2018-11-16 02:38

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
Line
Count
Source (jump to first uncovered line)
1
//===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
/// \file This file declares the API for the instruction selector.
11
/// This class is responsible for selecting machine instructions.
12
/// It's implemented by the target. It's used by the InstructionSelect pass.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17
#define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
18
19
#include "llvm/ADT/SmallVector.h"
20
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
21
#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
22
#include "llvm/CodeGen/GlobalISel/Utils.h"
23
#include "llvm/CodeGen/MachineInstrBuilder.h"
24
#include "llvm/CodeGen/MachineOperand.h"
25
#include "llvm/CodeGen/MachineRegisterInfo.h"
26
#include "llvm/CodeGen/TargetInstrInfo.h"
27
#include "llvm/CodeGen/TargetOpcodes.h"
28
#include "llvm/CodeGen/TargetRegisterInfo.h"
29
#include "llvm/IR/Constants.h"
30
#include "llvm/Support/Debug.h"
31
#include "llvm/Support/ErrorHandling.h"
32
#include "llvm/Support/raw_ostream.h"
33
#include <cassert>
34
#include <cstddef>
35
#include <cstdint>
36
37
namespace llvm {
38
39
/// GlobalISel PatFrag Predicates
40
enum {
41
  GIPFP_I64_Invalid = 0,
42
  GIPFP_APInt_Invalid = 0,
43
  GIPFP_APFloat_Invalid = 0,
44
  GIPFP_MI_Invalid = 0,
45
};
46
47
template <class TgtInstructionSelector, class PredicateBitset,
48
          class ComplexMatcherMemFn, class CustomRendererFn>
49
bool InstructionSelector::executeMatchTable(
50
    TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
51
    const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52
        &ISelInfo,
53
    const int64_t *MatchTable, const TargetInstrInfo &TII,
54
    MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
55
    const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
56
4.53M
    CodeGenCoverage &CoverageInfo) const {
57
4.53M
58
4.53M
  uint64_t CurrentIdx = 0;
59
4.53M
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
4.53M
61
4.53M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
6.65M
  auto handleReject = [&]() -> RejectAction {
63
6.65M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
6.65M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
6.65M
    if (OnFailResumeAt.empty())
66
2.68M
      return RejectAndGiveUp;
67
3.97M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
3.97M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
3.97M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
3.97M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
3.97M
    return RejectAndResume;
72
3.97M
  };
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<19ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<19ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<19ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
6.64M
  auto handleReject = [&]() -> RejectAction {
63
6.64M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
6.64M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
6.64M
    if (OnFailResumeAt.empty())
66
2.68M
      return RejectAndGiveUp;
67
3.96M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
3.96M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
3.96M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
3.96M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
3.96M
    return RejectAndResume;
72
3.96M
  };
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<32ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
155
  auto handleReject = [&]() -> RejectAction {
63
155
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
155
                    dbgs() << CurrentIdx << ": Rejected\n");
65
155
    if (OnFailResumeAt.empty())
66
0
      return RejectAndGiveUp;
67
155
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
155
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
155
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
155
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
155
    return RejectAndResume;
72
155
  };
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<64ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
2.27k
  auto handleReject = [&]() -> RejectAction {
63
2.27k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
2.27k
                    dbgs() << CurrentIdx << ": Rejected\n");
65
2.27k
    if (OnFailResumeAt.empty())
66
473
      return RejectAndGiveUp;
67
1.80k
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
1.80k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
1.80k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
1.80k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
1.80k
    return RejectAndResume;
72
1.80k
  };
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<41ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
420
  auto handleReject = [&]() -> RejectAction {
63
420
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
420
                    dbgs() << CurrentIdx << ": Rejected\n");
65
420
    if (OnFailResumeAt.empty())
66
80
      return RejectAndGiveUp;
67
340
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
340
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
340
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
340
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
340
    return RejectAndResume;
72
340
  };
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<113ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<113ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<113ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
3.86k
  auto handleReject = [&]() -> RejectAction {
63
3.86k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
3.86k
                    dbgs() << CurrentIdx << ": Rejected\n");
65
3.86k
    if (OnFailResumeAt.empty())
66
1.37k
      return RejectAndGiveUp;
67
2.49k
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
2.49k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
2.49k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
2.49k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
2.49k
    return RejectAndResume;
72
2.49k
  };
73
4.53M
74
38.5M
  while (true) {
75
38.5M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
38.5M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
38.5M
    switch (MatcherOpcode) {
78
38.5M
    case GIM_Try: {
79
5.17M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
5.17M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
5.17M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
5.17M
      break;
83
38.5M
    }
84
38.5M
85
38.5M
    case GIM_RecordInsn: {
86
1.71M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
1.71M
      int64_t InsnID = MatchTable[CurrentIdx++];
88
1.71M
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
1.71M
90
1.71M
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
1.71M
      // any attempt to modify it.
92
1.71M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
1.71M
94
1.71M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
1.71M
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
1.71M
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
1.71M
110
1.71M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
1.71M
      if ((size_t)NewInsnID < State.MIs.size())
112
1.15M
        State.MIs[NewInsnID] = NewMI;
113
558k
      else {
114
558k
        assert((size_t)NewInsnID == State.MIs.size() &&
115
558k
               "Expected to store MIs in order");
116
558k
        State.MIs.push_back(NewMI);
117
558k
      }
118
1.71M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
1.71M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
1.71M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
1.71M
                             << ")\n");
122
1.71M
      break;
123
1.71M
    }
124
1.71M
125
1.71M
    case GIM_CheckFeatures: {
126
423k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
423k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
423k
                      dbgs() << CurrentIdx
129
423k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
423k
                             << ExpectedBitsetID << ")\n");
131
423k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
423k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
47.8k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
423k
      }
136
423k
      break;
137
423k
    }
138
423k
139
1.71M
    case GIM_CheckOpcode: {
140
1.71M
      int64_t InsnID = MatchTable[CurrentIdx++];
141
1.71M
      int64_t Expected = MatchTable[CurrentIdx++];
142
1.71M
143
1.71M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
1.71M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
1.71M
146
1.71M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
1.71M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
1.71M
                             << "], ExpectedOpcode=" << Expected
149
1.71M
                             << ") // Got=" << Opcode << "\n");
150
1.71M
      if (Opcode != Expected) {
151
1.64M
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
1.71M
      }
154
1.71M
      break;
155
1.71M
    }
156
1.71M
157
4.53M
    case GIM_SwitchOpcode: {
158
4.53M
      int64_t InsnID = MatchTable[CurrentIdx++];
159
4.53M
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
4.53M
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
4.53M
      int64_t Default = MatchTable[CurrentIdx++];
162
4.53M
163
4.53M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
4.53M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
4.53M
166
4.53M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
4.53M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
4.53M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
4.53M
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
4.53M
      });
171
4.53M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
2
        CurrentIdx = Default;
173
2
        break;
174
2
      }
175
4.53M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
4.53M
      if (!CurrentIdx) {
177
1.46M
        CurrentIdx = Default;
178
1.46M
  break;
179
1.46M
      }
180
3.06M
      OnFailResumeAt.push_back(Default);
181
3.06M
      break;
182
3.06M
    }
183
3.06M
184
3.06M
    case GIM_SwitchType: {
185
2.65M
      int64_t InsnID = MatchTable[CurrentIdx++];
186
2.65M
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
2.65M
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
2.65M
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
2.65M
      int64_t Default = MatchTable[CurrentIdx++];
190
2.65M
191
2.65M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
2.65M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
2.65M
194
2.65M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
2.65M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
2.65M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
2.65M
               << UpperBound << "), Default=" << Default
198
2.65M
               << ", JumpTable...) // Got=";
199
2.65M
        if (!MO.isReg())
200
2.65M
          dbgs() << "Not a VReg\n";
201
2.65M
        else
202
2.65M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
2.65M
      });
204
2.65M
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
2.65M
      const LLT Ty = MRI.getType(MO.getReg());
209
2.65M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
2.65M
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
979k
        CurrentIdx = Default;
212
979k
        break;
213
979k
      }
214
1.67M
      const int64_t TypeID = TyI->second;
215
1.67M
      if (TypeID < LowerBound || 
UpperBound <= TypeID1.56M
) {
216
113k
        CurrentIdx = Default;
217
113k
        break;
218
113k
      }
219
1.56M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
1.56M
      if (!CurrentIdx) {
221
338
        CurrentIdx = Default;
222
338
        break;
223
338
      }
224
1.56M
      OnFailResumeAt.push_back(Default);
225
1.56M
      break;
226
1.56M
    }
227
1.56M
228
1.56M
    case GIM_CheckNumOperands: {
229
51.5k
      int64_t InsnID = MatchTable[CurrentIdx++];
230
51.5k
      int64_t Expected = MatchTable[CurrentIdx++];
231
51.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
51.5k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
51.5k
                             << InsnID << "], Expected=" << Expected << ")\n");
234
51.5k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
51.5k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
46.2k
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
51.5k
      }
239
51.5k
      break;
240
51.5k
    }
241
51.5k
    case GIM_CheckI64ImmPredicate: {
242
23.7k
      int64_t InsnID = MatchTable[CurrentIdx++];
243
23.7k
      int64_t Predicate = MatchTable[CurrentIdx++];
244
23.7k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
23.7k
                      dbgs()
246
23.7k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
23.7k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
23.7k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
23.7k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
23.7k
             "Expected G_CONSTANT");
251
23.7k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
23.7k
      int64_t Value = 0;
253
23.7k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
23.7k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
2
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
2
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
2
      else
258
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
259
23.7k
260
23.7k
      if (!testImmPredicate_I64(Predicate, Value))
261
20
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
23.7k
      break;
264
23.7k
    }
265
23.7k
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
2.71k
    case GIM_CheckAPFloatImmPredicate: {
288
2.71k
      int64_t InsnID = MatchTable[CurrentIdx++];
289
2.71k
      int64_t Predicate = MatchTable[CurrentIdx++];
290
2.71k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
2.71k
                      dbgs()
292
2.71k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
2.71k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
2.71k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
2.71k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
2.71k
             "Expected G_FCONSTANT");
297
2.71k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
2.71k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
2.71k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
2.71k
301
2.71k
      if (!testImmPredicate_APFloat(Predicate, Value))
302
1.81k
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
2.71k
      break;
305
2.71k
    }
306
2.71k
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
451k
    case GIM_CheckAtomicOrdering: {
322
451k
      int64_t InsnID = MatchTable[CurrentIdx++];
323
451k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
451k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
451k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
451k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
451k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
451k
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
451k
332
451k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
451k
        if (MMO->getOrdering() != Ordering)
334
2.84k
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
451k
      break;
337
451k
    }
338
451k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
25.1k
    case GIM_CheckMemorySizeEqualTo: {
375
25.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
376
25.1k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
25.1k
      uint64_t Size = MatchTable[CurrentIdx++];
378
25.1k
379
25.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
25.1k
                      dbgs() << CurrentIdx
381
25.1k
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
25.1k
                             << "]->memoperands() + " << MMOIdx
383
25.1k
                             << ", Size=" << Size << ")\n");
384
25.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
25.1k
386
25.1k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
25.1k
392
25.1k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
25.1k
394
25.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
25.1k
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
25.1k
                             << " bytes\n");
397
25.1k
      if (MMO->getSize() != Size)
398
11.5k
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
25.1k
401
25.1k
      break;
402
25.1k
    }
403
281k
    case GIM_CheckMemorySizeEqualToLLT:
404
281k
    case GIM_CheckMemorySizeLessThanLLT:
405
281k
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
281k
      int64_t InsnID = MatchTable[CurrentIdx++];
407
281k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
281k
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
281k
410
281k
      DEBUG_WITH_TYPE(
411
281k
          TgtInstructionSelector::getName(),
412
281k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
281k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
281k
                         ? "EqualTo"
415
281k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
281k
                               ? "GreaterThan"
417
281k
                               : "LessThan")
418
281k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
281k
                 << ", OpIdx=" << OpIdx << ")\n");
420
281k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
281k
422
281k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
281k
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
281k
431
281k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
281k
437
281k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
281k
439
281k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
281k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
281k
          
MMO->getSize() * 8 != Size281k
) {
442
80
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
281k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
281k
                 
MMO->getSize() * 8 >= Size104
) {
446
36
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
281k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
281k
                 
MMO->getSize() * 8 <= Size0
)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
281k
453
281k
      break;
454
281k
    }
455
1.05M
    case GIM_CheckType: {
456
1.05M
      int64_t InsnID = MatchTable[CurrentIdx++];
457
1.05M
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
1.05M
      int64_t TypeID = MatchTable[CurrentIdx++];
459
1.05M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
1.05M
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
1.05M
                             << "]->getOperand(" << OpIdx
462
1.05M
                             << "), TypeID=" << TypeID << ")\n");
463
1.05M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
1.05M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
1.05M
      if (!MO.isReg() ||
466
1.05M
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
26.9k
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
1.05M
      }
470
1.05M
      break;
471
1.05M
    }
472
1.05M
    case GIM_CheckPointerToAny: {
473
441k
      int64_t InsnID = MatchTable[CurrentIdx++];
474
441k
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
441k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
441k
477
441k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
441k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
441k
                             << InsnID << "]->getOperand(" << OpIdx
480
441k
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
441k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
441k
      // iPTR must be looked up in the target.
483
441k
      if (SizeInBits == 0) {
484
440k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
440k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
440k
      }
487
441k
488
441k
      assert(SizeInBits != 0 && "Pointer size must be known");
489
441k
490
441k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
441k
      if (MO.isReg()) {
492
441k
        const LLT &Ty = MRI.getType(MO.getReg());
493
441k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
441k
499
441k
      break;
500
441k
    }
501
4.05M
    case GIM_CheckRegBankForClass: {
502
4.05M
      int64_t InsnID = MatchTable[CurrentIdx++];
503
4.05M
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
4.05M
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
4.05M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
4.05M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
4.05M
                             << InsnID << "]->getOperand(" << OpIdx
508
4.05M
                             << "), RCEnum=" << RCEnum << ")\n");
509
4.05M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
4.05M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
4.05M
      if (!MO.isReg() ||
512
4.05M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
4.05M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
345k
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
4.05M
      }
517
4.05M
      break;
518
4.05M
    }
519
4.05M
520
4.05M
    case GIM_CheckComplexPattern: {
521
730k
      int64_t InsnID = MatchTable[CurrentIdx++];
522
730k
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
730k
      int64_t RendererID = MatchTable[CurrentIdx++];
524
730k
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
730k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
730k
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
730k
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
730k
                             << "]->getOperand(" << OpIdx
529
730k
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
730k
                             << ")\n");
531
730k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
730k
      // FIXME: Use std::invoke() when it's available.
533
730k
      ComplexRendererFns Renderer =
534
730k
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
730k
              State.MIs[InsnID]->getOperand(OpIdx));
536
730k
      if (Renderer.hasValue())
537
492k
        State.Renderers[RendererID] = Renderer.getValue();
538
237k
      else
539
237k
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
730k
      break;
542
730k
    }
543
730k
544
730k
    case GIM_CheckConstantInt: {
545
82.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
546
82.2k
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
82.2k
      int64_t Value = MatchTable[CurrentIdx++];
548
82.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
82.2k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
82.2k
                             << InsnID << "]->getOperand(" << OpIdx
551
82.2k
                             << "), Value=" << Value << ")\n");
552
82.2k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
82.2k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
82.2k
      if (MO.isReg()) {
555
82.2k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
82.2k
        LLT Ty = MRI.getType(MO.getReg());
557
82.2k
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
82.2k
559
82.2k
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
64.7k
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
82.2k
566
82.2k
      break;
567
82.2k
    }
568
82.2k
569
82.2k
    case GIM_CheckLiteralInt: {
570
54
      int64_t InsnID = MatchTable[CurrentIdx++];
571
54
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
54
      int64_t Value = MatchTable[CurrentIdx++];
573
54
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
54
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
54
                             << InsnID << "]->getOperand(" << OpIdx
576
54
                             << "), Value=" << Value << ")\n");
577
54
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
54
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
54
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
32
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
54
      }
583
54
      break;
584
54
    }
585
54
586
222k
    case GIM_CheckIntrinsicID: {
587
222k
      int64_t InsnID = MatchTable[CurrentIdx++];
588
222k
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
222k
      int64_t Value = MatchTable[CurrentIdx++];
590
222k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
222k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
222k
                             << InsnID << "]->getOperand(" << OpIdx
593
222k
                             << "), Value=" << Value << ")\n");
594
222k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
222k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
222k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value218k
)
597
219k
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
222k
      break;
600
222k
    }
601
222k
602
377k
    case GIM_CheckIsMBB: {
603
377k
      int64_t InsnID = MatchTable[CurrentIdx++];
604
377k
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
377k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
377k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
377k
                             << "]->getOperand(" << OpIdx << "))\n");
608
377k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
377k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
377k
      }
613
377k
      break;
614
377k
    }
615
377k
616
377k
    case GIM_CheckIsSafeToFold: {
617
46.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
618
46.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
46.0k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
46.0k
                             << InsnID << "])\n");
621
46.0k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
46.0k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
46.0k
      }
626
46.0k
      break;
627
46.0k
    }
628
46.0k
    case GIM_CheckIsSameOperand: {
629
6
      int64_t InsnID = MatchTable[CurrentIdx++];
630
6
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
6
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
6
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
6
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
6
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
6
                             << InsnID << "][" << OpIdx << "], MIs["
636
6
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
6
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
6
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
6
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
6
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
2
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
6
      }
644
6
      break;
645
6
    }
646
4.00M
    case GIM_Reject:
647
4.00M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
4.00M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
4.00M
      if (handleReject() == RejectAndGiveUp)
650
2.68M
        return false;
651
1.31M
      break;
652
1.31M
653
1.31M
    case GIR_MutateOpcode: {
654
625k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
625k
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
625k
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
625k
      if (NewInsnID >= OutMIs.size())
658
625k
        OutMIs.resize(NewInsnID + 1);
659
625k
660
625k
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
625k
                                              State.MIs[OldInsnID]);
662
625k
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
625k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
625k
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
625k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
625k
                             << NewOpcode << ")\n");
667
625k
      break;
668
1.31M
    }
669
1.31M
670
1.31M
    case GIR_BuildMI: {
671
1.23M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
1.23M
      int64_t Opcode = MatchTable[CurrentIdx++];
673
1.23M
      if (NewInsnID >= OutMIs.size())
674
1.22M
        OutMIs.resize(NewInsnID + 1);
675
1.23M
676
1.23M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
1.23M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
1.23M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
1.23M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
1.23M
                             << NewInsnID << "], " << Opcode << ")\n");
681
1.23M
      break;
682
1.31M
    }
683
1.31M
684
1.34M
    case GIR_Copy: {
685
1.34M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
1.34M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
1.34M
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
1.34M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
1.34M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
1.34M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
1.34M
                      dbgs()
692
1.34M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
1.34M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
1.34M
      break;
695
1.31M
    }
696
1.31M
697
1.31M
    case GIR_CopyOrAddZeroReg: {
698
140k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
140k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
140k
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
140k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
140k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
140k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
140k
      if (isOperandImmEqual(MO, 0, MRI))
705
15.2k
        OutMIs[NewInsnID].addReg(ZeroReg);
706
125k
      else
707
125k
        OutMIs[NewInsnID].add(MO);
708
140k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
140k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
140k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
140k
                             << OpIdx << ", " << ZeroReg << ")\n");
712
140k
      break;
713
1.31M
    }
714
1.31M
715
1.31M
    case GIR_CopySubReg: {
716
52.0k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
52.0k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
52.0k
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
52.0k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
52.0k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
52.0k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
52.0k
                               0, SubRegIdx);
723
52.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
52.0k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
52.0k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
52.0k
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
52.0k
      break;
728
1.31M
    }
729
1.31M
730
1.31M
    case GIR_AddImplicitDef: {
731
23.7k
      int64_t InsnID = MatchTable[CurrentIdx++];
732
23.7k
      int64_t RegNum = MatchTable[CurrentIdx++];
733
23.7k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
23.7k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
23.7k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
23.7k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
23.7k
                             << InsnID << "], " << RegNum << ")\n");
738
23.7k
      break;
739
1.31M
    }
740
1.31M
741
1.31M
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
1.31M
    }
751
1.31M
752
1.31M
    case GIR_AddRegister: {
753
85.8k
      int64_t InsnID = MatchTable[CurrentIdx++];
754
85.8k
      int64_t RegNum = MatchTable[CurrentIdx++];
755
85.8k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
85.8k
      OutMIs[InsnID].addReg(RegNum);
757
85.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
85.8k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
85.8k
                             << InsnID << "], " << RegNum << ")\n");
760
85.8k
      break;
761
1.31M
    }
762
1.31M
763
1.31M
    case GIR_AddTempRegister: {
764
7.85k
      int64_t InsnID = MatchTable[CurrentIdx++];
765
7.85k
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
7.85k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
7.85k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
7.85k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
7.85k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
7.85k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
7.85k
                             << InsnID << "], TempRegisters[" << TempRegID
772
7.85k
                             << "], " << TempRegFlags << ")\n");
773
7.85k
      break;
774
1.31M
    }
775
1.31M
776
1.31M
    case GIR_AddImm: {
777
18.4k
      int64_t InsnID = MatchTable[CurrentIdx++];
778
18.4k
      int64_t Imm = MatchTable[CurrentIdx++];
779
18.4k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
18.4k
      OutMIs[InsnID].addImm(Imm);
781
18.4k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
18.4k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
18.4k
                             << "], " << Imm << ")\n");
784
18.4k
      break;
785
1.31M
    }
786
1.31M
787
1.31M
    case GIR_ComplexRenderer: {
788
59.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
789
59.0k
      int64_t RendererID = MatchTable[CurrentIdx++];
790
59.0k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
59.0k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
118k
        RenderOpFn(OutMIs[InsnID]);
793
59.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
59.0k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
59.0k
                             << InsnID << "], " << RendererID << ")\n");
796
59.0k
      break;
797
1.31M
    }
798
1.31M
    case GIR_ComplexSubOperandRenderer: {
799
867k
      int64_t InsnID = MatchTable[CurrentIdx++];
800
867k
      int64_t RendererID = MatchTable[CurrentIdx++];
801
867k
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
867k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
867k
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
867k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
867k
                      dbgs() << CurrentIdx
806
867k
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
867k
                             << InsnID << "], " << RendererID << ", "
808
867k
                             << RenderOpID << ")\n");
809
867k
      break;
810
1.31M
    }
811
1.31M
812
1.31M
    case GIR_CopyConstantAsSImm: {
813
593k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
593k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
593k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
593k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
593k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
593k
        OutMIs[NewInsnID].addImm(
819
593k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
593k
      } else 
if (2
State.MIs[OldInsnID]->getOperand(1).isImm()2
)
821
2
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
2
      else
823
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
824
593k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
593k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
593k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
593k
      break;
828
593k
    }
829
593k
830
593k
    // TODO: Needs a test case once we have a pattern that uses this.
831
593k
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
3.91k
    case GIR_CustomRenderer: {
848
3.91k
      int64_t InsnID = MatchTable[CurrentIdx++];
849
3.91k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
3.91k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
3.91k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
3.91k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
3.91k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
3.91k
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
3.91k
                             << RendererFnID << ")\n");
856
3.91k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
3.91k
                                                     *State.MIs[OldInsnID]);
858
3.91k
      break;
859
0
    }
860
104k
    case GIR_ConstrainOperandRC: {
861
104k
      int64_t InsnID = MatchTable[CurrentIdx++];
862
104k
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
104k
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
104k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
104k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
104k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
104k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
104k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
104k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
104k
                             << ")\n");
871
104k
      break;
872
0
    }
873
0
874
1.80M
    case GIR_ConstrainSelectedInstOperands: {
875
1.80M
      int64_t InsnID = MatchTable[CurrentIdx++];
876
1.80M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
1.80M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
1.80M
                                       RBI);
879
1.80M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
1.80M
                      dbgs() << CurrentIdx
881
1.80M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
1.80M
                             << InsnID << "])\n");
883
1.80M
      break;
884
0
    }
885
0
886
434k
    case GIR_MergeMemOperands: {
887
434k
      int64_t InsnID = MatchTable[CurrentIdx++];
888
434k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
434k
890
434k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
434k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
434k
                             << InsnID << "]");
893
434k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
870k
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
870k
             GIU_MergeMemOperands_EndOfList) {
896
435k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
435k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
435k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
434k
          OutMIs[InsnID].addMemOperand(MMO);
900
435k
      }
901
434k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
434k
      break;
903
0
    }
904
0
905
1.22M
    case GIR_EraseFromParent: {
906
1.22M
      int64_t InsnID = MatchTable[CurrentIdx++];
907
1.22M
      assert(State.MIs[InsnID] &&
908
1.22M
             "Attempted to erase an undefined instruction");
909
1.22M
      State.MIs[InsnID]->eraseFromParent();
910
1.22M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
1.22M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
1.22M
                             << InsnID << "])\n");
913
1.22M
      break;
914
0
    }
915
0
916
3.92k
    case GIR_MakeTempReg: {
917
3.92k
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
3.92k
      int64_t TypeID = MatchTable[CurrentIdx++];
919
3.92k
920
3.92k
      State.TempRegisters[TempRegID] =
921
3.92k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
3.92k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
3.92k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
3.92k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
3.92k
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
1.85M
    case GIR_Done:
939
1.85M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
1.85M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
1.85M
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
38.5M
    }
946
38.5M
  }
947
4.53M
}
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<19ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<19ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<19ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
4.53M
    CodeGenCoverage &CoverageInfo) const {
57
4.53M
58
4.53M
  uint64_t CurrentIdx = 0;
59
4.53M
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
4.53M
61
4.53M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
4.53M
  auto handleReject = [&]() -> RejectAction {
63
4.53M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
4.53M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
4.53M
    if (OnFailResumeAt.empty())
66
4.53M
      return RejectAndGiveUp;
67
4.53M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
4.53M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
4.53M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
4.53M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
4.53M
    return RejectAndResume;
72
4.53M
  };
73
4.53M
74
38.5M
  while (true) {
75
38.5M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
38.5M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
38.5M
    switch (MatcherOpcode) {
78
38.5M
    case GIM_Try: {
79
5.17M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
5.17M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
5.17M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
5.17M
      break;
83
38.5M
    }
84
38.5M
85
38.5M
    case GIM_RecordInsn: {
86
1.71M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
1.71M
      int64_t InsnID = MatchTable[CurrentIdx++];
88
1.71M
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
1.71M
90
1.71M
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
1.71M
      // any attempt to modify it.
92
1.71M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
1.71M
94
1.71M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
1.71M
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
1.71M
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
1.71M
110
1.71M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
1.71M
      if ((size_t)NewInsnID < State.MIs.size())
112
1.15M
        State.MIs[NewInsnID] = NewMI;
113
557k
      else {
114
557k
        assert((size_t)NewInsnID == State.MIs.size() &&
115
557k
               "Expected to store MIs in order");
116
557k
        State.MIs.push_back(NewMI);
117
557k
      }
118
1.71M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
1.71M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
1.71M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
1.71M
                             << ")\n");
122
1.71M
      break;
123
1.71M
    }
124
1.71M
125
1.71M
    case GIM_CheckFeatures: {
126
419k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
419k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
419k
                      dbgs() << CurrentIdx
129
419k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
419k
                             << ExpectedBitsetID << ")\n");
131
419k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
419k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
45.6k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
419k
      }
136
419k
      break;
137
419k
    }
138
419k
139
1.71M
    case GIM_CheckOpcode: {
140
1.71M
      int64_t InsnID = MatchTable[CurrentIdx++];
141
1.71M
      int64_t Expected = MatchTable[CurrentIdx++];
142
1.71M
143
1.71M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
1.71M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
1.71M
146
1.71M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
1.71M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
1.71M
                             << "], ExpectedOpcode=" << Expected
149
1.71M
                             << ") // Got=" << Opcode << "\n");
150
1.71M
      if (Opcode != Expected) {
151
1.64M
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
1.71M
      }
154
1.71M
      break;
155
1.71M
    }
156
1.71M
157
4.53M
    case GIM_SwitchOpcode: {
158
4.53M
      int64_t InsnID = MatchTable[CurrentIdx++];
159
4.53M
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
4.53M
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
4.53M
      int64_t Default = MatchTable[CurrentIdx++];
162
4.53M
163
4.53M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
4.53M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
4.53M
166
4.53M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
4.53M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
4.53M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
4.53M
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
4.53M
      });
171
4.53M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
2
        CurrentIdx = Default;
173
2
        break;
174
2
      }
175
4.53M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
4.53M
      if (!CurrentIdx) {
177
1.46M
        CurrentIdx = Default;
178
1.46M
  break;
179
1.46M
      }
180
3.06M
      OnFailResumeAt.push_back(Default);
181
3.06M
      break;
182
3.06M
    }
183
3.06M
184
3.06M
    case GIM_SwitchType: {
185
2.65M
      int64_t InsnID = MatchTable[CurrentIdx++];
186
2.65M
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
2.65M
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
2.65M
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
2.65M
      int64_t Default = MatchTable[CurrentIdx++];
190
2.65M
191
2.65M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
2.65M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
2.65M
194
2.65M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
2.65M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
2.65M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
2.65M
               << UpperBound << "), Default=" << Default
198
2.65M
               << ", JumpTable...) // Got=";
199
2.65M
        if (!MO.isReg())
200
2.65M
          dbgs() << "Not a VReg\n";
201
2.65M
        else
202
2.65M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
2.65M
      });
204
2.65M
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
2.65M
      const LLT Ty = MRI.getType(MO.getReg());
209
2.65M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
2.65M
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
979k
        CurrentIdx = Default;
212
979k
        break;
213
979k
      }
214
1.67M
      const int64_t TypeID = TyI->second;
215
1.67M
      if (TypeID < LowerBound || 
UpperBound <= TypeID1.55M
) {
216
113k
        CurrentIdx = Default;
217
113k
        break;
218
113k
      }
219
1.55M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
1.55M
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
1.55M
      OnFailResumeAt.push_back(Default);
225
1.55M
      break;
226
1.55M
    }
227
1.55M
228
1.55M
    case GIM_CheckNumOperands: {
229
51.5k
      int64_t InsnID = MatchTable[CurrentIdx++];
230
51.5k
      int64_t Expected = MatchTable[CurrentIdx++];
231
51.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
51.5k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
51.5k
                             << InsnID << "], Expected=" << Expected << ")\n");
234
51.5k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
51.5k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
46.2k
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
51.5k
      }
239
51.5k
      break;
240
51.5k
    }
241
51.5k
    case GIM_CheckI64ImmPredicate: {
242
23.5k
      int64_t InsnID = MatchTable[CurrentIdx++];
243
23.5k
      int64_t Predicate = MatchTable[CurrentIdx++];
244
23.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
23.5k
                      dbgs()
246
23.5k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
23.5k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
23.5k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
23.5k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
23.5k
             "Expected G_CONSTANT");
251
23.5k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
23.5k
      int64_t Value = 0;
253
23.5k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
23.5k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
23.5k
260
23.5k
      if (!testImmPredicate_I64(Predicate, Value))
261
6
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
23.5k
      break;
264
23.5k
    }
265
23.5k
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
2.71k
    case GIM_CheckAPFloatImmPredicate: {
288
2.71k
      int64_t InsnID = MatchTable[CurrentIdx++];
289
2.71k
      int64_t Predicate = MatchTable[CurrentIdx++];
290
2.71k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
2.71k
                      dbgs()
292
2.71k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
2.71k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
2.71k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
2.71k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
2.71k
             "Expected G_FCONSTANT");
297
2.71k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
2.71k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
2.71k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
2.71k
301
2.71k
      if (!testImmPredicate_APFloat(Predicate, Value))
302
1.80k
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
2.71k
      break;
305
2.71k
    }
306
2.71k
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
451k
    case GIM_CheckAtomicOrdering: {
322
451k
      int64_t InsnID = MatchTable[CurrentIdx++];
323
451k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
451k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
451k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
451k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
451k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
451k
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
451k
332
451k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
451k
        if (MMO->getOrdering() != Ordering)
334
2.84k
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
451k
      break;
337
451k
    }
338
451k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
25.1k
    case GIM_CheckMemorySizeEqualTo: {
375
25.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
376
25.1k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
25.1k
      uint64_t Size = MatchTable[CurrentIdx++];
378
25.1k
379
25.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
25.1k
                      dbgs() << CurrentIdx
381
25.1k
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
25.1k
                             << "]->memoperands() + " << MMOIdx
383
25.1k
                             << ", Size=" << Size << ")\n");
384
25.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
25.1k
386
25.1k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
25.1k
392
25.1k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
25.1k
394
25.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
25.1k
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
25.1k
                             << " bytes\n");
397
25.1k
      if (MMO->getSize() != Size)
398
11.5k
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
25.1k
401
25.1k
      break;
402
25.1k
    }
403
281k
    case GIM_CheckMemorySizeEqualToLLT:
404
281k
    case GIM_CheckMemorySizeLessThanLLT:
405
281k
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
281k
      int64_t InsnID = MatchTable[CurrentIdx++];
407
281k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
281k
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
281k
410
281k
      DEBUG_WITH_TYPE(
411
281k
          TgtInstructionSelector::getName(),
412
281k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
281k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
281k
                         ? "EqualTo"
415
281k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
281k
                               ? "GreaterThan"
417
281k
                               : "LessThan")
418
281k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
281k
                 << ", OpIdx=" << OpIdx << ")\n");
420
281k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
281k
422
281k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
281k
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
281k
431
281k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
281k
437
281k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
281k
439
281k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
281k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
281k
          
MMO->getSize() * 8 != Size281k
) {
442
80
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
281k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
281k
                 
MMO->getSize() * 8 >= Size104
) {
446
36
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
281k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
281k
                 
MMO->getSize() * 8 <= Size0
)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
281k
453
281k
      break;
454
281k
    }
455
1.04M
    case GIM_CheckType: {
456
1.04M
      int64_t InsnID = MatchTable[CurrentIdx++];
457
1.04M
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
1.04M
      int64_t TypeID = MatchTable[CurrentIdx++];
459
1.04M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
1.04M
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
1.04M
                             << "]->getOperand(" << OpIdx
462
1.04M
                             << "), TypeID=" << TypeID << ")\n");
463
1.04M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
1.04M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
1.04M
      if (!MO.isReg() ||
466
1.04M
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
26.6k
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
1.04M
      }
470
1.04M
      break;
471
1.04M
    }
472
1.04M
    case GIM_CheckPointerToAny: {
473
441k
      int64_t InsnID = MatchTable[CurrentIdx++];
474
441k
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
441k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
441k
477
441k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
441k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
441k
                             << InsnID << "]->getOperand(" << OpIdx
480
441k
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
441k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
441k
      // iPTR must be looked up in the target.
483
441k
      if (SizeInBits == 0) {
484
440k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
440k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
440k
      }
487
441k
488
441k
      assert(SizeInBits != 0 && "Pointer size must be known");
489
441k
490
441k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
441k
      if (MO.isReg()) {
492
441k
        const LLT &Ty = MRI.getType(MO.getReg());
493
441k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
441k
499
441k
      break;
500
441k
    }
501
4.05M
    case GIM_CheckRegBankForClass: {
502
4.05M
      int64_t InsnID = MatchTable[CurrentIdx++];
503
4.05M
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
4.05M
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
4.05M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
4.05M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
4.05M
                             << InsnID << "]->getOperand(" << OpIdx
508
4.05M
                             << "), RCEnum=" << RCEnum << ")\n");
509
4.05M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
4.05M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
4.05M
      if (!MO.isReg() ||
512
4.05M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
4.05M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
345k
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
4.05M
      }
517
4.05M
      break;
518
4.05M
    }
519
4.05M
520
4.05M
    case GIM_CheckComplexPattern: {
521
730k
      int64_t InsnID = MatchTable[CurrentIdx++];
522
730k
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
730k
      int64_t RendererID = MatchTable[CurrentIdx++];
524
730k
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
730k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
730k
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
730k
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
730k
                             << "]->getOperand(" << OpIdx
529
730k
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
730k
                             << ")\n");
531
730k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
730k
      // FIXME: Use std::invoke() when it's available.
533
730k
      ComplexRendererFns Renderer =
534
730k
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
730k
              State.MIs[InsnID]->getOperand(OpIdx));
536
730k
      if (Renderer.hasValue())
537
492k
        State.Renderers[RendererID] = Renderer.getValue();
538
237k
      else
539
237k
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
730k
      break;
542
730k
    }
543
730k
544
730k
    case GIM_CheckConstantInt: {
545
81.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
546
81.9k
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
81.9k
      int64_t Value = MatchTable[CurrentIdx++];
548
81.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
81.9k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
81.9k
                             << InsnID << "]->getOperand(" << OpIdx
551
81.9k
                             << "), Value=" << Value << ")\n");
552
81.9k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
81.9k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
81.9k
      if (MO.isReg()) {
555
81.9k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
81.9k
        LLT Ty = MRI.getType(MO.getReg());
557
81.9k
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
81.9k
559
81.9k
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
64.4k
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
81.9k
566
81.9k
      break;
567
81.9k
    }
568
81.9k
569
81.9k
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
222k
    case GIM_CheckIntrinsicID: {
587
222k
      int64_t InsnID = MatchTable[CurrentIdx++];
588
222k
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
222k
      int64_t Value = MatchTable[CurrentIdx++];
590
222k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
222k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
222k
                             << InsnID << "]->getOperand(" << OpIdx
593
222k
                             << "), Value=" << Value << ")\n");
594
222k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
222k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
222k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value218k
)
597
219k
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
222k
      break;
600
222k
    }
601
222k
602
377k
    case GIM_CheckIsMBB: {
603
377k
      int64_t InsnID = MatchTable[CurrentIdx++];
604
377k
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
377k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
377k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
377k
                             << "]->getOperand(" << OpIdx << "))\n");
608
377k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
377k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
377k
      }
613
377k
      break;
614
377k
    }
615
377k
616
377k
    case GIM_CheckIsSafeToFold: {
617
45.8k
      int64_t InsnID = MatchTable[CurrentIdx++];
618
45.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
45.8k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
45.8k
                             << InsnID << "])\n");
621
45.8k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
45.8k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
45.8k
      }
626
45.8k
      break;
627
45.8k
    }
628
45.8k
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
3.99M
    case GIM_Reject:
647
3.99M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
3.99M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
3.99M
      if (handleReject() == RejectAndGiveUp)
650
2.68M
        return false;
651
1.31M
      break;
652
1.31M
653
1.31M
    case GIR_MutateOpcode: {
654
624k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
624k
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
624k
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
624k
      if (NewInsnID >= OutMIs.size())
658
624k
        OutMIs.resize(NewInsnID + 1);
659
624k
660
624k
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
624k
                                              State.MIs[OldInsnID]);
662
624k
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
624k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
624k
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
624k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
624k
                             << NewOpcode << ")\n");
667
624k
      break;
668
1.31M
    }
669
1.31M
670
1.31M
    case GIR_BuildMI: {
671
1.23M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
1.23M
      int64_t Opcode = MatchTable[CurrentIdx++];
673
1.23M
      if (NewInsnID >= OutMIs.size())
674
1.22M
        OutMIs.resize(NewInsnID + 1);
675
1.23M
676
1.23M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
1.23M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
1.23M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
1.23M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
1.23M
                             << NewInsnID << "], " << Opcode << ")\n");
681
1.23M
      break;
682
1.31M
    }
683
1.31M
684
1.33M
    case GIR_Copy: {
685
1.33M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
1.33M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
1.33M
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
1.33M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
1.33M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
1.33M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
1.33M
                      dbgs()
692
1.33M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
1.33M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
1.33M
      break;
695
1.31M
    }
696
1.31M
697
1.31M
    case GIR_CopyOrAddZeroReg: {
698
140k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
140k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
140k
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
140k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
140k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
140k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
140k
      if (isOperandImmEqual(MO, 0, MRI))
705
15.2k
        OutMIs[NewInsnID].addReg(ZeroReg);
706
125k
      else
707
125k
        OutMIs[NewInsnID].add(MO);
708
140k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
140k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
140k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
140k
                             << OpIdx << ", " << ZeroReg << ")\n");
712
140k
      break;
713
1.31M
    }
714
1.31M
715
1.31M
    case GIR_CopySubReg: {
716
51.8k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
51.8k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
51.8k
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
51.8k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
51.8k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
51.8k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
51.8k
                               0, SubRegIdx);
723
51.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
51.8k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
51.8k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
51.8k
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
51.8k
      break;
728
1.31M
    }
729
1.31M
730
1.31M
    case GIR_AddImplicitDef: {
731
23.6k
      int64_t InsnID = MatchTable[CurrentIdx++];
732
23.6k
      int64_t RegNum = MatchTable[CurrentIdx++];
733
23.6k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
23.6k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
23.6k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
23.6k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
23.6k
                             << InsnID << "], " << RegNum << ")\n");
738
23.6k
      break;
739
1.31M
    }
740
1.31M
741
1.31M
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
1.31M
    }
751
1.31M
752
1.31M
    case GIR_AddRegister: {
753
85.4k
      int64_t InsnID = MatchTable[CurrentIdx++];
754
85.4k
      int64_t RegNum = MatchTable[CurrentIdx++];
755
85.4k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
85.4k
      OutMIs[InsnID].addReg(RegNum);
757
85.4k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
85.4k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
85.4k
                             << InsnID << "], " << RegNum << ")\n");
760
85.4k
      break;
761
1.31M
    }
762
1.31M
763
1.31M
    case GIR_AddTempRegister: {
764
7.82k
      int64_t InsnID = MatchTable[CurrentIdx++];
765
7.82k
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
7.82k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
7.82k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
7.82k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
7.82k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
7.82k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
7.82k
                             << InsnID << "], TempRegisters[" << TempRegID
772
7.82k
                             << "], " << TempRegFlags << ")\n");
773
7.82k
      break;
774
1.31M
    }
775
1.31M
776
1.31M
    case GIR_AddImm: {
777
18.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
778
18.1k
      int64_t Imm = MatchTable[CurrentIdx++];
779
18.1k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
18.1k
      OutMIs[InsnID].addImm(Imm);
781
18.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
18.1k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
18.1k
                             << "], " << Imm << ")\n");
784
18.1k
      break;
785
1.31M
    }
786
1.31M
787
1.31M
    case GIR_ComplexRenderer: {
788
59.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
789
59.0k
      int64_t RendererID = MatchTable[CurrentIdx++];
790
59.0k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
59.0k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
118k
        RenderOpFn(OutMIs[InsnID]);
793
59.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
59.0k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
59.0k
                             << InsnID << "], " << RendererID << ")\n");
796
59.0k
      break;
797
1.31M
    }
798
1.31M
    case GIR_ComplexSubOperandRenderer: {
799
867k
      int64_t InsnID = MatchTable[CurrentIdx++];
800
867k
      int64_t RendererID = MatchTable[CurrentIdx++];
801
867k
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
867k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
867k
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
867k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
867k
                      dbgs() << CurrentIdx
806
867k
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
867k
                             << InsnID << "], " << RendererID << ", "
808
867k
                             << RenderOpID << ")\n");
809
867k
      break;
810
1.31M
    }
811
1.31M
812
1.31M
    case GIR_CopyConstantAsSImm: {
813
593k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
593k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
593k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
593k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
593k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
593k
        OutMIs[NewInsnID].addImm(
819
593k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
593k
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
593k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
593k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
593k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
593k
      break;
828
593k
    }
829
593k
830
593k
    // TODO: Needs a test case once we have a pattern that uses this.
831
593k
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
3.91k
    case GIR_CustomRenderer: {
848
3.91k
      int64_t InsnID = MatchTable[CurrentIdx++];
849
3.91k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
3.91k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
3.91k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
3.91k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
3.91k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
3.91k
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
3.91k
                             << RendererFnID << ")\n");
856
3.91k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
3.91k
                                                     *State.MIs[OldInsnID]);
858
3.91k
      break;
859
0
    }
860
104k
    case GIR_ConstrainOperandRC: {
861
104k
      int64_t InsnID = MatchTable[CurrentIdx++];
862
104k
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
104k
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
104k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
104k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
104k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
104k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
104k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
104k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
104k
                             << ")\n");
871
104k
      break;
872
0
    }
873
0
874
1.80M
    case GIR_ConstrainSelectedInstOperands: {
875
1.80M
      int64_t InsnID = MatchTable[CurrentIdx++];
876
1.80M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
1.80M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
1.80M
                                       RBI);
879
1.80M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
1.80M
                      dbgs() << CurrentIdx
881
1.80M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
1.80M
                             << InsnID << "])\n");
883
1.80M
      break;
884
0
    }
885
0
886
434k
    case GIR_MergeMemOperands: {
887
434k
      int64_t InsnID = MatchTable[CurrentIdx++];
888
434k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
434k
890
434k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
434k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
434k
                             << InsnID << "]");
893
434k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
870k
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
870k
             GIU_MergeMemOperands_EndOfList) {
896
435k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
435k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
435k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
434k
          OutMIs[InsnID].addMemOperand(MMO);
900
435k
      }
901
434k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
434k
      break;
903
0
    }
904
0
905
1.22M
    case GIR_EraseFromParent: {
906
1.22M
      int64_t InsnID = MatchTable[CurrentIdx++];
907
1.22M
      assert(State.MIs[InsnID] &&
908
1.22M
             "Attempted to erase an undefined instruction");
909
1.22M
      State.MIs[InsnID]->eraseFromParent();
910
1.22M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
1.22M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
1.22M
                             << InsnID << "])\n");
913
1.22M
      break;
914
0
    }
915
0
916
3.91k
    case GIR_MakeTempReg: {
917
3.91k
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
3.91k
      int64_t TypeID = MatchTable[CurrentIdx++];
919
3.91k
920
3.91k
      State.TempRegisters[TempRegID] =
921
3.91k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
3.91k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
3.91k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
3.91k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
3.91k
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
1.85M
    case GIR_Done:
939
1.85M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
1.85M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
1.85M
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
38.5M
    }
946
38.5M
  }
947
4.53M
}
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<32ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
56
    CodeGenCoverage &CoverageInfo) const {
57
56
58
56
  uint64_t CurrentIdx = 0;
59
56
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
56
61
56
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
56
  auto handleReject = [&]() -> RejectAction {
63
56
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
56
                    dbgs() << CurrentIdx << ": Rejected\n");
65
56
    if (OnFailResumeAt.empty())
66
56
      return RejectAndGiveUp;
67
56
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
56
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
56
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
56
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
56
    return RejectAndResume;
72
56
  };
73
56
74
1.51k
  while (true) {
75
1.51k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
1.51k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
1.51k
    switch (MatcherOpcode) {
78
1.51k
    case GIM_Try: {
79
280
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
280
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
280
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
280
      break;
83
1.51k
    }
84
1.51k
85
1.51k
    case GIM_RecordInsn: {
86
48
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
48
      int64_t InsnID = MatchTable[CurrentIdx++];
88
48
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
48
90
48
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
48
      // any attempt to modify it.
92
48
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
48
94
48
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
48
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
48
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
48
110
48
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
48
      if ((size_t)NewInsnID < State.MIs.size())
112
45
        State.MIs[NewInsnID] = NewMI;
113
3
      else {
114
3
        assert((size_t)NewInsnID == State.MIs.size() &&
115
3
               "Expected to store MIs in order");
116
3
        State.MIs.push_back(NewMI);
117
3
      }
118
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
48
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
48
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
48
                             << ")\n");
122
48
      break;
123
48
    }
124
48
125
160
    case GIM_CheckFeatures: {
126
160
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
160
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
160
                      dbgs() << CurrentIdx
129
160
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
160
                             << ExpectedBitsetID << ")\n");
131
160
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
160
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
7
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
160
      }
136
160
      break;
137
160
    }
138
160
139
160
    case GIM_CheckOpcode: {
140
48
      int64_t InsnID = MatchTable[CurrentIdx++];
141
48
      int64_t Expected = MatchTable[CurrentIdx++];
142
48
143
48
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
48
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
48
146
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
48
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
48
                             << "], ExpectedOpcode=" << Expected
149
48
                             << ") // Got=" << Opcode << "\n");
150
48
      if (Opcode != Expected) {
151
48
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
48
      }
154
48
      break;
155
48
    }
156
48
157
56
    case GIM_SwitchOpcode: {
158
56
      int64_t InsnID = MatchTable[CurrentIdx++];
159
56
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
56
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
56
      int64_t Default = MatchTable[CurrentIdx++];
162
56
163
56
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
56
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
56
166
56
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
56
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
56
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
56
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
56
      });
171
56
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
56
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
56
      if (!CurrentIdx) {
177
0
        CurrentIdx = Default;
178
0
  break;
179
0
      }
180
56
      OnFailResumeAt.push_back(Default);
181
56
      break;
182
56
    }
183
56
184
56
    case GIM_SwitchType: {
185
10
      int64_t InsnID = MatchTable[CurrentIdx++];
186
10
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
10
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
10
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
10
      int64_t Default = MatchTable[CurrentIdx++];
190
10
191
10
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
10
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
10
194
10
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
10
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
10
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
10
               << UpperBound << "), Default=" << Default
198
10
               << ", JumpTable...) // Got=";
199
10
        if (!MO.isReg())
200
10
          dbgs() << "Not a VReg\n";
201
10
        else
202
10
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
10
      });
204
10
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
10
      const LLT Ty = MRI.getType(MO.getReg());
209
10
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
10
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
0
        CurrentIdx = Default;
212
0
        break;
213
0
      }
214
10
      const int64_t TypeID = TyI->second;
215
10
      if (TypeID < LowerBound || UpperBound <= TypeID) {
216
0
        CurrentIdx = Default;
217
0
        break;
218
0
      }
219
10
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
10
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
10
      OnFailResumeAt.push_back(Default);
225
10
      break;
226
10
    }
227
10
228
54
    case GIM_CheckNumOperands: {
229
54
      int64_t InsnID = MatchTable[CurrentIdx++];
230
54
      int64_t Expected = MatchTable[CurrentIdx++];
231
54
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
54
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
54
                             << InsnID << "], Expected=" << Expected << ")\n");
234
54
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
54
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
36
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
54
      }
239
54
      break;
240
54
    }
241
54
    case GIM_CheckI64ImmPredicate: {
242
0
      int64_t InsnID = MatchTable[CurrentIdx++];
243
0
      int64_t Predicate = MatchTable[CurrentIdx++];
244
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
0
                      dbgs()
246
0
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
0
             "Expected G_CONSTANT");
251
0
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
0
      int64_t Value = 0;
253
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
0
260
0
      if (!testImmPredicate_I64(Predicate, Value))
261
0
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
0
      break;
264
0
    }
265
0
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
172
    case GIM_CheckType: {
456
172
      int64_t InsnID = MatchTable[CurrentIdx++];
457
172
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
172
      int64_t TypeID = MatchTable[CurrentIdx++];
459
172
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
172
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
172
                             << "]->getOperand(" << OpIdx
462
172
                             << "), TypeID=" << TypeID << ")\n");
463
172
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
172
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
172
      if (!MO.isReg() ||
466
172
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
12
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
172
      }
470
172
      break;
471
172
    }
472
172
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
120
    case GIM_CheckRegBankForClass: {
502
120
      int64_t InsnID = MatchTable[CurrentIdx++];
503
120
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
120
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
120
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
120
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
120
                             << InsnID << "]->getOperand(" << OpIdx
508
120
                             << "), RCEnum=" << RCEnum << ")\n");
509
120
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
120
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
120
      if (!MO.isReg() ||
512
120
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
120
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
20
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
120
      }
517
120
      break;
518
120
    }
519
120
520
120
    case GIM_CheckComplexPattern: {
521
66
      int64_t InsnID = MatchTable[CurrentIdx++];
522
66
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
66
      int64_t RendererID = MatchTable[CurrentIdx++];
524
66
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
66
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
66
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
66
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
66
                             << "]->getOperand(" << OpIdx
529
66
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
66
                             << ")\n");
531
66
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
66
      // FIXME: Use std::invoke() when it's available.
533
66
      ComplexRendererFns Renderer =
534
66
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
66
              State.MIs[InsnID]->getOperand(OpIdx));
536
66
      if (Renderer.hasValue())
537
66
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
66
      break;
542
66
    }
543
66
544
66
    case GIM_CheckConstantInt: {
545
0
      int64_t InsnID = MatchTable[CurrentIdx++];
546
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
0
      int64_t Value = MatchTable[CurrentIdx++];
548
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
0
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
0
                             << InsnID << "]->getOperand(" << OpIdx
551
0
                             << "), Value=" << Value << ")\n");
552
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
0
      if (MO.isReg()) {
555
0
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
0
        LLT Ty = MRI.getType(MO.getReg());
557
0
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
0
559
0
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
0
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
0
566
0
      break;
567
0
    }
568
0
569
0
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
64
    case GIM_CheckIntrinsicID: {
587
64
      int64_t InsnID = MatchTable[CurrentIdx++];
588
64
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
64
      int64_t Value = MatchTable[CurrentIdx++];
590
64
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
64
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
64
                             << InsnID << "]->getOperand(" << OpIdx
593
64
                             << "), Value=" << Value << ")\n");
594
64
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
64
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
64
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
32
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
64
      break;
600
64
    }
601
64
602
64
    case GIM_CheckIsMBB: {
603
0
      int64_t InsnID = MatchTable[CurrentIdx++];
604
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
0
                             << "]->getOperand(" << OpIdx << "))\n");
608
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
0
      }
613
0
      break;
614
0
    }
615
0
616
0
    case GIM_CheckIsSafeToFold: {
617
0
      int64_t InsnID = MatchTable[CurrentIdx++];
618
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
0
                             << InsnID << "])\n");
621
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
0
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
0
      }
626
0
      break;
627
0
    }
628
0
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
0
    case GIM_Reject:
647
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
0
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
0
      if (handleReject() == RejectAndGiveUp)
650
0
        return false;
651
0
      break;
652
0
653
11
    case GIR_MutateOpcode: {
654
11
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
11
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
11
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
11
      if (NewInsnID >= OutMIs.size())
658
11
        OutMIs.resize(NewInsnID + 1);
659
11
660
11
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
11
                                              State.MIs[OldInsnID]);
662
11
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
11
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
11
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
11
                             << NewOpcode << ")\n");
667
11
      break;
668
0
    }
669
0
670
45
    case GIR_BuildMI: {
671
45
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
45
      int64_t Opcode = MatchTable[CurrentIdx++];
673
45
      if (NewInsnID >= OutMIs.size())
674
45
        OutMIs.resize(NewInsnID + 1);
675
45
676
45
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
45
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
45
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
45
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
45
                             << NewInsnID << "], " << Opcode << ")\n");
681
45
      break;
682
0
    }
683
0
684
65
    case GIR_Copy: {
685
65
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
65
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
65
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
65
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
65
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
65
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
65
                      dbgs()
692
65
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
65
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
65
      break;
695
0
    }
696
0
697
0
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
0
    }
714
0
715
0
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
0
    }
729
0
730
11
    case GIR_AddImplicitDef: {
731
11
      int64_t InsnID = MatchTable[CurrentIdx++];
732
11
      int64_t RegNum = MatchTable[CurrentIdx++];
733
11
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
11
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
11
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
11
                             << InsnID << "], " << RegNum << ")\n");
738
11
      break;
739
0
    }
740
0
741
0
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
0
    }
751
0
752
0
    case GIR_AddRegister: {
753
0
      int64_t InsnID = MatchTable[CurrentIdx++];
754
0
      int64_t RegNum = MatchTable[CurrentIdx++];
755
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
0
      OutMIs[InsnID].addReg(RegNum);
757
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
0
                             << InsnID << "], " << RegNum << ")\n");
760
0
      break;
761
0
    }
762
0
763
0
    case GIR_AddTempRegister: {
764
0
      int64_t InsnID = MatchTable[CurrentIdx++];
765
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
0
                             << InsnID << "], TempRegisters[" << TempRegID
772
0
                             << "], " << TempRegFlags << ")\n");
773
0
      break;
774
0
    }
775
0
776
0
    case GIR_AddImm: {
777
0
      int64_t InsnID = MatchTable[CurrentIdx++];
778
0
      int64_t Imm = MatchTable[CurrentIdx++];
779
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
0
      OutMIs[InsnID].addImm(Imm);
781
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
0
                             << "], " << Imm << ")\n");
784
0
      break;
785
0
    }
786
0
787
0
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
0
    }
798
148
    case GIR_ComplexSubOperandRenderer: {
799
148
      int64_t InsnID = MatchTable[CurrentIdx++];
800
148
      int64_t RendererID = MatchTable[CurrentIdx++];
801
148
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
148
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
148
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
148
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
148
                      dbgs() << CurrentIdx
806
148
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
148
                             << InsnID << "], " << RendererID << ", "
808
148
                             << RenderOpID << ")\n");
809
148
      break;
810
0
    }
811
0
812
0
    case GIR_CopyConstantAsSImm: {
813
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
0
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
0
        OutMIs[NewInsnID].addImm(
819
0
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
0
      } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
0
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
0
      break;
828
0
    }
829
0
830
0
    // TODO: Needs a test case once we have a pattern that uses this.
831
0
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
0
    case GIR_ConstrainOperandRC: {
861
0
      int64_t InsnID = MatchTable[CurrentIdx++];
862
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
0
                             << ")\n");
871
0
      break;
872
0
    }
873
0
874
56
    case GIR_ConstrainSelectedInstOperands: {
875
56
      int64_t InsnID = MatchTable[CurrentIdx++];
876
56
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
56
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
56
                                       RBI);
879
56
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
56
                      dbgs() << CurrentIdx
881
56
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
56
                             << InsnID << "])\n");
883
56
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
0
                             << InsnID << "]");
893
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
0
             GIU_MergeMemOperands_EndOfList) {
896
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
0
          OutMIs[InsnID].addMemOperand(MMO);
900
0
      }
901
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
0
      break;
903
0
    }
904
0
905
45
    case GIR_EraseFromParent: {
906
45
      int64_t InsnID = MatchTable[CurrentIdx++];
907
45
      assert(State.MIs[InsnID] &&
908
45
             "Attempted to erase an undefined instruction");
909
45
      State.MIs[InsnID]->eraseFromParent();
910
45
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
45
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
45
                             << InsnID << "])\n");
913
45
      break;
914
0
    }
915
0
916
0
    case GIR_MakeTempReg: {
917
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
0
      int64_t TypeID = MatchTable[CurrentIdx++];
919
0
920
0
      State.TempRegisters[TempRegID] =
921
0
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
0
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
0
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
0
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
56
    case GIR_Done:
939
56
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
56
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
56
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
1.51k
    }
946
1.51k
  }
947
56
}
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<64ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
706
    CodeGenCoverage &CoverageInfo) const {
57
706
58
706
  uint64_t CurrentIdx = 0;
59
706
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
706
61
706
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
706
  auto handleReject = [&]() -> RejectAction {
63
706
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
706
                    dbgs() << CurrentIdx << ": Rejected\n");
65
706
    if (OnFailResumeAt.empty())
66
706
      return RejectAndGiveUp;
67
706
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
706
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
706
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
706
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
706
    return RejectAndResume;
72
706
  };
73
706
74
11.3k
  while (true) {
75
11.3k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
11.3k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
11.3k
    switch (MatcherOpcode) {
78
11.3k
    case GIM_Try: {
79
2.08k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
2.08k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
2.08k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
2.08k
      break;
83
11.3k
    }
84
11.3k
85
11.3k
    case GIM_RecordInsn: {
86
577
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
577
      int64_t InsnID = MatchTable[CurrentIdx++];
88
577
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
577
90
577
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
577
      // any attempt to modify it.
92
577
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
577
94
577
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
577
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
577
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
577
110
577
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
577
      if ((size_t)NewInsnID < State.MIs.size())
112
381
        State.MIs[NewInsnID] = NewMI;
113
196
      else {
114
196
        assert((size_t)NewInsnID == State.MIs.size() &&
115
196
               "Expected to store MIs in order");
116
196
        State.MIs.push_back(NewMI);
117
196
      }
118
577
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
577
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
577
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
577
                             << ")\n");
122
577
      break;
123
577
    }
124
577
125
1.75k
    case GIM_CheckFeatures: {
126
1.75k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
1.75k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
1.75k
                      dbgs() << CurrentIdx
129
1.75k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
1.75k
                             << ExpectedBitsetID << ")\n");
131
1.75k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
1.75k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
1.03k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
1.75k
      }
136
1.75k
      break;
137
1.75k
    }
138
1.75k
139
1.75k
    case GIM_CheckOpcode: {
140
577
      int64_t InsnID = MatchTable[CurrentIdx++];
141
577
      int64_t Expected = MatchTable[CurrentIdx++];
142
577
143
577
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
577
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
577
146
577
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
577
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
577
                             << "], ExpectedOpcode=" << Expected
149
577
                             << ") // Got=" << Opcode << "\n");
150
577
      if (Opcode != Expected) {
151
431
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
577
      }
154
577
      break;
155
577
    }
156
577
157
706
    case GIM_SwitchOpcode: {
158
706
      int64_t InsnID = MatchTable[CurrentIdx++];
159
706
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
706
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
706
      int64_t Default = MatchTable[CurrentIdx++];
162
706
163
706
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
706
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
706
166
706
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
706
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
706
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
706
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
706
      });
171
706
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
706
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
706
      if (!CurrentIdx) {
177
313
        CurrentIdx = Default;
178
313
  break;
179
313
      }
180
393
      OnFailResumeAt.push_back(Default);
181
393
      break;
182
393
    }
183
393
184
393
    case GIM_SwitchType: {
185
225
      int64_t InsnID = MatchTable[CurrentIdx++];
186
225
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
225
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
225
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
225
      int64_t Default = MatchTable[CurrentIdx++];
190
225
191
225
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
225
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
225
194
225
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
225
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
225
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
225
               << UpperBound << "), Default=" << Default
198
225
               << ", JumpTable...) // Got=";
199
225
        if (!MO.isReg())
200
225
          dbgs() << "Not a VReg\n";
201
225
        else
202
225
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
225
      });
204
225
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
225
      const LLT Ty = MRI.getType(MO.getReg());
209
225
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
225
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
13
        CurrentIdx = Default;
212
13
        break;
213
13
      }
214
212
      const int64_t TypeID = TyI->second;
215
212
      if (TypeID < LowerBound || 
UpperBound <= TypeID148
) {
216
64
        CurrentIdx = Default;
217
64
        break;
218
64
      }
219
148
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
148
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
148
      OnFailResumeAt.push_back(Default);
225
148
      break;
226
148
    }
227
148
228
148
    case GIM_CheckNumOperands: {
229
0
      int64_t InsnID = MatchTable[CurrentIdx++];
230
0
      int64_t Expected = MatchTable[CurrentIdx++];
231
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
0
                             << InsnID << "], Expected=" << Expected << ")\n");
234
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
0
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
0
      }
239
0
      break;
240
0
    }
241
71
    case GIM_CheckI64ImmPredicate: {
242
71
      int64_t InsnID = MatchTable[CurrentIdx++];
243
71
      int64_t Predicate = MatchTable[CurrentIdx++];
244
71
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
71
                      dbgs()
246
71
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
71
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
71
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
71
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
71
             "Expected G_CONSTANT");
251
71
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
71
      int64_t Value = 0;
253
71
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
69
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
2
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
2
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
2
      else
258
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
259
71
260
71
      if (!testImmPredicate_I64(Predicate, Value))
261
2
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
71
      break;
264
71
    }
265
71
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
899
    case GIM_CheckType: {
456
899
      int64_t InsnID = MatchTable[CurrentIdx++];
457
899
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
899
      int64_t TypeID = MatchTable[CurrentIdx++];
459
899
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
899
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
899
                             << "]->getOperand(" << OpIdx
462
899
                             << "), TypeID=" << TypeID << ")\n");
463
899
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
899
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
899
      if (!MO.isReg() ||
466
899
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
5
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
899
      }
470
899
      break;
471
899
    }
472
899
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
1.29k
    case GIM_CheckRegBankForClass: {
502
1.29k
      int64_t InsnID = MatchTable[CurrentIdx++];
503
1.29k
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
1.29k
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
1.29k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
1.29k
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
1.29k
                             << InsnID << "]->getOperand(" << OpIdx
508
1.29k
                             << "), RCEnum=" << RCEnum << ")\n");
509
1.29k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
1.29k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
1.29k
      if (!MO.isReg() ||
512
1.29k
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
1.29k
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
0
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
1.29k
      }
517
1.29k
      break;
518
1.29k
    }
519
1.29k
520
1.29k
    case GIM_CheckComplexPattern: {
521
0
      int64_t InsnID = MatchTable[CurrentIdx++];
522
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
0
      int64_t RendererID = MatchTable[CurrentIdx++];
524
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
0
                             << "]->getOperand(" << OpIdx
529
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
0
                             << ")\n");
531
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
0
      // FIXME: Use std::invoke() when it's available.
533
0
      ComplexRendererFns Renderer =
534
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
0
              State.MIs[InsnID]->getOperand(OpIdx));
536
0
      if (Renderer.hasValue())
537
0
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
0
      break;
542
0
    }
543
0
544
121
    case GIM_CheckConstantInt: {
545
121
      int64_t InsnID = MatchTable[CurrentIdx++];
546
121
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
121
      int64_t Value = MatchTable[CurrentIdx++];
548
121
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
121
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
121
                             << InsnID << "]->getOperand(" << OpIdx
551
121
                             << "), Value=" << Value << ")\n");
552
121
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
121
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
121
      if (MO.isReg()) {
555
121
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
121
        LLT Ty = MRI.getType(MO.getReg());
557
121
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
121
559
121
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
51
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
121
566
121
      break;
567
121
    }
568
121
569
121
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
0
    case GIM_CheckIntrinsicID: {
587
0
      int64_t InsnID = MatchTable[CurrentIdx++];
588
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
0
      int64_t Value = MatchTable[CurrentIdx++];
590
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
0
                             << InsnID << "]->getOperand(" << OpIdx
593
0
                             << "), Value=" << Value << ")\n");
594
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
0
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
0
      break;
600
0
    }
601
0
602
9
    case GIM_CheckIsMBB: {
603
9
      int64_t InsnID = MatchTable[CurrentIdx++];
604
9
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
9
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
9
                             << "]->getOperand(" << OpIdx << "))\n");
608
9
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
9
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
9
      }
613
9
      break;
614
9
    }
615
9
616
77
    case GIM_CheckIsSafeToFold: {
617
77
      int64_t InsnID = MatchTable[CurrentIdx++];
618
77
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
77
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
77
                             << InsnID << "])\n");
621
77
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
77
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
77
      }
626
77
      break;
627
77
    }
628
77
    case GIM_CheckIsSameOperand: {
629
2
      int64_t InsnID = MatchTable[CurrentIdx++];
630
2
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
2
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
2
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
2
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
2
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
2
                             << InsnID << "][" << OpIdx << "], MIs["
636
2
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
2
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
2
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
2
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
2
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
2
      }
644
2
      break;
645
2
    }
646
750
    case GIM_Reject:
647
750
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
750
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
750
      if (handleReject() == RejectAndGiveUp)
650
473
        return false;
651
277
      break;
652
277
653
277
    case GIR_MutateOpcode: {
654
9
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
9
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
9
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
9
      if (NewInsnID >= OutMIs.size())
658
9
        OutMIs.resize(NewInsnID + 1);
659
9
660
9
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
9
                                              State.MIs[OldInsnID]);
662
9
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
9
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
9
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
9
                             << NewOpcode << ")\n");
667
9
      break;
668
277
    }
669
277
670
277
    case GIR_BuildMI: {
671
233
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
233
      int64_t Opcode = MatchTable[CurrentIdx++];
673
233
      if (NewInsnID >= OutMIs.size())
674
224
        OutMIs.resize(NewInsnID + 1);
675
233
676
233
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
233
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
233
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
233
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
233
                             << NewInsnID << "], " << Opcode << ")\n");
681
233
      break;
682
277
    }
683
277
684
534
    case GIR_Copy: {
685
534
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
534
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
534
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
534
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
534
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
534
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
534
                      dbgs()
692
534
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
534
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
534
      break;
695
277
    }
696
277
697
277
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
277
    }
714
277
715
277
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
277
    }
729
277
730
277
    case GIR_AddImplicitDef: {
731
0
      int64_t InsnID = MatchTable[CurrentIdx++];
732
0
      int64_t RegNum = MatchTable[CurrentIdx++];
733
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
0
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
0
                             << InsnID << "], " << RegNum << ")\n");
738
0
      break;
739
277
    }
740
277
741
277
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
277
    }
751
277
752
324
    case GIR_AddRegister: {
753
324
      int64_t InsnID = MatchTable[CurrentIdx++];
754
324
      int64_t RegNum = MatchTable[CurrentIdx++];
755
324
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
324
      OutMIs[InsnID].addReg(RegNum);
757
324
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
324
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
324
                             << InsnID << "], " << RegNum << ")\n");
760
324
      break;
761
277
    }
762
277
763
277
    case GIR_AddTempRegister: {
764
18
      int64_t InsnID = MatchTable[CurrentIdx++];
765
18
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
18
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
18
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
18
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
18
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
18
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
18
                             << InsnID << "], TempRegisters[" << TempRegID
772
18
                             << "], " << TempRegFlags << ")\n");
773
18
      break;
774
277
    }
775
277
776
277
    case GIR_AddImm: {
777
260
      int64_t InsnID = MatchTable[CurrentIdx++];
778
260
      int64_t Imm = MatchTable[CurrentIdx++];
779
260
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
260
      OutMIs[InsnID].addImm(Imm);
781
260
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
260
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
260
                             << "], " << Imm << ")\n");
784
260
      break;
785
277
    }
786
277
787
277
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
277
    }
798
277
    case GIR_ComplexSubOperandRenderer: {
799
0
      int64_t InsnID = MatchTable[CurrentIdx++];
800
0
      int64_t RendererID = MatchTable[CurrentIdx++];
801
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
0
                      dbgs() << CurrentIdx
806
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
0
                             << InsnID << "], " << RendererID << ", "
808
0
                             << RenderOpID << ")\n");
809
0
      break;
810
277
    }
811
277
812
277
    case GIR_CopyConstantAsSImm: {
813
68
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
68
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
68
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
68
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
68
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
66
        OutMIs[NewInsnID].addImm(
819
66
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
66
      } else 
if (2
State.MIs[OldInsnID]->getOperand(1).isImm()2
)
821
2
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
2
      else
823
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
824
68
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
68
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
68
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
68
      break;
828
68
    }
829
68
830
68
    // TODO: Needs a test case once we have a pattern that uses this.
831
68
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
5
    case GIR_ConstrainOperandRC: {
861
5
      int64_t InsnID = MatchTable[CurrentIdx++];
862
5
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
5
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
5
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
5
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
5
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
5
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
5
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
5
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
5
                             << ")\n");
871
5
      break;
872
0
    }
873
0
874
237
    case GIR_ConstrainSelectedInstOperands: {
875
237
      int64_t InsnID = MatchTable[CurrentIdx++];
876
237
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
237
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
237
                                       RBI);
879
237
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
237
                      dbgs() << CurrentIdx
881
237
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
237
                             << InsnID << "])\n");
883
237
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
0
                             << InsnID << "]");
893
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
0
             GIU_MergeMemOperands_EndOfList) {
896
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
0
          OutMIs[InsnID].addMemOperand(MMO);
900
0
      }
901
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
0
      break;
903
0
    }
904
0
905
224
    case GIR_EraseFromParent: {
906
224
      int64_t InsnID = MatchTable[CurrentIdx++];
907
224
      assert(State.MIs[InsnID] &&
908
224
             "Attempted to erase an undefined instruction");
909
224
      State.MIs[InsnID]->eraseFromParent();
910
224
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
224
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
224
                             << InsnID << "])\n");
913
224
      break;
914
0
    }
915
0
916
9
    case GIR_MakeTempReg: {
917
9
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
9
      int64_t TypeID = MatchTable[CurrentIdx++];
919
9
920
9
      State.TempRegisters[TempRegID] =
921
9
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
9
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
9
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
9
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
233
    case GIR_Done:
939
233
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
233
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
233
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
11.3k
    }
946
11.3k
  }
947
706
}
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<41ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
144
    CodeGenCoverage &CoverageInfo) const {
57
144
58
144
  uint64_t CurrentIdx = 0;
59
144
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
144
61
144
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
144
  auto handleReject = [&]() -> RejectAction {
63
144
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
144
                    dbgs() << CurrentIdx << ": Rejected\n");
65
144
    if (OnFailResumeAt.empty())
66
144
      return RejectAndGiveUp;
67
144
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
144
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
144
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
144
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
144
    return RejectAndResume;
72
144
  };
73
144
74
1.79k
  while (true) {
75
1.79k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
1.79k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
1.79k
    switch (MatcherOpcode) {
78
1.79k
    case GIM_Try: {
79
419
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
419
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
419
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
419
      break;
83
1.79k
    }
84
1.79k
85
1.79k
    case GIM_RecordInsn: {
86
22
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
22
      int64_t InsnID = MatchTable[CurrentIdx++];
88
22
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
22
90
22
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
22
      // any attempt to modify it.
92
22
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
22
94
22
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
22
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
22
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
22
110
22
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
22
      if ((size_t)NewInsnID < State.MIs.size())
112
0
        State.MIs[NewInsnID] = NewMI;
113
22
      else {
114
22
        assert((size_t)NewInsnID == State.MIs.size() &&
115
22
               "Expected to store MIs in order");
116
22
        State.MIs.push_back(NewMI);
117
22
      }
118
22
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
22
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
22
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
22
                             << ")\n");
122
22
      break;
123
22
    }
124
22
125
314
    case GIM_CheckFeatures: {
126
314
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
314
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
314
                      dbgs() << CurrentIdx
129
314
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
314
                             << ExpectedBitsetID << ")\n");
131
314
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
314
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
240
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
314
      }
136
314
      break;
137
314
    }
138
314
139
314
    case GIM_CheckOpcode: {
140
22
      int64_t InsnID = MatchTable[CurrentIdx++];
141
22
      int64_t Expected = MatchTable[CurrentIdx++];
142
22
143
22
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
22
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
22
146
22
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
22
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
22
                             << "], ExpectedOpcode=" << Expected
149
22
                             << ") // Got=" << Opcode << "\n");
150
22
      if (Opcode != Expected) {
151
8
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
22
      }
154
22
      break;
155
22
    }
156
22
157
144
    case GIM_SwitchOpcode: {
158
144
      int64_t InsnID = MatchTable[CurrentIdx++];
159
144
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
144
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
144
      int64_t Default = MatchTable[CurrentIdx++];
162
144
163
144
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
144
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
144
166
144
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
144
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
144
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
144
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
144
      });
171
144
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
144
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
144
      if (!CurrentIdx) {
177
31
        CurrentIdx = Default;
178
31
  break;
179
31
      }
180
113
      OnFailResumeAt.push_back(Default);
181
113
      break;
182
113
    }
183
113
184
113
    case GIM_SwitchType: {
185
64
      int64_t InsnID = MatchTable[CurrentIdx++];
186
64
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
64
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
64
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
64
      int64_t Default = MatchTable[CurrentIdx++];
190
64
191
64
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
64
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
64
194
64
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
64
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
64
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
64
               << UpperBound << "), Default=" << Default
198
64
               << ", JumpTable...) // Got=";
199
64
        if (!MO.isReg())
200
64
          dbgs() << "Not a VReg\n";
201
64
        else
202
64
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
64
      });
204
64
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
64
      const LLT Ty = MRI.getType(MO.getReg());
209
64
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
64
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
0
        CurrentIdx = Default;
212
0
        break;
213
0
      }
214
64
      const int64_t TypeID = TyI->second;
215
64
      if (TypeID < LowerBound || UpperBound <= TypeID) {
216
0
        CurrentIdx = Default;
217
0
        break;
218
0
      }
219
64
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
64
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
64
      OnFailResumeAt.push_back(Default);
225
64
      break;
226
64
    }
227
64
228
64
    case GIM_CheckNumOperands: {
229
0
      int64_t InsnID = MatchTable[CurrentIdx++];
230
0
      int64_t Expected = MatchTable[CurrentIdx++];
231
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
0
                             << InsnID << "], Expected=" << Expected << ")\n");
234
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
0
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
0
      }
239
0
      break;
240
0
    }
241
14
    case GIM_CheckI64ImmPredicate: {
242
14
      int64_t InsnID = MatchTable[CurrentIdx++];
243
14
      int64_t Predicate = MatchTable[CurrentIdx++];
244
14
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
14
                      dbgs()
246
14
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
14
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
14
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
14
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
14
             "Expected G_CONSTANT");
251
14
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
14
      int64_t Value = 0;
253
14
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
14
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
14
260
14
      if (!testImmPredicate_I64(Predicate, Value))
261
0
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
14
      break;
264
14
    }
265
14
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
169
    case GIM_CheckType: {
456
169
      int64_t InsnID = MatchTable[CurrentIdx++];
457
169
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
169
      int64_t TypeID = MatchTable[CurrentIdx++];
459
169
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
169
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
169
                             << "]->getOperand(" << OpIdx
462
169
                             << "), TypeID=" << TypeID << ")\n");
463
169
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
169
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
169
      if (!MO.isReg() ||
466
169
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
0
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
169
      }
470
169
      break;
471
169
    }
472
169
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
196
    case GIM_CheckRegBankForClass: {
502
196
      int64_t InsnID = MatchTable[CurrentIdx++];
503
196
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
196
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
196
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
196
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
196
                             << InsnID << "]->getOperand(" << OpIdx
508
196
                             << "), RCEnum=" << RCEnum << ")\n");
509
196
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
196
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
196
      if (!MO.isReg() ||
512
196
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
196
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
0
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
196
      }
517
196
      break;
518
196
    }
519
196
520
196
    case GIM_CheckComplexPattern: {
521
0
      int64_t InsnID = MatchTable[CurrentIdx++];
522
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
0
      int64_t RendererID = MatchTable[CurrentIdx++];
524
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
0
                             << "]->getOperand(" << OpIdx
529
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
0
                             << ")\n");
531
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
0
      // FIXME: Use std::invoke() when it's available.
533
0
      ComplexRendererFns Renderer =
534
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
0
              State.MIs[InsnID]->getOperand(OpIdx));
536
0
      if (Renderer.hasValue())
537
0
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
0
      break;
542
0
    }
543
0
544
2
    case GIM_CheckConstantInt: {
545
2
      int64_t InsnID = MatchTable[CurrentIdx++];
546
2
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
2
      int64_t Value = MatchTable[CurrentIdx++];
548
2
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
2
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
2
                             << InsnID << "]->getOperand(" << OpIdx
551
2
                             << "), Value=" << Value << ")\n");
552
2
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
2
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
2
      if (MO.isReg()) {
555
2
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
2
        LLT Ty = MRI.getType(MO.getReg());
557
2
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
2
559
2
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
2
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
2
566
2
      break;
567
2
    }
568
2
569
2
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
0
    case GIM_CheckIntrinsicID: {
587
0
      int64_t InsnID = MatchTable[CurrentIdx++];
588
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
0
      int64_t Value = MatchTable[CurrentIdx++];
590
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
0
                             << InsnID << "]->getOperand(" << OpIdx
593
0
                             << "), Value=" << Value << ")\n");
594
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
0
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
0
      break;
600
0
    }
601
0
602
0
    case GIM_CheckIsMBB: {
603
0
      int64_t InsnID = MatchTable[CurrentIdx++];
604
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
0
                             << "]->getOperand(" << OpIdx << "))\n");
608
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
0
      }
613
0
      break;
614
0
    }
615
0
616
14
    case GIM_CheckIsSafeToFold: {
617
14
      int64_t InsnID = MatchTable[CurrentIdx++];
618
14
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
14
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
14
                             << InsnID << "])\n");
621
14
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
14
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
14
      }
626
14
      break;
627
14
    }
628
14
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
170
    case GIM_Reject:
647
170
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
170
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
170
      if (handleReject() == RejectAndGiveUp)
650
80
        return false;
651
90
      break;
652
90
653
90
    case GIR_MutateOpcode: {
654
50
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
50
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
50
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
50
      if (NewInsnID >= OutMIs.size())
658
50
        OutMIs.resize(NewInsnID + 1);
659
50
660
50
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
50
                                              State.MIs[OldInsnID]);
662
50
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
50
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
50
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
50
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
50
                             << NewOpcode << ")\n");
667
50
      break;
668
90
    }
669
90
670
90
    case GIR_BuildMI: {
671
14
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
14
      int64_t Opcode = MatchTable[CurrentIdx++];
673
14
      if (NewInsnID >= OutMIs.size())
674
14
        OutMIs.resize(NewInsnID + 1);
675
14
676
14
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
14
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
14
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
14
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
14
                             << NewInsnID << "], " << Opcode << ")\n");
681
14
      break;
682
90
    }
683
90
684
90
    case GIR_Copy: {
685
28
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
28
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
28
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
28
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
28
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
28
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
28
                      dbgs()
692
28
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
28
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
28
      break;
695
90
    }
696
90
697
90
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
90
    }
714
90
715
90
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
90
    }
729
90
730
90
    case GIR_AddImplicitDef: {
731
0
      int64_t InsnID = MatchTable[CurrentIdx++];
732
0
      int64_t RegNum = MatchTable[CurrentIdx++];
733
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
0
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
0
                             << InsnID << "], " << RegNum << ")\n");
738
0
      break;
739
90
    }
740
90
741
90
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
90
    }
751
90
752
90
    case GIR_AddRegister: {
753
0
      int64_t InsnID = MatchTable[CurrentIdx++];
754
0
      int64_t RegNum = MatchTable[CurrentIdx++];
755
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
0
      OutMIs[InsnID].addReg(RegNum);
757
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
0
                             << InsnID << "], " << RegNum << ")\n");
760
0
      break;
761
90
    }
762
90
763
90
    case GIR_AddTempRegister: {
764
0
      int64_t InsnID = MatchTable[CurrentIdx++];
765
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
0
                             << InsnID << "], TempRegisters[" << TempRegID
772
0
                             << "], " << TempRegFlags << ")\n");
773
0
      break;
774
90
    }
775
90
776
90
    case GIR_AddImm: {
777
0
      int64_t InsnID = MatchTable[CurrentIdx++];
778
0
      int64_t Imm = MatchTable[CurrentIdx++];
779
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
0
      OutMIs[InsnID].addImm(Imm);
781
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
0
                             << "], " << Imm << ")\n");
784
0
      break;
785
90
    }
786
90
787
90
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
90
    }
798
90
    case GIR_ComplexSubOperandRenderer: {
799
0
      int64_t InsnID = MatchTable[CurrentIdx++];
800
0
      int64_t RendererID = MatchTable[CurrentIdx++];
801
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
0
                      dbgs() << CurrentIdx
806
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
0
                             << InsnID << "], " << RendererID << ", "
808
0
                             << RenderOpID << ")\n");
809
0
      break;
810
90
    }
811
90
812
90
    case GIR_CopyConstantAsSImm: {
813
14
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
14
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
14
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
14
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
14
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
14
        OutMIs[NewInsnID].addImm(
819
14
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
14
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
14
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
14
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
14
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
14
      break;
828
14
    }
829
14
830
14
    // TODO: Needs a test case once we have a pattern that uses this.
831
14
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
0
    case GIR_ConstrainOperandRC: {
861
0
      int64_t InsnID = MatchTable[CurrentIdx++];
862
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
0
                             << ")\n");
871
0
      break;
872
0
    }
873
0
874
64
    case GIR_ConstrainSelectedInstOperands: {
875
64
      int64_t InsnID = MatchTable[CurrentIdx++];
876
64
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
64
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
64
                                       RBI);
879
64
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
64
                      dbgs() << CurrentIdx
881
64
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
64
                             << InsnID << "])\n");
883
64
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
0
                             << InsnID << "]");
893
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
0
             GIU_MergeMemOperands_EndOfList) {
896
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
0
          OutMIs[InsnID].addMemOperand(MMO);
900
0
      }
901
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
0
      break;
903
0
    }
904
0
905
14
    case GIR_EraseFromParent: {
906
14
      int64_t InsnID = MatchTable[CurrentIdx++];
907
14
      assert(State.MIs[InsnID] &&
908
14
             "Attempted to erase an undefined instruction");
909
14
      State.MIs[InsnID]->eraseFromParent();
910
14
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
14
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
14
                             << InsnID << "])\n");
913
14
      break;
914
0
    }
915
0
916
0
    case GIR_MakeTempReg: {
917
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
0
      int64_t TypeID = MatchTable[CurrentIdx++];