Coverage Report

Created: 2018-09-19 08:35

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
Line
Count
Source (jump to first uncovered line)
1
//===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
/// \file This file declares the API for the instruction selector.
11
/// This class is responsible for selecting machine instructions.
12
/// It's implemented by the target. It's used by the InstructionSelect pass.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17
#define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
18
19
#include "llvm/ADT/SmallVector.h"
20
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
21
#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
22
#include "llvm/CodeGen/GlobalISel/Utils.h"
23
#include "llvm/CodeGen/MachineInstrBuilder.h"
24
#include "llvm/CodeGen/MachineOperand.h"
25
#include "llvm/CodeGen/MachineRegisterInfo.h"
26
#include "llvm/CodeGen/TargetInstrInfo.h"
27
#include "llvm/CodeGen/TargetOpcodes.h"
28
#include "llvm/CodeGen/TargetRegisterInfo.h"
29
#include "llvm/IR/Constants.h"
30
#include "llvm/Support/Debug.h"
31
#include "llvm/Support/ErrorHandling.h"
32
#include "llvm/Support/raw_ostream.h"
33
#include <cassert>
34
#include <cstddef>
35
#include <cstdint>
36
37
namespace llvm {
38
39
/// GlobalISel PatFrag Predicates
40
enum {
41
  GIPFP_I64_Invalid = 0,
42
  GIPFP_APInt_Invalid = 0,
43
  GIPFP_APFloat_Invalid = 0,
44
  GIPFP_MI_Invalid = 0,
45
};
46
47
template <class TgtInstructionSelector, class PredicateBitset,
48
          class ComplexMatcherMemFn, class CustomRendererFn>
49
bool InstructionSelector::executeMatchTable(
50
    TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
51
    const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52
        &ISelInfo,
53
    const int64_t *MatchTable, const TargetInstrInfo &TII,
54
    MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
55
    const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
56
4.50M
    CodeGenCoverage &CoverageInfo) const {
57
4.50M
58
4.50M
  uint64_t CurrentIdx = 0;
59
4.50M
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
4.50M
61
4.50M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
5.78M
  auto handleReject = [&]() -> RejectAction {
63
5.78M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
5.78M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
5.78M
    if (OnFailResumeAt.empty())
66
2.69M
      return RejectAndGiveUp;
67
3.08M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
3.08M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
3.08M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
3.08M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
3.08M
    return RejectAndResume;
72
3.08M
  };
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<16ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<16ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<16ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
5.77M
  auto handleReject = [&]() -> RejectAction {
63
5.77M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
5.77M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
5.77M
    if (OnFailResumeAt.empty())
66
2.69M
      return RejectAndGiveUp;
67
3.08M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
3.08M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
3.08M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
3.08M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
3.08M
    return RejectAndResume;
72
3.08M
  };
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<32ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
197
  auto handleReject = [&]() -> RejectAction {
63
197
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
197
                    dbgs() << CurrentIdx << ": Rejected\n");
65
197
    if (OnFailResumeAt.empty())
66
0
      return RejectAndGiveUp;
67
197
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
197
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
197
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
197
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
197
    return RejectAndResume;
72
197
  };
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<64ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
2.27k
  auto handleReject = [&]() -> RejectAction {
63
2.27k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
2.27k
                    dbgs() << CurrentIdx << ": Rejected\n");
65
2.27k
    if (OnFailResumeAt.empty())
66
473
      return RejectAndGiveUp;
67
1.80k
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
1.80k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
1.80k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
1.80k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
1.80k
    return RejectAndResume;
72
1.80k
  };
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<41ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
331
  auto handleReject = [&]() -> RejectAction {
63
331
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
331
                    dbgs() << CurrentIdx << ": Rejected\n");
65
331
    if (OnFailResumeAt.empty())
66
66
      return RejectAndGiveUp;
67
265
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
265
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
265
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
265
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
265
    return RejectAndResume;
72
265
  };
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<113ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<113ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<113ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
3.74k
  auto handleReject = [&]() -> RejectAction {
63
3.74k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
3.74k
                    dbgs() << CurrentIdx << ": Rejected\n");
65
3.74k
    if (OnFailResumeAt.empty())
66
1.30k
      return RejectAndGiveUp;
67
2.43k
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
2.43k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
2.43k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
2.43k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
2.43k
    return RejectAndResume;
72
2.43k
  };
73
4.50M
74
35.4M
  while (true) {
75
35.4M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
35.4M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
35.4M
    switch (MatcherOpcode) {
78
35.4M
    case GIM_Try: {
79
4.26M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
4.26M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
4.26M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
4.26M
      break;
83
35.4M
    }
84
35.4M
85
35.4M
    case GIM_RecordInsn: {
86
1.14M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
1.14M
      int64_t InsnID = MatchTable[CurrentIdx++];
88
1.14M
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
1.14M
90
1.14M
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
1.14M
      // any attempt to modify it.
92
1.14M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
1.14M
94
1.14M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
1.14M
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
1.14M
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
1.14M
110
1.14M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
1.14M
      if ((size_t)NewInsnID < State.MIs.size())
112
815k
        State.MIs[NewInsnID] = NewMI;
113
324k
      else {
114
324k
        assert((size_t)NewInsnID == State.MIs.size() &&
115
324k
               "Expected to store MIs in order");
116
324k
        State.MIs.push_back(NewMI);
117
324k
      }
118
1.14M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
1.14M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
1.14M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
1.14M
                             << ")\n");
122
1.14M
      break;
123
1.14M
    }
124
1.14M
125
1.14M
    case GIM_CheckFeatures: {
126
339k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
339k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
339k
                      dbgs() << CurrentIdx
129
339k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
339k
                             << ExpectedBitsetID << ")\n");
131
339k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
339k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
33.2k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
339k
      }
136
339k
      break;
137
339k
    }
138
339k
139
1.14M
    case GIM_CheckOpcode: {
140
1.14M
      int64_t InsnID = MatchTable[CurrentIdx++];
141
1.14M
      int64_t Expected = MatchTable[CurrentIdx++];
142
1.14M
143
1.14M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
1.14M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
1.14M
146
1.14M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
1.14M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
1.14M
                             << "], ExpectedOpcode=" << Expected
149
1.14M
                             << ") // Got=" << Opcode << "\n");
150
1.14M
      if (Opcode != Expected) {
151
1.07M
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
1.14M
      }
154
1.14M
      break;
155
1.14M
    }
156
1.14M
157
4.50M
    case GIM_SwitchOpcode: {
158
4.50M
      int64_t InsnID = MatchTable[CurrentIdx++];
159
4.50M
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
4.50M
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
4.50M
      int64_t Default = MatchTable[CurrentIdx++];
162
4.50M
163
4.50M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
4.50M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
4.50M
166
4.50M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
4.50M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
4.50M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
4.50M
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
4.50M
      });
171
4.50M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
2
        CurrentIdx = Default;
173
2
        break;
174
2
      }
175
4.50M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
4.50M
      if (!CurrentIdx) {
177
1.45M
        CurrentIdx = Default;
178
1.45M
  break;
179
1.45M
      }
180
3.04M
      OnFailResumeAt.push_back(Default);
181
3.04M
      break;
182
3.04M
    }
183
3.04M
184
3.04M
    case GIM_SwitchType: {
185
2.64M
      int64_t InsnID = MatchTable[CurrentIdx++];
186
2.64M
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
2.64M
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
2.64M
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
2.64M
      int64_t Default = MatchTable[CurrentIdx++];
190
2.64M
191
2.64M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
2.64M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
2.64M
194
2.64M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
2.64M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
2.64M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
2.64M
               << UpperBound << "), Default=" << Default
198
2.64M
               << ", JumpTable...) // Got=";
199
2.64M
        if (!MO.isReg())
200
2.64M
          dbgs() << "Not a VReg\n";
201
2.64M
        else
202
2.64M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
2.64M
      });
204
2.64M
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
2.64M
      const LLT Ty = MRI.getType(MO.getReg());
209
2.64M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
2.64M
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
983k
        CurrentIdx = Default;
212
983k
        break;
213
983k
      }
214
1.66M
      const int64_t TypeID = TyI->second;
215
1.66M
      if (TypeID < LowerBound || 
UpperBound <= TypeID1.46M
) {
216
195k
        CurrentIdx = Default;
217
195k
        break;
218
195k
      }
219
1.46M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
1.46M
      if (!CurrentIdx) {
221
218
        CurrentIdx = Default;
222
218
        break;
223
218
      }
224
1.46M
      OnFailResumeAt.push_back(Default);
225
1.46M
      break;
226
1.46M
    }
227
1.46M
228
1.46M
    case GIM_CheckNumOperands: {
229
26.8k
      int64_t InsnID = MatchTable[CurrentIdx++];
230
26.8k
      int64_t Expected = MatchTable[CurrentIdx++];
231
26.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
26.8k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
26.8k
                             << InsnID << "], Expected=" << Expected << ")\n");
234
26.8k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
26.8k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
26.0k
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
26.8k
      }
239
26.8k
      break;
240
26.8k
    }
241
26.8k
    case GIM_CheckI64ImmPredicate: {
242
22.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
243
22.1k
      int64_t Predicate = MatchTable[CurrentIdx++];
244
22.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
22.1k
                      dbgs()
246
22.1k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
22.1k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
22.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
22.1k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
22.1k
             "Expected G_CONSTANT");
251
22.1k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
22.1k
      int64_t Value = 0;
253
22.1k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
22.1k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
2
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
2
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
2
      else
258
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
259
22.1k
260
22.1k
      if (!testImmPredicate_I64(Predicate, Value))
261
20
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
22.1k
      break;
264
22.1k
    }
265
22.1k
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
2.46k
    case GIM_CheckAPFloatImmPredicate: {
288
2.46k
      int64_t InsnID = MatchTable[CurrentIdx++];
289
2.46k
      int64_t Predicate = MatchTable[CurrentIdx++];
290
2.46k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
2.46k
                      dbgs()
292
2.46k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
2.46k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
2.46k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
2.46k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
2.46k
             "Expected G_FCONSTANT");
297
2.46k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
2.46k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
2.46k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
2.46k
301
2.46k
      if (!testImmPredicate_APFloat(Predicate, Value))
302
1.56k
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
2.46k
      break;
305
2.46k
    }
306
2.46k
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
436k
    case GIM_CheckAtomicOrdering: {
322
436k
      int64_t InsnID = MatchTable[CurrentIdx++];
323
436k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
436k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
436k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
436k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
436k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
436k
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
436k
332
436k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
436k
        if (MMO->getOrdering() != Ordering)
334
2.84k
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
436k
      break;
337
436k
    }
338
436k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
47
    case GIM_CheckMemorySizeEqualTo: {
375
47
      int64_t InsnID = MatchTable[CurrentIdx++];
376
47
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
47
      uint64_t Size = MatchTable[CurrentIdx++];
378
47
379
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
47
                      dbgs() << CurrentIdx
381
47
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
47
                             << "]->memoperands() + " << MMOIdx
383
47
                             << ", Size=" << Size << ")\n");
384
47
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
47
386
47
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
47
392
47
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
47
394
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
47
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
47
                             << " bytes\n");
397
47
      if (MMO->getSize() != Size)
398
2
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
47
401
47
      break;
402
47
    }
403
281k
    case GIM_CheckMemorySizeEqualToLLT:
404
281k
    case GIM_CheckMemorySizeLessThanLLT:
405
281k
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
281k
      int64_t InsnID = MatchTable[CurrentIdx++];
407
281k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
281k
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
281k
410
281k
      DEBUG_WITH_TYPE(
411
281k
          TgtInstructionSelector::getName(),
412
281k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
281k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
281k
                         ? "EqualTo"
415
281k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
281k
                               ? "GreaterThan"
417
281k
                               : "LessThan")
418
281k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
281k
                 << ", OpIdx=" << OpIdx << ")\n");
420
281k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
281k
422
281k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
281k
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
281k
431
281k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
281k
437
281k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
281k
439
281k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
281k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
281k
          
MMO->getSize() * 8 != Size281k
) {
442
16
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
281k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
281k
                 
MMO->getSize() * 8 >= Size42
) {
446
36
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
281k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
281k
                 
MMO->getSize() * 8 <= Size0
)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
281k
453
281k
      break;
454
281k
    }
455
946k
    case GIM_CheckType: {
456
946k
      int64_t InsnID = MatchTable[CurrentIdx++];
457
946k
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
946k
      int64_t TypeID = MatchTable[CurrentIdx++];
459
946k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
946k
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
946k
                             << "]->getOperand(" << OpIdx
462
946k
                             << "), TypeID=" << TypeID << ")\n");
463
946k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
946k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
946k
      if (!MO.isReg() ||
466
946k
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
22.7k
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
946k
      }
470
946k
      break;
471
946k
    }
472
946k
    case GIM_CheckPointerToAny: {
473
423k
      int64_t InsnID = MatchTable[CurrentIdx++];
474
423k
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
423k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
423k
477
423k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
423k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
423k
                             << InsnID << "]->getOperand(" << OpIdx
480
423k
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
423k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
423k
      // iPTR must be looked up in the target.
483
423k
      if (SizeInBits == 0) {
484
423k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
423k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
423k
      }
487
423k
488
423k
      assert(SizeInBits != 0 && "Pointer size must be known");
489
423k
490
423k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
423k
      if (MO.isReg()) {
492
423k
        const LLT &Ty = MRI.getType(MO.getReg());
493
423k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
423k
499
423k
      break;
500
423k
    }
501
3.75M
    case GIM_CheckRegBankForClass: {
502
3.75M
      int64_t InsnID = MatchTable[CurrentIdx++];
503
3.75M
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
3.75M
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
3.75M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
3.75M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
3.75M
                             << InsnID << "]->getOperand(" << OpIdx
508
3.75M
                             << "), RCEnum=" << RCEnum << ")\n");
509
3.75M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
3.75M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
3.75M
      if (!MO.isReg() ||
512
3.75M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
3.75M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
191k
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
3.75M
      }
517
3.75M
      break;
518
3.75M
    }
519
3.75M
520
3.75M
    case GIM_CheckComplexPattern: {
521
711k
      int64_t InsnID = MatchTable[CurrentIdx++];
522
711k
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
711k
      int64_t RendererID = MatchTable[CurrentIdx++];
524
711k
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
711k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
711k
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
711k
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
711k
                             << "]->getOperand(" << OpIdx
529
711k
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
711k
                             << ")\n");
531
711k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
711k
      // FIXME: Use std::invoke() when it's available.
533
711k
      ComplexRendererFns Renderer =
534
711k
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
711k
              State.MIs[InsnID]->getOperand(OpIdx));
536
711k
      if (Renderer.hasValue())
537
476k
        State.Renderers[RendererID] = Renderer.getValue();
538
235k
      else
539
235k
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
711k
      break;
542
711k
    }
543
711k
544
711k
    case GIM_CheckConstantInt: {
545
81.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
546
81.1k
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
81.1k
      int64_t Value = MatchTable[CurrentIdx++];
548
81.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
81.1k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
81.1k
                             << InsnID << "]->getOperand(" << OpIdx
551
81.1k
                             << "), Value=" << Value << ")\n");
552
81.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
81.1k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
81.1k
      if (MO.isReg()) {
555
81.1k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
81.1k
        LLT Ty = MRI.getType(MO.getReg());
557
81.1k
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
81.1k
559
81.1k
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
63.7k
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
81.1k
566
81.1k
      break;
567
81.1k
    }
568
81.1k
569
81.1k
    case GIM_CheckLiteralInt: {
570
54
      int64_t InsnID = MatchTable[CurrentIdx++];
571
54
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
54
      int64_t Value = MatchTable[CurrentIdx++];
573
54
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
54
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
54
                             << InsnID << "]->getOperand(" << OpIdx
576
54
                             << "), Value=" << Value << ")\n");
577
54
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
54
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
54
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
32
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
54
      }
583
54
      break;
584
54
    }
585
54
586
161k
    case GIM_CheckIntrinsicID: {
587
161k
      int64_t InsnID = MatchTable[CurrentIdx++];
588
161k
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
161k
      int64_t Value = MatchTable[CurrentIdx++];
590
161k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
161k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
161k
                             << InsnID << "]->getOperand(" << OpIdx
593
161k
                             << "), Value=" << Value << ")\n");
594
161k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
161k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
161k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value156k
)
597
159k
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
161k
      break;
600
161k
    }
601
161k
602
374k
    case GIM_CheckIsMBB: {
603
374k
      int64_t InsnID = MatchTable[CurrentIdx++];
604
374k
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
374k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
374k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
374k
                             << "]->getOperand(" << OpIdx << "))\n");
608
374k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
374k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
374k
      }
613
374k
      break;
614
374k
    }
615
374k
616
374k
    case GIM_CheckIsSafeToFold: {
617
43.8k
      int64_t InsnID = MatchTable[CurrentIdx++];
618
43.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
43.8k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
43.8k
                             << InsnID << "])\n");
621
43.8k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
43.8k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
43.8k
      }
626
43.8k
      break;
627
43.8k
    }
628
43.8k
    case GIM_CheckIsSameOperand: {
629
6
      int64_t InsnID = MatchTable[CurrentIdx++];
630
6
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
6
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
6
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
6
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
6
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
6
                             << InsnID << "][" << OpIdx << "], MIs["
636
6
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
6
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
6
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
6
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
6
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
2
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
6
      }
644
6
      break;
645
6
    }
646
3.96M
    case GIM_Reject:
647
3.96M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
3.96M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
3.96M
      if (handleReject() == RejectAndGiveUp)
650
2.69M
        return false;
651
1.27M
      break;
652
1.27M
653
1.27M
    case GIR_MutateOpcode: {
654
618k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
618k
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
618k
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
618k
      if (NewInsnID >= OutMIs.size())
658
618k
        OutMIs.resize(NewInsnID + 1);
659
618k
660
618k
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
618k
                                              State.MIs[OldInsnID]);
662
618k
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
618k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
618k
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
618k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
618k
                             << NewOpcode << ")\n");
667
618k
      break;
668
1.27M
    }
669
1.27M
670
1.27M
    case GIR_BuildMI: {
671
1.19M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
1.19M
      int64_t Opcode = MatchTable[CurrentIdx++];
673
1.19M
      if (NewInsnID >= OutMIs.size())
674
1.19M
        OutMIs.resize(NewInsnID + 1);
675
1.19M
676
1.19M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
1.19M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
1.19M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
1.19M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
1.19M
                             << NewInsnID << "], " << Opcode << ")\n");
681
1.19M
      break;
682
1.27M
    }
683
1.27M
684
1.30M
    case GIR_Copy: {
685
1.30M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
1.30M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
1.30M
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
1.30M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
1.30M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
1.30M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
1.30M
                      dbgs()
692
1.30M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
1.30M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
1.30M
      break;
695
1.27M
    }
696
1.27M
697
1.27M
    case GIR_CopyOrAddZeroReg: {
698
139k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
139k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
139k
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
139k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
139k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
139k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
139k
      if (isOperandImmEqual(MO, 0, MRI))
705
15.1k
        OutMIs[NewInsnID].addReg(ZeroReg);
706
124k
      else
707
124k
        OutMIs[NewInsnID].add(MO);
708
139k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
139k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
139k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
139k
                             << OpIdx << ", " << ZeroReg << ")\n");
712
139k
      break;
713
1.27M
    }
714
1.27M
715
1.27M
    case GIR_CopySubReg: {
716
42.0k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
42.0k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
42.0k
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
42.0k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
42.0k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
42.0k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
42.0k
                               0, SubRegIdx);
723
42.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
42.0k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
42.0k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
42.0k
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
42.0k
      break;
728
1.27M
    }
729
1.27M
730
1.27M
    case GIR_AddImplicitDef: {
731
23.3k
      int64_t InsnID = MatchTable[CurrentIdx++];
732
23.3k
      int64_t RegNum = MatchTable[CurrentIdx++];
733
23.3k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
23.3k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
23.3k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
23.3k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
23.3k
                             << InsnID << "], " << RegNum << ")\n");
738
23.3k
      break;
739
1.27M
    }
740
1.27M
741
1.27M
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
1.27M
    }
751
1.27M
752
1.27M
    case GIR_AddRegister: {
753
84.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
754
84.9k
      int64_t RegNum = MatchTable[CurrentIdx++];
755
84.9k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
84.9k
      OutMIs[InsnID].addReg(RegNum);
757
84.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
84.9k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
84.9k
                             << InsnID << "], " << RegNum << ")\n");
760
84.9k
      break;
761
1.27M
    }
762
1.27M
763
1.27M
    case GIR_AddTempRegister: {
764
7.86k
      int64_t InsnID = MatchTable[CurrentIdx++];
765
7.86k
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
7.86k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
7.86k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
7.86k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
7.86k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
7.86k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
7.86k
                             << InsnID << "], TempRegisters[" << TempRegID
772
7.86k
                             << "], " << TempRegFlags << ")\n");
773
7.86k
      break;
774
1.27M
    }
775
1.27M
776
1.27M
    case GIR_AddImm: {
777
18.3k
      int64_t InsnID = MatchTable[CurrentIdx++];
778
18.3k
      int64_t Imm = MatchTable[CurrentIdx++];
779
18.3k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
18.3k
      OutMIs[InsnID].addImm(Imm);
781
18.3k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
18.3k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
18.3k
                             << "], " << Imm << ")\n");
784
18.3k
      break;
785
1.27M
    }
786
1.27M
787
1.27M
    case GIR_ComplexRenderer: {
788
58.7k
      int64_t InsnID = MatchTable[CurrentIdx++];
789
58.7k
      int64_t RendererID = MatchTable[CurrentIdx++];
790
58.7k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
58.7k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
117k
        RenderOpFn(OutMIs[InsnID]);
793
58.7k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
58.7k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
58.7k
                             << InsnID << "], " << RendererID << ")\n");
796
58.7k
      break;
797
1.27M
    }
798
1.27M
    case GIR_ComplexSubOperandRenderer: {
799
834k
      int64_t InsnID = MatchTable[CurrentIdx++];
800
834k
      int64_t RendererID = MatchTable[CurrentIdx++];
801
834k
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
834k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
834k
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
834k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
834k
                      dbgs() << CurrentIdx
806
834k
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
834k
                             << InsnID << "], " << RendererID << ", "
808
834k
                             << RenderOpID << ")\n");
809
834k
      break;
810
1.27M
    }
811
1.27M
812
1.27M
    case GIR_CopyConstantAsSImm: {
813
586k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
586k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
586k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
586k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
586k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
586k
        OutMIs[NewInsnID].addImm(
819
586k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
586k
      } else 
if (2
State.MIs[OldInsnID]->getOperand(1).isImm()2
)
821
2
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
2
      else
823
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
824
586k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
586k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
586k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
586k
      break;
828
586k
    }
829
586k
830
586k
    // TODO: Needs a test case once we have a pattern that uses this.
831
586k
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
3.91k
    case GIR_CustomRenderer: {
848
3.91k
      int64_t InsnID = MatchTable[CurrentIdx++];
849
3.91k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
3.91k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
3.91k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
3.91k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
3.91k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
3.91k
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
3.91k
                             << RendererFnID << ")\n");
856
3.91k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
3.91k
                                                     *State.MIs[OldInsnID]);
858
3.91k
      break;
859
0
    }
860
84.2k
    case GIR_ConstrainOperandRC: {
861
84.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
862
84.2k
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
84.2k
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
84.2k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
84.2k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
84.2k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
84.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
84.2k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
84.2k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
84.2k
                             << ")\n");
871
84.2k
      break;
872
0
    }
873
0
874
1.77M
    case GIR_ConstrainSelectedInstOperands: {
875
1.77M
      int64_t InsnID = MatchTable[CurrentIdx++];
876
1.77M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
1.77M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
1.77M
                                       RBI);
879
1.77M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
1.77M
                      dbgs() << CurrentIdx
881
1.77M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
1.77M
                             << InsnID << "])\n");
883
1.77M
      break;
884
0
    }
885
0
886
417k
    case GIR_MergeMemOperands: {
887
417k
      int64_t InsnID = MatchTable[CurrentIdx++];
888
417k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
417k
890
417k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
417k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
417k
                             << InsnID << "]");
893
417k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
834k
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
834k
             GIU_MergeMemOperands_EndOfList) {
896
417k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
417k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
417k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
417k
          OutMIs[InsnID].addMemOperand(MMO);
900
417k
      }
901
417k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
417k
      break;
903
0
    }
904
0
905
1.19M
    case GIR_EraseFromParent: {
906
1.19M
      int64_t InsnID = MatchTable[CurrentIdx++];
907
1.19M
      assert(State.MIs[InsnID] &&
908
1.19M
             "Attempted to erase an undefined instruction");
909
1.19M
      State.MIs[InsnID]->eraseFromParent();
910
1.19M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
1.19M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
1.19M
                             << InsnID << "])\n");
913
1.19M
      break;
914
0
    }
915
0
916
3.93k
    case GIR_MakeTempReg: {
917
3.93k
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
3.93k
      int64_t TypeID = MatchTable[CurrentIdx++];
919
3.93k
920
3.93k
      State.TempRegisters[TempRegID] =
921
3.93k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
3.93k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
3.93k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
3.93k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
3.93k
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
1.81M
    case GIR_Done:
939
1.81M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
1.81M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
1.81M
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
35.4M
    }
946
35.4M
  }
947
4.50M
}
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<16ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<16ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<16ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
4.50M
    CodeGenCoverage &CoverageInfo) const {
57
4.50M
58
4.50M
  uint64_t CurrentIdx = 0;
59
4.50M
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
4.50M
61
4.50M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
4.50M
  auto handleReject = [&]() -> RejectAction {
63
4.50M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
4.50M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
4.50M
    if (OnFailResumeAt.empty())
66
4.50M
      return RejectAndGiveUp;
67
4.50M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
4.50M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
4.50M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
4.50M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
4.50M
    return RejectAndResume;
72
4.50M
  };
73
4.50M
74
35.4M
  while (true) {
75
35.4M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
35.4M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
35.4M
    switch (MatcherOpcode) {
78
35.4M
    case GIM_Try: {
79
4.25M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
4.25M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
4.25M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
4.25M
      break;
83
35.4M
    }
84
35.4M
85
35.4M
    case GIM_RecordInsn: {
86
1.13M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
1.13M
      int64_t InsnID = MatchTable[CurrentIdx++];
88
1.13M
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
1.13M
90
1.13M
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
1.13M
      // any attempt to modify it.
92
1.13M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
1.13M
94
1.13M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
1.13M
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
1.13M
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
1.13M
110
1.13M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
1.13M
      if ((size_t)NewInsnID < State.MIs.size())
112
815k
        State.MIs[NewInsnID] = NewMI;
113
324k
      else {
114
324k
        assert((size_t)NewInsnID == State.MIs.size() &&
115
324k
               "Expected to store MIs in order");
116
324k
        State.MIs.push_back(NewMI);
117
324k
      }
118
1.13M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
1.13M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
1.13M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
1.13M
                             << ")\n");
122
1.13M
      break;
123
1.13M
    }
124
1.13M
125
1.13M
    case GIM_CheckFeatures: {
126
336k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
336k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
336k
                      dbgs() << CurrentIdx
129
336k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
336k
                             << ExpectedBitsetID << ")\n");
131
336k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
336k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
31.0k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
336k
      }
136
336k
      break;
137
336k
    }
138
336k
139
1.13M
    case GIM_CheckOpcode: {
140
1.13M
      int64_t InsnID = MatchTable[CurrentIdx++];
141
1.13M
      int64_t Expected = MatchTable[CurrentIdx++];
142
1.13M
143
1.13M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
1.13M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
1.13M
146
1.13M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
1.13M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
1.13M
                             << "], ExpectedOpcode=" << Expected
149
1.13M
                             << ") // Got=" << Opcode << "\n");
150
1.13M
      if (Opcode != Expected) {
151
1.07M
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
1.13M
      }
154
1.13M
      break;
155
1.13M
    }
156
1.13M
157
4.50M
    case GIM_SwitchOpcode: {
158
4.50M
      int64_t InsnID = MatchTable[CurrentIdx++];
159
4.50M
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
4.50M
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
4.50M
      int64_t Default = MatchTable[CurrentIdx++];
162
4.50M
163
4.50M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
4.50M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
4.50M
166
4.50M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
4.50M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
4.50M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
4.50M
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
4.50M
      });
171
4.50M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
2
        CurrentIdx = Default;
173
2
        break;
174
2
      }
175
4.50M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
4.50M
      if (!CurrentIdx) {
177
1.45M
        CurrentIdx = Default;
178
1.45M
  break;
179
1.45M
      }
180
3.04M
      OnFailResumeAt.push_back(Default);
181
3.04M
      break;
182
3.04M
    }
183
3.04M
184
3.04M
    case GIM_SwitchType: {
185
2.64M
      int64_t InsnID = MatchTable[CurrentIdx++];
186
2.64M
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
2.64M
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
2.64M
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
2.64M
      int64_t Default = MatchTable[CurrentIdx++];
190
2.64M
191
2.64M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
2.64M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
2.64M
194
2.64M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
2.64M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
2.64M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
2.64M
               << UpperBound << "), Default=" << Default
198
2.64M
               << ", JumpTable...) // Got=";
199
2.64M
        if (!MO.isReg())
200
2.64M
          dbgs() << "Not a VReg\n";
201
2.64M
        else
202
2.64M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
2.64M
      });
204
2.64M
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
2.64M
      const LLT Ty = MRI.getType(MO.getReg());
209
2.64M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
2.64M
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
983k
        CurrentIdx = Default;
212
983k
        break;
213
983k
      }
214
1.66M
      const int64_t TypeID = TyI->second;
215
1.66M
      if (TypeID < LowerBound || 
UpperBound <= TypeID1.46M
) {
216
195k
        CurrentIdx = Default;
217
195k
        break;
218
195k
      }
219
1.46M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
1.46M
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
1.46M
      OnFailResumeAt.push_back(Default);
225
1.46M
      break;
226
1.46M
    }
227
1.46M
228
1.46M
    case GIM_CheckNumOperands: {
229
26.7k
      int64_t InsnID = MatchTable[CurrentIdx++];
230
26.7k
      int64_t Expected = MatchTable[CurrentIdx++];
231
26.7k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
26.7k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
26.7k
                             << InsnID << "], Expected=" << Expected << ")\n");
234
26.7k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
26.7k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
26.0k
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
26.7k
      }
239
26.7k
      break;
240
26.7k
    }
241
26.7k
    case GIM_CheckI64ImmPredicate: {
242
22.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
243
22.0k
      int64_t Predicate = MatchTable[CurrentIdx++];
244
22.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
22.0k
                      dbgs()
246
22.0k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
22.0k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
22.0k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
22.0k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
22.0k
             "Expected G_CONSTANT");
251
22.0k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
22.0k
      int64_t Value = 0;
253
22.0k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
22.0k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
22.0k
260
22.0k
      if (!testImmPredicate_I64(Predicate, Value))
261
6
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
22.0k
      break;
264
22.0k
    }
265
22.0k
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
2.45k
    case GIM_CheckAPFloatImmPredicate: {
288
2.45k
      int64_t InsnID = MatchTable[CurrentIdx++];
289
2.45k
      int64_t Predicate = MatchTable[CurrentIdx++];
290
2.45k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
2.45k
                      dbgs()
292
2.45k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
2.45k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
2.45k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
2.45k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
2.45k
             "Expected G_FCONSTANT");
297
2.45k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
2.45k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
2.45k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
2.45k
301
2.45k
      if (!testImmPredicate_APFloat(Predicate, Value))
302
1.56k
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
2.45k
      break;
305
2.45k
    }
306
2.45k
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
436k
    case GIM_CheckAtomicOrdering: {
322
436k
      int64_t InsnID = MatchTable[CurrentIdx++];
323
436k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
436k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
436k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
436k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
436k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
436k
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
436k
332
436k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
436k
        if (MMO->getOrdering() != Ordering)
334
2.84k
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
436k
      break;
337
436k
    }
338
436k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
47
    case GIM_CheckMemorySizeEqualTo: {
375
47
      int64_t InsnID = MatchTable[CurrentIdx++];
376
47
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
47
      uint64_t Size = MatchTable[CurrentIdx++];
378
47
379
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
47
                      dbgs() << CurrentIdx
381
47
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
47
                             << "]->memoperands() + " << MMOIdx
383
47
                             << ", Size=" << Size << ")\n");
384
47
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
47
386
47
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
47
392
47
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
47
394
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
47
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
47
                             << " bytes\n");
397
47
      if (MMO->getSize() != Size)
398
2
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
47
401
47
      break;
402
47
    }
403
281k
    case GIM_CheckMemorySizeEqualToLLT:
404
281k
    case GIM_CheckMemorySizeLessThanLLT:
405
281k
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
281k
      int64_t InsnID = MatchTable[CurrentIdx++];
407
281k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
281k
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
281k
410
281k
      DEBUG_WITH_TYPE(
411
281k
          TgtInstructionSelector::getName(),
412
281k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
281k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
281k
                         ? "EqualTo"
415
281k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
281k
                               ? "GreaterThan"
417
281k
                               : "LessThan")
418
281k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
281k
                 << ", OpIdx=" << OpIdx << ")\n");
420
281k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
281k
422
281k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
281k
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
281k
431
281k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
281k
437
281k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
281k
439
281k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
281k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
281k
          
MMO->getSize() * 8 != Size281k
) {
442
16
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
281k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
281k
                 
MMO->getSize() * 8 >= Size42
) {
446
36
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
281k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
281k
                 
MMO->getSize() * 8 <= Size0
)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
281k
453
281k
      break;
454
281k
    }
455
944k
    case GIM_CheckType: {
456
944k
      int64_t InsnID = MatchTable[CurrentIdx++];
457
944k
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
944k
      int64_t TypeID = MatchTable[CurrentIdx++];
459
944k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
944k
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
944k
                             << "]->getOperand(" << OpIdx
462
944k
                             << "), TypeID=" << TypeID << ")\n");
463
944k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
944k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
944k
      if (!MO.isReg() ||
466
944k
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
22.4k
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
944k
      }
470
944k
      break;
471
944k
    }
472
944k
    case GIM_CheckPointerToAny: {
473
423k
      int64_t InsnID = MatchTable[CurrentIdx++];
474
423k
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
423k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
423k
477
423k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
423k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
423k
                             << InsnID << "]->getOperand(" << OpIdx
480
423k
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
423k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
423k
      // iPTR must be looked up in the target.
483
423k
      if (SizeInBits == 0) {
484
423k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
423k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
423k
      }
487
423k
488
423k
      assert(SizeInBits != 0 && "Pointer size must be known");
489
423k
490
423k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
423k
      if (MO.isReg()) {
492
423k
        const LLT &Ty = MRI.getType(MO.getReg());
493
423k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
423k
499
423k
      break;
500
423k
    }
501
3.75M
    case GIM_CheckRegBankForClass: {
502
3.75M
      int64_t InsnID = MatchTable[CurrentIdx++];
503
3.75M
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
3.75M
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
3.75M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
3.75M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
3.75M
                             << InsnID << "]->getOperand(" << OpIdx
508
3.75M
                             << "), RCEnum=" << RCEnum << ")\n");
509
3.75M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
3.75M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
3.75M
      if (!MO.isReg() ||
512
3.75M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
3.75M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
191k
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
3.75M
      }
517
3.75M
      break;
518
3.75M
    }
519
3.75M
520
3.75M
    case GIM_CheckComplexPattern: {
521
711k
      int64_t InsnID = MatchTable[CurrentIdx++];
522
711k
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
711k
      int64_t RendererID = MatchTable[CurrentIdx++];
524
711k
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
711k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
711k
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
711k
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
711k
                             << "]->getOperand(" << OpIdx
529
711k
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
711k
                             << ")\n");
531
711k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
711k
      // FIXME: Use std::invoke() when it's available.
533
711k
      ComplexRendererFns Renderer =
534
711k
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
711k
              State.MIs[InsnID]->getOperand(OpIdx));
536
711k
      if (Renderer.hasValue())
537
476k
        State.Renderers[RendererID] = Renderer.getValue();
538
235k
      else
539
235k
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
711k
      break;
542
711k
    }
543
711k
544
711k
    case GIM_CheckConstantInt: {
545
80.8k
      int64_t InsnID = MatchTable[CurrentIdx++];
546
80.8k
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
80.8k
      int64_t Value = MatchTable[CurrentIdx++];
548
80.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
80.8k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
80.8k
                             << InsnID << "]->getOperand(" << OpIdx
551
80.8k
                             << "), Value=" << Value << ")\n");
552
80.8k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
80.8k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
80.8k
      if (MO.isReg()) {
555
80.8k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
80.8k
        LLT Ty = MRI.getType(MO.getReg());
557
80.8k
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
80.8k
559
80.8k
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
63.4k
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
80.8k
566
80.8k
      break;
567
80.8k
    }
568
80.8k
569
80.8k
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
161k
    case GIM_CheckIntrinsicID: {
587
161k
      int64_t InsnID = MatchTable[CurrentIdx++];
588
161k
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
161k
      int64_t Value = MatchTable[CurrentIdx++];
590
161k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
161k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
161k
                             << InsnID << "]->getOperand(" << OpIdx
593
161k
                             << "), Value=" << Value << ")\n");
594
161k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
161k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
161k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value156k
)
597
159k
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
161k
      break;
600
161k
    }
601
161k
602
374k
    case GIM_CheckIsMBB: {
603
374k
      int64_t InsnID = MatchTable[CurrentIdx++];
604
374k
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
374k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
374k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
374k
                             << "]->getOperand(" << OpIdx << "))\n");
608
374k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
374k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
374k
      }
613
374k
      break;
614
374k
    }
615
374k
616
374k
    case GIM_CheckIsSafeToFold: {
617
43.6k
      int64_t InsnID = MatchTable[CurrentIdx++];
618
43.6k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
43.6k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
43.6k
                             << InsnID << "])\n");
621
43.6k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
43.6k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
43.6k
      }
626
43.6k
      break;
627
43.6k
    }
628
43.6k
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
3.96M
    case GIM_Reject:
647
3.96M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
3.96M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
3.96M
      if (handleReject() == RejectAndGiveUp)
650
2.69M
        return false;
651
1.27M
      break;
652
1.27M
653
1.27M
    case GIR_MutateOpcode: {
654
617k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
617k
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
617k
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
617k
      if (NewInsnID >= OutMIs.size())
658
617k
        OutMIs.resize(NewInsnID + 1);
659
617k
660
617k
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
617k
                                              State.MIs[OldInsnID]);
662
617k
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
617k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
617k
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
617k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
617k
                             << NewOpcode << ")\n");
667
617k
      break;
668
1.27M
    }
669
1.27M
670
1.27M
    case GIR_BuildMI: {
671
1.19M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
1.19M
      int64_t Opcode = MatchTable[CurrentIdx++];
673
1.19M
      if (NewInsnID >= OutMIs.size())
674
1.19M
        OutMIs.resize(NewInsnID + 1);
675
1.19M
676
1.19M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
1.19M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
1.19M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
1.19M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
1.19M
                             << NewInsnID << "], " << Opcode << ")\n");
681
1.19M
      break;
682
1.27M
    }
683
1.27M
684
1.30M
    case GIR_Copy: {
685
1.30M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
1.30M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
1.30M
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
1.30M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
1.30M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
1.30M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
1.30M
                      dbgs()
692
1.30M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
1.30M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
1.30M
      break;
695
1.27M
    }
696
1.27M
697
1.27M
    case GIR_CopyOrAddZeroReg: {
698
139k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
139k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
139k
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
139k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
139k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
139k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
139k
      if (isOperandImmEqual(MO, 0, MRI))
705
15.1k
        OutMIs[NewInsnID].addReg(ZeroReg);
706
124k
      else
707
124k
        OutMIs[NewInsnID].add(MO);
708
139k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
139k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
139k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
139k
                             << OpIdx << ", " << ZeroReg << ")\n");
712
139k
      break;
713
1.27M
    }
714
1.27M
715
1.27M
    case GIR_CopySubReg: {
716
41.8k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
41.8k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
41.8k
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
41.8k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
41.8k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
41.8k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
41.8k
                               0, SubRegIdx);
723
41.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
41.8k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
41.8k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
41.8k
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
41.8k
      break;
728
1.27M
    }
729
1.27M
730
1.27M
    case GIR_AddImplicitDef: {
731
23.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
732
23.2k
      int64_t RegNum = MatchTable[CurrentIdx++];
733
23.2k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
23.2k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
23.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
23.2k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
23.2k
                             << InsnID << "], " << RegNum << ")\n");
738
23.2k
      break;
739
1.27M
    }
740
1.27M
741
1.27M
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
1.27M
    }
751
1.27M
752
1.27M
    case GIR_AddRegister: {
753
84.6k
      int64_t InsnID = MatchTable[CurrentIdx++];
754
84.6k
      int64_t RegNum = MatchTable[CurrentIdx++];
755
84.6k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
84.6k
      OutMIs[InsnID].addReg(RegNum);
757
84.6k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
84.6k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
84.6k
                             << InsnID << "], " << RegNum << ")\n");
760
84.6k
      break;
761
1.27M
    }
762
1.27M
763
1.27M
    case GIR_AddTempRegister: {
764
7.83k
      int64_t InsnID = MatchTable[CurrentIdx++];
765
7.83k
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
7.83k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
7.83k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
7.83k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
7.83k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
7.83k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
7.83k
                             << InsnID << "], TempRegisters[" << TempRegID
772
7.83k
                             << "], " << TempRegFlags << ")\n");
773
7.83k
      break;
774
1.27M
    }
775
1.27M
776
1.27M
    case GIR_AddImm: {
777
18.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
778
18.0k
      int64_t Imm = MatchTable[CurrentIdx++];
779
18.0k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
18.0k
      OutMIs[InsnID].addImm(Imm);
781
18.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
18.0k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
18.0k
                             << "], " << Imm << ")\n");
784
18.0k
      break;
785
1.27M
    }
786
1.27M
787
1.27M
    case GIR_ComplexRenderer: {
788
58.7k
      int64_t InsnID = MatchTable[CurrentIdx++];
789
58.7k
      int64_t RendererID = MatchTable[CurrentIdx++];
790
58.7k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
58.7k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
117k
        RenderOpFn(OutMIs[InsnID]);
793
58.7k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
58.7k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
58.7k
                             << InsnID << "], " << RendererID << ")\n");
796
58.7k
      break;
797
1.27M
    }
798
1.27M
    case GIR_ComplexSubOperandRenderer: {
799
834k
      int64_t InsnID = MatchTable[CurrentIdx++];
800
834k
      int64_t RendererID = MatchTable[CurrentIdx++];
801
834k
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
834k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
834k
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
834k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
834k
                      dbgs() << CurrentIdx
806
834k
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
834k
                             << InsnID << "], " << RendererID << ", "
808
834k
                             << RenderOpID << ")\n");
809
834k
      break;
810
1.27M
    }
811
1.27M
812
1.27M
    case GIR_CopyConstantAsSImm: {
813
586k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
586k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
586k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
586k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
586k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
586k
        OutMIs[NewInsnID].addImm(
819
586k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
586k
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
586k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
586k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
586k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
586k
      break;
828
586k
    }
829
586k
830
586k
    // TODO: Needs a test case once we have a pattern that uses this.
831
586k
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
3.91k
    case GIR_CustomRenderer: {
848
3.91k
      int64_t InsnID = MatchTable[CurrentIdx++];
849
3.91k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
3.91k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
3.91k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
3.91k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
3.91k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
3.91k
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
3.91k
                             << RendererFnID << ")\n");
856
3.91k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
3.91k
                                                     *State.MIs[OldInsnID]);
858
3.91k
      break;
859
0
    }
860
84.0k
    case GIR_ConstrainOperandRC: {
861
84.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
862
84.0k
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
84.0k
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
84.0k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
84.0k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
84.0k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
84.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
84.0k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
84.0k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
84.0k
                             << ")\n");
871
84.0k
      break;
872
0
    }
873
0
874
1.77M
    case GIR_ConstrainSelectedInstOperands: {
875
1.77M
      int64_t InsnID = MatchTable[CurrentIdx++];
876
1.77M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
1.77M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
1.77M
                                       RBI);
879
1.77M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
1.77M
                      dbgs() << CurrentIdx
881
1.77M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
1.77M
                             << InsnID << "])\n");
883
1.77M
      break;
884
0
    }
885
0
886
417k
    case GIR_MergeMemOperands: {
887
417k
      int64_t InsnID = MatchTable[CurrentIdx++];
888
417k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
417k
890
417k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
417k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
417k
                             << InsnID << "]");
893
417k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
834k
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
834k
             GIU_MergeMemOperands_EndOfList) {
896
417k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
417k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
417k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
417k
          OutMIs[InsnID].addMemOperand(MMO);
900
417k
      }
901
417k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
417k
      break;
903
0
    }
904
0
905
1.19M
    case GIR_EraseFromParent: {
906
1.19M
      int64_t InsnID = MatchTable[CurrentIdx++];
907
1.19M
      assert(State.MIs[InsnID] &&
908
1.19M
             "Attempted to erase an undefined instruction");
909
1.19M
      State.MIs[InsnID]->eraseFromParent();
910
1.19M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
1.19M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
1.19M
                             << InsnID << "])\n");
913
1.19M
      break;
914
0
    }
915
0
916
3.91k
    case GIR_MakeTempReg: {
917
3.91k
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
3.91k
      int64_t TypeID = MatchTable[CurrentIdx++];
919
3.91k
920
3.91k
      State.TempRegisters[TempRegID] =
921
3.91k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
3.91k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
3.91k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
3.91k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
3.91k
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
1.81M
    case GIR_Done:
939
1.81M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
1.81M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
1.81M
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
35.4M
    }
946
35.4M
  }
947
4.50M
}
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<32ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
53
    CodeGenCoverage &CoverageInfo) const {
57
53
58
53
  uint64_t CurrentIdx = 0;
59
53
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
53
61
53
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
53
  auto handleReject = [&]() -> RejectAction {
63
53
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
53
                    dbgs() << CurrentIdx << ": Rejected\n");
65
53
    if (OnFailResumeAt.empty())
66
53
      return RejectAndGiveUp;
67
53
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
53
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
53
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
53
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
53
    return RejectAndResume;
72
53
  };
73
53
74
1.58k
  while (true) {
75
1.58k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
1.58k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
1.58k
    switch (MatcherOpcode) {
78
1.58k
    case GIM_Try: {
79
299
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
299
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
299
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
299
      break;
83
1.58k
    }
84
1.58k
85
1.58k
    case GIM_RecordInsn: {
86
48
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
48
      int64_t InsnID = MatchTable[CurrentIdx++];
88
48
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
48
90
48
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
48
      // any attempt to modify it.
92
48
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
48
94
48
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
48
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
48
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
48
110
48
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
48
      if ((size_t)NewInsnID < State.MIs.size())
112
45
        State.MIs[NewInsnID] = NewMI;
113
3
      else {
114
3
        assert((size_t)NewInsnID == State.MIs.size() &&
115
3
               "Expected to store MIs in order");
116
3
        State.MIs.push_back(NewMI);
117
3
      }
118
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
48
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
48
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
48
                             << ")\n");
122
48
      break;
123
48
    }
124
48
125
210
    case GIM_CheckFeatures: {
126
210
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
210
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
210
                      dbgs() << CurrentIdx
129
210
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
210
                             << ExpectedBitsetID << ")\n");
131
210
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
210
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
12
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
210
      }
136
210
      break;
137
210
    }
138
210
139
210
    case GIM_CheckOpcode: {
140
48
      int64_t InsnID = MatchTable[CurrentIdx++];
141
48
      int64_t Expected = MatchTable[CurrentIdx++];
142
48
143
48
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
48
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
48
146
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
48
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
48
                             << "], ExpectedOpcode=" << Expected
149
48
                             << ") // Got=" << Opcode << "\n");
150
48
      if (Opcode != Expected) {
151
48
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
48
      }
154
48
      break;
155
48
    }
156
48
157
53
    case GIM_SwitchOpcode: {
158
53
      int64_t InsnID = MatchTable[CurrentIdx++];
159
53
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
53
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
53
      int64_t Default = MatchTable[CurrentIdx++];
162
53
163
53
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
53
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
53
166
53
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
53
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
53
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
53
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
53
      });
171
53
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
53
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
53
      if (!CurrentIdx) {
177
0
        CurrentIdx = Default;
178
0
  break;
179
0
      }
180
53
      OnFailResumeAt.push_back(Default);
181
53
      break;
182
53
    }
183
53
184
53
    case GIM_SwitchType: {
185
38
      int64_t InsnID = MatchTable[CurrentIdx++];
186
38
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
38
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
38
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
38
      int64_t Default = MatchTable[CurrentIdx++];
190
38
191
38
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
38
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
38
194
38
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
38
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
38
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
38
               << UpperBound << "), Default=" << Default
198
38
               << ", JumpTable...) // Got=";
199
38
        if (!MO.isReg())
200
38
          dbgs() << "Not a VReg\n";
201
38
        else
202
38
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
38
      });
204
38
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
38
      const LLT Ty = MRI.getType(MO.getReg());
209
38
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
38
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
0
        CurrentIdx = Default;
212
0
        break;
213
0
      }
214
38
      const int64_t TypeID = TyI->second;
215
38
      if (TypeID < LowerBound || UpperBound <= TypeID) {
216
0
        CurrentIdx = Default;
217
0
        break;
218
0
      }
219
38
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
38
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
38
      OnFailResumeAt.push_back(Default);
225
38
      break;
226
38
    }
227
38
228
45
    case GIM_CheckNumOperands: {
229
45
      int64_t InsnID = MatchTable[CurrentIdx++];
230
45
      int64_t Expected = MatchTable[CurrentIdx++];
231
45
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
45
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
45
                             << InsnID << "], Expected=" << Expected << ")\n");
234
45
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
45
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
30
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
45
      }
239
45
      break;
240
45
    }
241
45
    case GIM_CheckI64ImmPredicate: {
242
0
      int64_t InsnID = MatchTable[CurrentIdx++];
243
0
      int64_t Predicate = MatchTable[CurrentIdx++];
244
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
0
                      dbgs()
246
0
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
0
             "Expected G_CONSTANT");
251
0
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
0
      int64_t Value = 0;
253
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
0
260
0
      if (!testImmPredicate_I64(Predicate, Value))
261
0
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
0
      break;
264
0
    }
265
0
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
138
    case GIM_CheckType: {
456
138
      int64_t InsnID = MatchTable[CurrentIdx++];
457
138
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
138
      int64_t TypeID = MatchTable[CurrentIdx++];
459
138
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
138
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
138
                             << "]->getOperand(" << OpIdx
462
138
                             << "), TypeID=" << TypeID << ")\n");
463
138
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
138
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
138
      if (!MO.isReg() ||
466
138
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
12
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
138
      }
470
138
      break;
471
138
    }
472
138
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
189
    case GIM_CheckRegBankForClass: {
502
189
      int64_t InsnID = MatchTable[CurrentIdx++];
503
189
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
189
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
189
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
189
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
189
                             << InsnID << "]->getOperand(" << OpIdx
508
189
                             << "), RCEnum=" << RCEnum << ")\n");
509
189
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
189
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
189
      if (!MO.isReg() ||
512
189
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
189
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
53
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
189
      }
517
189
      break;
518
189
    }
519
189
520
189
    case GIM_CheckComplexPattern: {
521
57
      int64_t InsnID = MatchTable[CurrentIdx++];
522
57
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
57
      int64_t RendererID = MatchTable[CurrentIdx++];
524
57
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
57
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
57
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
57
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
57
                             << "]->getOperand(" << OpIdx
529
57
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
57
                             << ")\n");
531
57
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
57
      // FIXME: Use std::invoke() when it's available.
533
57
      ComplexRendererFns Renderer =
534
57
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
57
              State.MIs[InsnID]->getOperand(OpIdx));
536
57
      if (Renderer.hasValue())
537
57
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
57
      break;
542
57
    }
543
57
544
57
    case GIM_CheckConstantInt: {
545
0
      int64_t InsnID = MatchTable[CurrentIdx++];
546
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
0
      int64_t Value = MatchTable[CurrentIdx++];
548
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
0
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
0
                             << InsnID << "]->getOperand(" << OpIdx
551
0
                             << "), Value=" << Value << ")\n");
552
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
0
      if (MO.isReg()) {
555
0
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
0
        LLT Ty = MRI.getType(MO.getReg());
557
0
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
0
559
0
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
0
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
0
566
0
      break;
567
0
    }
568
0
569
0
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
72
    case GIM_CheckIntrinsicID: {
587
72
      int64_t InsnID = MatchTable[CurrentIdx++];
588
72
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
72
      int64_t Value = MatchTable[CurrentIdx++];
590
72
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
72
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
72
                             << InsnID << "]->getOperand(" << OpIdx
593
72
                             << "), Value=" << Value << ")\n");
594
72
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
72
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
72
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
42
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
72
      break;
600
72
    }
601
72
602
72
    case GIM_CheckIsMBB: {
603
0
      int64_t InsnID = MatchTable[CurrentIdx++];
604
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
0
                             << "]->getOperand(" << OpIdx << "))\n");
608
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
0
      }
613
0
      break;
614
0
    }
615
0
616
0
    case GIM_CheckIsSafeToFold: {
617
0
      int64_t InsnID = MatchTable[CurrentIdx++];
618
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
0
                             << InsnID << "])\n");
621
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
0
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
0
      }
626
0
      break;
627
0
    }
628
0
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
0
    case GIM_Reject:
647
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
0
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
0
      if (handleReject() == RejectAndGiveUp)
650
0
        return false;
651
0
      break;
652
0
653
11
    case GIR_MutateOpcode: {
654
11
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
11
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
11
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
11
      if (NewInsnID >= OutMIs.size())
658
11
        OutMIs.resize(NewInsnID + 1);
659
11
660
11
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
11
                                              State.MIs[OldInsnID]);
662
11
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
11
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
11
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
11
                             << NewOpcode << ")\n");
667
11
      break;
668
0
    }
669
0
670
42
    case GIR_BuildMI: {
671
42
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
42
      int64_t Opcode = MatchTable[CurrentIdx++];
673
42
      if (NewInsnID >= OutMIs.size())
674
42
        OutMIs.resize(NewInsnID + 1);
675
42
676
42
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
42
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
42
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
42
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
42
                             << NewInsnID << "], " << Opcode << ")\n");
681
42
      break;
682
0
    }
683
0
684
65
    case GIR_Copy: {
685
65
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
65
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
65
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
65
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
65
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
65
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
65
                      dbgs()
692
65
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
65
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
65
      break;
695
0
    }
696
0
697
0
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
0
    }
714
0
715
0
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
0
    }
729
0
730
11
    case GIR_AddImplicitDef: {
731
11
      int64_t InsnID = MatchTable[CurrentIdx++];
732
11
      int64_t RegNum = MatchTable[CurrentIdx++];
733
11
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
11
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
11
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
11
                             << InsnID << "], " << RegNum << ")\n");
738
11
      break;
739
0
    }
740
0
741
0
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
0
    }
751
0
752
0
    case GIR_AddRegister: {
753
0
      int64_t InsnID = MatchTable[CurrentIdx++];
754
0
      int64_t RegNum = MatchTable[CurrentIdx++];
755
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
0
      OutMIs[InsnID].addReg(RegNum);
757
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
0
                             << InsnID << "], " << RegNum << ")\n");
760
0
      break;
761
0
    }
762
0
763
0
    case GIR_AddTempRegister: {
764
0
      int64_t InsnID = MatchTable[CurrentIdx++];
765
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
0
                             << InsnID << "], TempRegisters[" << TempRegID
772
0
                             << "], " << TempRegFlags << ")\n");
773
0
      break;
774
0
    }
775
0
776
0
    case GIR_AddImm: {
777
0
      int64_t InsnID = MatchTable[CurrentIdx++];
778
0
      int64_t Imm = MatchTable[CurrentIdx++];
779
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
0
      OutMIs[InsnID].addImm(Imm);
781
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
0
                             << "], " << Imm << ")\n");
784
0
      break;
785
0
    }
786
0
787
0
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
0
    }
798
115
    case GIR_ComplexSubOperandRenderer: {
799
115
      int64_t InsnID = MatchTable[CurrentIdx++];
800
115
      int64_t RendererID = MatchTable[CurrentIdx++];
801
115
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
115
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
115
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
115
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
115
                      dbgs() << CurrentIdx
806
115
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
115
                             << InsnID << "], " << RendererID << ", "
808
115
                             << RenderOpID << ")\n");
809
115
      break;
810
0
    }
811
0
812
0
    case GIR_CopyConstantAsSImm: {
813
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
0
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
0
        OutMIs[NewInsnID].addImm(
819
0
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
0
      } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
0
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
0
      break;
828
0
    }
829
0
830
0
    // TODO: Needs a test case once we have a pattern that uses this.
831
0
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
0
    case GIR_ConstrainOperandRC: {
861
0
      int64_t InsnID = MatchTable[CurrentIdx++];
862
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
0
                             << ")\n");
871
0
      break;
872
0
    }
873
0
874
53
    case GIR_ConstrainSelectedInstOperands: {
875
53
      int64_t InsnID = MatchTable[CurrentIdx++];
876
53
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
53
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
53
                                       RBI);
879
53
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
53
                      dbgs() << CurrentIdx
881
53
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
53
                             << InsnID << "])\n");
883
53
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
0
                             << InsnID << "]");
893
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
0
             GIU_MergeMemOperands_EndOfList) {
896
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
0
          OutMIs[InsnID].addMemOperand(MMO);
900
0
      }
901
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
0
      break;
903
0
    }
904
0
905
42
    case GIR_EraseFromParent: {
906
42
      int64_t InsnID = MatchTable[CurrentIdx++];
907
42
      assert(State.MIs[InsnID] &&
908
42
             "Attempted to erase an undefined instruction");
909
42
      State.MIs[InsnID]->eraseFromParent();
910
42
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
42
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
42
                             << InsnID << "])\n");
913
42
      break;
914
0
    }
915
0
916
0
    case GIR_MakeTempReg: {
917
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
0
      int64_t TypeID = MatchTable[CurrentIdx++];
919
0
920
0
      State.TempRegisters[TempRegID] =
921
0
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
0
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
0
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
0
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
53
    case GIR_Done:
939
53
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
53
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
53
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
1.58k
    }
946
1.58k
  }
947
53
}
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<64ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
706
    CodeGenCoverage &CoverageInfo) const {
57
706
58
706
  uint64_t CurrentIdx = 0;
59
706
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
706
61
706
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
706
  auto handleReject = [&]() -> RejectAction {
63
706
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
706
                    dbgs() << CurrentIdx << ": Rejected\n");
65
706
    if (OnFailResumeAt.empty())
66
706
      return RejectAndGiveUp;
67
706
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
706
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
706
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
706
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
706
    return RejectAndResume;
72
706
  };
73
706
74
11.3k
  while (true) {
75
11.3k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
11.3k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
11.3k
    switch (MatcherOpcode) {
78
11.3k
    case GIM_Try: {
79
2.08k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
2.08k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
2.08k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
2.08k
      break;
83
11.3k
    }
84
11.3k
85
11.3k
    case GIM_RecordInsn: {
86
577
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
577
      int64_t InsnID = MatchTable[CurrentIdx++];
88
577
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
577
90
577
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
577
      // any attempt to modify it.
92
577
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
577
94
577
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
577
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
577
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
577
110
577
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
577
      if ((size_t)NewInsnID < State.MIs.size())
112
381
        State.MIs[NewInsnID] = NewMI;
113
196
      else {
114
196
        assert((size_t)NewInsnID == State.MIs.size() &&
115
196
               "Expected to store MIs in order");
116
196
        State.MIs.push_back(NewMI);
117
196
      }
118
577
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
577
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
577
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
577
                             << ")\n");
122
577
      break;
123
577
    }
124
577
125
1.75k
    case GIM_CheckFeatures: {
126
1.75k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
1.75k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
1.75k
                      dbgs() << CurrentIdx
129
1.75k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
1.75k
                             << ExpectedBitsetID << ")\n");
131
1.75k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
1.75k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
1.03k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
1.75k
      }
136
1.75k
      break;
137
1.75k
    }
138
1.75k
139
1.75k
    case GIM_CheckOpcode: {
140
577
      int64_t InsnID = MatchTable[CurrentIdx++];
141
577
      int64_t Expected = MatchTable[CurrentIdx++];
142
577
143
577
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
577
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
577
146
577
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
577
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
577
                             << "], ExpectedOpcode=" << Expected
149
577
                             << ") // Got=" << Opcode << "\n");
150
577
      if (Opcode != Expected) {
151
431
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
577
      }
154
577
      break;
155
577
    }
156
577
157
706
    case GIM_SwitchOpcode: {
158
706
      int64_t InsnID = MatchTable[CurrentIdx++];
159
706
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
706
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
706
      int64_t Default = MatchTable[CurrentIdx++];
162
706
163
706
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
706
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
706
166
706
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
706
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
706
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
706
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
706
      });
171
706
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
706
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
706
      if (!CurrentIdx) {
177
313
        CurrentIdx = Default;
178
313
  break;
179
313
      }
180
393
      OnFailResumeAt.push_back(Default);
181
393
      break;
182
393
    }
183
393
184
393
    case GIM_SwitchType: {
185
225
      int64_t InsnID = MatchTable[CurrentIdx++];
186
225
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
225
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
225
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
225
      int64_t Default = MatchTable[CurrentIdx++];
190
225
191
225
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
225
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
225
194
225
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
225
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
225
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
225
               << UpperBound << "), Default=" << Default
198
225
               << ", JumpTable...) // Got=";
199
225
        if (!MO.isReg())
200
225
          dbgs() << "Not a VReg\n";
201
225
        else
202
225
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
225
      });
204
225
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
225
      const LLT Ty = MRI.getType(MO.getReg());
209
225
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
225
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
13
        CurrentIdx = Default;
212
13
        break;
213
13
      }
214
212
      const int64_t TypeID = TyI->second;
215
212
      if (TypeID < LowerBound || 
UpperBound <= TypeID148
) {
216
64
        CurrentIdx = Default;
217
64
        break;
218
64
      }
219
148
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
148
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
148
      OnFailResumeAt.push_back(Default);
225
148
      break;
226
148
    }
227
148
228
148
    case GIM_CheckNumOperands: {
229
0
      int64_t InsnID = MatchTable[CurrentIdx++];
230
0
      int64_t Expected = MatchTable[CurrentIdx++];
231
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
0
                             << InsnID << "], Expected=" << Expected << ")\n");
234
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
0
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
0
      }
239
0
      break;
240
0
    }
241
71
    case GIM_CheckI64ImmPredicate: {
242
71
      int64_t InsnID = MatchTable[CurrentIdx++];
243
71
      int64_t Predicate = MatchTable[CurrentIdx++];
244
71
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
71
                      dbgs()
246
71
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
71
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
71
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
71
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
71
             "Expected G_CONSTANT");
251
71
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
71
      int64_t Value = 0;
253
71
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
69
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
2
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
2
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
2
      else
258
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
259
71
260
71
      if (!testImmPredicate_I64(Predicate, Value))
261
2
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
71
      break;
264
71
    }
265
71
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
899
    case GIM_CheckType: {
456
899
      int64_t InsnID = MatchTable[CurrentIdx++];
457
899
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
899
      int64_t TypeID = MatchTable[CurrentIdx++];
459
899
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
899
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
899
                             << "]->getOperand(" << OpIdx
462
899
                             << "), TypeID=" << TypeID << ")\n");
463
899
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
899
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
899
      if (!MO.isReg() ||
466
899
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
5
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
899
      }
470
899
      break;
471
899
    }
472
899
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
1.29k
    case GIM_CheckRegBankForClass: {
502
1.29k
      int64_t InsnID = MatchTable[CurrentIdx++];
503
1.29k
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
1.29k
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
1.29k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
1.29k
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
1.29k
                             << InsnID << "]->getOperand(" << OpIdx
508
1.29k
                             << "), RCEnum=" << RCEnum << ")\n");
509
1.29k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
1.29k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
1.29k
      if (!MO.isReg() ||
512
1.29k
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
1.29k
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
0
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
1.29k
      }
517
1.29k
      break;
518
1.29k
    }
519
1.29k
520
1.29k
    case GIM_CheckComplexPattern: {
521
0
      int64_t InsnID = MatchTable[CurrentIdx++];
522
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
0
      int64_t RendererID = MatchTable[CurrentIdx++];
524
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
0
                             << "]->getOperand(" << OpIdx
529
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
0
                             << ")\n");
531
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
0
      // FIXME: Use std::invoke() when it's available.
533
0
      ComplexRendererFns Renderer =
534
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
0
              State.MIs[InsnID]->getOperand(OpIdx));
536
0
      if (Renderer.hasValue())
537
0
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
0
      break;
542
0
    }
543
0
544
121
    case GIM_CheckConstantInt: {
545
121
      int64_t InsnID = MatchTable[CurrentIdx++];
546
121
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
121
      int64_t Value = MatchTable[CurrentIdx++];
548
121
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
121
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
121
                             << InsnID << "]->getOperand(" << OpIdx
551
121
                             << "), Value=" << Value << ")\n");
552
121
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
121
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
121
      if (MO.isReg()) {
555
121
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
121
        LLT Ty = MRI.getType(MO.getReg());
557
121
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
121
559
121
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
51
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
121
566
121
      break;
567
121
    }
568
121
569
121
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
0
    case GIM_CheckIntrinsicID: {
587
0
      int64_t InsnID = MatchTable[CurrentIdx++];
588
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
0
      int64_t Value = MatchTable[CurrentIdx++];
590
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
0
                             << InsnID << "]->getOperand(" << OpIdx
593
0
                             << "), Value=" << Value << ")\n");
594
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
0
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
0
      break;
600
0
    }
601
0
602
9
    case GIM_CheckIsMBB: {
603
9
      int64_t InsnID = MatchTable[CurrentIdx++];
604
9
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
9
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
9
                             << "]->getOperand(" << OpIdx << "))\n");
608
9
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
9
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
9
      }
613
9
      break;
614
9
    }
615
9
616
77
    case GIM_CheckIsSafeToFold: {
617
77
      int64_t InsnID = MatchTable[CurrentIdx++];
618
77
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
77
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
77
                             << InsnID << "])\n");
621
77
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
77
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
77
      }
626
77
      break;
627
77
    }
628
77
    case GIM_CheckIsSameOperand: {
629
2
      int64_t InsnID = MatchTable[CurrentIdx++];
630
2
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
2
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
2
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
2
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
2
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
2
                             << InsnID << "][" << OpIdx << "], MIs["
636
2
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
2
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
2
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
2
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
2
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
2
      }
644
2
      break;
645
2
    }
646
750
    case GIM_Reject:
647
750
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
750
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
750
      if (handleReject() == RejectAndGiveUp)
650
473
        return false;
651
277
      break;
652
277
653
277
    case GIR_MutateOpcode: {
654
9
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
9
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
9
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
9
      if (NewInsnID >= OutMIs.size())
658
9
        OutMIs.resize(NewInsnID + 1);
659
9
660
9
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
9
                                              State.MIs[OldInsnID]);
662
9
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
9
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
9
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
9
                             << NewOpcode << ")\n");
667
9
      break;
668
277
    }
669
277
670
277
    case GIR_BuildMI: {
671
233
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
233
      int64_t Opcode = MatchTable[CurrentIdx++];
673
233
      if (NewInsnID >= OutMIs.size())
674
224
        OutMIs.resize(NewInsnID + 1);
675
233
676
233
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
233
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
233
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
233
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
233
                             << NewInsnID << "], " << Opcode << ")\n");
681
233
      break;
682
277
    }
683
277
684
534
    case GIR_Copy: {
685
534
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
534
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
534
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
534
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
534
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
534
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
534
                      dbgs()
692
534
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
534
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
534
      break;
695
277
    }
696
277
697
277
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
277
    }
714
277
715
277
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
277
    }
729
277
730
277
    case GIR_AddImplicitDef: {
731
0
      int64_t InsnID = MatchTable[CurrentIdx++];
732
0
      int64_t RegNum = MatchTable[CurrentIdx++];
733
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
0
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
0
                             << InsnID << "], " << RegNum << ")\n");
738
0
      break;
739
277
    }
740
277
741
277
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
277
    }
751
277
752
324
    case GIR_AddRegister: {
753
324
      int64_t InsnID = MatchTable[CurrentIdx++];
754
324
      int64_t RegNum = MatchTable[CurrentIdx++];
755
324
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
324
      OutMIs[InsnID].addReg(RegNum);
757
324
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
324
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
324
                             << InsnID << "], " << RegNum << ")\n");
760
324
      break;
761
277
    }
762
277
763
277
    case GIR_AddTempRegister: {
764
18
      int64_t InsnID = MatchTable[CurrentIdx++];
765
18
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
18
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
18
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
18
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
18
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
18
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
18
                             << InsnID << "], TempRegisters[" << TempRegID
772
18
                             << "], " << TempRegFlags << ")\n");
773
18
      break;
774
277
    }
775
277
776
277
    case GIR_AddImm: {
777
260
      int64_t InsnID = MatchTable[CurrentIdx++];
778
260
      int64_t Imm = MatchTable[CurrentIdx++];
779
260
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
260
      OutMIs[InsnID].addImm(Imm);
781
260
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
260
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
260
                             << "], " << Imm << ")\n");
784
260
      break;
785
277
    }
786
277
787
277
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
277
    }
798
277
    case GIR_ComplexSubOperandRenderer: {
799
0
      int64_t InsnID = MatchTable[CurrentIdx++];
800
0
      int64_t RendererID = MatchTable[CurrentIdx++];
801
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
0
                      dbgs() << CurrentIdx
806
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
0
                             << InsnID << "], " << RendererID << ", "
808
0
                             << RenderOpID << ")\n");
809
0
      break;
810
277
    }
811
277
812
277
    case GIR_CopyConstantAsSImm: {
813
68
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
68
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
68
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
68
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
68
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
66
        OutMIs[NewInsnID].addImm(
819
66
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
66
      } else 
if (2
State.MIs[OldInsnID]->getOperand(1).isImm()2
)
821
2
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
2
      else
823
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
824
68
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
68
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
68
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
68
      break;
828
68
    }
829
68
830
68
    // TODO: Needs a test case once we have a pattern that uses this.
831
68
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
5
    case GIR_ConstrainOperandRC: {
861
5
      int64_t InsnID = MatchTable[CurrentIdx++];
862
5
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
5
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
5
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
5
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
5
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
5
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
5
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
5
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
5
                             << ")\n");
871
5
      break;
872
0
    }
873
0
874
237
    case GIR_ConstrainSelectedInstOperands: {
875
237
      int64_t InsnID = MatchTable[CurrentIdx++];
876
237
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
237
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
237
                                       RBI);
879
237
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
237
                      dbgs() << CurrentIdx
881
237
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
237
                             << InsnID << "])\n");
883
237
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
0
                             << InsnID << "]");
893
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
0
             GIU_MergeMemOperands_EndOfList) {
896
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
0
          OutMIs[InsnID].addMemOperand(MMO);
900
0
      }
901
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
0
      break;
903
0
    }
904
0
905
224
    case GIR_EraseFromParent: {
906
224
      int64_t InsnID = MatchTable[CurrentIdx++];
907
224
      assert(State.MIs[InsnID] &&
908
224
             "Attempted to erase an undefined instruction");
909
224
      State.MIs[InsnID]->eraseFromParent();
910
224
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
224
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
224
                             << InsnID << "])\n");
913
224
      break;
914
0
    }
915
0
916
9
    case GIR_MakeTempReg: {
917
9
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
9
      int64_t TypeID = MatchTable[CurrentIdx++];
919
9
920
9
      State.TempRegisters[TempRegID] =
921
9
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
9
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
9
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
9
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
233
    case GIR_Done:
939
233
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
233
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
233
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
11.3k
    }
946
11.3k
  }
947
706
}
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<41ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
119
    CodeGenCoverage &CoverageInfo) const {
57
119
58
119
  uint64_t CurrentIdx = 0;
59
119
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
119
61
119
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
119
  auto handleReject = [&]() -> RejectAction {
63
119
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
119
                    dbgs() << CurrentIdx << ": Rejected\n");
65
119
    if (OnFailResumeAt.empty())
66
119
      return RejectAndGiveUp;
67
119
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
119
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
119
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
119
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
119
    return RejectAndResume;
72
119
  };
73
119
74
1.44k
  while (true) {
75
1.44k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
1.44k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
1.44k
    switch (MatcherOpcode) {
78
1.44k
    case GIM_Try: {
79
335
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
335
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
335
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
335
      break;
83
1.44k
    }
84
1.44k
85
1.44k
    case GIM_RecordInsn: {
86
18
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
18
      int64_t InsnID = MatchTable[CurrentIdx++];
88
18
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
18
90
18
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
18
      // any attempt to modify it.
92
18
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
18
94
18
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
18
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
18
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
18
110
18
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
18
      if ((size_t)NewInsnID < State.MIs.size())
112
0
        State.MIs[NewInsnID] = NewMI;
113
18
      else {
114
18
        assert((size_t)NewInsnID == State.MIs.size() &&
115
18
               "Expected to store MIs in order");
116
18
        State.MIs.push_back(NewMI);
117
18
      }
118
18
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
18
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
18
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
18
                             << ")\n");
122
18
      break;
123
18
    }
124
18
125
254
    case GIM_CheckFeatures: {
126
254
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
254
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
254
                      dbgs() << CurrentIdx
129
254
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
254
                             << ExpectedBitsetID << ")\n");
131
254
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
254
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
191
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
254
      }
136
254
      break;
137
254
    }
138
254
139
254
    case GIM_CheckOpcode: {
140
18
      int64_t InsnID = MatchTable[CurrentIdx++];
141
18
      int64_t Expected = MatchTable[CurrentIdx++];
142
18
143
18
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
18
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
18
146
18
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
18
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
18
                             << "], ExpectedOpcode=" << Expected
149
18
                             << ") // Got=" << Opcode << "\n");
150
18
      if (Opcode != Expected) {
151
8
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
18
      }
154
18
      break;
155
18
    }
156
18
157
119
    case GIM_SwitchOpcode: {
158
119
      int64_t InsnID = MatchTable[CurrentIdx++];
159
119
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
119
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
119
      int64_t Default = MatchTable[CurrentIdx++];
162
119
163
119
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
119
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
119
166
119
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
119
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
119
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
119
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
119
      });
171
119
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
119
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
119
      if (!CurrentIdx) {
177
30
        CurrentIdx = Default;
178
30
  break;
179
30
      }
180
89
      OnFailResumeAt.push_back(Default);
181
89
      break;
182
89
    }
183
89
184
89
    case GIM_SwitchType: {
185
53
      int64_t InsnID = MatchTable[CurrentIdx++];
186
53
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
53
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
53
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
53
      int64_t Default = MatchTable[CurrentIdx++];
190
53
191
53
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
53
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
53
194
53
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
53
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
53
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
53
               << UpperBound << "), Default=" << Default
198
53
               << ", JumpTable...) // Got=";
199
53
        if (!MO.isReg())
200
53
          dbgs() << "Not a VReg\n";
201
53
        else
202
53
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
53
      });
204
53
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
53
      const LLT Ty = MRI.getType(MO.getReg());
209
53
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
53
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
0
        CurrentIdx = Default;
212
0
        break;
213
0
      }
214
53
      const int64_t TypeID = TyI->second;
215
53
      if (TypeID < LowerBound || UpperBound <= TypeID) {
216
0
        CurrentIdx = Default;
217
0
        break;
218
0
      }
219
53
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
53
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
53
      OnFailResumeAt.push_back(Default);
225
53
      break;
226
53
    }
227
53
228
53
    case GIM_CheckNumOperands: {
229
0
      int64_t InsnID = MatchTable[CurrentIdx++];
230
0
      int64_t Expected = MatchTable[CurrentIdx++];
231
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
0
                             << InsnID << "], Expected=" << Expected << ")\n");
234
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
0
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
0
      }
239
0
      break;
240
0
    }
241
10
    case GIM_CheckI64ImmPredicate: {
242
10
      int64_t InsnID = MatchTable[CurrentIdx++];
243
10
      int64_t Predicate = MatchTable[CurrentIdx++];
244
10
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
10
                      dbgs()
246
10
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
10
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
10
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
10
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
10
             "Expected G_CONSTANT");
251
10
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
10
      int64_t Value = 0;
253
10
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
10
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
10
260
10
      if (!testImmPredicate_I64(Predicate, Value))
261
0
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
10
      break;
264
10
    }
265
10
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
134
    case GIM_CheckType: {
456
134
      int64_t InsnID = MatchTable[CurrentIdx++];
457
134
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
134
      int64_t TypeID = MatchTable[CurrentIdx++];
459
134
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
134
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
134
                             << "]->getOperand(" << OpIdx
462
134
                             << "), TypeID=" << TypeID << ")\n");
463
134
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
134
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
134
      if (!MO.isReg() ||
466
134
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
0
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
134
      }
470
134
      break;
471
134
    }
472
134
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
167
    case GIM_CheckRegBankForClass: {
502
167
      int64_t InsnID = MatchTable[CurrentIdx++];
503
167
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
167
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
167
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
167
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
167
                             << InsnID << "]->getOperand(" << OpIdx
508
167
                             << "), RCEnum=" << RCEnum << ")\n");
509
167
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
167
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
167
      if (!MO.isReg() ||
512
167
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
167
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
0
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
167
      }
517
167
      break;
518
167
    }
519
167
520
167
    case GIM_CheckComplexPattern: {
521
0
      int64_t InsnID = MatchTable[CurrentIdx++];
522
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
0
      int64_t RendererID = MatchTable[CurrentIdx++];
524
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
0
                             << "]->getOperand(" << OpIdx
529
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
0
                             << ")\n");
531
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
0
      // FIXME: Use std::invoke() when it's available.
533
0
      ComplexRendererFns Renderer =
534
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
0
              State.MIs[InsnID]->getOperand(OpIdx));
536
0
      if (Renderer.hasValue())
537
0
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
0
      break;
542
0
    }
543
0
544
2
    case GIM_CheckConstantInt: {
545
2
      int64_t InsnID = MatchTable[CurrentIdx++];
546
2
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
2
      int64_t Value = MatchTable[CurrentIdx++];
548
2
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
2
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
2
                             << InsnID << "]->getOperand(" << OpIdx
551
2
                             << "), Value=" << Value << ")\n");
552
2
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
2
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
2
      if (MO.isReg()) {
555
2
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
2
        LLT Ty = MRI.getType(MO.getReg());
557
2
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
2
559
2
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
2
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
2
566
2
      break;
567
2
    }
568
2
569
2
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
0
    case GIM_CheckIntrinsicID: {
587
0
      int64_t InsnID = MatchTable[CurrentIdx++];
588
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
0
      int64_t Value = MatchTable[CurrentIdx++];
590
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
0
                             << InsnID << "]->getOperand(" << OpIdx
593
0
                             << "), Value=" << Value << ")\n");
594
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
0
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
0
      break;
600
0
    }
601
0
602
0
    case GIM_CheckIsMBB: {
603
0
      int64_t InsnID = MatchTable[CurrentIdx++];
604
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
0
                             << "]->getOperand(" << OpIdx << "))\n");
608
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
0
      }
613
0
      break;
614
0
    }
615
0
616
10
    case GIM_CheckIsSafeToFold: {
617
10
      int64_t InsnID = MatchTable[CurrentIdx++];
618
10
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
10
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
10
                             << InsnID << "])\n");
621
10
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
10
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
10
      }
626
10
      break;
627
10
    }
628
10
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
130
    case GIM_Reject:
647
130
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
130
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
130
      if (handleReject() == RejectAndGiveUp)
650
66
        return false;
651
64
      break;
652
64
653
64
    case GIR_MutateOpcode: {
654
43
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
43
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
43
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
43
      if (NewInsnID >= OutMIs.size())
658
43
        OutMIs.resize(NewInsnID + 1);
659
43
660
43
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
43
                                              State.MIs[OldInsnID]);
662
43
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
43
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
43
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
43
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
43
                             << NewOpcode << ")\n");
667
43
      break;
668
64
    }
669
64
670
64
    case GIR_BuildMI: {
671
10
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
10
      int64_t Opcode = MatchTable[CurrentIdx++];
673
10
      if (NewInsnID >= OutMIs.size())
674
10
        OutMIs.resize(NewInsnID + 1);
675
10
676
10
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
10
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
10
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
10
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
10
                             << NewInsnID << "], " << Opcode << ")\n");
681
10
      break;
682
64
    }
683
64
684
64
    case GIR_Copy: {
685
20
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
20
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
20
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
20
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
20
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
20
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
20
                      dbgs()
692
20
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
20
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
20
      break;
695
64
    }
696
64
697
64
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
64
    }
714
64
715
64
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
64
    }
729
64
730
64
    case GIR_AddImplicitDef: {
731
0
      int64_t InsnID = MatchTable[CurrentIdx++];
732
0
      int64_t RegNum = MatchTable[CurrentIdx++];
733
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
0
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
0
                             << InsnID << "], " << RegNum << ")\n");
738
0
      break;
739
64
    }
740
64
741
64
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
64
    }
751
64
752
64
    case GIR_AddRegister: {
753
0
      int64_t InsnID = MatchTable[CurrentIdx++];
754
0
      int64_t RegNum = MatchTable[CurrentIdx++];
755
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
0
      OutMIs[InsnID].addReg(RegNum);
757
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
0
                             << InsnID << "], " << RegNum << ")\n");
760
0
      break;
761
64
    }
762
64
763
64
    case GIR_AddTempRegister: {
764
0
      int64_t InsnID = MatchTable[CurrentIdx++];
765
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
0
                             << InsnID << "], TempRegisters[" << TempRegID
772
0
                             << "], " << TempRegFlags << ")\n");
773
0
      break;
774
64
    }
775
64
776
64
    case GIR_AddImm: {
777
0
      int64_t InsnID = MatchTable[CurrentIdx++];
778
0
      int64_t Imm = MatchTable[CurrentIdx++];
779
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
0
      OutMIs[InsnID].addImm(Imm);
781
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
0
                             << "], " << Imm << ")\n");
784
0
      break;
785
64
    }
786
64
787
64
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
64
    }
798
64
    case GIR_ComplexSubOperandRenderer: {
799
0
      int64_t InsnID = MatchTable[CurrentIdx++];
800
0
      int64_t RendererID = MatchTable[CurrentIdx++];
801
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
0
                      dbgs() << CurrentIdx
806
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
0
                             << InsnID << "], " << RendererID << ", "
808
0
                             << RenderOpID << ")\n");
809
0
      break;
810
64
    }
811
64
812
64
    case GIR_CopyConstantAsSImm: {
813
10
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
10
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
10
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
10
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
10
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
10
        OutMIs[NewInsnID].addImm(
819
10
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
10
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
10
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
10
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
10
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
10
      break;
828
10
    }
829
10
830
10
    // TODO: Needs a test case once we have a pattern that uses this.
831
10
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
0
    case GIR_ConstrainOperandRC: {
861
0
      int64_t InsnID = MatchTable[CurrentIdx++];
862
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
0
                             << ")\n");
871
0
      break;
872
0
    }
873
0
874
53
    case GIR_ConstrainSelectedInstOperands: {
875
53
      int64_t InsnID = MatchTable[CurrentIdx++];
876
53
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
53
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
53
                                       RBI);
879
53
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
53
                      dbgs() << CurrentIdx
881
53
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
53
                             << InsnID << "])\n");
883
53
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
0
                             << InsnID << "]");
893
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
0
             GIU_MergeMemOperands_EndOfList) {
896
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
0
          OutMIs[InsnID].addMemOperand(MMO);
900
0
      }
901
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
0
      break;
903
0
    }
904
0
905
10
    case GIR_EraseFromParent: {
906
10
      int64_t InsnID = MatchTable[CurrentIdx++];
907
10
      assert(State.MIs[InsnID] &&
908
10
             "Attempted to erase an undefined instruction");
909
10
      State.MIs[InsnID]->eraseFromParent();
910
10
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
10
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
10
                             << InsnID << "])\n");
913
10
      break;
914
0
    }
915
0
916
0
    case GIR_MakeTempReg: {
917
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
0
      int64_t TypeID = MatchTable[CurrentIdx++];
919
0