Coverage Report

Created: 2018-07-12 09:57

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
Line
Count
Source (jump to first uncovered line)
1
//===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
/// \file This file declares the API for the instruction selector.
11
/// This class is responsible for selecting machine instructions.
12
/// It's implemented by the target. It's used by the InstructionSelect pass.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17
#define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
18
19
#include "llvm/ADT/SmallVector.h"
20
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
21
#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
22
#include "llvm/CodeGen/GlobalISel/Utils.h"
23
#include "llvm/CodeGen/MachineInstrBuilder.h"
24
#include "llvm/CodeGen/MachineOperand.h"
25
#include "llvm/CodeGen/MachineRegisterInfo.h"
26
#include "llvm/CodeGen/TargetInstrInfo.h"
27
#include "llvm/CodeGen/TargetOpcodes.h"
28
#include "llvm/CodeGen/TargetRegisterInfo.h"
29
#include "llvm/IR/Constants.h"
30
#include "llvm/Support/Debug.h"
31
#include "llvm/Support/ErrorHandling.h"
32
#include "llvm/Support/raw_ostream.h"
33
#include <cassert>
34
#include <cstddef>
35
#include <cstdint>
36
37
namespace llvm {
38
39
/// GlobalISel PatFrag Predicates
40
enum {
41
  GIPFP_I64_Invalid = 0,
42
  GIPFP_APInt_Invalid = 0,
43
  GIPFP_APFloat_Invalid = 0,
44
  GIPFP_MI_Invalid = 0,
45
};
46
47
template <class TgtInstructionSelector, class PredicateBitset,
48
          class ComplexMatcherMemFn, class CustomRendererFn>
49
bool InstructionSelector::executeMatchTable(
50
    TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
51
    const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
52
        &ISelInfo,
53
    const int64_t *MatchTable, const TargetInstrInfo &TII,
54
    MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
55
    const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
56
4.44M
    CodeGenCoverage &CoverageInfo) const {
57
4.44M
58
4.44M
  uint64_t CurrentIdx = 0;
59
4.44M
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
4.44M
61
4.44M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
6.31M
  auto handleReject = [&]() -> RejectAction {
63
6.31M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
6.31M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
6.31M
    if (OnFailResumeAt.empty())
66
2.66M
      return RejectAndGiveUp;
67
3.65M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
3.65M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
3.65M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
3.65M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
3.65M
    return RejectAndResume;
72
3.65M
  };
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<15ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<15ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<15ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
6.31M
  auto handleReject = [&]() -> RejectAction {
63
6.31M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
6.31M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
6.31M
    if (OnFailResumeAt.empty())
66
2.66M
      return RejectAndGiveUp;
67
3.64M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
3.64M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
3.64M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
3.64M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
3.64M
    return RejectAndResume;
72
3.64M
  };
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<32ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
117
  auto handleReject = [&]() -> RejectAction {
63
117
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
117
                    dbgs() << CurrentIdx << ": Rejected\n");
65
117
    if (OnFailResumeAt.empty())
66
0
      return RejectAndGiveUp;
67
117
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
117
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
117
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
117
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
117
    return RejectAndResume;
72
117
  };
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<64ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
2.27k
  auto handleReject = [&]() -> RejectAction {
63
2.27k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
2.27k
                    dbgs() << CurrentIdx << ": Rejected\n");
65
2.27k
    if (OnFailResumeAt.empty())
66
473
      return RejectAndGiveUp;
67
1.80k
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
1.80k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
1.80k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
1.80k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
1.80k
    return RejectAndResume;
72
1.80k
  };
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<41ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
15
  auto handleReject = [&]() -> RejectAction {
63
15
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
15
                    dbgs() << CurrentIdx << ": Rejected\n");
65
15
    if (OnFailResumeAt.empty())
66
0
      return RejectAndGiveUp;
67
15
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
15
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
15
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
15
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
15
    return RejectAndResume;
72
15
  };
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<114ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
62
3.54k
  auto handleReject = [&]() -> RejectAction {
63
3.54k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
3.54k
                    dbgs() << CurrentIdx << ": Rejected\n");
65
3.54k
    if (OnFailResumeAt.empty())
66
1.21k
      return RejectAndGiveUp;
67
2.33k
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
2.33k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
2.33k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
2.33k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
2.33k
    return RejectAndResume;
72
2.33k
  };
73
4.44M
74
36.5M
  while (true) {
75
36.5M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
36.5M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
36.5M
    switch (MatcherOpcode) {
78
36.5M
    case GIM_Try: {
79
4.79M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
4.79M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
4.79M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
4.79M
      break;
83
36.5M
    }
84
36.5M
85
36.5M
    case GIM_RecordInsn: {
86
1.11M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
1.11M
      int64_t InsnID = MatchTable[CurrentIdx++];
88
1.11M
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
1.11M
90
1.11M
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
1.11M
      // any attempt to modify it.
92
1.11M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
1.11M
94
1.11M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
1.11M
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
1.11M
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
1.11M
110
1.11M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
1.11M
      if ((size_t)NewInsnID < State.MIs.size())
112
794k
        State.MIs[NewInsnID] = NewMI;
113
321k
      else {
114
321k
        assert((size_t)NewInsnID == State.MIs.size() &&
115
321k
               "Expected to store MIs in order");
116
321k
        State.MIs.push_back(NewMI);
117
321k
      }
118
1.11M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
1.11M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
1.11M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
1.11M
                             << ")\n");
122
1.11M
      break;
123
1.11M
    }
124
1.11M
125
1.11M
    case GIM_CheckFeatures: {
126
891k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
891k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
891k
                      dbgs() << CurrentIdx
129
891k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
891k
                             << ExpectedBitsetID << ")\n");
131
891k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
891k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
106k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
891k
      }
136
891k
      break;
137
891k
    }
138
891k
139
1.11M
    case GIM_CheckOpcode: {
140
1.11M
      int64_t InsnID = MatchTable[CurrentIdx++];
141
1.11M
      int64_t Expected = MatchTable[CurrentIdx++];
142
1.11M
143
1.11M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
1.11M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
1.11M
146
1.11M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
1.11M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
1.11M
                             << "], ExpectedOpcode=" << Expected
149
1.11M
                             << ") // Got=" << Opcode << "\n");
150
1.11M
      if (Opcode != Expected) {
151
1.05M
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
1.11M
      }
154
1.11M
      break;
155
1.11M
    }
156
1.11M
157
4.44M
    case GIM_SwitchOpcode: {
158
4.44M
      int64_t InsnID = MatchTable[CurrentIdx++];
159
4.44M
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
4.44M
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
4.44M
      int64_t Default = MatchTable[CurrentIdx++];
162
4.44M
163
4.44M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
4.44M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
4.44M
166
4.44M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
4.44M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
4.44M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
4.44M
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
4.44M
      });
171
4.44M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
4.44M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
4.44M
      if (!CurrentIdx) {
177
1.43M
        CurrentIdx = Default;
178
1.43M
  break;
179
1.43M
      }
180
3.01M
      OnFailResumeAt.push_back(Default);
181
3.01M
      break;
182
3.01M
    }
183
3.01M
184
3.01M
    case GIM_SwitchType: {
185
2.61M
      int64_t InsnID = MatchTable[CurrentIdx++];
186
2.61M
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
2.61M
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
2.61M
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
2.61M
      int64_t Default = MatchTable[CurrentIdx++];
190
2.61M
191
2.61M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
2.61M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
2.61M
194
2.61M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
2.61M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
2.61M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
2.61M
               << UpperBound << "), Default=" << Default
198
2.61M
               << ", JumpTable...) // Got=";
199
2.61M
        if (!MO.isReg())
200
2.61M
          dbgs() << "Not a VReg\n";
201
2.61M
        else
202
2.61M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
2.61M
      });
204
2.61M
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
2.61M
      const LLT Ty = MRI.getType(MO.getReg());
209
2.61M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
2.61M
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
971k
        CurrentIdx = Default;
212
971k
        break;
213
971k
      }
214
1.64M
      const int64_t TypeID = TyI->second;
215
1.64M
      if (TypeID < LowerBound || 
UpperBound <= TypeID1.44M
) {
216
194k
        CurrentIdx = Default;
217
194k
        break;
218
194k
      }
219
1.44M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
1.44M
      if (!CurrentIdx) {
221
153
        CurrentIdx = Default;
222
153
        break;
223
153
      }
224
1.44M
      OnFailResumeAt.push_back(Default);
225
1.44M
      break;
226
1.44M
    }
227
1.44M
228
1.44M
    case GIM_CheckNumOperands: {
229
33.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
230
33.2k
      int64_t Expected = MatchTable[CurrentIdx++];
231
33.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
33.2k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
33.2k
                             << InsnID << "], Expected=" << Expected << ")\n");
234
33.2k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
33.2k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
31.1k
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
33.2k
      }
239
33.2k
      break;
240
33.2k
    }
241
33.2k
    case GIM_CheckI64ImmPredicate: {
242
21.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
243
21.2k
      int64_t Predicate = MatchTable[CurrentIdx++];
244
21.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
21.2k
                      dbgs()
246
21.2k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
21.2k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
21.2k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
21.2k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
21.2k
             "Expected G_CONSTANT");
251
21.2k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
21.2k
      int64_t Value = 0;
253
21.2k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
21.2k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
2
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
2
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
2
      else
258
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
259
21.2k
260
21.2k
      if (!testImmPredicate_I64(Predicate, Value))
261
19
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
21.2k
      break;
264
21.2k
    }
265
21.2k
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
2.34k
    case GIM_CheckAPFloatImmPredicate: {
288
2.34k
      int64_t InsnID = MatchTable[CurrentIdx++];
289
2.34k
      int64_t Predicate = MatchTable[CurrentIdx++];
290
2.34k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
2.34k
                      dbgs()
292
2.34k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
2.34k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
2.34k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
2.34k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
2.34k
             "Expected G_FCONSTANT");
297
2.34k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
2.34k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
2.34k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
2.34k
301
2.34k
      if (!testImmPredicate_APFloat(Predicate, Value))
302
1.46k
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
2.34k
      break;
305
2.34k
    }
306
2.34k
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
422k
    case GIM_CheckAtomicOrdering: {
322
422k
      int64_t InsnID = MatchTable[CurrentIdx++];
323
422k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
422k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
422k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
422k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
422k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
422k
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
422k
332
422k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
422k
        if (MMO->getOrdering() != Ordering)
334
2.84k
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
422k
      break;
337
422k
    }
338
422k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
47
    case GIM_CheckMemorySizeEqualTo: {
375
47
      int64_t InsnID = MatchTable[CurrentIdx++];
376
47
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
47
      uint64_t Size = MatchTable[CurrentIdx++];
378
47
379
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
47
                      dbgs() << CurrentIdx
381
47
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
47
                             << "]->memoperands() + " << MMOIdx
383
47
                             << ", Size=" << Size << ")\n");
384
47
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
47
386
47
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
47
392
47
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
47
394
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
47
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
47
                             << " bytes\n");
397
47
      if (MMO->getSize() != Size)
398
2
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
47
401
47
      break;
402
47
    }
403
273k
    case GIM_CheckMemorySizeEqualToLLT:
404
273k
    case GIM_CheckMemorySizeLessThanLLT:
405
273k
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
273k
      int64_t InsnID = MatchTable[CurrentIdx++];
407
273k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
273k
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
273k
410
273k
      DEBUG_WITH_TYPE(
411
273k
          TgtInstructionSelector::getName(),
412
273k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
273k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
273k
                         ? "EqualTo"
415
273k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
273k
                               ? "GreaterThan"
417
273k
                               : "LessThan")
418
273k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
273k
                 << ", OpIdx=" << OpIdx << ")\n");
420
273k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
273k
422
273k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
273k
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
273k
431
273k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
273k
437
273k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
273k
439
273k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
273k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
273k
          
MMO->getSize() * 8 != Size273k
) {
442
16
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
273k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
273k
                 
MMO->getSize() * 8 >= Size42
) {
446
36
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
273k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
273k
                 
MMO->getSize() * 8 <= Size0
)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
273k
453
273k
      break;
454
273k
    }
455
943k
    case GIM_CheckType: {
456
943k
      int64_t InsnID = MatchTable[CurrentIdx++];
457
943k
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
943k
      int64_t TypeID = MatchTable[CurrentIdx++];
459
943k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
943k
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
943k
                             << "]->getOperand(" << OpIdx
462
943k
                             << "), TypeID=" << TypeID << ")\n");
463
943k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
943k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
943k
      if (!MO.isReg() ||
466
943k
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
26.6k
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
943k
      }
470
943k
      break;
471
943k
    }
472
943k
    case GIM_CheckPointerToAny: {
473
409k
      int64_t InsnID = MatchTable[CurrentIdx++];
474
409k
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
409k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
409k
477
409k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
409k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
409k
                             << InsnID << "]->getOperand(" << OpIdx
480
409k
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
409k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
409k
      // iPTR must be looked up in the target.
483
409k
      if (SizeInBits == 0) {
484
409k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
409k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
409k
      }
487
409k
488
409k
      assert(SizeInBits != 0 && "Pointer size must be known");
489
409k
490
409k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
409k
      if (MO.isReg()) {
492
409k
        const LLT &Ty = MRI.getType(MO.getReg());
493
409k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
409k
499
409k
      break;
500
409k
    }
501
3.68M
    case GIM_CheckRegBankForClass: {
502
3.68M
      int64_t InsnID = MatchTable[CurrentIdx++];
503
3.68M
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
3.68M
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
3.68M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
3.68M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
3.68M
                             << InsnID << "]->getOperand(" << OpIdx
508
3.68M
                             << "), RCEnum=" << RCEnum << ")\n");
509
3.68M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
3.68M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
3.68M
      if (!MO.isReg() ||
512
3.68M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
3.68M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
186k
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
3.68M
      }
517
3.68M
      break;
518
3.68M
    }
519
3.68M
520
3.68M
    case GIM_CheckComplexPattern: {
521
694k
      int64_t InsnID = MatchTable[CurrentIdx++];
522
694k
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
694k
      int64_t RendererID = MatchTable[CurrentIdx++];
524
694k
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
694k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
694k
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
694k
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
694k
                             << "]->getOperand(" << OpIdx
529
694k
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
694k
                             << ")\n");
531
694k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
694k
      // FIXME: Use std::invoke() when it's available.
533
694k
      ComplexRendererFns Renderer =
534
694k
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
694k
              State.MIs[InsnID]->getOperand(OpIdx));
536
694k
      if (Renderer.hasValue())
537
461k
        State.Renderers[RendererID] = Renderer.getValue();
538
233k
      else
539
233k
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
694k
      break;
542
694k
    }
543
694k
544
694k
    case GIM_CheckConstantInt: {
545
75.4k
      int64_t InsnID = MatchTable[CurrentIdx++];
546
75.4k
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
75.4k
      int64_t Value = MatchTable[CurrentIdx++];
548
75.4k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
75.4k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
75.4k
                             << InsnID << "]->getOperand(" << OpIdx
551
75.4k
                             << "), Value=" << Value << ")\n");
552
75.4k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
75.4k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
75.4k
      if (MO.isReg()) {
555
75.4k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
75.4k
        LLT Ty = MRI.getType(MO.getReg());
557
75.4k
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
75.4k
559
75.4k
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
56.7k
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
75.4k
566
75.4k
      break;
567
75.4k
    }
568
75.4k
569
75.4k
    case GIM_CheckLiteralInt: {
570
54
      int64_t InsnID = MatchTable[CurrentIdx++];
571
54
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
54
      int64_t Value = MatchTable[CurrentIdx++];
573
54
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
54
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
54
                             << InsnID << "]->getOperand(" << OpIdx
576
54
                             << "), Value=" << Value << ")\n");
577
54
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
54
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
54
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
32
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
54
      }
583
54
      break;
584
54
    }
585
54
586
686k
    case GIM_CheckIntrinsicID: {
587
686k
      int64_t InsnID = MatchTable[CurrentIdx++];
588
686k
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
686k
      int64_t Value = MatchTable[CurrentIdx++];
590
686k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
686k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
686k
                             << InsnID << "]->getOperand(" << OpIdx
593
686k
                             << "), Value=" << Value << ")\n");
594
686k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
686k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
686k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value682k
)
597
684k
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
686k
      break;
600
686k
    }
601
686k
602
686k
    case GIM_CheckIsMBB: {
603
372k
      int64_t InsnID = MatchTable[CurrentIdx++];
604
372k
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
372k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
372k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
372k
                             << "]->getOperand(" << OpIdx << "))\n");
608
372k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
372k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
372k
      }
613
372k
      break;
614
372k
    }
615
372k
616
372k
    case GIM_CheckIsSafeToFold: {
617
44.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
618
44.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
44.1k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
44.1k
                             << InsnID << "])\n");
621
44.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
44.1k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
44.1k
      }
626
44.1k
      break;
627
44.1k
    }
628
44.1k
    case GIM_CheckIsSameOperand: {
629
6
      int64_t InsnID = MatchTable[CurrentIdx++];
630
6
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
6
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
6
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
6
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
6
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
6
                             << InsnID << "][" << OpIdx << "], MIs["
636
6
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
6
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
6
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
6
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
6
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
2
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
6
      }
644
6
      break;
645
6
    }
646
3.93M
    case GIM_Reject:
647
3.93M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
3.93M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
3.93M
      if (handleReject() == RejectAndGiveUp)
650
2.66M
        return false;
651
1.26M
      break;
652
1.26M
653
1.26M
    case GIR_MutateOpcode: {
654
615k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
615k
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
615k
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
615k
      if (NewInsnID >= OutMIs.size())
658
615k
        OutMIs.resize(NewInsnID + 1);
659
615k
660
615k
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
615k
                                              State.MIs[OldInsnID]);
662
615k
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
615k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
615k
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
615k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
615k
                             << NewOpcode << ")\n");
667
615k
      break;
668
1.26M
    }
669
1.26M
670
1.26M
    case GIR_BuildMI: {
671
1.17M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
1.17M
      int64_t Opcode = MatchTable[CurrentIdx++];
673
1.17M
      if (NewInsnID >= OutMIs.size())
674
1.16M
        OutMIs.resize(NewInsnID + 1);
675
1.17M
676
1.17M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
1.17M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
1.17M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
1.17M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
1.17M
                             << NewInsnID << "], " << Opcode << ")\n");
681
1.17M
      break;
682
1.26M
    }
683
1.26M
684
1.27M
    case GIR_Copy: {
685
1.27M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
1.27M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
1.27M
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
1.27M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
1.27M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
1.27M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
1.27M
                      dbgs()
692
1.27M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
1.27M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
1.27M
      break;
695
1.26M
    }
696
1.26M
697
1.26M
    case GIR_CopyOrAddZeroReg: {
698
134k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
134k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
134k
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
134k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
134k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
134k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
134k
      if (isOperandImmEqual(MO, 0, MRI))
705
14.8k
        OutMIs[NewInsnID].addReg(ZeroReg);
706
119k
      else
707
119k
        OutMIs[NewInsnID].add(MO);
708
134k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
134k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
134k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
134k
                             << OpIdx << ", " << ZeroReg << ")\n");
712
134k
      break;
713
1.26M
    }
714
1.26M
715
1.26M
    case GIR_CopySubReg: {
716
41.1k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
41.1k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
41.1k
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
41.1k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
41.1k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
41.1k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
41.1k
                               0, SubRegIdx);
723
41.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
41.1k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
41.1k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
41.1k
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
41.1k
      break;
728
1.26M
    }
729
1.26M
730
1.26M
    case GIR_AddImplicitDef: {
731
22.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
732
22.9k
      int64_t RegNum = MatchTable[CurrentIdx++];
733
22.9k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
22.9k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
22.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
22.9k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
22.9k
                             << InsnID << "], " << RegNum << ")\n");
738
22.9k
      break;
739
1.26M
    }
740
1.26M
741
1.26M
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
1.26M
    }
751
1.26M
752
1.26M
    case GIR_AddRegister: {
753
81.3k
      int64_t InsnID = MatchTable[CurrentIdx++];
754
81.3k
      int64_t RegNum = MatchTable[CurrentIdx++];
755
81.3k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
81.3k
      OutMIs[InsnID].addReg(RegNum);
757
81.3k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
81.3k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
81.3k
                             << InsnID << "], " << RegNum << ")\n");
760
81.3k
      break;
761
1.26M
    }
762
1.26M
763
1.26M
    case GIR_AddTempRegister: {
764
7.67k
      int64_t InsnID = MatchTable[CurrentIdx++];
765
7.67k
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
7.67k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
7.67k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
7.67k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
7.67k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
7.67k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
7.67k
                             << InsnID << "], TempRegisters[" << TempRegID
772
7.67k
                             << "], " << TempRegFlags << ")\n");
773
7.67k
      break;
774
1.26M
    }
775
1.26M
776
1.26M
    case GIR_AddImm: {
777
17.5k
      int64_t InsnID = MatchTable[CurrentIdx++];
778
17.5k
      int64_t Imm = MatchTable[CurrentIdx++];
779
17.5k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
17.5k
      OutMIs[InsnID].addImm(Imm);
781
17.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
17.5k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
17.5k
                             << "], " << Imm << ")\n");
784
17.5k
      break;
785
1.26M
    }
786
1.26M
787
1.26M
    case GIR_ComplexRenderer: {
788
57.6k
      int64_t InsnID = MatchTable[CurrentIdx++];
789
57.6k
      int64_t RendererID = MatchTable[CurrentIdx++];
790
57.6k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
57.6k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
115k
        RenderOpFn(OutMIs[InsnID]);
793
57.6k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
57.6k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
57.6k
                             << InsnID << "], " << RendererID << ")\n");
796
57.6k
      break;
797
1.26M
    }
798
1.26M
    case GIR_ComplexSubOperandRenderer: {
799
806k
      int64_t InsnID = MatchTable[CurrentIdx++];
800
806k
      int64_t RendererID = MatchTable[CurrentIdx++];
801
806k
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
806k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
806k
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
806k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
806k
                      dbgs() << CurrentIdx
806
806k
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
806k
                             << InsnID << "], " << RendererID << ", "
808
806k
                             << RenderOpID << ")\n");
809
806k
      break;
810
1.26M
    }
811
1.26M
812
1.26M
    case GIR_CopyConstantAsSImm: {
813
580k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
580k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
580k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
580k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
580k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
580k
        OutMIs[NewInsnID].addImm(
819
580k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
580k
      } else 
if (2
State.MIs[OldInsnID]->getOperand(1).isImm()2
)
821
2
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
2
      else
823
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
824
580k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
580k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
580k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
580k
      break;
828
580k
    }
829
580k
830
580k
    // TODO: Needs a test case once we have a pattern that uses this.
831
580k
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
3.82k
    case GIR_CustomRenderer: {
848
3.82k
      int64_t InsnID = MatchTable[CurrentIdx++];
849
3.82k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
3.82k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
3.82k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
3.82k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
3.82k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
3.82k
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
3.82k
                             << RendererFnID << ")\n");
856
3.82k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
3.82k
                                                     *State.MIs[OldInsnID]);
858
3.82k
      break;
859
0
    }
860
82.4k
    case GIR_ConstrainOperandRC: {
861
82.4k
      int64_t InsnID = MatchTable[CurrentIdx++];
862
82.4k
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
82.4k
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
82.4k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
82.4k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
82.4k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
82.4k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
82.4k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
82.4k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
82.4k
                             << ")\n");
871
82.4k
      break;
872
0
    }
873
0
874
1.74M
    case GIR_ConstrainSelectedInstOperands: {
875
1.74M
      int64_t InsnID = MatchTable[CurrentIdx++];
876
1.74M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
1.74M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
1.74M
                                       RBI);
879
1.74M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
1.74M
                      dbgs() << CurrentIdx
881
1.74M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
1.74M
                             << InsnID << "])\n");
883
1.74M
      break;
884
0
    }
885
0
886
403k
    case GIR_MergeMemOperands: {
887
403k
      int64_t InsnID = MatchTable[CurrentIdx++];
888
403k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
403k
890
403k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
403k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
403k
                             << InsnID << "]");
893
403k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
806k
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
806k
             GIU_MergeMemOperands_EndOfList) {
896
403k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
403k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
403k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
403k
          OutMIs[InsnID].addMemOperand(MMO);
900
403k
      }
901
403k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
403k
      break;
903
0
    }
904
0
905
1.16M
    case GIR_EraseFromParent: {
906
1.16M
      int64_t InsnID = MatchTable[CurrentIdx++];
907
1.16M
      assert(State.MIs[InsnID] &&
908
1.16M
             "Attempted to erase an undefined instruction");
909
1.16M
      State.MIs[InsnID]->eraseFromParent();
910
1.16M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
1.16M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
1.16M
                             << InsnID << "])\n");
913
1.16M
      break;
914
0
    }
915
0
916
3.83k
    case GIR_MakeTempReg: {
917
3.83k
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
3.83k
      int64_t TypeID = MatchTable[CurrentIdx++];
919
3.83k
920
3.83k
      State.TempRegisters[TempRegID] =
921
3.83k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
3.83k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
3.83k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
3.83k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
3.83k
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
1.78M
    case GIR_Done:
939
1.78M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
1.78M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
1.78M
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
36.5M
    }
946
36.5M
  }
947
4.44M
}
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<15ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<15ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<15ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
4.44M
    CodeGenCoverage &CoverageInfo) const {
57
4.44M
58
4.44M
  uint64_t CurrentIdx = 0;
59
4.44M
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
4.44M
61
4.44M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
4.44M
  auto handleReject = [&]() -> RejectAction {
63
4.44M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
4.44M
                    dbgs() << CurrentIdx << ": Rejected\n");
65
4.44M
    if (OnFailResumeAt.empty())
66
4.44M
      return RejectAndGiveUp;
67
4.44M
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
4.44M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
4.44M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
4.44M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
4.44M
    return RejectAndResume;
72
4.44M
  };
73
4.44M
74
36.5M
  while (true) {
75
36.5M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
36.5M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
36.5M
    switch (MatcherOpcode) {
78
36.5M
    case GIM_Try: {
79
4.78M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
4.78M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
4.78M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
4.78M
      break;
83
36.5M
    }
84
36.5M
85
36.5M
    case GIM_RecordInsn: {
86
1.11M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
1.11M
      int64_t InsnID = MatchTable[CurrentIdx++];
88
1.11M
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
1.11M
90
1.11M
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
1.11M
      // any attempt to modify it.
92
1.11M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
1.11M
94
1.11M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
1.11M
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
1.11M
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
1.11M
110
1.11M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
1.11M
      if ((size_t)NewInsnID < State.MIs.size())
112
793k
        State.MIs[NewInsnID] = NewMI;
113
320k
      else {
114
320k
        assert((size_t)NewInsnID == State.MIs.size() &&
115
320k
               "Expected to store MIs in order");
116
320k
        State.MIs.push_back(NewMI);
117
320k
      }
118
1.11M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
1.11M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
1.11M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
1.11M
                             << ")\n");
122
1.11M
      break;
123
1.11M
    }
124
1.11M
125
1.11M
    case GIM_CheckFeatures: {
126
888k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
888k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
888k
                      dbgs() << CurrentIdx
129
888k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
888k
                             << ExpectedBitsetID << ")\n");
131
888k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
888k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
104k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
888k
      }
136
888k
      break;
137
888k
    }
138
888k
139
1.11M
    case GIM_CheckOpcode: {
140
1.11M
      int64_t InsnID = MatchTable[CurrentIdx++];
141
1.11M
      int64_t Expected = MatchTable[CurrentIdx++];
142
1.11M
143
1.11M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
1.11M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
1.11M
146
1.11M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
1.11M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
1.11M
                             << "], ExpectedOpcode=" << Expected
149
1.11M
                             << ") // Got=" << Opcode << "\n");
150
1.11M
      if (Opcode != Expected) {
151
1.05M
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
1.11M
      }
154
1.11M
      break;
155
1.11M
    }
156
1.11M
157
4.44M
    case GIM_SwitchOpcode: {
158
4.44M
      int64_t InsnID = MatchTable[CurrentIdx++];
159
4.44M
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
4.44M
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
4.44M
      int64_t Default = MatchTable[CurrentIdx++];
162
4.44M
163
4.44M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
4.44M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
4.44M
166
4.44M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
4.44M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
4.44M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
4.44M
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
4.44M
      });
171
4.44M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
4.44M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
4.44M
      if (!CurrentIdx) {
177
1.43M
        CurrentIdx = Default;
178
1.43M
  break;
179
1.43M
      }
180
3.01M
      OnFailResumeAt.push_back(Default);
181
3.01M
      break;
182
3.01M
    }
183
3.01M
184
3.01M
    case GIM_SwitchType: {
185
2.61M
      int64_t InsnID = MatchTable[CurrentIdx++];
186
2.61M
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
2.61M
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
2.61M
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
2.61M
      int64_t Default = MatchTable[CurrentIdx++];
190
2.61M
191
2.61M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
2.61M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
2.61M
194
2.61M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
2.61M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
2.61M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
2.61M
               << UpperBound << "), Default=" << Default
198
2.61M
               << ", JumpTable...) // Got=";
199
2.61M
        if (!MO.isReg())
200
2.61M
          dbgs() << "Not a VReg\n";
201
2.61M
        else
202
2.61M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
2.61M
      });
204
2.61M
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
2.61M
      const LLT Ty = MRI.getType(MO.getReg());
209
2.61M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
2.61M
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
971k
        CurrentIdx = Default;
212
971k
        break;
213
971k
      }
214
1.64M
      const int64_t TypeID = TyI->second;
215
1.64M
      if (TypeID < LowerBound || 
UpperBound <= TypeID1.44M
) {
216
194k
        CurrentIdx = Default;
217
194k
        break;
218
194k
      }
219
1.44M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
1.44M
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
1.44M
      OnFailResumeAt.push_back(Default);
225
1.44M
      break;
226
1.44M
    }
227
1.44M
228
1.44M
    case GIM_CheckNumOperands: {
229
33.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
230
33.2k
      int64_t Expected = MatchTable[CurrentIdx++];
231
33.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
33.2k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
33.2k
                             << InsnID << "], Expected=" << Expected << ")\n");
234
33.2k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
33.2k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
31.1k
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
33.2k
      }
239
33.2k
      break;
240
33.2k
    }
241
33.2k
    case GIM_CheckI64ImmPredicate: {
242
21.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
243
21.1k
      int64_t Predicate = MatchTable[CurrentIdx++];
244
21.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
21.1k
                      dbgs()
246
21.1k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
21.1k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
21.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
21.1k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
21.1k
             "Expected G_CONSTANT");
251
21.1k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
21.1k
      int64_t Value = 0;
253
21.1k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
21.1k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
21.1k
260
21.1k
      if (!testImmPredicate_I64(Predicate, Value))
261
5
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
21.1k
      break;
264
21.1k
    }
265
21.1k
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
2.33k
    case GIM_CheckAPFloatImmPredicate: {
288
2.33k
      int64_t InsnID = MatchTable[CurrentIdx++];
289
2.33k
      int64_t Predicate = MatchTable[CurrentIdx++];
290
2.33k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
2.33k
                      dbgs()
292
2.33k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
2.33k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
2.33k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
2.33k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
2.33k
             "Expected G_FCONSTANT");
297
2.33k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
2.33k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
2.33k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
2.33k
301
2.33k
      if (!testImmPredicate_APFloat(Predicate, Value))
302
1.46k
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
2.33k
      break;
305
2.33k
    }
306
2.33k
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
422k
    case GIM_CheckAtomicOrdering: {
322
422k
      int64_t InsnID = MatchTable[CurrentIdx++];
323
422k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
422k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
422k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
422k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
422k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
422k
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
422k
332
422k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
422k
        if (MMO->getOrdering() != Ordering)
334
2.84k
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
422k
      break;
337
422k
    }
338
422k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
47
    case GIM_CheckMemorySizeEqualTo: {
375
47
      int64_t InsnID = MatchTable[CurrentIdx++];
376
47
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
47
      uint64_t Size = MatchTable[CurrentIdx++];
378
47
379
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
47
                      dbgs() << CurrentIdx
381
47
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
47
                             << "]->memoperands() + " << MMOIdx
383
47
                             << ", Size=" << Size << ")\n");
384
47
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
47
386
47
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
47
392
47
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
47
394
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
47
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
47
                             << " bytes\n");
397
47
      if (MMO->getSize() != Size)
398
2
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
47
401
47
      break;
402
47
    }
403
273k
    case GIM_CheckMemorySizeEqualToLLT:
404
273k
    case GIM_CheckMemorySizeLessThanLLT:
405
273k
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
273k
      int64_t InsnID = MatchTable[CurrentIdx++];
407
273k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
273k
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
273k
410
273k
      DEBUG_WITH_TYPE(
411
273k
          TgtInstructionSelector::getName(),
412
273k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
273k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
273k
                         ? "EqualTo"
415
273k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
273k
                               ? "GreaterThan"
417
273k
                               : "LessThan")
418
273k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
273k
                 << ", OpIdx=" << OpIdx << ")\n");
420
273k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
273k
422
273k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
273k
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
273k
431
273k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
273k
437
273k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
273k
439
273k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
273k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
273k
          
MMO->getSize() * 8 != Size273k
) {
442
16
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
273k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
273k
                 
MMO->getSize() * 8 >= Size42
) {
446
36
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
273k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
273k
                 
MMO->getSize() * 8 <= Size0
)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
273k
453
273k
      break;
454
273k
    }
455
940k
    case GIM_CheckType: {
456
940k
      int64_t InsnID = MatchTable[CurrentIdx++];
457
940k
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
940k
      int64_t TypeID = MatchTable[CurrentIdx++];
459
940k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
940k
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
940k
                             << "]->getOperand(" << OpIdx
462
940k
                             << "), TypeID=" << TypeID << ")\n");
463
940k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
940k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
940k
      if (!MO.isReg() ||
466
940k
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
26.3k
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
940k
      }
470
940k
      break;
471
940k
    }
472
940k
    case GIM_CheckPointerToAny: {
473
409k
      int64_t InsnID = MatchTable[CurrentIdx++];
474
409k
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
409k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
409k
477
409k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
409k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
409k
                             << InsnID << "]->getOperand(" << OpIdx
480
409k
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
409k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
409k
      // iPTR must be looked up in the target.
483
409k
      if (SizeInBits == 0) {
484
409k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
409k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
409k
      }
487
409k
488
409k
      assert(SizeInBits != 0 && "Pointer size must be known");
489
409k
490
409k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
409k
      if (MO.isReg()) {
492
409k
        const LLT &Ty = MRI.getType(MO.getReg());
493
409k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
409k
499
409k
      break;
500
409k
    }
501
3.67M
    case GIM_CheckRegBankForClass: {
502
3.67M
      int64_t InsnID = MatchTable[CurrentIdx++];
503
3.67M
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
3.67M
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
3.67M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
3.67M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
3.67M
                             << InsnID << "]->getOperand(" << OpIdx
508
3.67M
                             << "), RCEnum=" << RCEnum << ")\n");
509
3.67M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
3.67M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
3.67M
      if (!MO.isReg() ||
512
3.67M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
3.67M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
186k
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
3.67M
      }
517
3.67M
      break;
518
3.67M
    }
519
3.67M
520
3.67M
    case GIM_CheckComplexPattern: {
521
694k
      int64_t InsnID = MatchTable[CurrentIdx++];
522
694k
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
694k
      int64_t RendererID = MatchTable[CurrentIdx++];
524
694k
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
694k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
694k
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
694k
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
694k
                             << "]->getOperand(" << OpIdx
529
694k
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
694k
                             << ")\n");
531
694k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
694k
      // FIXME: Use std::invoke() when it's available.
533
694k
      ComplexRendererFns Renderer =
534
694k
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
694k
              State.MIs[InsnID]->getOperand(OpIdx));
536
694k
      if (Renderer.hasValue())
537
461k
        State.Renderers[RendererID] = Renderer.getValue();
538
233k
      else
539
233k
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
694k
      break;
542
694k
    }
543
694k
544
694k
    case GIM_CheckConstantInt: {
545
75.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
546
75.1k
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
75.1k
      int64_t Value = MatchTable[CurrentIdx++];
548
75.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
75.1k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
75.1k
                             << InsnID << "]->getOperand(" << OpIdx
551
75.1k
                             << "), Value=" << Value << ")\n");
552
75.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
75.1k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
75.1k
      if (MO.isReg()) {
555
75.1k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
75.1k
        LLT Ty = MRI.getType(MO.getReg());
557
75.1k
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
75.1k
559
75.1k
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
56.4k
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
75.1k
566
75.1k
      break;
567
75.1k
    }
568
75.1k
569
75.1k
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
686k
    case GIM_CheckIntrinsicID: {
587
686k
      int64_t InsnID = MatchTable[CurrentIdx++];
588
686k
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
686k
      int64_t Value = MatchTable[CurrentIdx++];
590
686k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
686k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
686k
                             << InsnID << "]->getOperand(" << OpIdx
593
686k
                             << "), Value=" << Value << ")\n");
594
686k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
686k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
686k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value682k
)
597
684k
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
686k
      break;
600
686k
    }
601
686k
602
686k
    case GIM_CheckIsMBB: {
603
372k
      int64_t InsnID = MatchTable[CurrentIdx++];
604
372k
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
372k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
372k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
372k
                             << "]->getOperand(" << OpIdx << "))\n");
608
372k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
372k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
372k
      }
613
372k
      break;
614
372k
    }
615
372k
616
372k
    case GIM_CheckIsSafeToFold: {
617
43.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
618
43.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
43.9k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
43.9k
                             << InsnID << "])\n");
621
43.9k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
43.9k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
43.9k
      }
626
43.9k
      break;
627
43.9k
    }
628
43.9k
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
3.93M
    case GIM_Reject:
647
3.93M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
3.93M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
3.93M
      if (handleReject() == RejectAndGiveUp)
650
2.66M
        return false;
651
1.26M
      break;
652
1.26M
653
1.26M
    case GIR_MutateOpcode: {
654
615k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
615k
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
615k
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
615k
      if (NewInsnID >= OutMIs.size())
658
615k
        OutMIs.resize(NewInsnID + 1);
659
615k
660
615k
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
615k
                                              State.MIs[OldInsnID]);
662
615k
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
615k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
615k
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
615k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
615k
                             << NewOpcode << ")\n");
667
615k
      break;
668
1.26M
    }
669
1.26M
670
1.26M
    case GIR_BuildMI: {
671
1.17M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
1.17M
      int64_t Opcode = MatchTable[CurrentIdx++];
673
1.17M
      if (NewInsnID >= OutMIs.size())
674
1.16M
        OutMIs.resize(NewInsnID + 1);
675
1.17M
676
1.17M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
1.17M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
1.17M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
1.17M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
1.17M
                             << NewInsnID << "], " << Opcode << ")\n");
681
1.17M
      break;
682
1.26M
    }
683
1.26M
684
1.27M
    case GIR_Copy: {
685
1.27M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
1.27M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
1.27M
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
1.27M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
1.27M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
1.27M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
1.27M
                      dbgs()
692
1.27M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
1.27M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
1.27M
      break;
695
1.26M
    }
696
1.26M
697
1.26M
    case GIR_CopyOrAddZeroReg: {
698
134k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
134k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
134k
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
134k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
134k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
134k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
134k
      if (isOperandImmEqual(MO, 0, MRI))
705
14.8k
        OutMIs[NewInsnID].addReg(ZeroReg);
706
119k
      else
707
119k
        OutMIs[NewInsnID].add(MO);
708
134k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
134k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
134k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
134k
                             << OpIdx << ", " << ZeroReg << ")\n");
712
134k
      break;
713
1.26M
    }
714
1.26M
715
1.26M
    case GIR_CopySubReg: {
716
40.9k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
40.9k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
40.9k
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
40.9k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
40.9k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
40.9k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
40.9k
                               0, SubRegIdx);
723
40.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
40.9k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
40.9k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
40.9k
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
40.9k
      break;
728
1.26M
    }
729
1.26M
730
1.26M
    case GIR_AddImplicitDef: {
731
22.8k
      int64_t InsnID = MatchTable[CurrentIdx++];
732
22.8k
      int64_t RegNum = MatchTable[CurrentIdx++];
733
22.8k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
22.8k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
22.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
22.8k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
22.8k
                             << InsnID << "], " << RegNum << ")\n");
738
22.8k
      break;
739
1.26M
    }
740
1.26M
741
1.26M
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
1.26M
    }
751
1.26M
752
1.26M
    case GIR_AddRegister: {
753
81.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
754
81.0k
      int64_t RegNum = MatchTable[CurrentIdx++];
755
81.0k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
81.0k
      OutMIs[InsnID].addReg(RegNum);
757
81.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
81.0k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
81.0k
                             << InsnID << "], " << RegNum << ")\n");
760
81.0k
      break;
761
1.26M
    }
762
1.26M
763
1.26M
    case GIR_AddTempRegister: {
764
7.65k
      int64_t InsnID = MatchTable[CurrentIdx++];
765
7.65k
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
7.65k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
7.65k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
7.65k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
7.65k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
7.65k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
7.65k
                             << InsnID << "], TempRegisters[" << TempRegID
772
7.65k
                             << "], " << TempRegFlags << ")\n");
773
7.65k
      break;
774
1.26M
    }
775
1.26M
776
1.26M
    case GIR_AddImm: {
777
17.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
778
17.2k
      int64_t Imm = MatchTable[CurrentIdx++];
779
17.2k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
17.2k
      OutMIs[InsnID].addImm(Imm);
781
17.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
17.2k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
17.2k
                             << "], " << Imm << ")\n");
784
17.2k
      break;
785
1.26M
    }
786
1.26M
787
1.26M
    case GIR_ComplexRenderer: {
788
57.6k
      int64_t InsnID = MatchTable[CurrentIdx++];
789
57.6k
      int64_t RendererID = MatchTable[CurrentIdx++];
790
57.6k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
57.6k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
115k
        RenderOpFn(OutMIs[InsnID]);
793
57.6k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
57.6k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
57.6k
                             << InsnID << "], " << RendererID << ")\n");
796
57.6k
      break;
797
1.26M
    }
798
1.26M
    case GIR_ComplexSubOperandRenderer: {
799
806k
      int64_t InsnID = MatchTable[CurrentIdx++];
800
806k
      int64_t RendererID = MatchTable[CurrentIdx++];
801
806k
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
806k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
806k
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
806k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
806k
                      dbgs() << CurrentIdx
806
806k
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
806k
                             << InsnID << "], " << RendererID << ", "
808
806k
                             << RenderOpID << ")\n");
809
806k
      break;
810
1.26M
    }
811
1.26M
812
1.26M
    case GIR_CopyConstantAsSImm: {
813
580k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
580k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
580k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
580k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
580k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
580k
        OutMIs[NewInsnID].addImm(
819
580k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
580k
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
580k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
580k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
580k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
580k
      break;
828
580k
    }
829
580k
830
580k
    // TODO: Needs a test case once we have a pattern that uses this.
831
580k
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
3.82k
    case GIR_CustomRenderer: {
848
3.82k
      int64_t InsnID = MatchTable[CurrentIdx++];
849
3.82k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
3.82k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
3.82k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
3.82k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
3.82k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
3.82k
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
3.82k
                             << RendererFnID << ")\n");
856
3.82k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
3.82k
                                                     *State.MIs[OldInsnID]);
858
3.82k
      break;
859
0
    }
860
82.2k
    case GIR_ConstrainOperandRC: {
861
82.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
862
82.2k
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
82.2k
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
82.2k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
82.2k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
82.2k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
82.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
82.2k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
82.2k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
82.2k
                             << ")\n");
871
82.2k
      break;
872
0
    }
873
0
874
1.74M
    case GIR_ConstrainSelectedInstOperands: {
875
1.74M
      int64_t InsnID = MatchTable[CurrentIdx++];
876
1.74M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
1.74M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
1.74M
                                       RBI);
879
1.74M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
1.74M
                      dbgs() << CurrentIdx
881
1.74M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
1.74M
                             << InsnID << "])\n");
883
1.74M
      break;
884
0
    }
885
0
886
403k
    case GIR_MergeMemOperands: {
887
403k
      int64_t InsnID = MatchTable[CurrentIdx++];
888
403k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
403k
890
403k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
403k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
403k
                             << InsnID << "]");
893
403k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
806k
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
806k
             GIU_MergeMemOperands_EndOfList) {
896
403k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
403k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
403k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
403k
          OutMIs[InsnID].addMemOperand(MMO);
900
403k
      }
901
403k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
403k
      break;
903
0
    }
904
0
905
1.16M
    case GIR_EraseFromParent: {
906
1.16M
      int64_t InsnID = MatchTable[CurrentIdx++];
907
1.16M
      assert(State.MIs[InsnID] &&
908
1.16M
             "Attempted to erase an undefined instruction");
909
1.16M
      State.MIs[InsnID]->eraseFromParent();
910
1.16M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
1.16M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
1.16M
                             << InsnID << "])\n");
913
1.16M
      break;
914
0
    }
915
0
916
3.82k
    case GIR_MakeTempReg: {
917
3.82k
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
3.82k
      int64_t TypeID = MatchTable[CurrentIdx++];
919
3.82k
920
3.82k
      State.TempRegisters[TempRegID] =
921
3.82k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
3.82k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
3.82k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
3.82k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
3.82k
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
1.78M
    case GIR_Done:
939
1.78M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
1.78M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
1.78M
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
36.5M
    }
946
36.5M
  }
947
4.44M
}
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<32ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<32ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
41
    CodeGenCoverage &CoverageInfo) const {
57
41
58
41
  uint64_t CurrentIdx = 0;
59
41
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
41
61
41
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
41
  auto handleReject = [&]() -> RejectAction {
63
41
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
41
                    dbgs() << CurrentIdx << ": Rejected\n");
65
41
    if (OnFailResumeAt.empty())
66
41
      return RejectAndGiveUp;
67
41
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
41
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
41
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
41
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
41
    return RejectAndResume;
72
41
  };
73
41
74
1.09k
  while (true) {
75
1.09k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
1.09k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
1.09k
    switch (MatcherOpcode) {
78
1.09k
    case GIM_Try: {
79
195
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
195
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
195
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
195
      break;
83
1.09k
    }
84
1.09k
85
1.09k
    case GIM_RecordInsn: {
86
48
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
48
      int64_t InsnID = MatchTable[CurrentIdx++];
88
48
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
48
90
48
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
48
      // any attempt to modify it.
92
48
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
48
94
48
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
48
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
48
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
48
110
48
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
48
      if ((size_t)NewInsnID < State.MIs.size())
112
45
        State.MIs[NewInsnID] = NewMI;
113
3
      else {
114
3
        assert((size_t)NewInsnID == State.MIs.size() &&
115
3
               "Expected to store MIs in order");
116
3
        State.MIs.push_back(NewMI);
117
3
      }
118
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
48
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
48
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
48
                             << ")\n");
122
48
      break;
123
48
    }
124
48
125
142
    case GIM_CheckFeatures: {
126
142
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
142
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
142
                      dbgs() << CurrentIdx
129
142
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
142
                             << ExpectedBitsetID << ")\n");
131
142
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
142
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
12
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
142
      }
136
142
      break;
137
142
    }
138
142
139
142
    case GIM_CheckOpcode: {
140
48
      int64_t InsnID = MatchTable[CurrentIdx++];
141
48
      int64_t Expected = MatchTable[CurrentIdx++];
142
48
143
48
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
48
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
48
146
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
48
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
48
                             << "], ExpectedOpcode=" << Expected
149
48
                             << ") // Got=" << Opcode << "\n");
150
48
      if (Opcode != Expected) {
151
48
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
48
      }
154
48
      break;
155
48
    }
156
48
157
48
    case GIM_SwitchOpcode: {
158
41
      int64_t InsnID = MatchTable[CurrentIdx++];
159
41
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
41
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
41
      int64_t Default = MatchTable[CurrentIdx++];
162
41
163
41
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
41
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
41
166
41
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
41
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
41
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
41
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
41
      });
171
41
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
41
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
41
      if (!CurrentIdx) {
177
0
        CurrentIdx = Default;
178
0
  break;
179
0
      }
180
41
      OnFailResumeAt.push_back(Default);
181
41
      break;
182
41
    }
183
41
184
41
    case GIM_SwitchType: {
185
38
      int64_t InsnID = MatchTable[CurrentIdx++];
186
38
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
38
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
38
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
38
      int64_t Default = MatchTable[CurrentIdx++];
190
38
191
38
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
38
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
38
194
38
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
38
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
38
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
38
               << UpperBound << "), Default=" << Default
198
38
               << ", JumpTable...) // Got=";
199
38
        if (!MO.isReg())
200
38
          dbgs() << "Not a VReg\n";
201
38
        else
202
38
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
38
      });
204
38
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
38
      const LLT Ty = MRI.getType(MO.getReg());
209
38
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
38
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
0
        CurrentIdx = Default;
212
0
        break;
213
0
      }
214
38
      const int64_t TypeID = TyI->second;
215
38
      if (TypeID < LowerBound || UpperBound <= TypeID) {
216
0
        CurrentIdx = Default;
217
0
        break;
218
0
      }
219
38
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
38
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
38
      OnFailResumeAt.push_back(Default);
225
38
      break;
226
38
    }
227
38
228
38
    case GIM_CheckNumOperands: {
229
9
      int64_t InsnID = MatchTable[CurrentIdx++];
230
9
      int64_t Expected = MatchTable[CurrentIdx++];
231
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
9
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
9
                             << InsnID << "], Expected=" << Expected << ")\n");
234
9
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
9
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
6
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
9
      }
239
9
      break;
240
9
    }
241
9
    case GIM_CheckI64ImmPredicate: {
242
0
      int64_t InsnID = MatchTable[CurrentIdx++];
243
0
      int64_t Predicate = MatchTable[CurrentIdx++];
244
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
0
                      dbgs()
246
0
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
0
             "Expected G_CONSTANT");
251
0
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
0
      int64_t Value = 0;
253
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
0
260
0
      if (!testImmPredicate_I64(Predicate, Value))
261
0
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
0
      break;
264
0
    }
265
0
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
84
    case GIM_CheckType: {
456
84
      int64_t InsnID = MatchTable[CurrentIdx++];
457
84
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
84
      int64_t TypeID = MatchTable[CurrentIdx++];
459
84
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
84
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
84
                             << "]->getOperand(" << OpIdx
462
84
                             << "), TypeID=" << TypeID << ")\n");
463
84
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
84
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
84
      if (!MO.isReg() ||
466
84
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
0
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
84
      }
470
84
      break;
471
84
    }
472
84
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
167
    case GIM_CheckRegBankForClass: {
502
167
      int64_t InsnID = MatchTable[CurrentIdx++];
503
167
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
167
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
167
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
167
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
167
                             << InsnID << "]->getOperand(" << OpIdx
508
167
                             << "), RCEnum=" << RCEnum << ")\n");
509
167
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
167
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
167
      if (!MO.isReg() ||
512
167
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
167
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
51
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
167
      }
517
167
      break;
518
167
    }
519
167
520
167
    case GIM_CheckComplexPattern: {
521
39
      int64_t InsnID = MatchTable[CurrentIdx++];
522
39
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
39
      int64_t RendererID = MatchTable[CurrentIdx++];
524
39
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
39
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
39
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
39
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
39
                             << "]->getOperand(" << OpIdx
529
39
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
39
                             << ")\n");
531
39
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
39
      // FIXME: Use std::invoke() when it's available.
533
39
      ComplexRendererFns Renderer =
534
39
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
39
              State.MIs[InsnID]->getOperand(OpIdx));
536
39
      if (Renderer.hasValue())
537
39
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
39
      break;
542
39
    }
543
39
544
39
    case GIM_CheckConstantInt: {
545
0
      int64_t InsnID = MatchTable[CurrentIdx++];
546
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
0
      int64_t Value = MatchTable[CurrentIdx++];
548
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
0
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
0
                             << InsnID << "]->getOperand(" << OpIdx
551
0
                             << "), Value=" << Value << ")\n");
552
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
0
      if (MO.isReg()) {
555
0
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
0
        LLT Ty = MRI.getType(MO.getReg());
557
0
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
0
559
0
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
0
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
0
566
0
      break;
567
0
    }
568
0
569
0
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
4
    case GIM_CheckIntrinsicID: {
587
4
      int64_t InsnID = MatchTable[CurrentIdx++];
588
4
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
4
      int64_t Value = MatchTable[CurrentIdx++];
590
4
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
4
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
4
                             << InsnID << "]->getOperand(" << OpIdx
593
4
                             << "), Value=" << Value << ")\n");
594
4
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
4
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
4
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
0
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
4
      break;
600
4
    }
601
4
602
4
    case GIM_CheckIsMBB: {
603
0
      int64_t InsnID = MatchTable[CurrentIdx++];
604
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
0
                             << "]->getOperand(" << OpIdx << "))\n");
608
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
0
      }
613
0
      break;
614
0
    }
615
0
616
0
    case GIM_CheckIsSafeToFold: {
617
0
      int64_t InsnID = MatchTable[CurrentIdx++];
618
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
0
                             << InsnID << "])\n");
621
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
0
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
0
      }
626
0
      break;
627
0
    }
628
0
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
0
    case GIM_Reject:
647
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
0
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
0
      if (handleReject() == RejectAndGiveUp)
650
0
        return false;
651
0
      break;
652
0
653
11
    case GIR_MutateOpcode: {
654
11
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
11
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
11
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
11
      if (NewInsnID >= OutMIs.size())
658
11
        OutMIs.resize(NewInsnID + 1);
659
11
660
11
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
11
                                              State.MIs[OldInsnID]);
662
11
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
11
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
11
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
11
                             << NewOpcode << ")\n");
667
11
      break;
668
0
    }
669
0
670
30
    case GIR_BuildMI: {
671
30
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
30
      int64_t Opcode = MatchTable[CurrentIdx++];
673
30
      if (NewInsnID >= OutMIs.size())
674
30
        OutMIs.resize(NewInsnID + 1);
675
30
676
30
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
30
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
30
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
30
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
30
                             << NewInsnID << "], " << Opcode << ")\n");
681
30
      break;
682
0
    }
683
0
684
47
    case GIR_Copy: {
685
47
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
47
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
47
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
47
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
47
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
47
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
47
                      dbgs()
692
47
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
47
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
47
      break;
695
0
    }
696
0
697
0
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
0
    }
714
0
715
0
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
0
    }
729
0
730
11
    case GIR_AddImplicitDef: {
731
11
      int64_t InsnID = MatchTable[CurrentIdx++];
732
11
      int64_t RegNum = MatchTable[CurrentIdx++];
733
11
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
11
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
11
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
11
                             << InsnID << "], " << RegNum << ")\n");
738
11
      break;
739
0
    }
740
0
741
0
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
0
    }
751
0
752
0
    case GIR_AddRegister: {
753
0
      int64_t InsnID = MatchTable[CurrentIdx++];
754
0
      int64_t RegNum = MatchTable[CurrentIdx++];
755
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
0
      OutMIs[InsnID].addReg(RegNum);
757
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
0
                             << InsnID << "], " << RegNum << ")\n");
760
0
      break;
761
0
    }
762
0
763
0
    case GIR_AddTempRegister: {
764
0
      int64_t InsnID = MatchTable[CurrentIdx++];
765
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
0
                             << InsnID << "], TempRegisters[" << TempRegID
772
0
                             << "], " << TempRegFlags << ")\n");
773
0
      break;
774
0
    }
775
0
776
0
    case GIR_AddImm: {
777
0
      int64_t InsnID = MatchTable[CurrentIdx++];
778
0
      int64_t Imm = MatchTable[CurrentIdx++];
779
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
0
      OutMIs[InsnID].addImm(Imm);
781
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
0
                             << "], " << Imm << ")\n");
784
0
      break;
785
0
    }
786
0
787
0
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
0
    }
798
73
    case GIR_ComplexSubOperandRenderer: {
799
73
      int64_t InsnID = MatchTable[CurrentIdx++];
800
73
      int64_t RendererID = MatchTable[CurrentIdx++];
801
73
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
73
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
73
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
73
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
73
                      dbgs() << CurrentIdx
806
73
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
73
                             << InsnID << "], " << RendererID << ", "
808
73
                             << RenderOpID << ")\n");
809
73
      break;
810
0
    }
811
0
812
0
    case GIR_CopyConstantAsSImm: {
813
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
0
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
0
        OutMIs[NewInsnID].addImm(
819
0
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
0
      } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
0
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
0
      break;
828
0
    }
829
0
830
0
    // TODO: Needs a test case once we have a pattern that uses this.
831
0
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
0
    case GIR_ConstrainOperandRC: {
861
0
      int64_t InsnID = MatchTable[CurrentIdx++];
862
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
0
                             << ")\n");
871
0
      break;
872
0
    }
873
0
874
41
    case GIR_ConstrainSelectedInstOperands: {
875
41
      int64_t InsnID = MatchTable[CurrentIdx++];
876
41
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
41
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
41
                                       RBI);
879
41
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
41
                      dbgs() << CurrentIdx
881
41
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
41
                             << InsnID << "])\n");
883
41
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
0
                             << InsnID << "]");
893
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
0
             GIU_MergeMemOperands_EndOfList) {
896
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
0
          OutMIs[InsnID].addMemOperand(MMO);
900
0
      }
901
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
0
      break;
903
0
    }
904
0
905
30
    case GIR_EraseFromParent: {
906
30
      int64_t InsnID = MatchTable[CurrentIdx++];
907
30
      assert(State.MIs[InsnID] &&
908
30
             "Attempted to erase an undefined instruction");
909
30
      State.MIs[InsnID]->eraseFromParent();
910
30
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
30
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
30
                             << InsnID << "])\n");
913
30
      break;
914
0
    }
915
0
916
0
    case GIR_MakeTempReg: {
917
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
0
      int64_t TypeID = MatchTable[CurrentIdx++];
919
0
920
0
      State.TempRegisters[TempRegID] =
921
0
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
0
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
0
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
0
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
41
    case GIR_Done:
939
41
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
41
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
41
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
1.09k
    }
946
1.09k
  }
947
41
}
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<64ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
706
    CodeGenCoverage &CoverageInfo) const {
57
706
58
706
  uint64_t CurrentIdx = 0;
59
706
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
706
61
706
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
706
  auto handleReject = [&]() -> RejectAction {
63
706
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
706
                    dbgs() << CurrentIdx << ": Rejected\n");
65
706
    if (OnFailResumeAt.empty())
66
706
      return RejectAndGiveUp;
67
706
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
706
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
706
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
706
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
706
    return RejectAndResume;
72
706
  };
73
706
74
11.3k
  while (true) {
75
11.3k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
11.3k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
11.3k
    switch (MatcherOpcode) {
78
11.3k
    case GIM_Try: {
79
2.08k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
2.08k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
2.08k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
2.08k
      break;
83
11.3k
    }
84
11.3k
85
11.3k
    case GIM_RecordInsn: {
86
577
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
577
      int64_t InsnID = MatchTable[CurrentIdx++];
88
577
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
577
90
577
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
577
      // any attempt to modify it.
92
577
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
577
94
577
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
577
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
577
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
577
110
577
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
577
      if ((size_t)NewInsnID < State.MIs.size())
112
381
        State.MIs[NewInsnID] = NewMI;
113
196
      else {
114
196
        assert((size_t)NewInsnID == State.MIs.size() &&
115
196
               "Expected to store MIs in order");
116
196
        State.MIs.push_back(NewMI);
117
196
      }
118
577
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
577
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
577
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
577
                             << ")\n");
122
577
      break;
123
577
    }
124
577
125
1.75k
    case GIM_CheckFeatures: {
126
1.75k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
1.75k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
1.75k
                      dbgs() << CurrentIdx
129
1.75k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
1.75k
                             << ExpectedBitsetID << ")\n");
131
1.75k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
1.75k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
1.03k
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
1.75k
      }
136
1.75k
      break;
137
1.75k
    }
138
1.75k
139
1.75k
    case GIM_CheckOpcode: {
140
577
      int64_t InsnID = MatchTable[CurrentIdx++];
141
577
      int64_t Expected = MatchTable[CurrentIdx++];
142
577
143
577
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
577
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
577
146
577
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
577
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
577
                             << "], ExpectedOpcode=" << Expected
149
577
                             << ") // Got=" << Opcode << "\n");
150
577
      if (Opcode != Expected) {
151
431
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
577
      }
154
577
      break;
155
577
    }
156
577
157
706
    case GIM_SwitchOpcode: {
158
706
      int64_t InsnID = MatchTable[CurrentIdx++];
159
706
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
706
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
706
      int64_t Default = MatchTable[CurrentIdx++];
162
706
163
706
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
706
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
706
166
706
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
706
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
706
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
706
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
706
      });
171
706
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
706
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
706
      if (!CurrentIdx) {
177
313
        CurrentIdx = Default;
178
313
  break;
179
313
      }
180
393
      OnFailResumeAt.push_back(Default);
181
393
      break;
182
393
    }
183
393
184
393
    case GIM_SwitchType: {
185
225
      int64_t InsnID = MatchTable[CurrentIdx++];
186
225
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
225
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
225
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
225
      int64_t Default = MatchTable[CurrentIdx++];
190
225
191
225
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
225
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
225
194
225
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
225
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
225
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
225
               << UpperBound << "), Default=" << Default
198
225
               << ", JumpTable...) // Got=";
199
225
        if (!MO.isReg())
200
225
          dbgs() << "Not a VReg\n";
201
225
        else
202
225
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
225
      });
204
225
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
225
      const LLT Ty = MRI.getType(MO.getReg());
209
225
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
225
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
13
        CurrentIdx = Default;
212
13
        break;
213
13
      }
214
212
      const int64_t TypeID = TyI->second;
215
212
      if (TypeID < LowerBound || 
UpperBound <= TypeID148
) {
216
64
        CurrentIdx = Default;
217
64
        break;
218
64
      }
219
148
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
148
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
148
      OnFailResumeAt.push_back(Default);
225
148
      break;
226
148
    }
227
148
228
148
    case GIM_CheckNumOperands: {
229
0
      int64_t InsnID = MatchTable[CurrentIdx++];
230
0
      int64_t Expected = MatchTable[CurrentIdx++];
231
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
0
                             << InsnID << "], Expected=" << Expected << ")\n");
234
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
0
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
0
      }
239
0
      break;
240
0
    }
241
71
    case GIM_CheckI64ImmPredicate: {
242
71
      int64_t InsnID = MatchTable[CurrentIdx++];
243
71
      int64_t Predicate = MatchTable[CurrentIdx++];
244
71
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
71
                      dbgs()
246
71
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
71
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
71
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
71
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
71
             "Expected G_CONSTANT");
251
71
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
71
      int64_t Value = 0;
253
71
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
69
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
2
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
2
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
2
      else
258
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
259
71
260
71
      if (!testImmPredicate_I64(Predicate, Value))
261
2
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
71
      break;
264
71
    }
265
71
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
899
    case GIM_CheckType: {
456
899
      int64_t InsnID = MatchTable[CurrentIdx++];
457
899
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
899
      int64_t TypeID = MatchTable[CurrentIdx++];
459
899
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
899
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
899
                             << "]->getOperand(" << OpIdx
462
899
                             << "), TypeID=" << TypeID << ")\n");
463
899
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
899
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
899
      if (!MO.isReg() ||
466
899
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
5
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
899
      }
470
899
      break;
471
899
    }
472
899
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
1.29k
    case GIM_CheckRegBankForClass: {
502
1.29k
      int64_t InsnID = MatchTable[CurrentIdx++];
503
1.29k
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
1.29k
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
1.29k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
1.29k
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
1.29k
                             << InsnID << "]->getOperand(" << OpIdx
508
1.29k
                             << "), RCEnum=" << RCEnum << ")\n");
509
1.29k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
1.29k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
1.29k
      if (!MO.isReg() ||
512
1.29k
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
1.29k
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
0
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
1.29k
      }
517
1.29k
      break;
518
1.29k
    }
519
1.29k
520
1.29k
    case GIM_CheckComplexPattern: {
521
0
      int64_t InsnID = MatchTable[CurrentIdx++];
522
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
0
      int64_t RendererID = MatchTable[CurrentIdx++];
524
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
0
                             << "]->getOperand(" << OpIdx
529
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
0
                             << ")\n");
531
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
0
      // FIXME: Use std::invoke() when it's available.
533
0
      ComplexRendererFns Renderer =
534
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
0
              State.MIs[InsnID]->getOperand(OpIdx));
536
0
      if (Renderer.hasValue())
537
0
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
0
      break;
542
0
    }
543
0
544
121
    case GIM_CheckConstantInt: {
545
121
      int64_t InsnID = MatchTable[CurrentIdx++];
546
121
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
121
      int64_t Value = MatchTable[CurrentIdx++];
548
121
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
121
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
121
                             << InsnID << "]->getOperand(" << OpIdx
551
121
                             << "), Value=" << Value << ")\n");
552
121
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
121
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
121
      if (MO.isReg()) {
555
121
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
121
        LLT Ty = MRI.getType(MO.getReg());
557
121
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
121
559
121
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
51
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
121
566
121
      break;
567
121
    }
568
121
569
121
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
0
    case GIM_CheckIntrinsicID: {
587
0
      int64_t InsnID = MatchTable[CurrentIdx++];
588
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
0
      int64_t Value = MatchTable[CurrentIdx++];
590
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
0
                             << InsnID << "]->getOperand(" << OpIdx
593
0
                             << "), Value=" << Value << ")\n");
594
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
0
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
0
      break;
600
0
    }
601
0
602
9
    case GIM_CheckIsMBB: {
603
9
      int64_t InsnID = MatchTable[CurrentIdx++];
604
9
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
9
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
9
                             << "]->getOperand(" << OpIdx << "))\n");
608
9
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
9
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
9
      }
613
9
      break;
614
9
    }
615
9
616
77
    case GIM_CheckIsSafeToFold: {
617
77
      int64_t InsnID = MatchTable[CurrentIdx++];
618
77
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
77
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
77
                             << InsnID << "])\n");
621
77
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
77
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
77
      }
626
77
      break;
627
77
    }
628
77
    case GIM_CheckIsSameOperand: {
629
2
      int64_t InsnID = MatchTable[CurrentIdx++];
630
2
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
2
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
2
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
2
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
2
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
2
                             << InsnID << "][" << OpIdx << "], MIs["
636
2
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
2
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
2
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
2
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
2
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
2
      }
644
2
      break;
645
2
    }
646
750
    case GIM_Reject:
647
750
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
750
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
750
      if (handleReject() == RejectAndGiveUp)
650
473
        return false;
651
277
      break;
652
277
653
277
    case GIR_MutateOpcode: {
654
9
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
9
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
9
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
9
      if (NewInsnID >= OutMIs.size())
658
9
        OutMIs.resize(NewInsnID + 1);
659
9
660
9
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
9
                                              State.MIs[OldInsnID]);
662
9
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
9
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
9
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
9
                             << NewOpcode << ")\n");
667
9
      break;
668
277
    }
669
277
670
277
    case GIR_BuildMI: {
671
233
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
233
      int64_t Opcode = MatchTable[CurrentIdx++];
673
233
      if (NewInsnID >= OutMIs.size())
674
224
        OutMIs.resize(NewInsnID + 1);
675
233
676
233
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
233
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
233
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
233
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
233
                             << NewInsnID << "], " << Opcode << ")\n");
681
233
      break;
682
277
    }
683
277
684
534
    case GIR_Copy: {
685
534
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
534
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
534
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
534
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
534
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
534
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
534
                      dbgs()
692
534
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
534
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
534
      break;
695
277
    }
696
277
697
277
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
277
    }
714
277
715
277
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
277
    }
729
277
730
277
    case GIR_AddImplicitDef: {
731
0
      int64_t InsnID = MatchTable[CurrentIdx++];
732
0
      int64_t RegNum = MatchTable[CurrentIdx++];
733
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
0
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
0
                             << InsnID << "], " << RegNum << ")\n");
738
0
      break;
739
277
    }
740
277
741
277
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
277
    }
751
277
752
324
    case GIR_AddRegister: {
753
324
      int64_t InsnID = MatchTable[CurrentIdx++];
754
324
      int64_t RegNum = MatchTable[CurrentIdx++];
755
324
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
324
      OutMIs[InsnID].addReg(RegNum);
757
324
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
324
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
324
                             << InsnID << "], " << RegNum << ")\n");
760
324
      break;
761
277
    }
762
277
763
277
    case GIR_AddTempRegister: {
764
18
      int64_t InsnID = MatchTable[CurrentIdx++];
765
18
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
18
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
18
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
18
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
18
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
18
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
18
                             << InsnID << "], TempRegisters[" << TempRegID
772
18
                             << "], " << TempRegFlags << ")\n");
773
18
      break;
774
277
    }
775
277
776
277
    case GIR_AddImm: {
777
260
      int64_t InsnID = MatchTable[CurrentIdx++];
778
260
      int64_t Imm = MatchTable[CurrentIdx++];
779
260
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
260
      OutMIs[InsnID].addImm(Imm);
781
260
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
260
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
260
                             << "], " << Imm << ")\n");
784
260
      break;
785
277
    }
786
277
787
277
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
277
    }
798
277
    case GIR_ComplexSubOperandRenderer: {
799
0
      int64_t InsnID = MatchTable[CurrentIdx++];
800
0
      int64_t RendererID = MatchTable[CurrentIdx++];
801
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
0
                      dbgs() << CurrentIdx
806
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
0
                             << InsnID << "], " << RendererID << ", "
808
0
                             << RenderOpID << ")\n");
809
0
      break;
810
277
    }
811
277
812
277
    case GIR_CopyConstantAsSImm: {
813
68
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
68
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
68
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
68
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
68
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
66
        OutMIs[NewInsnID].addImm(
819
66
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
66
      } else 
if (2
State.MIs[OldInsnID]->getOperand(1).isImm()2
)
821
2
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
2
      else
823
2
        
llvm_unreachable0
("Expected Imm or CImm operand");
824
68
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
68
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
68
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
68
      break;
828
68
    }
829
68
830
68
    // TODO: Needs a test case once we have a pattern that uses this.
831
68
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
5
    case GIR_ConstrainOperandRC: {
861
5
      int64_t InsnID = MatchTable[CurrentIdx++];
862
5
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
5
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
5
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
5
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
5
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
5
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
5
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
5
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
5
                             << ")\n");
871
5
      break;
872
0
    }
873
0
874
237
    case GIR_ConstrainSelectedInstOperands: {
875
237
      int64_t InsnID = MatchTable[CurrentIdx++];
876
237
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
237
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
237
                                       RBI);
879
237
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
237
                      dbgs() << CurrentIdx
881
237
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
237
                             << InsnID << "])\n");
883
237
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
892
0
                             << InsnID << "]");
893
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
894
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
895
0
             GIU_MergeMemOperands_EndOfList) {
896
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
897
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
898
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
899
0
          OutMIs[InsnID].addMemOperand(MMO);
900
0
      }
901
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
902
0
      break;
903
0
    }
904
0
905
224
    case GIR_EraseFromParent: {
906
224
      int64_t InsnID = MatchTable[CurrentIdx++];
907
224
      assert(State.MIs[InsnID] &&
908
224
             "Attempted to erase an undefined instruction");
909
224
      State.MIs[InsnID]->eraseFromParent();
910
224
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
911
224
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
912
224
                             << InsnID << "])\n");
913
224
      break;
914
0
    }
915
0
916
9
    case GIR_MakeTempReg: {
917
9
      int64_t TempRegID = MatchTable[CurrentIdx++];
918
9
      int64_t TypeID = MatchTable[CurrentIdx++];
919
9
920
9
      State.TempRegisters[TempRegID] =
921
9
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
922
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
923
9
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
924
9
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
925
9
      break;
926
0
    }
927
0
928
0
    case GIR_Coverage: {
929
0
      int64_t RuleID = MatchTable[CurrentIdx++];
930
0
      CoverageInfo.setCovered(RuleID);
931
0
932
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
933
0
                      dbgs()
934
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
935
0
      break;
936
0
    }
937
0
938
233
    case GIR_Done:
939
233
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
940
233
                      dbgs() << CurrentIdx << ": GIR_Done\n");
941
233
      return true;
942
0
943
0
    default:
944
0
      llvm_unreachable("Unexpected command");
945
11.3k
    }
946
11.3k
  }
947
706
}
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<41ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<41ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
56
3
    CodeGenCoverage &CoverageInfo) const {
57
3
58
3
  uint64_t CurrentIdx = 0;
59
3
  SmallVector<uint64_t, 4> OnFailResumeAt;
60
3
61
3
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
62
3
  auto handleReject = [&]() -> RejectAction {
63
3
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
64
3
                    dbgs() << CurrentIdx << ": Rejected\n");
65
3
    if (OnFailResumeAt.empty())
66
3
      return RejectAndGiveUp;
67
3
    CurrentIdx = OnFailResumeAt.pop_back_val();
68
3
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
69
3
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
70
3
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
71
3
    return RejectAndResume;
72
3
  };
73
3
74
69
  while (true) {
75
69
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
76
69
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
77
69
    switch (MatcherOpcode) {
78
69
    case GIM_Try: {
79
21
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
80
21
                      dbgs() << CurrentIdx << ": Begin try-block\n");
81
21
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
82
21
      break;
83
69
    }
84
69
85
69
    case GIM_RecordInsn: {
86
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
87
0
      int64_t InsnID = MatchTable[CurrentIdx++];
88
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
89
0
90
0
      // As an optimisation we require that MIs[0] is always the root. Refuse
91
0
      // any attempt to modify it.
92
0
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
93
0
94
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
95
0
      if (!MO.isReg()) {
96
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
97
0
                        dbgs() << CurrentIdx << ": Not a register\n");
98
0
        if (handleReject() == RejectAndGiveUp)
99
0
          return false;
100
0
        break;
101
0
      }
102
0
      if (TRI.isPhysicalRegister(MO.getReg())) {
103
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
104
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
105
0
        if (handleReject() == RejectAndGiveUp)
106
0
          return false;
107
0
        break;
108
0
      }
109
0
110
0
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
111
0
      if ((size_t)NewInsnID < State.MIs.size())
112
0
        State.MIs[NewInsnID] = NewMI;
113
0
      else {
114
0
        assert((size_t)NewInsnID == State.MIs.size() &&
115
0
               "Expected to store MIs in order");
116
0
        State.MIs.push_back(NewMI);
117
0
      }
118
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
119
0
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
120
0
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
121
0
                             << ")\n");
122
0
      break;
123
0
    }
124
0
125
18
    case GIM_CheckFeatures: {
126
18
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
127
18
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
128
18
                      dbgs() << CurrentIdx
129
18
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
130
18
                             << ExpectedBitsetID << ")\n");
131
18
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
132
18
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
133
15
        if (handleReject() == RejectAndGiveUp)
134
0
          return false;
135
18
      }
136
18
      break;
137
18
    }
138
18
139
18
    case GIM_CheckOpcode: {
140
0
      int64_t InsnID = MatchTable[CurrentIdx++];
141
0
      int64_t Expected = MatchTable[CurrentIdx++];
142
0
143
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
144
0
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
145
0
146
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
147
0
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
148
0
                             << "], ExpectedOpcode=" << Expected
149
0
                             << ") // Got=" << Opcode << "\n");
150
0
      if (Opcode != Expected) {
151
0
        if (handleReject() == RejectAndGiveUp)
152
0
          return false;
153
0
      }
154
0
      break;
155
0
    }
156
0
157
3
    case GIM_SwitchOpcode: {
158
3
      int64_t InsnID = MatchTable[CurrentIdx++];
159
3
      int64_t LowerBound = MatchTable[CurrentIdx++];
160
3
      int64_t UpperBound = MatchTable[CurrentIdx++];
161
3
      int64_t Default = MatchTable[CurrentIdx++];
162
3
163
3
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
164
3
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
165
3
166
3
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
167
3
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
168
3
               << LowerBound << ", " << UpperBound << "), Default=" << Default
169
3
               << ", JumpTable...) // Got=" << Opcode << "\n";
170
3
      });
171
3
      if (Opcode < LowerBound || UpperBound <= Opcode) {
172
0
        CurrentIdx = Default;
173
0
        break;
174
0
      }
175
3
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
176
3
      if (!CurrentIdx) {
177
0
        CurrentIdx = Default;
178
0
  break;
179
0
      }
180
3
      OnFailResumeAt.push_back(Default);
181
3
      break;
182
3
    }
183
3
184
3
    case GIM_SwitchType: {
185
3
      int64_t InsnID = MatchTable[CurrentIdx++];
186
3
      int64_t OpIdx = MatchTable[CurrentIdx++];
187
3
      int64_t LowerBound = MatchTable[CurrentIdx++];
188
3
      int64_t UpperBound = MatchTable[CurrentIdx++];
189
3
      int64_t Default = MatchTable[CurrentIdx++];
190
3
191
3
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
192
3
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
193
3
194
3
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
195
3
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
196
3
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
197
3
               << UpperBound << "), Default=" << Default
198
3
               << ", JumpTable...) // Got=";
199
3
        if (!MO.isReg())
200
3
          dbgs() << "Not a VReg\n";
201
3
        else
202
3
          dbgs() << MRI.getType(MO.getReg()) << "\n";
203
3
      });
204
3
      if (!MO.isReg()) {
205
0
        CurrentIdx = Default;
206
0
        break;
207
0
      }
208
3
      const LLT Ty = MRI.getType(MO.getReg());
209
3
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
210
3
      if (TyI == ISelInfo.TypeIDMap.end()) {
211
0
        CurrentIdx = Default;
212
0
        break;
213
0
      }
214
3
      const int64_t TypeID = TyI->second;
215
3
      if (TypeID < LowerBound || UpperBound <= TypeID) {
216
0
        CurrentIdx = Default;
217
0
        break;
218
0
      }
219
3
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
220
3
      if (!CurrentIdx) {
221
0
        CurrentIdx = Default;
222
0
        break;
223
0
      }
224
3
      OnFailResumeAt.push_back(Default);
225
3
      break;
226
3
    }
227
3
228
3
    case GIM_CheckNumOperands: {
229
0
      int64_t InsnID = MatchTable[CurrentIdx++];
230
0
      int64_t Expected = MatchTable[CurrentIdx++];
231
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
232
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
233
0
                             << InsnID << "], Expected=" << Expected << ")\n");
234
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
235
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
236
0
        if (handleReject() == RejectAndGiveUp)
237
0
          return false;
238
0
      }
239
0
      break;
240
0
    }
241
0
    case GIM_CheckI64ImmPredicate: {
242
0
      int64_t InsnID = MatchTable[CurrentIdx++];
243
0
      int64_t Predicate = MatchTable[CurrentIdx++];
244
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
245
0
                      dbgs()
246
0
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
247
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
248
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
249
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
250
0
             "Expected G_CONSTANT");
251
0
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
252
0
      int64_t Value = 0;
253
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
254
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
255
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
256
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
257
0
      else
258
0
        llvm_unreachable("Expected Imm or CImm operand");
259
0
260
0
      if (!testImmPredicate_I64(Predicate, Value))
261
0
        if (handleReject() == RejectAndGiveUp)
262
0
          return false;
263
0
      break;
264
0
    }
265
0
    case GIM_CheckAPIntImmPredicate: {
266
0
      int64_t InsnID = MatchTable[CurrentIdx++];
267
0
      int64_t Predicate = MatchTable[CurrentIdx++];
268
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
269
0
                      dbgs()
270
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
271
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
272
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
273
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
274
0
             "Expected G_CONSTANT");
275
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
276
0
      APInt Value;
277
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
278
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
279
0
      else
280
0
        llvm_unreachable("Expected Imm or CImm operand");
281
0
282
0
      if (!testImmPredicate_APInt(Predicate, Value))
283
0
        if (handleReject() == RejectAndGiveUp)
284
0
          return false;
285
0
      break;
286
0
    }
287
0
    case GIM_CheckAPFloatImmPredicate: {
288
0
      int64_t InsnID = MatchTable[CurrentIdx++];
289
0
      int64_t Predicate = MatchTable[CurrentIdx++];
290
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
291
0
                      dbgs()
292
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
293
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
294
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
295
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
296
0
             "Expected G_FCONSTANT");
297
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
298
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
299
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
300
0
301
0
      if (!testImmPredicate_APFloat(Predicate, Value))
302
0
        if (handleReject() == RejectAndGiveUp)
303
0
          return false;
304
0
      break;
305
0
    }
306
0
    case GIM_CheckCxxInsnPredicate: {
307
0
      int64_t InsnID = MatchTable[CurrentIdx++];
308
0
      int64_t Predicate = MatchTable[CurrentIdx++];
309
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
310
0
                      dbgs()
311
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
312
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
313
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
314
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
315
0
316
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
317
0
        if (handleReject() == RejectAndGiveUp)
318
0
          return false;
319
0
      break;
320
0
    }
321
0
    case GIM_CheckAtomicOrdering: {
322
0
      int64_t InsnID = MatchTable[CurrentIdx++];
323
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
324
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
325
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
326
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
327
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
328
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
329
0
        if (handleReject() == RejectAndGiveUp)
330
0
          return false;
331
0
332
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
333
0
        if (MMO->getOrdering() != Ordering)
334
0
          if (handleReject() == RejectAndGiveUp)
335
0
            return false;
336
0
      break;
337
0
    }
338
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
339
0
      int64_t InsnID = MatchTable[CurrentIdx++];
340
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
341
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
342
0
                      dbgs() << CurrentIdx
343
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
344
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
345
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
346
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
347
0
        if (handleReject() == RejectAndGiveUp)
348
0
          return false;
349
0
350
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
351
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
352
0
          if (handleReject() == RejectAndGiveUp)
353
0
            return false;
354
0
      break;
355
0
    }
356
0
    case GIM_CheckAtomicOrderingWeakerThan: {
357
0
      int64_t InsnID = MatchTable[CurrentIdx++];
358
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
359
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
360
0
                      dbgs() << CurrentIdx
361
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
362
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
363
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
364
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
365
0
        if (handleReject() == RejectAndGiveUp)
366
0
          return false;
367
0
368
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
369
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
370
0
          if (handleReject() == RejectAndGiveUp)
371
0
            return false;
372
0
      break;
373
0
    }
374
0
    case GIM_CheckMemorySizeEqualTo: {
375
0
      int64_t InsnID = MatchTable[CurrentIdx++];
376
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
377
0
      uint64_t Size = MatchTable[CurrentIdx++];
378
0
379
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
380
0
                      dbgs() << CurrentIdx
381
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
382
0
                             << "]->memoperands() + " << MMOIdx
383
0
                             << ", Size=" << Size << ")\n");
384
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
385
0
386
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
387
0
        if (handleReject() == RejectAndGiveUp)
388
0
          return false;
389
0
        break;
390
0
      }
391
0
392
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
393
0
394
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
395
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
396
0
                             << " bytes\n");
397
0
      if (MMO->getSize() != Size)
398
0
        if (handleReject() == RejectAndGiveUp)
399
0
          return false;
400
0
401
0
      break;
402
0
    }
403
0
    case GIM_CheckMemorySizeEqualToLLT:
404
0
    case GIM_CheckMemorySizeLessThanLLT:
405
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
406
0
      int64_t InsnID = MatchTable[CurrentIdx++];
407
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
408
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
409
0
410
0
      DEBUG_WITH_TYPE(
411
0
          TgtInstructionSelector::getName(),
412
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
413
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
414
0
                         ? "EqualTo"
415
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
416
0
                               ? "GreaterThan"
417
0
                               : "LessThan")
418
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
419
0
                 << ", OpIdx=" << OpIdx << ")\n");
420
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
421
0
422
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
423
0
      if (!MO.isReg()) {
424
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
425
0
                        dbgs() << CurrentIdx << ": Not a register\n");
426
0
        if (handleReject() == RejectAndGiveUp)
427
0
          return false;
428
0
        break;
429
0
      }
430
0
431
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
432
0
        if (handleReject() == RejectAndGiveUp)
433
0
          return false;
434
0
        break;
435
0
      }
436
0
437
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
438
0
439
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
440
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
441
0
          MMO->getSize() * 8 != Size) {
442
0
        if (handleReject() == RejectAndGiveUp)
443
0
          return false;
444
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
445
0
                 MMO->getSize() * 8 >= Size) {
446
0
        if (handleReject() == RejectAndGiveUp)
447
0
          return false;
448
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
449
0
                 MMO->getSize() * 8 <= Size)
450
0
        if (handleReject() == RejectAndGiveUp)
451
0
          return false;
452
0
453
0
      break;
454
0
    }
455
6
    case GIM_CheckType: {
456
6
      int64_t InsnID = MatchTable[CurrentIdx++];
457
6
      int64_t OpIdx = MatchTable[CurrentIdx++];
458
6
      int64_t TypeID = MatchTable[CurrentIdx++];
459
6
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
460
6
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
461
6
                             << "]->getOperand(" << OpIdx
462
6
                             << "), TypeID=" << TypeID << ")\n");
463
6
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
464
6
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
465
6
      if (!MO.isReg() ||
466
6
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
467
0
        if (handleReject() == RejectAndGiveUp)
468
0
          return false;
469
6
      }
470
6
      break;
471
6
    }
472
6
    case GIM_CheckPointerToAny: {
473
0
      int64_t InsnID = MatchTable[CurrentIdx++];
474
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
475
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
476
0
477
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
478
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
479
0
                             << InsnID << "]->getOperand(" << OpIdx
480
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
481
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
482
0
      // iPTR must be looked up in the target.
483
0
      if (SizeInBits == 0) {
484
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
485
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
486
0
      }
487
0
488
0
      assert(SizeInBits != 0 && "Pointer size must be known");
489
0
490
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
491
0
      if (MO.isReg()) {
492
0
        const LLT &Ty = MRI.getType(MO.getReg());
493
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
494
0
          if (handleReject() == RejectAndGiveUp)
495
0
            return false;
496
0
      } else if (handleReject() == RejectAndGiveUp)
497
0
        return false;
498
0
499
0
      break;
500
0
    }
501
9
    case GIM_CheckRegBankForClass: {
502
9
      int64_t InsnID = MatchTable[CurrentIdx++];
503
9
      int64_t OpIdx = MatchTable[CurrentIdx++];
504
9
      int64_t RCEnum = MatchTable[CurrentIdx++];
505
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
506
9
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
507
9
                             << InsnID << "]->getOperand(" << OpIdx
508
9
                             << "), RCEnum=" << RCEnum << ")\n");
509
9
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
510
9
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
511
9
      if (!MO.isReg() ||
512
9
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
513
9
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
514
0
        if (handleReject() == RejectAndGiveUp)
515
0
          return false;
516
9
      }
517
9
      break;
518
9
    }
519
9
520
9
    case GIM_CheckComplexPattern: {
521
0
      int64_t InsnID = MatchTable[CurrentIdx++];
522
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
523
0
      int64_t RendererID = MatchTable[CurrentIdx++];
524
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
525
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
526
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
527
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
528
0
                             << "]->getOperand(" << OpIdx
529
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
530
0
                             << ")\n");
531
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
532
0
      // FIXME: Use std::invoke() when it's available.
533
0
      ComplexRendererFns Renderer =
534
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
535
0
              State.MIs[InsnID]->getOperand(OpIdx));
536
0
      if (Renderer.hasValue())
537
0
        State.Renderers[RendererID] = Renderer.getValue();
538
0
      else
539
0
        if (handleReject() == RejectAndGiveUp)
540
0
          return false;
541
0
      break;
542
0
    }
543
0
544
0
    case GIM_CheckConstantInt: {
545
0
      int64_t InsnID = MatchTable[CurrentIdx++];
546
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
547
0
      int64_t Value = MatchTable[CurrentIdx++];
548
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
549
0
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
550
0
                             << InsnID << "]->getOperand(" << OpIdx
551
0
                             << "), Value=" << Value << ")\n");
552
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
553
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
554
0
      if (MO.isReg()) {
555
0
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
556
0
        LLT Ty = MRI.getType(MO.getReg());
557
0
        Value = SignExtend64(Value, Ty.getSizeInBits());
558
0
559
0
        if (!isOperandImmEqual(MO, Value, MRI)) {
560
0
          if (handleReject() == RejectAndGiveUp)
561
0
            return false;
562
0
        }
563
0
      } else if (handleReject() == RejectAndGiveUp)
564
0
        return false;
565
0
566
0
      break;
567
0
    }
568
0
569
0
    case GIM_CheckLiteralInt: {
570
0
      int64_t InsnID = MatchTable[CurrentIdx++];
571
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
572
0
      int64_t Value = MatchTable[CurrentIdx++];
573
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
574
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
575
0
                             << InsnID << "]->getOperand(" << OpIdx
576
0
                             << "), Value=" << Value << ")\n");
577
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
578
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
579
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
580
0
        if (handleReject() == RejectAndGiveUp)
581
0
          return false;
582
0
      }
583
0
      break;
584
0
    }
585
0
586
0
    case GIM_CheckIntrinsicID: {
587
0
      int64_t InsnID = MatchTable[CurrentIdx++];
588
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
589
0
      int64_t Value = MatchTable[CurrentIdx++];
590
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
591
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
592
0
                             << InsnID << "]->getOperand(" << OpIdx
593
0
                             << "), Value=" << Value << ")\n");
594
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
595
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
596
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
597
0
        if (handleReject() == RejectAndGiveUp)
598
0
          return false;
599
0
      break;
600
0
    }
601
0
602
0
    case GIM_CheckIsMBB: {
603
0
      int64_t InsnID = MatchTable[CurrentIdx++];
604
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
605
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
606
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
607
0
                             << "]->getOperand(" << OpIdx << "))\n");
608
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
609
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
610
0
        if (handleReject() == RejectAndGiveUp)
611
0
          return false;
612
0
      }
613
0
      break;
614
0
    }
615
0
616
0
    case GIM_CheckIsSafeToFold: {
617
0
      int64_t InsnID = MatchTable[CurrentIdx++];
618
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
619
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
620
0
                             << InsnID << "])\n");
621
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
622
0
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
623
0
        if (handleReject() == RejectAndGiveUp)
624
0
          return false;
625
0
      }
626
0
      break;
627
0
    }
628
0
    case GIM_CheckIsSameOperand: {
629
0
      int64_t InsnID = MatchTable[CurrentIdx++];
630
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
631
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
632
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
633
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
634
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
635
0
                             << InsnID << "][" << OpIdx << "], MIs["
636
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
637
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
638
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
639
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
640
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
641
0
        if (handleReject() == RejectAndGiveUp)
642
0
          return false;
643
0
      }
644
0
      break;
645
0
    }
646
0
    case GIM_Reject:
647
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
648
0
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
649
0
      if (handleReject() == RejectAndGiveUp)
650
0
        return false;
651
0
      break;
652
0
653
3
    case GIR_MutateOpcode: {
654
3
      int64_t OldInsnID = MatchTable[CurrentIdx++];
655
3
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
656
3
      int64_t NewOpcode = MatchTable[CurrentIdx++];
657
3
      if (NewInsnID >= OutMIs.size())
658
3
        OutMIs.resize(NewInsnID + 1);
659
3
660
3
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
661
3
                                              State.MIs[OldInsnID]);
662
3
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
663
3
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
664
3
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
665
3
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
666
3
                             << NewOpcode << ")\n");
667
3
      break;
668
0
    }
669
0
670
0
    case GIR_BuildMI: {
671
0
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
672
0
      int64_t Opcode = MatchTable[CurrentIdx++];
673
0
      if (NewInsnID >= OutMIs.size())
674
0
        OutMIs.resize(NewInsnID + 1);
675
0
676
0
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
677
0
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
678
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
679
0
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
680
0
                             << NewInsnID << "], " << Opcode << ")\n");
681
0
      break;
682
0
    }
683
0
684
0
    case GIR_Copy: {
685
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
686
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
687
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
688
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
689
0
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
690
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
691
0
                      dbgs()
692
0
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
693
0
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
694
0
      break;
695
0
    }
696
0
697
0
    case GIR_CopyOrAddZeroReg: {
698
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
700
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
701
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
702
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
703
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
704
0
      if (isOperandImmEqual(MO, 0, MRI))
705
0
        OutMIs[NewInsnID].addReg(ZeroReg);
706
0
      else
707
0
        OutMIs[NewInsnID].add(MO);
708
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
709
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
710
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
711
0
                             << OpIdx << ", " << ZeroReg << ")\n");
712
0
      break;
713
0
    }
714
0
715
0
    case GIR_CopySubReg: {
716
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
718
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
719
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
720
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
721
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
722
0
                               0, SubRegIdx);
723
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
724
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
725
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
726
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
727
0
      break;
728
0
    }
729
0
730
0
    case GIR_AddImplicitDef: {
731
0
      int64_t InsnID = MatchTable[CurrentIdx++];
732
0
      int64_t RegNum = MatchTable[CurrentIdx++];
733
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
734
0
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
735
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
736
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
737
0
                             << InsnID << "], " << RegNum << ")\n");
738
0
      break;
739
0
    }
740
0
741
0
    case GIR_AddImplicitUse: {
742
0
      int64_t InsnID = MatchTable[CurrentIdx++];
743
0
      int64_t RegNum = MatchTable[CurrentIdx++];
744
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
745
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
746
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
747
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
748
0
                             << InsnID << "], " << RegNum << ")\n");
749
0
      break;
750
0
    }
751
0
752
0
    case GIR_AddRegister: {
753
0
      int64_t InsnID = MatchTable[CurrentIdx++];
754
0
      int64_t RegNum = MatchTable[CurrentIdx++];
755
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
756
0
      OutMIs[InsnID].addReg(RegNum);
757
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
758
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
759
0
                             << InsnID << "], " << RegNum << ")\n");
760
0
      break;
761
0
    }
762
0
763
0
    case GIR_AddTempRegister: {
764
0
      int64_t InsnID = MatchTable[CurrentIdx++];
765
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
766
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
767
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
768
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
769
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
770
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
771
0
                             << InsnID << "], TempRegisters[" << TempRegID
772
0
                             << "], " << TempRegFlags << ")\n");
773
0
      break;
774
0
    }
775
0
776
0
    case GIR_AddImm: {
777
0
      int64_t InsnID = MatchTable[CurrentIdx++];
778
0
      int64_t Imm = MatchTable[CurrentIdx++];
779
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
780
0
      OutMIs[InsnID].addImm(Imm);
781
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
782
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
783
0
                             << "], " << Imm << ")\n");
784
0
      break;
785
0
    }
786
0
787
0
    case GIR_ComplexRenderer: {
788
0
      int64_t InsnID = MatchTable[CurrentIdx++];
789
0
      int64_t RendererID = MatchTable[CurrentIdx++];
790
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
791
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
792
0
        RenderOpFn(OutMIs[InsnID]);
793
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
794
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
795
0
                             << InsnID << "], " << RendererID << ")\n");
796
0
      break;
797
0
    }
798
0
    case GIR_ComplexSubOperandRenderer: {
799
0
      int64_t InsnID = MatchTable[CurrentIdx++];
800
0
      int64_t RendererID = MatchTable[CurrentIdx++];
801
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
802
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
803
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
804
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
805
0
                      dbgs() << CurrentIdx
806
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
807
0
                             << InsnID << "], " << RendererID << ", "
808
0
                             << RenderOpID << ")\n");
809
0
      break;
810
0
    }
811
0
812
0
    case GIR_CopyConstantAsSImm: {
813
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
814
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
815
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
816
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
817
0
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
818
0
        OutMIs[NewInsnID].addImm(
819
0
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
820
0
      } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
821
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
822
0
      else
823
0
        llvm_unreachable("Expected Imm or CImm operand");
824
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
825
0
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
826
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
827
0
      break;
828
0
    }
829
0
830
0
    // TODO: Needs a test case once we have a pattern that uses this.
831
0
    case GIR_CopyFConstantAsFPImm: {
832
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
833
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
834
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
835
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
836
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
837
0
        OutMIs[NewInsnID].addFPImm(
838
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
839
0
      else
840
0
        llvm_unreachable("Expected FPImm operand");
841
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
842
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
843
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
844
0
      break;
845
0
    }
846
0
847
0
    case GIR_CustomRenderer: {
848
0
      int64_t InsnID = MatchTable[CurrentIdx++];
849
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
850
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
851
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
852
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
853
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
854
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
855
0
                             << RendererFnID << ")\n");
856
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
857
0
                                                     *State.MIs[OldInsnID]);
858
0
      break;
859
0
    }
860
0
    case GIR_ConstrainOperandRC: {
861
0
      int64_t InsnID = MatchTable[CurrentIdx++];
862
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
863
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
864
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
865
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
866
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
867
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
868
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
869
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
870
0
                             << ")\n");
871
0
      break;
872
0
    }
873
0
874
3
    case GIR_ConstrainSelectedInstOperands: {
875
3
      int64_t InsnID = MatchTable[CurrentIdx++];
876
3
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
877
3
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
878
3
                                       RBI);
879
3
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
880
3
                      dbgs() << CurrentIdx
881
3
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
882
3
                             << InsnID << "])\n");
883
3
      break;
884
0
    }
885
0
886
0
    case GIR_MergeMemOperands: {
887
0
      int64_t InsnID = MatchTable[CurrentIdx++];
888
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
889
0
890
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
891
0