Coverage Report

Created: 2019-02-21 13:17

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
Line
Count
Source (jump to first uncovered line)
1
//===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file This file declares the API for the instruction selector.
10
/// This class is responsible for selecting machine instructions.
11
/// It's implemented by the target. It's used by the InstructionSelect pass.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
16
#define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17
18
#include "llvm/ADT/SmallVector.h"
19
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
20
#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
21
#include "llvm/CodeGen/GlobalISel/Utils.h"
22
#include "llvm/CodeGen/MachineInstrBuilder.h"
23
#include "llvm/CodeGen/MachineOperand.h"
24
#include "llvm/CodeGen/MachineRegisterInfo.h"
25
#include "llvm/CodeGen/TargetInstrInfo.h"
26
#include "llvm/CodeGen/TargetOpcodes.h"
27
#include "llvm/CodeGen/TargetRegisterInfo.h"
28
#include "llvm/IR/Constants.h"
29
#include "llvm/Support/Debug.h"
30
#include "llvm/Support/ErrorHandling.h"
31
#include "llvm/Support/raw_ostream.h"
32
#include <cassert>
33
#include <cstddef>
34
#include <cstdint>
35
36
namespace llvm {
37
38
/// GlobalISel PatFrag Predicates
39
enum {
40
  GIPFP_I64_Invalid = 0,
41
  GIPFP_APInt_Invalid = 0,
42
  GIPFP_APFloat_Invalid = 0,
43
  GIPFP_MI_Invalid = 0,
44
};
45
46
template <class TgtInstructionSelector, class PredicateBitset,
47
          class ComplexMatcherMemFn, class CustomRendererFn>
48
bool InstructionSelector::executeMatchTable(
49
    TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
50
    const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
51
        &ISelInfo,
52
    const int64_t *MatchTable, const TargetInstrInfo &TII,
53
    MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
54
    const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
55
6.77M
    CodeGenCoverage &CoverageInfo) const {
56
6.77M
57
6.77M
  uint64_t CurrentIdx = 0;
58
6.77M
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
6.77M
60
6.77M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
9.86M
  auto handleReject = [&]() -> RejectAction {
62
9.86M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
9.86M
                    dbgs() << CurrentIdx << ": Rejected\n");
64
9.86M
    if (OnFailResumeAt.empty())
65
3.87M
      return RejectAndGiveUp;
66
5.99M
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
5.99M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
5.99M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
5.99M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
5.99M
    return RejectAndResume;
71
5.99M
  };
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<19ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<19ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<19ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
9.85M
  auto handleReject = [&]() -> RejectAction {
62
9.85M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
9.85M
                    dbgs() << CurrentIdx << ": Rejected\n");
64
9.85M
    if (OnFailResumeAt.empty())
65
3.87M
      return RejectAndGiveUp;
66
5.98M
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
5.98M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
5.98M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
5.98M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
5.98M
    return RejectAndResume;
71
5.98M
  };
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<33ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<33ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<33ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
251
  auto handleReject = [&]() -> RejectAction {
62
251
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
251
                    dbgs() << CurrentIdx << ": Rejected\n");
64
251
    if (OnFailResumeAt.empty())
65
3
      return RejectAndGiveUp;
66
248
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
248
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
248
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
248
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
248
    return RejectAndResume;
71
248
  };
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<64ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
2.78k
  auto handleReject = [&]() -> RejectAction {
62
2.78k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
2.78k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
2.78k
    if (OnFailResumeAt.empty())
65
531
      return RejectAndGiveUp;
66
2.25k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
2.25k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
2.25k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
2.25k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
2.25k
    return RejectAndResume;
71
2.25k
  };
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<42ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
1.11k
  auto handleReject = [&]() -> RejectAction {
62
1.11k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
1.11k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
1.11k
    if (OnFailResumeAt.empty())
65
206
      return RejectAndGiveUp;
66
911
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
911
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
911
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
911
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
911
    return RejectAndResume;
71
911
  };
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<112ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<112ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<112ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
3.95k
  auto handleReject = [&]() -> RejectAction {
62
3.95k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
3.95k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
3.95k
    if (OnFailResumeAt.empty())
65
1.36k
      return RejectAndGiveUp;
66
2.59k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
2.59k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
2.59k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
2.59k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
2.59k
    return RejectAndResume;
71
2.59k
  };
72
6.77M
73
59.8M
  while (true) {
74
59.8M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
59.8M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
59.8M
    switch (MatcherOpcode) {
77
59.8M
    case GIM_Try: {
78
8.12M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
8.12M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
8.12M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
8.12M
      break;
82
59.8M
    }
83
59.8M
84
59.8M
    case GIM_RecordInsn: {
85
2.70M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
2.70M
      int64_t InsnID = MatchTable[CurrentIdx++];
87
2.70M
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
2.70M
89
2.70M
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
2.70M
      // any attempt to modify it.
91
2.70M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
2.70M
93
2.70M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
2.70M
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
2.70M
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
2.70M
109
2.70M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
2.70M
      if ((size_t)NewInsnID < State.MIs.size())
111
1.80M
        State.MIs[NewInsnID] = NewMI;
112
902k
      else {
113
902k
        assert((size_t)NewInsnID == State.MIs.size() &&
114
902k
               "Expected to store MIs in order");
115
902k
        State.MIs.push_back(NewMI);
116
902k
      }
117
2.70M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
2.70M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
2.70M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
2.70M
                             << ")\n");
121
2.70M
      break;
122
2.70M
    }
123
2.70M
124
2.70M
    case GIM_CheckFeatures: {
125
642k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
642k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
642k
                      dbgs() << CurrentIdx
128
642k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
642k
                             << ExpectedBitsetID << ")\n");
130
642k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
642k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
40.2k
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
642k
      }
135
642k
      break;
136
642k
    }
137
642k
138
2.70M
    case GIM_CheckOpcode: {
139
2.70M
      int64_t InsnID = MatchTable[CurrentIdx++];
140
2.70M
      int64_t Expected = MatchTable[CurrentIdx++];
141
2.70M
142
2.70M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
2.70M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
2.70M
145
2.70M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
2.70M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
2.70M
                             << "], ExpectedOpcode=" << Expected
148
2.70M
                             << ") // Got=" << Opcode << "\n");
149
2.70M
      if (Opcode != Expected) {
150
2.59M
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
2.70M
      }
153
2.70M
      break;
154
2.70M
    }
155
2.70M
156
6.77M
    case GIM_SwitchOpcode: {
157
6.77M
      int64_t InsnID = MatchTable[CurrentIdx++];
158
6.77M
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
6.77M
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
6.77M
      int64_t Default = MatchTable[CurrentIdx++];
161
6.77M
162
6.77M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
6.77M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
6.77M
165
6.77M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
6.77M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
6.77M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
6.77M
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
6.77M
      });
170
6.77M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
2
        CurrentIdx = Default;
172
2
        break;
173
2
      }
174
6.77M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
6.77M
      if (!CurrentIdx) {
176
2.18M
        CurrentIdx = Default;
177
2.18M
  break;
178
2.18M
      }
179
4.59M
      OnFailResumeAt.push_back(Default);
180
4.59M
      break;
181
4.59M
    }
182
4.59M
183
4.59M
    case GIM_SwitchType: {
184
3.99M
      int64_t InsnID = MatchTable[CurrentIdx++];
185
3.99M
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
3.99M
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
3.99M
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
3.99M
      int64_t Default = MatchTable[CurrentIdx++];
189
3.99M
190
3.99M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
3.99M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
3.99M
193
3.99M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
3.99M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
3.99M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
3.99M
               << UpperBound << "), Default=" << Default
197
3.99M
               << ", JumpTable...) // Got=";
198
3.99M
        if (!MO.isReg())
199
3.99M
          dbgs() << "Not a VReg\n";
200
3.99M
        else
201
3.99M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
3.99M
      });
203
3.99M
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
3.99M
      const LLT Ty = MRI.getType(MO.getReg());
208
3.99M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
3.99M
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
1.33M
        CurrentIdx = Default;
211
1.33M
        break;
212
1.33M
      }
213
2.66M
      const int64_t TypeID = TyI->second;
214
2.66M
      if (TypeID < LowerBound || 
UpperBound <= TypeID2.50M
) {
215
153k
        CurrentIdx = Default;
216
153k
        break;
217
153k
      }
218
2.50M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
2.50M
      if (!CurrentIdx) {
220
339
        CurrentIdx = Default;
221
339
        break;
222
339
      }
223
2.50M
      OnFailResumeAt.push_back(Default);
224
2.50M
      break;
225
2.50M
    }
226
2.50M
227
2.50M
    case GIM_CheckNumOperands: {
228
12.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
229
12.1k
      int64_t Expected = MatchTable[CurrentIdx++];
230
12.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
12.1k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
12.1k
                             << InsnID << "], Expected=" << Expected << ")\n");
233
12.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
12.1k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
7.00k
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
12.1k
      }
238
12.1k
      break;
239
12.1k
    }
240
43.8k
    case GIM_CheckI64ImmPredicate: {
241
43.8k
      int64_t InsnID = MatchTable[CurrentIdx++];
242
43.8k
      int64_t Predicate = MatchTable[CurrentIdx++];
243
43.8k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
43.8k
                      dbgs()
245
43.8k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
43.8k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
43.8k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
43.8k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
43.8k
             "Expected G_CONSTANT");
250
43.8k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
43.8k
      int64_t Value = 0;
252
43.8k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
43.8k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
43.8k
259
43.8k
      if (!testImmPredicate_I64(Predicate, Value))
260
39
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
43.8k
      break;
263
43.8k
    }
264
43.8k
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
22.1k
    case GIM_CheckAPFloatImmPredicate: {
287
22.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
288
22.1k
      int64_t Predicate = MatchTable[CurrentIdx++];
289
22.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
22.1k
                      dbgs()
291
22.1k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
22.1k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
22.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
22.1k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
22.1k
             "Expected G_FCONSTANT");
296
22.1k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
22.1k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
22.1k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
22.1k
300
22.1k
      if (!testImmPredicate_APFloat(Predicate, Value))
301
19.0k
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
22.1k
      break;
304
22.1k
    }
305
22.1k
    case GIM_CheckCxxInsnPredicate: {
306
67
      int64_t InsnID = MatchTable[CurrentIdx++];
307
67
      int64_t Predicate = MatchTable[CurrentIdx++];
308
67
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
67
                      dbgs()
310
67
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
67
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
67
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
67
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
67
315
67
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
67
      break;
319
67
    }
320
709k
    case GIM_CheckAtomicOrdering: {
321
709k
      int64_t InsnID = MatchTable[CurrentIdx++];
322
709k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
709k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
709k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
709k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
709k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
709k
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
709k
331
709k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
709k
        if (MMO->getOrdering() != Ordering)
333
4.24k
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
709k
      break;
336
709k
    }
337
709k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
32.5k
    case GIM_CheckMemorySizeEqualTo: {
374
32.5k
      int64_t InsnID = MatchTable[CurrentIdx++];
375
32.5k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
32.5k
      uint64_t Size = MatchTable[CurrentIdx++];
377
32.5k
378
32.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
379
32.5k
                      dbgs() << CurrentIdx
380
32.5k
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
381
32.5k
                             << "]->memoperands() + " << MMOIdx
382
32.5k
                             << ", Size=" << Size << ")\n");
383
32.5k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
384
32.5k
385
32.5k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
386
0
        if (handleReject() == RejectAndGiveUp)
387
0
          return false;
388
0
        break;
389
0
      }
390
32.5k
391
32.5k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
392
32.5k
393
32.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
394
32.5k
                      dbgs() << MMO->getSize() << " bytes vs " << Size
395
32.5k
                             << " bytes\n");
396
32.5k
      if (MMO->getSize() != Size)
397
14.3k
        if (handleReject() == RejectAndGiveUp)
398
0
          return false;
399
32.5k
400
32.5k
      break;
401
32.5k
    }
402
432k
    case GIM_CheckMemorySizeEqualToLLT:
403
432k
    case GIM_CheckMemorySizeLessThanLLT:
404
432k
    case GIM_CheckMemorySizeGreaterThanLLT: {
405
432k
      int64_t InsnID = MatchTable[CurrentIdx++];
406
432k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
407
432k
      int64_t OpIdx = MatchTable[CurrentIdx++];
408
432k
409
432k
      DEBUG_WITH_TYPE(
410
432k
          TgtInstructionSelector::getName(),
411
432k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
412
432k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
413
432k
                         ? "EqualTo"
414
432k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
415
432k
                               ? "GreaterThan"
416
432k
                               : "LessThan")
417
432k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
418
432k
                 << ", OpIdx=" << OpIdx << ")\n");
419
432k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
420
432k
421
432k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
422
432k
      if (!MO.isReg()) {
423
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
424
0
                        dbgs() << CurrentIdx << ": Not a register\n");
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
432k
430
432k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
431
0
        if (handleReject() == RejectAndGiveUp)
432
0
          return false;
433
0
        break;
434
0
      }
435
432k
436
432k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
437
432k
438
432k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
439
432k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
440
432k
          
MMO->getSize() * 8 != Size432k
) {
441
120
        if (handleReject() == RejectAndGiveUp)
442
0
          return false;
443
432k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
444
432k
                 
MMO->getSize() * 8 >= Size118
) {
445
36
        if (handleReject() == RejectAndGiveUp)
446
0
          return false;
447
432k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
448
432k
                 
MMO->getSize() * 8 <= Size0
)
449
0
        if (handleReject() == RejectAndGiveUp)
450
0
          return false;
451
432k
452
432k
      break;
453
432k
    }
454
1.96M
    case GIM_CheckType: {
455
1.96M
      int64_t InsnID = MatchTable[CurrentIdx++];
456
1.96M
      int64_t OpIdx = MatchTable[CurrentIdx++];
457
1.96M
      int64_t TypeID = MatchTable[CurrentIdx++];
458
1.96M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
459
1.96M
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
460
1.96M
                             << "]->getOperand(" << OpIdx
461
1.96M
                             << "), TypeID=" << TypeID << ")\n");
462
1.96M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
463
1.96M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
464
1.96M
      if (!MO.isReg() ||
465
1.96M
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
466
103k
        if (handleReject() == RejectAndGiveUp)
467
0
          return false;
468
1.96M
      }
469
1.96M
      break;
470
1.96M
    }
471
1.96M
    case GIM_CheckPointerToAny: {
472
697k
      int64_t InsnID = MatchTable[CurrentIdx++];
473
697k
      int64_t OpIdx = MatchTable[CurrentIdx++];
474
697k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
475
697k
476
697k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
477
697k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
478
697k
                             << InsnID << "]->getOperand(" << OpIdx
479
697k
                             << "), SizeInBits=" << SizeInBits << ")\n");
480
697k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
481
697k
      // iPTR must be looked up in the target.
482
697k
      if (SizeInBits == 0) {
483
696k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
484
696k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
485
696k
      }
486
697k
487
697k
      assert(SizeInBits != 0 && "Pointer size must be known");
488
697k
489
697k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
490
697k
      if (MO.isReg()) {
491
697k
        const LLT &Ty = MRI.getType(MO.getReg());
492
697k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
493
0
          if (handleReject() == RejectAndGiveUp)
494
0
            return false;
495
0
      } else if (handleReject() == RejectAndGiveUp)
496
0
        return false;
497
697k
498
697k
      break;
499
697k
    }
500
6.81M
    case GIM_CheckRegBankForClass: {
501
6.81M
      int64_t InsnID = MatchTable[CurrentIdx++];
502
6.81M
      int64_t OpIdx = MatchTable[CurrentIdx++];
503
6.81M
      int64_t RCEnum = MatchTable[CurrentIdx++];
504
6.81M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
505
6.81M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
506
6.81M
                             << InsnID << "]->getOperand(" << OpIdx
507
6.81M
                             << "), RCEnum=" << RCEnum << ")\n");
508
6.81M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
509
6.81M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
510
6.81M
      if (!MO.isReg() ||
511
6.81M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
512
6.81M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
513
531k
        if (handleReject() == RejectAndGiveUp)
514
0
          return false;
515
6.81M
      }
516
6.81M
      break;
517
6.81M
    }
518
6.81M
519
6.81M
    case GIM_CheckComplexPattern: {
520
1.18M
      int64_t InsnID = MatchTable[CurrentIdx++];
521
1.18M
      int64_t OpIdx = MatchTable[CurrentIdx++];
522
1.18M
      int64_t RendererID = MatchTable[CurrentIdx++];
523
1.18M
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
524
1.18M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
525
1.18M
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
526
1.18M
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
527
1.18M
                             << "]->getOperand(" << OpIdx
528
1.18M
                             << "), ComplexPredicateID=" << ComplexPredicateID
529
1.18M
                             << ")\n");
530
1.18M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
531
1.18M
      // FIXME: Use std::invoke() when it's available.
532
1.18M
      ComplexRendererFns Renderer =
533
1.18M
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
534
1.18M
              State.MIs[InsnID]->getOperand(OpIdx));
535
1.18M
      if (Renderer.hasValue())
536
778k
        State.Renderers[RendererID] = Renderer.getValue();
537
406k
      else
538
406k
        if (handleReject() == RejectAndGiveUp)
539
0
          return false;
540
1.18M
      break;
541
1.18M
    }
542
1.18M
543
1.18M
    case GIM_CheckConstantInt: {
544
242k
      int64_t InsnID = MatchTable[CurrentIdx++];
545
242k
      int64_t OpIdx = MatchTable[CurrentIdx++];
546
242k
      int64_t Value = MatchTable[CurrentIdx++];
547
242k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
548
242k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
549
242k
                             << InsnID << "]->getOperand(" << OpIdx
550
242k
                             << "), Value=" << Value << ")\n");
551
242k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
552
242k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
553
242k
      if (MO.isReg()) {
554
242k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
555
242k
        LLT Ty = MRI.getType(MO.getReg());
556
242k
        Value = SignExtend64(Value, Ty.getSizeInBits());
557
242k
558
242k
        if (!isOperandImmEqual(MO, Value, MRI)) {
559
220k
          if (handleReject() == RejectAndGiveUp)
560
0
            return false;
561
0
        }
562
0
      } else if (handleReject() == RejectAndGiveUp)
563
0
        return false;
564
242k
565
242k
      break;
566
242k
    }
567
242k
568
242k
    case GIM_CheckLiteralInt: {
569
48
      int64_t InsnID = MatchTable[CurrentIdx++];
570
48
      int64_t OpIdx = MatchTable[CurrentIdx++];
571
48
      int64_t Value = MatchTable[CurrentIdx++];
572
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
573
48
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
574
48
                             << InsnID << "]->getOperand(" << OpIdx
575
48
                             << "), Value=" << Value << ")\n");
576
48
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
577
48
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
578
48
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
579
26
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
48
      }
582
48
      break;
583
48
    }
584
48
585
167k
    case GIM_CheckIntrinsicID: {
586
167k
      int64_t InsnID = MatchTable[CurrentIdx++];
587
167k
      int64_t OpIdx = MatchTable[CurrentIdx++];
588
167k
      int64_t Value = MatchTable[CurrentIdx++];
589
167k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
590
167k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
591
167k
                             << InsnID << "]->getOperand(" << OpIdx
592
167k
                             << "), Value=" << Value << ")\n");
593
167k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
594
167k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
595
167k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value163k
)
596
163k
        if (handleReject() == RejectAndGiveUp)
597
0
          return false;
598
167k
      break;
599
167k
    }
600
167k
601
576k
    case GIM_CheckIsMBB: {
602
576k
      int64_t InsnID = MatchTable[CurrentIdx++];
603
576k
      int64_t OpIdx = MatchTable[CurrentIdx++];
604
576k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
605
576k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
606
576k
                             << "]->getOperand(" << OpIdx << "))\n");
607
576k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
608
576k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
609
0
        if (handleReject() == RejectAndGiveUp)
610
0
          return false;
611
576k
      }
612
576k
      break;
613
576k
    }
614
576k
615
576k
    case GIM_CheckIsSafeToFold: {
616
67.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
617
67.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
618
67.9k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
619
67.9k
                             << InsnID << "])\n");
620
67.9k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
621
67.9k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
622
0
        if (handleReject() == RejectAndGiveUp)
623
0
          return false;
624
67.9k
      }
625
67.9k
      break;
626
67.9k
    }
627
67.9k
    case GIM_CheckIsSameOperand: {
628
9
      int64_t InsnID = MatchTable[CurrentIdx++];
629
9
      int64_t OpIdx = MatchTable[CurrentIdx++];
630
9
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
631
9
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
632
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
633
9
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
634
9
                             << InsnID << "][" << OpIdx << "], MIs["
635
9
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
636
9
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
637
9
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
638
9
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
639
9
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
640
2
        if (handleReject() == RejectAndGiveUp)
641
0
          return false;
642
9
      }
643
9
      break;
644
9
    }
645
5.75M
    case GIM_Reject:
646
5.75M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
647
5.75M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
648
5.75M
      if (handleReject() == RejectAndGiveUp)
649
3.87M
        return false;
650
1.88M
      break;
651
1.88M
652
1.88M
    case GIR_MutateOpcode: {
653
1.06M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
654
1.06M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
655
1.06M
      int64_t NewOpcode = MatchTable[CurrentIdx++];
656
1.06M
      if (NewInsnID >= OutMIs.size())
657
1.06M
        OutMIs.resize(NewInsnID + 1);
658
1.06M
659
1.06M
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
660
1.06M
                                              State.MIs[OldInsnID]);
661
1.06M
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
662
1.06M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
663
1.06M
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
664
1.06M
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
665
1.06M
                             << NewOpcode << ")\n");
666
1.06M
      break;
667
1.88M
    }
668
1.88M
669
1.88M
    case GIR_BuildMI: {
670
1.84M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
671
1.84M
      int64_t Opcode = MatchTable[CurrentIdx++];
672
1.84M
      if (NewInsnID >= OutMIs.size())
673
1.83M
        OutMIs.resize(NewInsnID + 1);
674
1.84M
675
1.84M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
676
1.84M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
677
1.84M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
678
1.84M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
679
1.84M
                             << NewInsnID << "], " << Opcode << ")\n");
680
1.84M
      break;
681
1.88M
    }
682
1.88M
683
2.02M
    case GIR_Copy: {
684
2.02M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
685
2.02M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
686
2.02M
      int64_t OpIdx = MatchTable[CurrentIdx++];
687
2.02M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
688
2.02M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
689
2.02M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
690
2.02M
                      dbgs()
691
2.02M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
692
2.02M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
693
2.02M
      break;
694
1.88M
    }
695
1.88M
696
1.88M
    case GIR_CopyOrAddZeroReg: {
697
217k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
698
217k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
699
217k
      int64_t OpIdx = MatchTable[CurrentIdx++];
700
217k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
701
217k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
702
217k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
703
217k
      if (isOperandImmEqual(MO, 0, MRI))
704
22.3k
        OutMIs[NewInsnID].addReg(ZeroReg);
705
195k
      else
706
195k
        OutMIs[NewInsnID].add(MO);
707
217k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
708
217k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
709
217k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
710
217k
                             << OpIdx << ", " << ZeroReg << ")\n");
711
217k
      break;
712
1.88M
    }
713
1.88M
714
1.88M
    case GIR_CopySubReg: {
715
86.1k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
716
86.1k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
717
86.1k
      int64_t OpIdx = MatchTable[CurrentIdx++];
718
86.1k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
719
86.1k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
720
86.1k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
721
86.1k
                               0, SubRegIdx);
722
86.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
723
86.1k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
724
86.1k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
725
86.1k
                             << OpIdx << ", " << SubRegIdx << ")\n");
726
86.1k
      break;
727
1.88M
    }
728
1.88M
729
1.88M
    case GIR_AddImplicitDef: {
730
61.7k
      int64_t InsnID = MatchTable[CurrentIdx++];
731
61.7k
      int64_t RegNum = MatchTable[CurrentIdx++];
732
61.7k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
733
61.7k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
734
61.7k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
735
61.7k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
736
61.7k
                             << InsnID << "], " << RegNum << ")\n");
737
61.7k
      break;
738
1.88M
    }
739
1.88M
740
1.88M
    case GIR_AddImplicitUse: {
741
0
      int64_t InsnID = MatchTable[CurrentIdx++];
742
0
      int64_t RegNum = MatchTable[CurrentIdx++];
743
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
744
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
745
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
746
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
747
0
                             << InsnID << "], " << RegNum << ")\n");
748
0
      break;
749
1.88M
    }
750
1.88M
751
1.88M
    case GIR_AddRegister: {
752
131k
      int64_t InsnID = MatchTable[CurrentIdx++];
753
131k
      int64_t RegNum = MatchTable[CurrentIdx++];
754
131k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
755
131k
      OutMIs[InsnID].addReg(RegNum);
756
131k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
757
131k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
758
131k
                             << InsnID << "], " << RegNum << ")\n");
759
131k
      break;
760
1.88M
    }
761
1.88M
762
1.88M
    case GIR_AddTempRegister: {
763
8.96k
      int64_t InsnID = MatchTable[CurrentIdx++];
764
8.96k
      int64_t TempRegID = MatchTable[CurrentIdx++];
765
8.96k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
766
8.96k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
767
8.96k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
768
8.96k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
769
8.96k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
770
8.96k
                             << InsnID << "], TempRegisters[" << TempRegID
771
8.96k
                             << "], " << TempRegFlags << ")\n");
772
8.96k
      break;
773
1.88M
    }
774
1.88M
775
1.88M
    case GIR_AddImm: {
776
37.3k
      int64_t InsnID = MatchTable[CurrentIdx++];
777
37.3k
      int64_t Imm = MatchTable[CurrentIdx++];
778
37.3k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
779
37.3k
      OutMIs[InsnID].addImm(Imm);
780
37.3k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
781
37.3k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
782
37.3k
                             << "], " << Imm << ")\n");
783
37.3k
      break;
784
1.88M
    }
785
1.88M
786
1.88M
    case GIR_ComplexRenderer: {
787
99.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
788
99.2k
      int64_t RendererID = MatchTable[CurrentIdx++];
789
99.2k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
790
99.2k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
791
198k
        RenderOpFn(OutMIs[InsnID]);
792
99.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
793
99.2k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
794
99.2k
                             << InsnID << "], " << RendererID << ")\n");
795
99.2k
      break;
796
1.88M
    }
797
1.88M
    case GIR_ComplexSubOperandRenderer: {
798
1.35M
      int64_t InsnID = MatchTable[CurrentIdx++];
799
1.35M
      int64_t RendererID = MatchTable[CurrentIdx++];
800
1.35M
      int64_t RenderOpID = MatchTable[CurrentIdx++];
801
1.35M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
802
1.35M
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
803
1.35M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
804
1.35M
                      dbgs() << CurrentIdx
805
1.35M
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
806
1.35M
                             << InsnID << "], " << RendererID << ", "
807
1.35M
                             << RenderOpID << ")\n");
808
1.35M
      break;
809
1.88M
    }
810
1.88M
811
1.88M
    case GIR_CopyConstantAsSImm: {
812
836k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
813
836k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
814
836k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
815
836k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
816
836k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
817
836k
        OutMIs[NewInsnID].addImm(
818
836k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
819
836k
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
820
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
821
0
      else
822
0
        llvm_unreachable("Expected Imm or CImm operand");
823
836k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
824
836k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
825
836k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
826
836k
      break;
827
836k
    }
828
836k
829
836k
    // TODO: Needs a test case once we have a pattern that uses this.
830
836k
    case GIR_CopyFConstantAsFPImm: {
831
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
832
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
833
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
834
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
835
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
836
0
        OutMIs[NewInsnID].addFPImm(
837
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
838
0
      else
839
0
        llvm_unreachable("Expected FPImm operand");
840
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
841
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
842
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
843
0
      break;
844
0
    }
845
0
846
4.46k
    case GIR_CustomRenderer: {
847
4.46k
      int64_t InsnID = MatchTable[CurrentIdx++];
848
4.46k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
849
4.46k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
850
4.46k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
851
4.46k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
852
4.46k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
853
4.46k
                             << InsnID << "], MIs[" << OldInsnID << "], "
854
4.46k
                             << RendererFnID << ")\n");
855
4.46k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
856
4.46k
                                                     *State.MIs[OldInsnID]);
857
4.46k
      break;
858
0
    }
859
172k
    case GIR_ConstrainOperandRC: {
860
172k
      int64_t InsnID = MatchTable[CurrentIdx++];
861
172k
      int64_t OpIdx = MatchTable[CurrentIdx++];
862
172k
      int64_t RCEnum = MatchTable[CurrentIdx++];
863
172k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
864
172k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
865
172k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
866
172k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
867
172k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
868
172k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
869
172k
                             << ")\n");
870
172k
      break;
871
0
    }
872
0
873
2.82M
    case GIR_ConstrainSelectedInstOperands: {
874
2.82M
      int64_t InsnID = MatchTable[CurrentIdx++];
875
2.82M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
876
2.82M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
877
2.82M
                                       RBI);
878
2.82M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
879
2.82M
                      dbgs() << CurrentIdx
880
2.82M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
881
2.82M
                             << InsnID << "])\n");
882
2.82M
      break;
883
0
    }
884
0
885
679k
    case GIR_MergeMemOperands: {
886
679k
      int64_t InsnID = MatchTable[CurrentIdx++];
887
679k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
888
679k
889
679k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
890
679k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
891
679k
                             << InsnID << "]");
892
679k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
893
1.36M
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
894
1.36M
             GIU_MergeMemOperands_EndOfList) {
895
681k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
896
681k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
897
681k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
898
679k
          OutMIs[InsnID].addMemOperand(MMO);
899
681k
      }
900
679k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
901
679k
      break;
902
0
    }
903
0
904
1.83M
    case GIR_EraseFromParent: {
905
1.83M
      int64_t InsnID = MatchTable[CurrentIdx++];
906
1.83M
      assert(State.MIs[InsnID] &&
907
1.83M
             "Attempted to erase an undefined instruction");
908
1.83M
      State.MIs[InsnID]->eraseFromParent();
909
1.83M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
910
1.83M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
911
1.83M
                             << InsnID << "])\n");
912
1.83M
      break;
913
0
    }
914
0
915
4.48k
    case GIR_MakeTempReg: {
916
4.48k
      int64_t TempRegID = MatchTable[CurrentIdx++];
917
4.48k
      int64_t TypeID = MatchTable[CurrentIdx++];
918
4.48k
919
4.48k
      State.TempRegisters[TempRegID] =
920
4.48k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
921
4.48k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
922
4.48k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
923
4.48k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
924
4.48k
      break;
925
0
    }
926
0
927
0
    case GIR_Coverage: {
928
0
      int64_t RuleID = MatchTable[CurrentIdx++];
929
0
      CoverageInfo.setCovered(RuleID);
930
0
931
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
932
0
                      dbgs()
933
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
934
0
      break;
935
0
    }
936
0
937
2.90M
    case GIR_Done:
938
2.90M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
939
2.90M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
940
2.90M
      return true;
941
0
942
0
    default:
943
0
      llvm_unreachable("Unexpected command");
944
59.8M
    }
945
59.8M
  }
946
6.77M
}
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<19ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<19ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<19ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
6.77M
    CodeGenCoverage &CoverageInfo) const {
56
6.77M
57
6.77M
  uint64_t CurrentIdx = 0;
58
6.77M
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
6.77M
60
6.77M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
6.77M
  auto handleReject = [&]() -> RejectAction {
62
6.77M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
6.77M
                    dbgs() << CurrentIdx << ": Rejected\n");
64
6.77M
    if (OnFailResumeAt.empty())
65
6.77M
      return RejectAndGiveUp;
66
6.77M
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
6.77M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
6.77M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
6.77M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
6.77M
    return RejectAndResume;
71
6.77M
  };
72
6.77M
73
59.8M
  while (true) {
74
59.8M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
59.8M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
59.8M
    switch (MatcherOpcode) {
77
59.8M
    case GIM_Try: {
78
8.11M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
8.11M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
8.11M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
8.11M
      break;
82
59.8M
    }
83
59.8M
84
59.8M
    case GIM_RecordInsn: {
85
2.70M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
2.70M
      int64_t InsnID = MatchTable[CurrentIdx++];
87
2.70M
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
2.70M
89
2.70M
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
2.70M
      // any attempt to modify it.
91
2.70M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
2.70M
93
2.70M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
2.70M
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
2.70M
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
2.70M
109
2.70M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
2.70M
      if ((size_t)NewInsnID < State.MIs.size())
111
1.80M
        State.MIs[NewInsnID] = NewMI;
112
901k
      else {
113
901k
        assert((size_t)NewInsnID == State.MIs.size() &&
114
901k
               "Expected to store MIs in order");
115
901k
        State.MIs.push_back(NewMI);
116
901k
      }
117
2.70M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
2.70M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
2.70M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
2.70M
                             << ")\n");
121
2.70M
      break;
122
2.70M
    }
123
2.70M
124
2.70M
    case GIM_CheckFeatures: {
125
637k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
637k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
637k
                      dbgs() << CurrentIdx
128
637k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
637k
                             << ExpectedBitsetID << ")\n");
130
637k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
637k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
37.3k
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
637k
      }
135
637k
      break;
136
637k
    }
137
637k
138
2.70M
    case GIM_CheckOpcode: {
139
2.70M
      int64_t InsnID = MatchTable[CurrentIdx++];
140
2.70M
      int64_t Expected = MatchTable[CurrentIdx++];
141
2.70M
142
2.70M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
2.70M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
2.70M
145
2.70M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
2.70M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
2.70M
                             << "], ExpectedOpcode=" << Expected
148
2.70M
                             << ") // Got=" << Opcode << "\n");
149
2.70M
      if (Opcode != Expected) {
150
2.59M
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
2.70M
      }
153
2.70M
      break;
154
2.70M
    }
155
2.70M
156
6.77M
    case GIM_SwitchOpcode: {
157
6.77M
      int64_t InsnID = MatchTable[CurrentIdx++];
158
6.77M
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
6.77M
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
6.77M
      int64_t Default = MatchTable[CurrentIdx++];
161
6.77M
162
6.77M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
6.77M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
6.77M
165
6.77M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
6.77M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
6.77M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
6.77M
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
6.77M
      });
170
6.77M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
2
        CurrentIdx = Default;
172
2
        break;
173
2
      }
174
6.77M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
6.77M
      if (!CurrentIdx) {
176
2.18M
        CurrentIdx = Default;
177
2.18M
  break;
178
2.18M
      }
179
4.58M
      OnFailResumeAt.push_back(Default);
180
4.58M
      break;
181
4.58M
    }
182
4.58M
183
4.58M
    case GIM_SwitchType: {
184
3.99M
      int64_t InsnID = MatchTable[CurrentIdx++];
185
3.99M
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
3.99M
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
3.99M
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
3.99M
      int64_t Default = MatchTable[CurrentIdx++];
189
3.99M
190
3.99M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
3.99M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
3.99M
193
3.99M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
3.99M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
3.99M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
3.99M
               << UpperBound << "), Default=" << Default
197
3.99M
               << ", JumpTable...) // Got=";
198
3.99M
        if (!MO.isReg())
199
3.99M
          dbgs() << "Not a VReg\n";
200
3.99M
        else
201
3.99M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
3.99M
      });
203
3.99M
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
3.99M
      const LLT Ty = MRI.getType(MO.getReg());
208
3.99M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
3.99M
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
1.33M
        CurrentIdx = Default;
211
1.33M
        break;
212
1.33M
      }
213
2.65M
      const int64_t TypeID = TyI->second;
214
2.65M
      if (TypeID < LowerBound || 
UpperBound <= TypeID2.50M
) {
215
153k
        CurrentIdx = Default;
216
153k
        break;
217
153k
      }
218
2.50M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
2.50M
      if (!CurrentIdx) {
220
1
        CurrentIdx = Default;
221
1
        break;
222
1
      }
223
2.50M
      OnFailResumeAt.push_back(Default);
224
2.50M
      break;
225
2.50M
    }
226
2.50M
227
2.50M
    case GIM_CheckNumOperands: {
228
12.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
229
12.0k
      int64_t Expected = MatchTable[CurrentIdx++];
230
12.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
12.0k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
12.0k
                             << InsnID << "], Expected=" << Expected << ")\n");
233
12.0k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
12.0k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
6.96k
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
12.0k
      }
238
12.0k
      break;
239
12.0k
    }
240
43.6k
    case GIM_CheckI64ImmPredicate: {
241
43.6k
      int64_t InsnID = MatchTable[CurrentIdx++];
242
43.6k
      int64_t Predicate = MatchTable[CurrentIdx++];
243
43.6k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
43.6k
                      dbgs()
245
43.6k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
43.6k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
43.6k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
43.6k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
43.6k
             "Expected G_CONSTANT");
250
43.6k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
43.6k
      int64_t Value = 0;
252
43.6k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
43.6k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
43.6k
259
43.6k
      if (!testImmPredicate_I64(Predicate, Value))
260
12
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
43.6k
      break;
263
43.6k
    }
264
43.6k
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
22.1k
    case GIM_CheckAPFloatImmPredicate: {
287
22.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
288
22.1k
      int64_t Predicate = MatchTable[CurrentIdx++];
289
22.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
22.1k
                      dbgs()
291
22.1k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
22.1k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
22.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
22.1k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
22.1k
             "Expected G_FCONSTANT");
296
22.1k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
22.1k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
22.1k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
22.1k
300
22.1k
      if (!testImmPredicate_APFloat(Predicate, Value))
301
19.0k
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
22.1k
      break;
304
22.1k
    }
305
22.1k
    case GIM_CheckCxxInsnPredicate: {
306
0
      int64_t InsnID = MatchTable[CurrentIdx++];
307
0
      int64_t Predicate = MatchTable[CurrentIdx++];
308
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
0
                      dbgs()
310
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
0
315
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
0
      break;
319
0
    }
320
709k
    case GIM_CheckAtomicOrdering: {
321
709k
      int64_t InsnID = MatchTable[CurrentIdx++];
322
709k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
709k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
709k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
709k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
709k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
709k
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
709k
331
709k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
709k
        if (MMO->getOrdering() != Ordering)
333
4.24k
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
709k
      break;
336
709k
    }
337
709k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
32.5k
    case GIM_CheckMemorySizeEqualTo: {
374
32.5k
      int64_t InsnID = MatchTable[CurrentIdx++];
375
32.5k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
32.5k
      uint64_t Size = MatchTable[CurrentIdx++];
377
32.5k
378
32.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
379
32.5k
                      dbgs() << CurrentIdx
380
32.5k
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
381
32.5k
                             << "]->memoperands() + " << MMOIdx
382
32.5k
                             << ", Size=" << Size << ")\n");
383
32.5k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
384
32.5k
385
32.5k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
386
0
        if (handleReject() == RejectAndGiveUp)
387
0
          return false;
388
0
        break;
389
0
      }
390
32.5k
391
32.5k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
392
32.5k
393
32.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
394
32.5k
                      dbgs() << MMO->getSize() << " bytes vs " << Size
395
32.5k
                             << " bytes\n");
396
32.5k
      if (MMO->getSize() != Size)
397
14.3k
        if (handleReject() == RejectAndGiveUp)
398
0
          return false;
399
32.5k
400
32.5k
      break;
401
32.5k
    }
402
432k
    case GIM_CheckMemorySizeEqualToLLT:
403
432k
    case GIM_CheckMemorySizeLessThanLLT:
404
432k
    case GIM_CheckMemorySizeGreaterThanLLT: {
405
432k
      int64_t InsnID = MatchTable[CurrentIdx++];
406
432k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
407
432k
      int64_t OpIdx = MatchTable[CurrentIdx++];
408
432k
409
432k
      DEBUG_WITH_TYPE(
410
432k
          TgtInstructionSelector::getName(),
411
432k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
412
432k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
413
432k
                         ? "EqualTo"
414
432k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
415
432k
                               ? "GreaterThan"
416
432k
                               : "LessThan")
417
432k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
418
432k
                 << ", OpIdx=" << OpIdx << ")\n");
419
432k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
420
432k
421
432k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
422
432k
      if (!MO.isReg()) {
423
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
424
0
                        dbgs() << CurrentIdx << ": Not a register\n");
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
432k
430
432k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
431
0
        if (handleReject() == RejectAndGiveUp)
432
0
          return false;
433
0
        break;
434
0
      }
435
432k
436
432k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
437
432k
438
432k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
439
432k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
440
432k
          
MMO->getSize() * 8 != Size432k
) {
441
120
        if (handleReject() == RejectAndGiveUp)
442
0
          return false;
443
432k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
444
432k
                 
MMO->getSize() * 8 >= Size118
) {
445
36
        if (handleReject() == RejectAndGiveUp)
446
0
          return false;
447
432k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
448
432k
                 
MMO->getSize() * 8 <= Size0
)
449
0
        if (handleReject() == RejectAndGiveUp)
450
0
          return false;
451
432k
452
432k
      break;
453
432k
    }
454
1.95M
    case GIM_CheckType: {
455
1.95M
      int64_t InsnID = MatchTable[CurrentIdx++];
456
1.95M
      int64_t OpIdx = MatchTable[CurrentIdx++];
457
1.95M
      int64_t TypeID = MatchTable[CurrentIdx++];
458
1.95M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
459
1.95M
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
460
1.95M
                             << "]->getOperand(" << OpIdx
461
1.95M
                             << "), TypeID=" << TypeID << ")\n");
462
1.95M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
463
1.95M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
464
1.95M
      if (!MO.isReg() ||
465
1.95M
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
466
103k
        if (handleReject() == RejectAndGiveUp)
467
0
          return false;
468
1.95M
      }
469
1.95M
      break;
470
1.95M
    }
471
1.95M
    case GIM_CheckPointerToAny: {
472
697k
      int64_t InsnID = MatchTable[CurrentIdx++];
473
697k
      int64_t OpIdx = MatchTable[CurrentIdx++];
474
697k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
475
697k
476
697k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
477
697k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
478
697k
                             << InsnID << "]->getOperand(" << OpIdx
479
697k
                             << "), SizeInBits=" << SizeInBits << ")\n");
480
697k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
481
697k
      // iPTR must be looked up in the target.
482
697k
      if (SizeInBits == 0) {
483
696k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
484
696k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
485
696k
      }
486
697k
487
697k
      assert(SizeInBits != 0 && "Pointer size must be known");
488
697k
489
697k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
490
697k
      if (MO.isReg()) {
491
697k
        const LLT &Ty = MRI.getType(MO.getReg());
492
697k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
493
0
          if (handleReject() == RejectAndGiveUp)
494
0
            return false;
495
0
      } else if (handleReject() == RejectAndGiveUp)
496
0
        return false;
497
697k
498
697k
      break;
499
697k
    }
500
6.81M
    case GIM_CheckRegBankForClass: {
501
6.81M
      int64_t InsnID = MatchTable[CurrentIdx++];
502
6.81M
      int64_t OpIdx = MatchTable[CurrentIdx++];
503
6.81M
      int64_t RCEnum = MatchTable[CurrentIdx++];
504
6.81M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
505
6.81M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
506
6.81M
                             << InsnID << "]->getOperand(" << OpIdx
507
6.81M
                             << "), RCEnum=" << RCEnum << ")\n");
508
6.81M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
509
6.81M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
510
6.81M
      if (!MO.isReg() ||
511
6.81M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
512
6.81M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
513
531k
        if (handleReject() == RejectAndGiveUp)
514
0
          return false;
515
6.81M
      }
516
6.81M
      break;
517
6.81M
    }
518
6.81M
519
6.81M
    case GIM_CheckComplexPattern: {
520
1.18M
      int64_t InsnID = MatchTable[CurrentIdx++];
521
1.18M
      int64_t OpIdx = MatchTable[CurrentIdx++];
522
1.18M
      int64_t RendererID = MatchTable[CurrentIdx++];
523
1.18M
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
524
1.18M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
525
1.18M
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
526
1.18M
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
527
1.18M
                             << "]->getOperand(" << OpIdx
528
1.18M
                             << "), ComplexPredicateID=" << ComplexPredicateID
529
1.18M
                             << ")\n");
530
1.18M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
531
1.18M
      // FIXME: Use std::invoke() when it's available.
532
1.18M
      ComplexRendererFns Renderer =
533
1.18M
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
534
1.18M
              State.MIs[InsnID]->getOperand(OpIdx));
535
1.18M
      if (Renderer.hasValue())
536
778k
        State.Renderers[RendererID] = Renderer.getValue();
537
406k
      else
538
406k
        if (handleReject() == RejectAndGiveUp)
539
0
          return false;
540
1.18M
      break;
541
1.18M
    }
542
1.18M
543
1.18M
    case GIM_CheckConstantInt: {
544
242k
      int64_t InsnID = MatchTable[CurrentIdx++];
545
242k
      int64_t OpIdx = MatchTable[CurrentIdx++];
546
242k
      int64_t Value = MatchTable[CurrentIdx++];
547
242k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
548
242k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
549
242k
                             << InsnID << "]->getOperand(" << OpIdx
550
242k
                             << "), Value=" << Value << ")\n");
551
242k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
552
242k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
553
242k
      if (MO.isReg()) {
554
242k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
555
242k
        LLT Ty = MRI.getType(MO.getReg());
556
242k
        Value = SignExtend64(Value, Ty.getSizeInBits());
557
242k
558
242k
        if (!isOperandImmEqual(MO, Value, MRI)) {
559
220k
          if (handleReject() == RejectAndGiveUp)
560
0
            return false;
561
0
        }
562
0
      } else if (handleReject() == RejectAndGiveUp)
563
0
        return false;
564
242k
565
242k
      break;
566
242k
    }
567
242k
568
242k
    case GIM_CheckLiteralInt: {
569
0
      int64_t InsnID = MatchTable[CurrentIdx++];
570
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
571
0
      int64_t Value = MatchTable[CurrentIdx++];
572
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
573
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
574
0
                             << InsnID << "]->getOperand(" << OpIdx
575
0
                             << "), Value=" << Value << ")\n");
576
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
577
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
578
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
579
0
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
0
      }
582
0
      break;
583
0
    }
584
0
585
167k
    case GIM_CheckIntrinsicID: {
586
167k
      int64_t InsnID = MatchTable[CurrentIdx++];
587
167k
      int64_t OpIdx = MatchTable[CurrentIdx++];
588
167k
      int64_t Value = MatchTable[CurrentIdx++];
589
167k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
590
167k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
591
167k
                             << InsnID << "]->getOperand(" << OpIdx
592
167k
                             << "), Value=" << Value << ")\n");
593
167k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
594
167k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
595
167k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value162k
)
596
163k
        if (handleReject() == RejectAndGiveUp)
597
0
          return false;
598
167k
      break;
599
167k
    }
600
167k
601
576k
    case GIM_CheckIsMBB: {
602
576k
      int64_t InsnID = MatchTable[CurrentIdx++];
603
576k
      int64_t OpIdx = MatchTable[CurrentIdx++];
604
576k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
605
576k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
606
576k
                             << "]->getOperand(" << OpIdx << "))\n");
607
576k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
608
576k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
609
0
        if (handleReject() == RejectAndGiveUp)
610
0
          return false;
611
576k
      }
612
576k
      break;
613
576k
    }
614
576k
615
576k
    case GIM_CheckIsSafeToFold: {
616
67.6k
      int64_t InsnID = MatchTable[CurrentIdx++];
617
67.6k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
618
67.6k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
619
67.6k
                             << InsnID << "])\n");
620
67.6k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
621
67.6k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
622
0
        if (handleReject() == RejectAndGiveUp)
623
0
          return false;
624
67.6k
      }
625
67.6k
      break;
626
67.6k
    }
627
67.6k
    case GIM_CheckIsSameOperand: {
628
1
      int64_t InsnID = MatchTable[CurrentIdx++];
629
1
      int64_t OpIdx = MatchTable[CurrentIdx++];
630
1
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
631
1
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
632
1
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
633
1
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
634
1
                             << InsnID << "][" << OpIdx << "], MIs["
635
1
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
636
1
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
637
1
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
638
1
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
639
1
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
640
0
        if (handleReject() == RejectAndGiveUp)
641
0
          return false;
642
1
      }
643
1
      break;
644
1
    }
645
5.75M
    case GIM_Reject:
646
5.75M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
647
5.75M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
648
5.75M
      if (handleReject() == RejectAndGiveUp)
649
3.87M
        return false;
650
1.88M
      break;
651
1.88M
652
1.88M
    case GIR_MutateOpcode: {
653
1.06M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
654
1.06M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
655
1.06M
      int64_t NewOpcode = MatchTable[CurrentIdx++];
656
1.06M
      if (NewInsnID >= OutMIs.size())
657
1.06M
        OutMIs.resize(NewInsnID + 1);
658
1.06M
659
1.06M
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
660
1.06M
                                              State.MIs[OldInsnID]);
661
1.06M
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
662
1.06M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
663
1.06M
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
664
1.06M
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
665
1.06M
                             << NewOpcode << ")\n");
666
1.06M
      break;
667
1.88M
    }
668
1.88M
669
1.88M
    case GIR_BuildMI: {
670
1.84M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
671
1.84M
      int64_t Opcode = MatchTable[CurrentIdx++];
672
1.84M
      if (NewInsnID >= OutMIs.size())
673
1.83M
        OutMIs.resize(NewInsnID + 1);
674
1.84M
675
1.84M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
676
1.84M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
677
1.84M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
678
1.84M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
679
1.84M
                             << NewInsnID << "], " << Opcode << ")\n");
680
1.84M
      break;
681
1.88M
    }
682
1.88M
683
2.02M
    case GIR_Copy: {
684
2.02M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
685
2.02M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
686
2.02M
      int64_t OpIdx = MatchTable[CurrentIdx++];
687
2.02M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
688
2.02M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
689
2.02M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
690
2.02M
                      dbgs()
691
2.02M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
692
2.02M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
693
2.02M
      break;
694
1.88M
    }
695
1.88M
696
1.88M
    case GIR_CopyOrAddZeroReg: {
697
217k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
698
217k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
699
217k
      int64_t OpIdx = MatchTable[CurrentIdx++];
700
217k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
701
217k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
702
217k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
703
217k
      if (isOperandImmEqual(MO, 0, MRI))
704
22.3k
        OutMIs[NewInsnID].addReg(ZeroReg);
705
195k
      else
706
195k
        OutMIs[NewInsnID].add(MO);
707
217k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
708
217k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
709
217k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
710
217k
                             << OpIdx << ", " << ZeroReg << ")\n");
711
217k
      break;
712
1.88M
    }
713
1.88M
714
1.88M
    case GIR_CopySubReg: {
715
85.9k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
716
85.9k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
717
85.9k
      int64_t OpIdx = MatchTable[CurrentIdx++];
718
85.9k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
719
85.9k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
720
85.9k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
721
85.9k
                               0, SubRegIdx);
722
85.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
723
85.9k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
724
85.9k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
725
85.9k
                             << OpIdx << ", " << SubRegIdx << ")\n");
726
85.9k
      break;
727
1.88M
    }
728
1.88M
729
1.88M
    case GIR_AddImplicitDef: {
730
61.6k
      int64_t InsnID = MatchTable[CurrentIdx++];
731
61.6k
      int64_t RegNum = MatchTable[CurrentIdx++];
732
61.6k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
733
61.6k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
734
61.6k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
735
61.6k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
736
61.6k
                             << InsnID << "], " << RegNum << ")\n");
737
61.6k
      break;
738
1.88M
    }
739
1.88M
740
1.88M
    case GIR_AddImplicitUse: {
741
0
      int64_t InsnID = MatchTable[CurrentIdx++];
742
0
      int64_t RegNum = MatchTable[CurrentIdx++];
743
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
744
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
745
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
746
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
747
0
                             << InsnID << "], " << RegNum << ")\n");
748
0
      break;
749
1.88M
    }
750
1.88M
751
1.88M
    case GIR_AddRegister: {
752
131k
      int64_t InsnID = MatchTable[CurrentIdx++];
753
131k
      int64_t RegNum = MatchTable[CurrentIdx++];
754
131k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
755
131k
      OutMIs[InsnID].addReg(RegNum);
756
131k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
757
131k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
758
131k
                             << InsnID << "], " << RegNum << ")\n");
759
131k
      break;
760
1.88M
    }
761
1.88M
762
1.88M
    case GIR_AddTempRegister: {
763
8.93k
      int64_t InsnID = MatchTable[CurrentIdx++];
764
8.93k
      int64_t TempRegID = MatchTable[CurrentIdx++];
765
8.93k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
766
8.93k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
767
8.93k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
768
8.93k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
769
8.93k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
770
8.93k
                             << InsnID << "], TempRegisters[" << TempRegID
771
8.93k
                             << "], " << TempRegFlags << ")\n");
772
8.93k
      break;
773
1.88M
    }
774
1.88M
775
1.88M
    case GIR_AddImm: {
776
36.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
777
36.9k
      int64_t Imm = MatchTable[CurrentIdx++];
778
36.9k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
779
36.9k
      OutMIs[InsnID].addImm(Imm);
780
36.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
781
36.9k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
782
36.9k
                             << "], " << Imm << ")\n");
783
36.9k
      break;
784
1.88M
    }
785
1.88M
786
1.88M
    case GIR_ComplexRenderer: {
787
99.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
788
99.2k
      int64_t RendererID = MatchTable[CurrentIdx++];
789
99.2k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
790
99.2k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
791
198k
        RenderOpFn(OutMIs[InsnID]);
792
99.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
793
99.2k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
794
99.2k
                             << InsnID << "], " << RendererID << ")\n");
795
99.2k
      break;
796
1.88M
    }
797
1.88M
    case GIR_ComplexSubOperandRenderer: {
798
1.35M
      int64_t InsnID = MatchTable[CurrentIdx++];
799
1.35M
      int64_t RendererID = MatchTable[CurrentIdx++];
800
1.35M
      int64_t RenderOpID = MatchTable[CurrentIdx++];
801
1.35M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
802
1.35M
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
803
1.35M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
804
1.35M
                      dbgs() << CurrentIdx
805
1.35M
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
806
1.35M
                             << InsnID << "], " << RendererID << ", "
807
1.35M
                             << RenderOpID << ")\n");
808
1.35M
      break;
809
1.88M
    }
810
1.88M
811
1.88M
    case GIR_CopyConstantAsSImm: {
812
836k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
813
836k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
814
836k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
815
836k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
816
836k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
817
836k
        OutMIs[NewInsnID].addImm(
818
836k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
819
836k
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
820
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
821
0
      else
822
0
        llvm_unreachable("Expected Imm or CImm operand");
823
836k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
824
836k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
825
836k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
826
836k
      break;
827
836k
    }
828
836k
829
836k
    // TODO: Needs a test case once we have a pattern that uses this.
830
836k
    case GIR_CopyFConstantAsFPImm: {
831
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
832
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
833
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
834
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
835
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
836
0
        OutMIs[NewInsnID].addFPImm(
837
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
838
0
      else
839
0
        llvm_unreachable("Expected FPImm operand");
840
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
841
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
842
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
843
0
      break;
844
0
    }
845
0
846
4.46k
    case GIR_CustomRenderer: {
847
4.46k
      int64_t InsnID = MatchTable[CurrentIdx++];
848
4.46k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
849
4.46k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
850
4.46k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
851
4.46k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
852
4.46k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
853
4.46k
                             << InsnID << "], MIs[" << OldInsnID << "], "
854
4.46k
                             << RendererFnID << ")\n");
855
4.46k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
856
4.46k
                                                     *State.MIs[OldInsnID]);
857
4.46k
      break;
858
0
    }
859
172k
    case GIR_ConstrainOperandRC: {
860
172k
      int64_t InsnID = MatchTable[CurrentIdx++];
861
172k
      int64_t OpIdx = MatchTable[CurrentIdx++];
862
172k
      int64_t RCEnum = MatchTable[CurrentIdx++];
863
172k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
864
172k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
865
172k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
866
172k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
867
172k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
868
172k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
869
172k
                             << ")\n");
870
172k
      break;
871
0
    }
872
0
873
2.82M
    case GIR_ConstrainSelectedInstOperands: {
874
2.82M
      int64_t InsnID = MatchTable[CurrentIdx++];
875
2.82M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
876
2.82M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
877
2.82M
                                       RBI);
878
2.82M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
879
2.82M
                      dbgs() << CurrentIdx
880
2.82M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
881
2.82M
                             << InsnID << "])\n");
882
2.82M
      break;
883
0
    }
884
0
885
679k
    case GIR_MergeMemOperands: {
886
679k
      int64_t InsnID = MatchTable[CurrentIdx++];
887
679k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
888
679k
889
679k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
890
679k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
891
679k
                             << InsnID << "]");
892
679k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
893
1.36M
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
894
1.36M
             GIU_MergeMemOperands_EndOfList) {
895
681k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
896
681k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
897
681k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
898
679k
          OutMIs[InsnID].addMemOperand(MMO);
899
681k
      }
900
679k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
901
679k
      break;
902
0
    }
903
0
904
1.83M
    case GIR_EraseFromParent: {
905
1.83M
      int64_t InsnID = MatchTable[CurrentIdx++];
906
1.83M
      assert(State.MIs[InsnID] &&
907
1.83M
             "Attempted to erase an undefined instruction");
908
1.83M
      State.MIs[InsnID]->eraseFromParent();
909
1.83M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
910
1.83M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
911
1.83M
                             << InsnID << "])\n");
912
1.83M
      break;
913
0
    }
914
0
915
4.46k
    case GIR_MakeTempReg: {
916
4.46k
      int64_t TempRegID = MatchTable[CurrentIdx++];
917
4.46k
      int64_t TypeID = MatchTable[CurrentIdx++];
918
4.46k
919
4.46k
      State.TempRegisters[TempRegID] =
920
4.46k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
921
4.46k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
922
4.46k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
923
4.46k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
924
4.46k
      break;
925
0
    }
926
0
927
0
    case GIR_Coverage: {
928
0
      int64_t RuleID = MatchTable[CurrentIdx++];
929
0
      CoverageInfo.setCovered(RuleID);
930
0
931
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
932
0
                      dbgs()
933
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
934
0
      break;
935
0
    }
936
0
937
2.90M
    case GIR_Done:
938
2.90M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
939
2.90M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
940
2.90M
      return true;
941
0
942
0
    default:
943
0
      llvm_unreachable("Unexpected command");
944
59.8M
    }
945
59.8M
  }
946
6.77M
}
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<33ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<33ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<33ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
120
    CodeGenCoverage &CoverageInfo) const {
56
120
57
120
  uint64_t CurrentIdx = 0;
58
120
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
120
60
120
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
120
  auto handleReject = [&]() -> RejectAction {
62
120
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
120
                    dbgs() << CurrentIdx << ": Rejected\n");
64
120
    if (OnFailResumeAt.empty())
65
120
      return RejectAndGiveUp;
66
120
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
120
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
120
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
120
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
120
    return RejectAndResume;
71
120
  };
72
120
73
2.99k
  while (true) {
74
2.99k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
2.99k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
2.99k
    switch (MatcherOpcode) {
77
2.99k
    case GIM_Try: {
78
489
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
489
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
489
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
489
      break;
82
2.99k
    }
83
2.99k
84
2.99k
    case GIM_RecordInsn: {
85
48
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
48
      int64_t InsnID = MatchTable[CurrentIdx++];
87
48
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
48
89
48
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
48
      // any attempt to modify it.
91
48
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
48
93
48
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
48
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
48
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
48
109
48
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
48
      if ((size_t)NewInsnID < State.MIs.size())
111
45
        State.MIs[NewInsnID] = NewMI;
112
3
      else {
113
3
        assert((size_t)NewInsnID == State.MIs.size() &&
114
3
               "Expected to store MIs in order");
115
3
        State.MIs.push_back(NewMI);
116
3
      }
117
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
48
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
48
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
48
                             << ")\n");
121
48
      break;
122
48
    }
123
48
124
299
    case GIM_CheckFeatures: {
125
299
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
299
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
299
                      dbgs() << CurrentIdx
128
299
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
299
                             << ExpectedBitsetID << ")\n");
130
299
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
299
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
29
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
299
      }
135
299
      break;
136
299
    }
137
299
138
299
    case GIM_CheckOpcode: {
139
48
      int64_t InsnID = MatchTable[CurrentIdx++];
140
48
      int64_t Expected = MatchTable[CurrentIdx++];
141
48
142
48
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
48
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
48
145
48
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
48
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
48
                             << "], ExpectedOpcode=" << Expected
148
48
                             << ") // Got=" << Opcode << "\n");
149
48
      if (Opcode != Expected) {
150
48
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
48
      }
153
48
      break;
154
48
    }
155
48
156
120
    case GIM_SwitchOpcode: {
157
120
      int64_t InsnID = MatchTable[CurrentIdx++];
158
120
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
120
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
120
      int64_t Default = MatchTable[CurrentIdx++];
161
120
162
120
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
120
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
120
165
120
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
120
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
120
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
120
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
120
      });
170
120
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
0
        CurrentIdx = Default;
172
0
        break;
173
0
      }
174
120
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
120
      if (!CurrentIdx) {
176
0
        CurrentIdx = Default;
177
0
  break;
178
0
      }
179
120
      OnFailResumeAt.push_back(Default);
180
120
      break;
181
120
    }
182
120
183
120
    case GIM_SwitchType: {
184
74
      int64_t InsnID = MatchTable[CurrentIdx++];
185
74
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
74
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
74
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
74
      int64_t Default = MatchTable[CurrentIdx++];
189
74
190
74
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
74
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
74
193
74
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
74
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
74
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
74
               << UpperBound << "), Default=" << Default
197
74
               << ", JumpTable...) // Got=";
198
74
        if (!MO.isReg())
199
74
          dbgs() << "Not a VReg\n";
200
74
        else
201
74
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
74
      });
203
74
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
74
      const LLT Ty = MRI.getType(MO.getReg());
208
74
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
74
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
0
        CurrentIdx = Default;
211
0
        break;
212
0
      }
213
74
      const int64_t TypeID = TyI->second;
214
74
      if (TypeID < LowerBound || UpperBound <= TypeID) {
215
0
        CurrentIdx = Default;
216
0
        break;
217
0
      }
218
74
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
74
      if (!CurrentIdx) {
220
0
        CurrentIdx = Default;
221
0
        break;
222
0
      }
223
74
      OnFailResumeAt.push_back(Default);
224
74
      break;
225
74
    }
226
74
227
74
    case GIM_CheckNumOperands: {
228
54
      int64_t InsnID = MatchTable[CurrentIdx++];
229
54
      int64_t Expected = MatchTable[CurrentIdx++];
230
54
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
54
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
54
                             << InsnID << "], Expected=" << Expected << ")\n");
233
54
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
54
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
36
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
54
      }
238
54
      break;
239
54
    }
240
54
    case GIM_CheckI64ImmPredicate: {
241
0
      int64_t InsnID = MatchTable[CurrentIdx++];
242
0
      int64_t Predicate = MatchTable[CurrentIdx++];
243
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
0
                      dbgs()
245
0
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
0
             "Expected G_CONSTANT");
250
0
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
0
      int64_t Value = 0;
252
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
0
259
0
      if (!testImmPredicate_I64(Predicate, Value))
260
0
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
0
      break;
263
0
    }
264
0
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
0
    case GIM_CheckAPFloatImmPredicate: {
287
0
      int64_t InsnID = MatchTable[CurrentIdx++];
288
0
      int64_t Predicate = MatchTable[CurrentIdx++];
289
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
0
                      dbgs()
291
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
0
             "Expected G_FCONSTANT");
296
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
0
300
0
      if (!testImmPredicate_APFloat(Predicate, Value))
301
0
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
0
      break;
304
0
    }
305
64
    case GIM_CheckCxxInsnPredicate: {
306
64
      int64_t InsnID = MatchTable[CurrentIdx++];
307
64
      int64_t Predicate = MatchTable[CurrentIdx++];
308
64
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
64
                      dbgs()
310
64
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
64
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
64
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
64
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
64
315
64
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
64
      break;
319
64
    }
320
64
    case GIM_CheckAtomicOrdering: {
321
64
      int64_t InsnID = MatchTable[CurrentIdx++];
322
64
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
64
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
64
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
64
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
64
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
64
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
64
331
64
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
64
        if (MMO->getOrdering() != Ordering)
333
0
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
64
      break;
336
64
    }
337
64
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
0
    case GIM_CheckMemorySizeEqualTo: {
374
0
      int64_t InsnID = MatchTable[CurrentIdx++];
375
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
0
      uint64_t Size = MatchTable[CurrentIdx++];
377
0
378
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
379
0
                      dbgs() << CurrentIdx
380
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
381
0
                             << "]->memoperands() + " << MMOIdx
382
0
                             << ", Size=" << Size << ")\n");
383
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
384
0
385
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
386
0
        if (handleReject() == RejectAndGiveUp)
387
0
          return false;
388
0
        break;
389
0
      }
390
0
391
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
392
0
393
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
394
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
395
0
                             << " bytes\n");
396
0
      if (MMO->getSize() != Size)
397
0
        if (handleReject() == RejectAndGiveUp)
398
0
          return false;
399
0
400
0
      break;
401
0
    }
402
64
    case GIM_CheckMemorySizeEqualToLLT:
403
64
    case GIM_CheckMemorySizeLessThanLLT:
404
64
    case GIM_CheckMemorySizeGreaterThanLLT: {
405
64
      int64_t InsnID = MatchTable[CurrentIdx++];
406
64
      int64_t MMOIdx = MatchTable[CurrentIdx++];
407
64
      int64_t OpIdx = MatchTable[CurrentIdx++];
408
64
409
64
      DEBUG_WITH_TYPE(
410
64
          TgtInstructionSelector::getName(),
411
64
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
412
64
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
413
64
                         ? "EqualTo"
414
64
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
415
64
                               ? "GreaterThan"
416
64
                               : "LessThan")
417
64
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
418
64
                 << ", OpIdx=" << OpIdx << ")\n");
419
64
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
420
64
421
64
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
422
64
      if (!MO.isReg()) {
423
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
424
0
                        dbgs() << CurrentIdx << ": Not a register\n");
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
64
430
64
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
431
0
        if (handleReject() == RejectAndGiveUp)
432
0
          return false;
433
0
        break;
434
0
      }
435
64
436
64
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
437
64
438
64
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
439
64
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
440
64
          MMO->getSize() * 8 != Size) {
441
0
        if (handleReject() == RejectAndGiveUp)
442
0
          return false;
443
64
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
444
64
                 
MMO->getSize() * 8 >= Size0
) {
445
0
        if (handleReject() == RejectAndGiveUp)
446
0
          return false;
447
64
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
448
64
                 
MMO->getSize() * 8 <= Size0
)
449
0
        if (handleReject() == RejectAndGiveUp)
450
0
          return false;
451
64
452
64
      break;
453
64
    }
454
172
    case GIM_CheckType: {
455
172
      int64_t InsnID = MatchTable[CurrentIdx++];
456
172
      int64_t OpIdx = MatchTable[CurrentIdx++];
457
172
      int64_t TypeID = MatchTable[CurrentIdx++];
458
172
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
459
172
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
460
172
                             << "]->getOperand(" << OpIdx
461
172
                             << "), TypeID=" << TypeID << ")\n");
462
172
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
463
172
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
464
172
      if (!MO.isReg() ||
465
172
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
466
12
        if (handleReject() == RejectAndGiveUp)
467
0
          return false;
468
172
      }
469
172
      break;
470
172
    }
471
172
    case GIM_CheckPointerToAny: {
472
62
      int64_t InsnID = MatchTable[CurrentIdx++];
473
62
      int64_t OpIdx = MatchTable[CurrentIdx++];
474
62
      int64_t SizeInBits = MatchTable[CurrentIdx++];
475
62
476
62
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
477
62
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
478
62
                             << InsnID << "]->getOperand(" << OpIdx
479
62
                             << "), SizeInBits=" << SizeInBits << ")\n");
480
62
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
481
62
      // iPTR must be looked up in the target.
482
62
      if (SizeInBits == 0) {
483
62
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
484
62
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
485
62
      }
486
62
487
62
      assert(SizeInBits != 0 && "Pointer size must be known");
488
62
489
62
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
490
62
      if (MO.isReg()) {
491
62
        const LLT &Ty = MRI.getType(MO.getReg());
492
62
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
493
0
          if (handleReject() == RejectAndGiveUp)
494
0
            return false;
495
0
      } else if (handleReject() == RejectAndGiveUp)
496
0
        return false;
497
62
498
62
      break;
499
62
    }
500
204
    case GIM_CheckRegBankForClass: {
501
204
      int64_t InsnID = MatchTable[CurrentIdx++];
502
204
      int64_t OpIdx = MatchTable[CurrentIdx++];
503
204
      int64_t RCEnum = MatchTable[CurrentIdx++];
504
204
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
505
204
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
506
204
                             << InsnID << "]->getOperand(" << OpIdx
507
204
                             << "), RCEnum=" << RCEnum << ")\n");
508
204
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
509
204
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
510
204
      if (!MO.isReg() ||
511
204
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
512
204
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
513
23
        if (handleReject() == RejectAndGiveUp)
514
0
          return false;
515
204
      }
516
204
      break;
517
204
    }
518
204
519
204
    case GIM_CheckComplexPattern: {
520
169
      int64_t InsnID = MatchTable[CurrentIdx++];
521
169
      int64_t OpIdx = MatchTable[CurrentIdx++];
522
169
      int64_t RendererID = MatchTable[CurrentIdx++];
523
169
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
524
169
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
525
169
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
526
169
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
527
169
                             << "]->getOperand(" << OpIdx
528
169
                             << "), ComplexPredicateID=" << ComplexPredicateID
529
169
                             << ")\n");
530
169
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
531
169
      // FIXME: Use std::invoke() when it's available.
532
169
      ComplexRendererFns Renderer =
533
169
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
534
169
              State.MIs[InsnID]->getOperand(OpIdx));
535
169
      if (Renderer.hasValue())
536
108
        State.Renderers[RendererID] = Renderer.getValue();
537
61
      else
538
61
        if (handleReject() == RejectAndGiveUp)
539
0
          return false;
540
169
      break;
541
169
    }
542
169
543
169
    case GIM_CheckConstantInt: {
544
0
      int64_t InsnID = MatchTable[CurrentIdx++];
545
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
546
0
      int64_t Value = MatchTable[CurrentIdx++];
547
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
548
0
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
549
0
                             << InsnID << "]->getOperand(" << OpIdx
550
0
                             << "), Value=" << Value << ")\n");
551
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
552
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
553
0
      if (MO.isReg()) {
554
0
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
555
0
        LLT Ty = MRI.getType(MO.getReg());
556
0
        Value = SignExtend64(Value, Ty.getSizeInBits());
557
0
558
0
        if (!isOperandImmEqual(MO, Value, MRI)) {
559
0
          if (handleReject() == RejectAndGiveUp)
560
0
            return false;
561
0
        }
562
0
      } else if (handleReject() == RejectAndGiveUp)
563
0
        return false;
564
0
565
0
      break;
566
0
    }
567
0
568
0
    case GIM_CheckLiteralInt: {
569
0
      int64_t InsnID = MatchTable[CurrentIdx++];
570
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
571
0
      int64_t Value = MatchTable[CurrentIdx++];
572
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
573
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
574
0
                             << InsnID << "]->getOperand(" << OpIdx
575
0
                             << "), Value=" << Value << ")\n");
576
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
577
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
578
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
579
0
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
0
      }
582
0
      break;
583
0
    }
584
0
585
64
    case GIM_CheckIntrinsicID: {
586
64
      int64_t InsnID = MatchTable[CurrentIdx++];
587
64
      int64_t OpIdx = MatchTable[CurrentIdx++];
588
64
      int64_t Value = MatchTable[CurrentIdx++];
589
64
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
590
64
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
591
64
                             << InsnID << "]->getOperand(" << OpIdx
592
64
                             << "), Value=" << Value << ")\n");
593
64
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
594
64
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
595
64
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
596
32
        if (handleReject() == RejectAndGiveUp)
597
0
          return false;
598
64
      break;
599
64
    }
600
64
601
64
    case GIM_CheckIsMBB: {
602
0
      int64_t InsnID = MatchTable[CurrentIdx++];
603
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
604
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
605
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
606
0
                             << "]->getOperand(" << OpIdx << "))\n");
607
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
608
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
609
0
        if (handleReject() == RejectAndGiveUp)
610
0
          return false;
611
0
      }
612
0
      break;
613
0
    }
614
0
615
0
    case GIM_CheckIsSafeToFold: {
616
0
      int64_t InsnID = MatchTable[CurrentIdx++];
617
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
618
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
619
0
                             << InsnID << "])\n");
620
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
621
0
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
622
0
        if (handleReject() == RejectAndGiveUp)
623
0
          return false;
624
0
      }
625
0
      break;
626
0
    }
627
0
    case GIM_CheckIsSameOperand: {
628
0
      int64_t InsnID = MatchTable[CurrentIdx++];
629
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
630
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
631
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
632
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
633
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
634
0
                             << InsnID << "][" << OpIdx << "], MIs["
635
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
636
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
637
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
638
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
639
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
640
0
        if (handleReject() == RejectAndGiveUp)
641
0
          return false;
642
0
      }
643
0
      break;
644
0
    }
645
10
    case GIM_Reject:
646
10
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
647
10
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
648
10
      if (handleReject() == RejectAndGiveUp)
649
3
        return false;
650
7
      break;
651
7
652
11
    case GIR_MutateOpcode: {
653
11
      int64_t OldInsnID = MatchTable[CurrentIdx++];
654
11
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
655
11
      int64_t NewOpcode = MatchTable[CurrentIdx++];
656
11
      if (NewInsnID >= OutMIs.size())
657
11
        OutMIs.resize(NewInsnID + 1);
658
11
659
11
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
660
11
                                              State.MIs[OldInsnID]);
661
11
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
662
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
663
11
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
664
11
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
665
11
                             << NewOpcode << ")\n");
666
11
      break;
667
7
    }
668
7
669
106
    case GIR_BuildMI: {
670
106
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
671
106
      int64_t Opcode = MatchTable[CurrentIdx++];
672
106
      if (NewInsnID >= OutMIs.size())
673
106
        OutMIs.resize(NewInsnID + 1);
674
106
675
106
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
676
106
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
677
106
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
678
106
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
679
106
                             << NewInsnID << "], " << Opcode << ")\n");
680
106
      break;
681
7
    }
682
7
683
145
    case GIR_Copy: {
684
145
      int64_t NewInsnID = MatchTable[CurrentIdx++];
685
145
      int64_t OldInsnID = MatchTable[CurrentIdx++];
686
145
      int64_t OpIdx = MatchTable[CurrentIdx++];
687
145
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
688
145
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
689
145
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
690
145
                      dbgs()
691
145
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
692
145
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
693
145
      break;
694
7
    }
695
7
696
7
    case GIR_CopyOrAddZeroReg: {
697
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
698
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
700
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
701
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
702
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
703
0
      if (isOperandImmEqual(MO, 0, MRI))
704
0
        OutMIs[NewInsnID].addReg(ZeroReg);
705
0
      else
706
0
        OutMIs[NewInsnID].add(MO);
707
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
708
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
709
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
710
0
                             << OpIdx << ", " << ZeroReg << ")\n");
711
0
      break;
712
7
    }
713
7
714
7
    case GIR_CopySubReg: {
715
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
716
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
718
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
719
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
720
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
721
0
                               0, SubRegIdx);
722
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
723
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
724
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
725
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
726
0
      break;
727
7
    }
728
7
729
11
    case GIR_AddImplicitDef: {
730
11
      int64_t InsnID = MatchTable[CurrentIdx++];
731
11
      int64_t RegNum = MatchTable[CurrentIdx++];
732
11
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
733
11
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
734
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
735
11
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
736
11
                             << InsnID << "], " << RegNum << ")\n");
737
11
      break;
738
7
    }
739
7
740
7
    case GIR_AddImplicitUse: {
741
0
      int64_t InsnID = MatchTable[CurrentIdx++];
742
0
      int64_t RegNum = MatchTable[CurrentIdx++];
743
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
744
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
745
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
746
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
747
0
                             << InsnID << "], " << RegNum << ")\n");
748
0
      break;
749
7
    }
750
7
751
7
    case GIR_AddRegister: {
752
0
      int64_t InsnID = MatchTable[CurrentIdx++];
753
0
      int64_t RegNum = MatchTable[CurrentIdx++];
754
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
755
0
      OutMIs[InsnID].addReg(RegNum);
756
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
757
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
758
0
                             << InsnID << "], " << RegNum << ")\n");
759
0
      break;
760
7
    }
761
7
762
7
    case GIR_AddTempRegister: {
763
0
      int64_t InsnID = MatchTable[CurrentIdx++];
764
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
765
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
766
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
767
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
768
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
769
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
770
0
                             << InsnID << "], TempRegisters[" << TempRegID
771
0
                             << "], " << TempRegFlags << ")\n");
772
0
      break;
773
7
    }
774
7
775
80
    case GIR_AddImm: {
776
80
      int64_t InsnID = MatchTable[CurrentIdx++];
777
80
      int64_t Imm = MatchTable[CurrentIdx++];
778
80
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
779
80
      OutMIs[InsnID].addImm(Imm);
780
80
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
781
80
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
782
80
                             << "], " << Imm << ")\n");
783
80
      break;
784
7
    }
785
7
786
7
    case GIR_ComplexRenderer: {
787
0
      int64_t InsnID = MatchTable[CurrentIdx++];
788
0
      int64_t RendererID = MatchTable[CurrentIdx++];
789
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
790
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
791
0
        RenderOpFn(OutMIs[InsnID]);
792
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
793
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
794
0
                             << InsnID << "], " << RendererID << ")\n");
795
0
      break;
796
7
    }
797
232
    case GIR_ComplexSubOperandRenderer: {
798
232
      int64_t InsnID = MatchTable[CurrentIdx++];
799
232
      int64_t RendererID = MatchTable[CurrentIdx++];
800
232
      int64_t RenderOpID = MatchTable[CurrentIdx++];
801
232
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
802
232
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
803
232
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
804
232
                      dbgs() << CurrentIdx
805
232
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
806
232
                             << InsnID << "], " << RendererID << ", "
807
232
                             << RenderOpID << ")\n");
808
232
      break;
809
7
    }
810
7
811
7
    case GIR_CopyConstantAsSImm: {
812
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
813
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
814
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
815
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
816
0
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
817
0
        OutMIs[NewInsnID].addImm(
818
0
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
819
0
      } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
820
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
821
0
      else
822
0
        llvm_unreachable("Expected Imm or CImm operand");
823
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
824
0
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
825
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
826
0
      break;
827
0
    }
828
0
829
0
    // TODO: Needs a test case once we have a pattern that uses this.
830
0
    case GIR_CopyFConstantAsFPImm: {
831
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
832
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
833
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
834
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
835
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
836
0
        OutMIs[NewInsnID].addFPImm(
837
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
838
0
      else
839
0
        llvm_unreachable("Expected FPImm operand");
840
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
841
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
842
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
843
0
      break;
844
0
    }
845
0
846
0
    case GIR_CustomRenderer: {
847
0
      int64_t InsnID = MatchTable[CurrentIdx++];
848
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
849
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
850
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
851
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
852
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
853
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
854
0
                             << RendererFnID << ")\n");
855
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
856
0
                                                     *State.MIs[OldInsnID]);
857
0
      break;
858
0
    }
859
0
    case GIR_ConstrainOperandRC: {
860
0
      int64_t InsnID = MatchTable[CurrentIdx++];
861
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
862
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
863
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
864
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
865
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
866
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
867
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
868
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
869
0
                             << ")\n");
870
0
      break;
871
0
    }
872
0
873
117
    case GIR_ConstrainSelectedInstOperands: {
874
117
      int64_t InsnID = MatchTable[CurrentIdx++];
875
117
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
876
117
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
877
117
                                       RBI);
878
117
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
879
117
                      dbgs() << CurrentIdx
880
117
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
881
117
                             << InsnID << "])\n");
882
117
      break;
883
0
    }
884
0
885
61
    case GIR_MergeMemOperands: {
886
61
      int64_t InsnID = MatchTable[CurrentIdx++];
887
61
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
888
61
889
61
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
890
61
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
891
61
                             << InsnID << "]");
892
61
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
893
122
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
894
122
             GIU_MergeMemOperands_EndOfList) {
895
61
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
896
61
                        dbgs() << ", MIs[" << MergeInsnID << "]");
897
61
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
898
61
          OutMIs[InsnID].addMemOperand(MMO);
899
61
      }
900
61
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
901
61
      break;
902
0
    }
903
0
904
106
    case GIR_EraseFromParent: {
905
106
      int64_t InsnID = MatchTable[CurrentIdx++];
906
106
      assert(State.MIs[InsnID] &&
907
106
             "Attempted to erase an undefined instruction");
908
106
      State.MIs[InsnID]->eraseFromParent();
909
106
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
910
106
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
911
106
                             << InsnID << "])\n");
912
106
      break;
913
0
    }
914
0
915
0
    case GIR_MakeTempReg: {
916
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
917
0
      int64_t TypeID = MatchTable[CurrentIdx++];
918
0
919
0
      State.TempRegisters[TempRegID] =
920
0
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
921
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
922
0
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
923
0
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
924
0
      break;
925
0
    }
926
0
927
0
    case GIR_Coverage: {
928
0
      int64_t RuleID = MatchTable[CurrentIdx++];
929
0
      CoverageInfo.setCovered(RuleID);
930
0
931
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
932
0
                      dbgs()
933
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
934
0
      break;
935
0
    }
936
0
937
117
    case GIR_Done:
938
117
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
939
117
                      dbgs() << CurrentIdx << ": GIR_Done\n");
940
117
      return true;
941
0
942
0
    default:
943
0
      llvm_unreachable("Unexpected command");
944
2.99k
    }
945
2.99k
  }
946
120
}
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<64ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<64ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
788
    CodeGenCoverage &CoverageInfo) const {
56
788
57
788
  uint64_t CurrentIdx = 0;
58
788
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
788
60
788
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
788
  auto handleReject = [&]() -> RejectAction {
62
788
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
788
                    dbgs() << CurrentIdx << ": Rejected\n");
64
788
    if (OnFailResumeAt.empty())
65
788
      return RejectAndGiveUp;
66
788
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
788
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
788
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
788
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
788
    return RejectAndResume;
71
788
  };
72
788
73
13.6k
  while (true) {
74
13.6k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
13.6k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
13.6k
    switch (MatcherOpcode) {
77
13.6k
    case GIM_Try: {
78
2.55k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
2.55k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
2.55k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
2.55k
      break;
82
13.6k
    }
83
13.6k
84
13.6k
    case GIM_RecordInsn: {
85
775
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
775
      int64_t InsnID = MatchTable[CurrentIdx++];
87
775
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
775
89
775
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
775
      // any attempt to modify it.
91
775
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
775
93
775
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
775
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
775
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
775
109
775
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
775
      if ((size_t)NewInsnID < State.MIs.size())
111
527
        State.MIs[NewInsnID] = NewMI;
112
248
      else {
113
248
        assert((size_t)NewInsnID == State.MIs.size() &&
114
248
               "Expected to store MIs in order");
115
248
        State.MIs.push_back(NewMI);
116
248
      }
117
775
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
775
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
775
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
775
                             << ")\n");
121
775
      break;
122
775
    }
123
775
124
2.20k
    case GIM_CheckFeatures: {
125
2.20k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
2.20k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
2.20k
                      dbgs() << CurrentIdx
128
2.20k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
2.20k
                             << ExpectedBitsetID << ")\n");
130
2.20k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
2.20k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
1.31k
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
2.20k
      }
135
2.20k
      break;
136
2.20k
    }
137
2.20k
138
2.20k
    case GIM_CheckOpcode: {
139
775
      int64_t InsnID = MatchTable[CurrentIdx++];
140
775
      int64_t Expected = MatchTable[CurrentIdx++];
141
775
142
775
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
775
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
775
145
775
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
775
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
775
                             << "], ExpectedOpcode=" << Expected
148
775
                             << ") // Got=" << Opcode << "\n");
149
775
      if (Opcode != Expected) {
150
532
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
775
      }
153
775
      break;
154
775
    }
155
775
156
788
    case GIM_SwitchOpcode: {
157
788
      int64_t InsnID = MatchTable[CurrentIdx++];
158
788
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
788
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
788
      int64_t Default = MatchTable[CurrentIdx++];
161
788
162
788
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
788
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
788
165
788
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
788
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
788
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
788
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
788
      });
170
788
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
0
        CurrentIdx = Default;
172
0
        break;
173
0
      }
174
788
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
788
      if (!CurrentIdx) {
176
341
        CurrentIdx = Default;
177
341
  break;
178
341
      }
179
447
      OnFailResumeAt.push_back(Default);
180
447
      break;
181
447
    }
182
447
183
447
    case GIM_SwitchType: {
184
283
      int64_t InsnID = MatchTable[CurrentIdx++];
185
283
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
283
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
283
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
283
      int64_t Default = MatchTable[CurrentIdx++];
189
283
190
283
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
283
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
283
193
283
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
283
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
283
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
283
               << UpperBound << "), Default=" << Default
197
283
               << ", JumpTable...) // Got=";
198
283
        if (!MO.isReg())
199
283
          dbgs() << "Not a VReg\n";
200
283
        else
201
283
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
283
      });
203
283
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
283
      const LLT Ty = MRI.getType(MO.getReg());
208
283
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
283
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
23
        CurrentIdx = Default;
211
23
        break;
212
23
      }
213
260
      const int64_t TypeID = TyI->second;
214
260
      if (TypeID < LowerBound || 
UpperBound <= TypeID173
) {
215
87
        CurrentIdx = Default;
216
87
        break;
217
87
      }
218
173
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
173
      if (!CurrentIdx) {
220
0
        CurrentIdx = Default;
221
0
        break;
222
0
      }
223
173
      OnFailResumeAt.push_back(Default);
224
173
      break;
225
173
    }
226
173
227
173
    case GIM_CheckNumOperands: {
228
0
      int64_t InsnID = MatchTable[CurrentIdx++];
229
0
      int64_t Expected = MatchTable[CurrentIdx++];
230
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
0
                             << InsnID << "], Expected=" << Expected << ")\n");
233
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
0
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
0
      }
238
0
      break;
239
0
    }
240
81
    case GIM_CheckI64ImmPredicate: {
241
81
      int64_t InsnID = MatchTable[CurrentIdx++];
242
81
      int64_t Predicate = MatchTable[CurrentIdx++];
243
81
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
81
                      dbgs()
245
81
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
81
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
81
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
81
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
81
             "Expected G_CONSTANT");
250
81
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
81
      int64_t Value = 0;
252
81
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
81
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
81
259
81
      if (!testImmPredicate_I64(Predicate, Value))
260
15
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
81
      break;
263
81
    }
264
81
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
0
    case GIM_CheckAPFloatImmPredicate: {
287
0
      int64_t InsnID = MatchTable[CurrentIdx++];
288
0
      int64_t Predicate = MatchTable[CurrentIdx++];
289
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
0
                      dbgs()
291
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
0
             "Expected G_FCONSTANT");
296
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
0
300
0
      if (!testImmPredicate_APFloat(Predicate, Value))
301
0
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
0
      break;
304
0
    }
305
3
    case GIM_CheckCxxInsnPredicate: {
306
3
      int64_t InsnID = MatchTable[CurrentIdx++];
307
3
      int64_t Predicate = MatchTable[CurrentIdx++];
308
3
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
3
                      dbgs()
310
3
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
3
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
3
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
3
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
3
315
3
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
3
      break;
319
3
    }
320
3
    case GIM_CheckAtomicOrdering: {
321
0
      int64_t InsnID = MatchTable[CurrentIdx++];
322
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
0
331
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
0
        if (MMO->getOrdering() != Ordering)
333
0
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
0
      break;
336
0
    }
337
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
0
    case GIM_CheckMemorySizeEqualTo: {
374
0
      int64_t InsnID = MatchTable[CurrentIdx++];
375
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
0
      uint64_t Size = MatchTable[CurrentIdx++];
377
0
378
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
379
0
                      dbgs() << CurrentIdx
380
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
381
0
                             << "]->memoperands() + " << MMOIdx
382
0
                             << ", Size=" << Size << ")\n");
383
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
384
0
385
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
386
0
        if (handleReject() == RejectAndGiveUp)
387
0
          return false;
388
0
        break;
389
0
      }
390
0
391
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
392
0
393
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
394
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
395
0
                             << " bytes\n");
396
0
      if (MMO->getSize() != Size)
397
0
        if (handleReject() == RejectAndGiveUp)
398
0
          return false;
399
0
400
0
      break;
401
0
    }
402
0
    case GIM_CheckMemorySizeEqualToLLT:
403
0
    case GIM_CheckMemorySizeLessThanLLT:
404
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
405
0
      int64_t InsnID = MatchTable[CurrentIdx++];
406
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
407
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
408
0
409
0
      DEBUG_WITH_TYPE(
410
0
          TgtInstructionSelector::getName(),
411
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
412
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
413
0
                         ? "EqualTo"
414
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
415
0
                               ? "GreaterThan"
416
0
                               : "LessThan")
417
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
418
0
                 << ", OpIdx=" << OpIdx << ")\n");
419
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
420
0
421
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
422
0
      if (!MO.isReg()) {
423
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
424
0
                        dbgs() << CurrentIdx << ": Not a register\n");
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
0
430
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
431
0
        if (handleReject() == RejectAndGiveUp)
432
0
          return false;
433
0
        break;
434
0
      }
435
0
436
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
437
0
438
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
439
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
440
0
          MMO->getSize() * 8 != Size) {
441
0
        if (handleReject() == RejectAndGiveUp)
442
0
          return false;
443
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
444
0
                 MMO->getSize() * 8 >= Size) {
445
0
        if (handleReject() == RejectAndGiveUp)
446
0
          return false;
447
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
448
0
                 MMO->getSize() * 8 <= Size)
449
0
        if (handleReject() == RejectAndGiveUp)
450
0
          return false;
451
0
452
0
      break;
453
0
    }
454
1.10k
    case GIM_CheckType: {
455
1.10k
      int64_t InsnID = MatchTable[CurrentIdx++];
456
1.10k
      int64_t OpIdx = MatchTable[CurrentIdx++];
457
1.10k
      int64_t TypeID = MatchTable[CurrentIdx++];
458
1.10k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
459
1.10k
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
460
1.10k
                             << "]->getOperand(" << OpIdx
461
1.10k
                             << "), TypeID=" << TypeID << ")\n");
462
1.10k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
463
1.10k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
464
1.10k
      if (!MO.isReg() ||
465
1.10k
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
466
5
        if (handleReject() == RejectAndGiveUp)
467
0
          return false;
468
1.10k
      }
469
1.10k
      break;
470
1.10k
    }
471
1.10k
    case GIM_CheckPointerToAny: {
472
0
      int64_t InsnID = MatchTable[CurrentIdx++];
473
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
474
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
475
0
476
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
477
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
478
0
                             << InsnID << "]->getOperand(" << OpIdx
479
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
480
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
481
0
      // iPTR must be looked up in the target.
482
0
      if (SizeInBits == 0) {
483
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
484
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
485
0
      }
486
0
487
0
      assert(SizeInBits != 0 && "Pointer size must be known");
488
0
489
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
490
0
      if (MO.isReg()) {
491
0
        const LLT &Ty = MRI.getType(MO.getReg());
492
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
493
0
          if (handleReject() == RejectAndGiveUp)
494
0
            return false;
495
0
      } else if (handleReject() == RejectAndGiveUp)
496
0
        return false;
497
0
498
0
      break;
499
0
    }
500
1.57k
    case GIM_CheckRegBankForClass: {
501
1.57k
      int64_t InsnID = MatchTable[CurrentIdx++];
502
1.57k
      int64_t OpIdx = MatchTable[CurrentIdx++];
503
1.57k
      int64_t RCEnum = MatchTable[CurrentIdx++];
504
1.57k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
505
1.57k
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
506
1.57k
                             << InsnID << "]->getOperand(" << OpIdx
507
1.57k
                             << "), RCEnum=" << RCEnum << ")\n");
508
1.57k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
509
1.57k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
510
1.57k
      if (!MO.isReg() ||
511
1.57k
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
512
1.57k
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
513
0
        if (handleReject() == RejectAndGiveUp)
514
0
          return false;
515
1.57k
      }
516
1.57k
      break;
517
1.57k
    }
518
1.57k
519
1.57k
    case GIM_CheckComplexPattern: {
520
0
      int64_t InsnID = MatchTable[CurrentIdx++];
521
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
522
0
      int64_t RendererID = MatchTable[CurrentIdx++];
523
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
524
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
525
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
526
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
527
0
                             << "]->getOperand(" << OpIdx
528
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
529
0
                             << ")\n");
530
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
531
0
      // FIXME: Use std::invoke() when it's available.
532
0
      ComplexRendererFns Renderer =
533
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
534
0
              State.MIs[InsnID]->getOperand(OpIdx));
535
0
      if (Renderer.hasValue())
536
0
        State.Renderers[RendererID] = Renderer.getValue();
537
0
      else
538
0
        if (handleReject() == RejectAndGiveUp)
539
0
          return false;
540
0
      break;
541
0
    }
542
0
543
183
    case GIM_CheckConstantInt: {
544
183
      int64_t InsnID = MatchTable[CurrentIdx++];
545
183
      int64_t OpIdx = MatchTable[CurrentIdx++];
546
183
      int64_t Value = MatchTable[CurrentIdx++];
547
183
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
548
183
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
549
183
                             << InsnID << "]->getOperand(" << OpIdx
550
183
                             << "), Value=" << Value << ")\n");
551
183
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
552
183
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
553
183
      if (MO.isReg()) {
554
183
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
555
183
        LLT Ty = MRI.getType(MO.getReg());
556
183
        Value = SignExtend64(Value, Ty.getSizeInBits());
557
183
558
183
        if (!isOperandImmEqual(MO, Value, MRI)) {
559
83
          if (handleReject() == RejectAndGiveUp)
560
0
            return false;
561
0
        }
562
0
      } else if (handleReject() == RejectAndGiveUp)
563
0
        return false;
564
183
565
183
      break;
566
183
    }
567
183
568
183
    case GIM_CheckLiteralInt: {
569
0
      int64_t InsnID = MatchTable[CurrentIdx++];
570
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
571
0
      int64_t Value = MatchTable[CurrentIdx++];
572
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
573
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
574
0
                             << InsnID << "]->getOperand(" << OpIdx
575
0
                             << "), Value=" << Value << ")\n");
576
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
577
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
578
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
579
0
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
0
      }
582
0
      break;
583
0
    }
584
0
585
0
    case GIM_CheckIntrinsicID: {
586
0
      int64_t InsnID = MatchTable[CurrentIdx++];
587
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
588
0
      int64_t Value = MatchTable[CurrentIdx++];
589
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
590
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
591
0
                             << InsnID << "]->getOperand(" << OpIdx
592
0
                             << "), Value=" << Value << ")\n");
593
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
594
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
595
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
596
0
        if (handleReject() == RejectAndGiveUp)
597
0
          return false;
598
0
      break;
599
0
    }
600
0
601
13
    case GIM_CheckIsMBB: {
602
13
      int64_t InsnID = MatchTable[CurrentIdx++];
603
13
      int64_t OpIdx = MatchTable[CurrentIdx++];
604
13
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
605
13
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
606
13
                             << "]->getOperand(" << OpIdx << "))\n");
607
13
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
608
13
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
609
0
        if (handleReject() == RejectAndGiveUp)
610
0
          return false;
611
13
      }
612
13
      break;
613
13
    }
614
13
615
118
    case GIM_CheckIsSafeToFold: {
616
118
      int64_t InsnID = MatchTable[CurrentIdx++];
617
118
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
618
118
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
619
118
                             << InsnID << "])\n");
620
118
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
621
118
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
622
0
        if (handleReject() == RejectAndGiveUp)
623
0
          return false;
624
118
      }
625
118
      break;
626
118
    }
627
118
    case GIM_CheckIsSameOperand: {
628
4
      int64_t InsnID = MatchTable[CurrentIdx++];
629
4
      int64_t OpIdx = MatchTable[CurrentIdx++];
630
4
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
631
4
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
632
4
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
633
4
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
634
4
                             << InsnID << "][" << OpIdx << "], MIs["
635
4
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
636
4
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
637
4
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
638
4
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
639
4
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
640
0
        if (handleReject() == RejectAndGiveUp)
641
0
          return false;
642
4
      }
643
4
      break;
644
4
    }
645
835
    case GIM_Reject:
646
835
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
647
835
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
648
835
      if (handleReject() == RejectAndGiveUp)
649
531
        return false;
650
304
      break;
651
304
652
304
    case GIR_MutateOpcode: {
653
9
      int64_t OldInsnID = MatchTable[CurrentIdx++];
654
9
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
655
9
      int64_t NewOpcode = MatchTable[CurrentIdx++];
656
9
      if (NewInsnID >= OutMIs.size())
657
9
        OutMIs.resize(NewInsnID + 1);
658
9
659
9
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
660
9
                                              State.MIs[OldInsnID]);
661
9
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
662
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
663
9
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
664
9
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
665
9
                             << NewOpcode << ")\n");
666
9
      break;
667
304
    }
668
304
669
304
    case GIR_BuildMI: {
670
257
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
671
257
      int64_t Opcode = MatchTable[CurrentIdx++];
672
257
      if (NewInsnID >= OutMIs.size())
673
248
        OutMIs.resize(NewInsnID + 1);
674
257
675
257
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
676
257
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
677
257
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
678
257
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
679
257
                             << NewInsnID << "], " << Opcode << ")\n");
680
257
      break;
681
304
    }
682
304
683
611
    case GIR_Copy: {
684
611
      int64_t NewInsnID = MatchTable[CurrentIdx++];
685
611
      int64_t OldInsnID = MatchTable[CurrentIdx++];
686
611
      int64_t OpIdx = MatchTable[CurrentIdx++];
687
611
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
688
611
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
689
611
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
690
611
                      dbgs()
691
611
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
692
611
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
693
611
      break;
694
304
    }
695
304
696
304
    case GIR_CopyOrAddZeroReg: {
697
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
698
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
700
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
701
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
702
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
703
0
      if (isOperandImmEqual(MO, 0, MRI))
704
0
        OutMIs[NewInsnID].addReg(ZeroReg);
705
0
      else
706
0
        OutMIs[NewInsnID].add(MO);
707
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
708
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
709
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
710
0
                             << OpIdx << ", " << ZeroReg << ")\n");
711
0
      break;
712
304
    }
713
304
714
304
    case GIR_CopySubReg: {
715
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
716
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
718
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
719
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
720
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
721
0
                               0, SubRegIdx);
722
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
723
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
724
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
725
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
726
0
      break;
727
304
    }
728
304
729
304
    case GIR_AddImplicitDef: {
730
0
      int64_t InsnID = MatchTable[CurrentIdx++];
731
0
      int64_t RegNum = MatchTable[CurrentIdx++];
732
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
733
0
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
734
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
735
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
736
0
                             << InsnID << "], " << RegNum << ")\n");
737
0
      break;
738
304
    }
739
304
740
304
    case GIR_AddImplicitUse: {
741
0
      int64_t InsnID = MatchTable[CurrentIdx++];
742
0
      int64_t RegNum = MatchTable[CurrentIdx++];
743
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
744
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
745
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
746
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
747
0
                             << InsnID << "], " << RegNum << ")\n");
748
0
      break;
749
304
    }
750
304
751
345
    case GIR_AddRegister: {
752
345
      int64_t InsnID = MatchTable[CurrentIdx++];
753
345
      int64_t RegNum = MatchTable[CurrentIdx++];
754
345
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
755
345
      OutMIs[InsnID].addReg(RegNum);
756
345
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
757
345
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
758
345
                             << InsnID << "], " << RegNum << ")\n");
759
345
      break;
760
304
    }
761
304
762
304
    case GIR_AddTempRegister: {
763
18
      int64_t InsnID = MatchTable[CurrentIdx++];
764
18
      int64_t TempRegID = MatchTable[CurrentIdx++];
765
18
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
766
18
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
767
18
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
768
18
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
769
18
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
770
18
                             << InsnID << "], TempRegisters[" << TempRegID
771
18
                             << "], " << TempRegFlags << ")\n");
772
18
      break;
773
304
    }
774
304
775
304
    case GIR_AddImm: {
776
283
      int64_t InsnID = MatchTable[CurrentIdx++];
777
283
      int64_t Imm = MatchTable[CurrentIdx++];
778
283
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
779
283
      OutMIs[InsnID].addImm(Imm);
780
283
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
781
283
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
782
283
                             << "], " << Imm << ")\n");
783
283
      break;
784
304
    }
785
304
786
304
    case GIR_ComplexRenderer: {
787
0
      int64_t InsnID = MatchTable[CurrentIdx++];
788
0
      int64_t RendererID = MatchTable[CurrentIdx++];
789
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
790
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
791
0
        RenderOpFn(OutMIs[InsnID]);
792
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
793
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
794
0
                             << InsnID << "], " << RendererID << ")\n");
795
0
      break;
796
304
    }
797
304
    case GIR_ComplexSubOperandRenderer: {
798
0
      int64_t InsnID = MatchTable[CurrentIdx++];
799
0
      int64_t RendererID = MatchTable[CurrentIdx++];
800
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
801
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
802
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
803
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
804
0
                      dbgs() << CurrentIdx
805
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
806
0
                             << InsnID << "], " << RendererID << ", "
807
0
                             << RenderOpID << ")\n");
808
0
      break;
809
304
    }
810
304
811
304
    case GIR_CopyConstantAsSImm: {
812
70
      int64_t NewInsnID = MatchTable[CurrentIdx++];
813
70
      int64_t OldInsnID = MatchTable[CurrentIdx++];
814
70
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
815
70
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
816
70
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
817
70
        OutMIs[NewInsnID].addImm(
818
70
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
819
70
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
820
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
821
0
      else
822
0
        llvm_unreachable("Expected Imm or CImm operand");
823
70
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
824
70
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
825
70
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
826
70
      break;
827
70
    }
828
70
829
70
    // TODO: Needs a test case once we have a pattern that uses this.
830
70
    case GIR_CopyFConstantAsFPImm: {
831
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
832
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
833
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
834
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
835
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
836
0
        OutMIs[NewInsnID].addFPImm(
837
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
838
0
      else
839
0
        llvm_unreachable("Expected FPImm operand");
840
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
841
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
842
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
843
0
      break;
844
0
    }
845
0
846
0
    case GIR_CustomRenderer: {
847
0
      int64_t InsnID = MatchTable[CurrentIdx++];
848
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
849
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
850
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
851
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
852
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
853
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
854
0
                             << RendererFnID << ")\n");
855
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
856
0
                                                     *State.MIs[OldInsnID]);
857
0
      break;
858
0
    }
859
5
    case GIR_ConstrainOperandRC: {
860
5
      int64_t InsnID = MatchTable[CurrentIdx++];
861
5
      int64_t OpIdx = MatchTable[CurrentIdx++];
862
5
      int64_t RCEnum = MatchTable[CurrentIdx++];
863
5
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
864
5
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
865
5
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
866
5
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
867
5
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
868
5
                             << InsnID << "], " << OpIdx << ", " << RCEnum
869
5
                             << ")\n");
870
5
      break;
871
0
    }
872
0
873
261
    case GIR_ConstrainSelectedInstOperands: {
874
261
      int64_t InsnID = MatchTable[CurrentIdx++];
875
261
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
876
261
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
877
261
                                       RBI);
878
261
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
879
261
                      dbgs() << CurrentIdx
880
261
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
881
261
                             << InsnID << "])\n");
882
261
      break;
883
0
    }
884
0
885
0
    case GIR_MergeMemOperands: {
886
0
      int64_t InsnID = MatchTable[CurrentIdx++];
887
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
888
0
889
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
890
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
891
0
                             << InsnID << "]");
892
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
893
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
894
0
             GIU_MergeMemOperands_EndOfList) {
895
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
896
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
897
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
898
0
          OutMIs[InsnID].addMemOperand(MMO);
899
0
      }
900
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
901
0
      break;
902
0
    }
903
0
904
248
    case GIR_EraseFromParent: {
905
248
      int64_t InsnID = MatchTable[CurrentIdx++];
906
248
      assert(State.MIs[InsnID] &&
907
248
             "Attempted to erase an undefined instruction");
908
248
      State.MIs[InsnID]->eraseFromParent();
909
248
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
910
248
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
911
248
                             << InsnID << "])\n");
912
248
      break;
913
0
    }
914
0
915
9
    case GIR_MakeTempReg: {
916
9
      int64_t TempRegID = MatchTable[CurrentIdx++];
917
9
      int64_t TypeID = MatchTable[CurrentIdx++];
918
9
919
9
      State.TempRegisters[TempRegID] =
920
9
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
921
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
922
9
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
923
9
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
924
9
      break;
925
0
    }
926
0
927
0
    case GIR_Coverage: {
928
0
      int64_t RuleID = MatchTable[CurrentIdx++];
929
0
      CoverageInfo.setCovered(RuleID);
930
0
931
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
932
0
                      dbgs()
933
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
934
0
      break;
935
0
    }
936
0
937
257
    case GIR_Done:
938
257
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
939
257
                      dbgs() << CurrentIdx << ": GIR_Done\n");
940
257
      return true;
941
0
942
0
    default:
943
0
      llvm_unreachable("Unexpected command");
944
13.6k
    }
945
13.6k
  }
946
788
}
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<42ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
423
    CodeGenCoverage &CoverageInfo) const {
56
423
57
423
  uint64_t CurrentIdx = 0;
58
423
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
423
60
423
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
423
  auto handleReject = [&]() -> RejectAction {
62
423
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
423
                    dbgs() << CurrentIdx << ": Rejected\n");
64
423
    if (OnFailResumeAt.empty())
65
423
      return RejectAndGiveUp;
66
423
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
423
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
423
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
423
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
423
    return RejectAndResume;
71
423
  };
72
423
73
5.52k
  while (true) {
74
5.52k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
5.52k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
5.52k
    switch (MatcherOpcode) {
77
5.52k
    case GIM_Try: {
78
1.18k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
1.18k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
1.18k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
1.18k
      break;
82
5.52k
    }
83
5.52k
84
5.52k
    case GIM_RecordInsn: {
85
68
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
68
      int64_t InsnID = MatchTable[CurrentIdx++];
87
68
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
68
89
68
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
68
      // any attempt to modify it.
91
68
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
68
93
68
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
68
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
68
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
68
109
68
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
68
      if ((size_t)NewInsnID < State.MIs.size())
111
0
        State.MIs[NewInsnID] = NewMI;
112
68
      else {
113
68
        assert((size_t)NewInsnID == State.MIs.size() &&
114
68
               "Expected to store MIs in order");
115
68
        State.MIs.push_back(NewMI);
116
68
      }
117
68
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
68
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
68
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
68
                             << ")\n");
121
68
      break;
122
68
    }
123
68
124
875
    case GIM_CheckFeatures: {
125
875
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
875
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
875
                      dbgs() << CurrentIdx
128
875
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
875
                             << ExpectedBitsetID << ")\n");
130
875
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
875
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
636
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
875
      }
135
875
      break;
136
875
    }
137
875
138
875
    case GIM_CheckOpcode: {
139
68
      int64_t InsnID = MatchTable[CurrentIdx++];
140
68
      int64_t Expected = MatchTable[CurrentIdx++];
141
68
142
68
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
68
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
68
145
68
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
68
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
68
                             << "], ExpectedOpcode=" << Expected
148
68
                             << ") // Got=" << Opcode << "\n");
149
68
      if (Opcode != Expected) {
150
14
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
68
      }
153
68
      break;
154
68
    }
155
68
156
423
    case GIM_SwitchOpcode: {
157
423
      int64_t InsnID = MatchTable[CurrentIdx++];
158
423
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
423
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
423
      int64_t Default = MatchTable[CurrentIdx++];
161
423
162
423
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
423
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
423
165
423
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
423
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
423
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
423
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
423
      });
170
423
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
0
        CurrentIdx = Default;
172
0
        break;
173
0
      }
174
423
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
423
      if (!CurrentIdx) {
176
68
        CurrentIdx = Default;
177
68
  break;
178
68
      }
179
355
      OnFailResumeAt.push_back(Default);
180
355
      break;
181
355
    }
182
355
183
355
    case GIM_SwitchType: {
184
219
      int64_t InsnID = MatchTable[CurrentIdx++];
185
219
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
219
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
219
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
219
      int64_t Default = MatchTable[CurrentIdx++];
189
219
190
219
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
219
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
219
193
219
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
219
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
219
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
219
               << UpperBound << "), Default=" << Default
197
219
               << ", JumpTable...) // Got=";
198
219
        if (!MO.isReg())
199
219
          dbgs() << "Not a VReg\n";
200
219
        else
201
219
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
219
      });
203
219
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
219
      const LLT Ty = MRI.getType(MO.getReg());
208
219
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
219
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
2
        CurrentIdx = Default;
211
2
        break;
212
2
      }
213
217
      const int64_t TypeID = TyI->second;
214
217
      if (TypeID < LowerBound || UpperBound <= TypeID) {
215
0
        CurrentIdx = Default;
216
0
        break;
217
0
      }
218
217
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
217
      if (!CurrentIdx) {
220
0
        CurrentIdx = Default;
221
0
        break;
222
0
      }
223
217
      OnFailResumeAt.push_back(Default);
224
217
      break;
225
217
    }
226
217
227
217
    case GIM_CheckNumOperands: {
228
0
      int64_t InsnID = MatchTable[CurrentIdx++];
229
0
      int64_t Expected = MatchTable[CurrentIdx++];
230
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
0
                             << InsnID << "], Expected=" << Expected << ")\n");
233
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
0
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
0
      }
238
0
      break;
239
0
    }
240
54
    case GIM_CheckI64ImmPredicate: {
241
54
      int64_t InsnID = MatchTable[CurrentIdx++];
242
54
      int64_t Predicate = MatchTable[CurrentIdx++];
243
54
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
54
                      dbgs()
245
54
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
54
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
54
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
54
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
54
             "Expected G_CONSTANT");
250
54
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
54
      int64_t Value = 0;
252
54
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
54
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
54
259
54
      if (!testImmPredicate_I64(Predicate, Value))
260
0
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
54
      break;
263
54
    }
264
54
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
0
    case GIM_CheckAPFloatImmPredicate: {
287
0
      int64_t InsnID = MatchTable[CurrentIdx++];
288
0
      int64_t Predicate = MatchTable[CurrentIdx++];
289
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
0
                      dbgs()
291
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
0
             "Expected G_FCONSTANT");
296
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
0
300
0
      if (!testImmPredicate_APFloat(Predicate, Value))
301
0
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
0
      break;
304
0
    }
305
0
    case GIM_CheckCxxInsnPredicate: {
306
0
      int64_t InsnID = MatchTable[CurrentIdx++];
307
0
      int64_t Predicate = MatchTable[CurrentIdx++];
308
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
0
                      dbgs()
310
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
0
315
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
0
      break;
319
0
    }
320
0
    case GIM_CheckAtomicOrdering: {
321
0
      int64_t InsnID = MatchTable[CurrentIdx++];
322
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
0
331
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
0
        if (MMO->getOrdering() != Ordering)
333
0
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
0
      break;
336
0
    }
337
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
0
    case GIM_CheckMemorySizeEqualTo: {
374
0
      int64_t InsnID = MatchTable[CurrentIdx++];
375
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
0
      uint64_t Size = MatchTable[CurrentIdx++];
377
0
378
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
379
0
                      dbgs() << CurrentIdx
380
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
381
0
                             << "]->memoperands() + " << MMOIdx
382
0
                             << ", Size=" << Size << ")\n");
383
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
384
0
385
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
386
0
        if (handleReject() == RejectAndGiveUp)
387
0
          return false;
388
0
        break;
389
0
      }
390
0
391
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
392
0
393
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
394
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
395
0
                             << " bytes\n");
396
0
      if (MMO->getSize() != Size)
397
0
        if (handleReject() == RejectAndGiveUp)
398
0
          return false;
399
0
400
0
      break;
401
0
    }
402
0
    case GIM_CheckMemorySizeEqualToLLT:
403
0
    case GIM_CheckMemorySizeLessThanLLT:
404
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
405
0
      int64_t InsnID = MatchTable[CurrentIdx++];
406
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
407
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
408
0
409
0
      DEBUG_WITH_TYPE(
410
0
          TgtInstructionSelector::getName(),
411
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
412
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
413
0
                         ? "EqualTo"
414
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
415
0
                               ? "GreaterThan"
416
0
                               : "LessThan")
417
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
418
0
                 << ", OpIdx=" << OpIdx << ")\n");
419
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
420
0
421
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
422
0
      if (!MO.isReg()) {
423
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
424
0
                        dbgs() << CurrentIdx << ": Not a register\n");
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
0
430
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
431
0
        if (handleReject() == RejectAndGiveUp)
432
0
          return false;
433
0
        break;
434
0
      }
435
0
436
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
437
0
438
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
439
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
440
0
          MMO->getSize() * 8 != Size) {
441
0
        if (handleReject() == RejectAndGiveUp)
442
0
          return false;
443
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
444
0
                 MMO->getSize() * 8 >= Size) {
445
0
        if (handleReject() == RejectAndGiveUp)
446
0
          return false;
447
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
448
0
                 MMO->getSize() * 8 <= Size)
449
0
        if (handleReject() == RejectAndGiveUp)
450
0
          return false;
451
0
452
0
      break;
453
0
    }
454
524
    case GIM_CheckType: {
455
524
      int64_t InsnID = MatchTable[CurrentIdx++];
456
524
      int64_t OpIdx = MatchTable[CurrentIdx++];
457
524
      int64_t TypeID = MatchTable[CurrentIdx++];
458
524
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
459
524
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
460
524
                             << "]->getOperand(" << OpIdx
461
524
                             << "), TypeID=" << TypeID << ")\n");
462
524
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
463
524
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
464
524
      if (!MO.isReg() ||
465
524
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
466
0
        if (handleReject() == RejectAndGiveUp)
467
0
          return false;
468
524
      }
469
524
      break;
470
524
    }
471
524
    case GIM_CheckPointerToAny: {
472
0
      int64_t InsnID = MatchTable[CurrentIdx++];
473
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
474
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
475
0
476
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
477
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
478
0
                             << InsnID << "]->getOperand(" << OpIdx
479
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
480
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
481
0
      // iPTR must be looked up in the target.
482
0
      if (SizeInBits == 0) {
483
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
484
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(0);
485
0
      }
486
0
487
0
      assert(SizeInBits != 0 && "Pointer size must be known");
488
0
489
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
490
0
      if (MO.isReg()) {
491
0
        const LLT &Ty = MRI.getType(MO.getReg());
492
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
493
0
          if (handleReject() == RejectAndGiveUp)
494
0
            return false;
495
0
      } else if (handleReject() == RejectAndGiveUp)
496
0
        return false;
497
0
498
0
      break;
499
0
    }
500
640
    case GIM_CheckRegBankForClass: {
501
640
      int64_t InsnID = MatchTable[CurrentIdx++];
502
640
      int64_t OpIdx = MatchTable[CurrentIdx++];
503
640
      int64_t RCEnum = MatchTable[CurrentIdx++];
504
640
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
505
640
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
506
640
                             << InsnID << "]->getOperand(" << OpIdx
507
640
                             << "), RCEnum=" << RCEnum << ")\n");
508
640
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
509
640
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
510
640
      if (!MO.isReg() ||
511
640
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
512
640
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
513
0
        if (handleReject() == RejectAndGiveUp)
514
0
          return false;
515
640
      }
516
640
      break;
517
640
    }
518
640
519
640
    case GIM_CheckComplexPattern: {
520
0
      int64_t InsnID = MatchTable[CurrentIdx++];
521
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
522
0
      int64_t RendererID = MatchTable[CurrentIdx++];
523
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
524
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
525
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
526
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
527
0
                             << "]->getOperand(" << OpIdx
528
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
529
0
                             << ")\n");
530
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
531
0
      // FIXME: Use std::invoke() when it's available.
532
0
      ComplexRendererFns Renderer =
533
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
534
0
              State.MIs[InsnID]->getOperand(OpIdx));
535
0
      if (Renderer.hasValue())
536
0
        State.Renderers[RendererID] = Renderer.getValue();
537
0
      else
538
0
        if (handleReject() == RejectAndGiveUp)
539
0
          return false;
540
0
      break;
541
0
    }
542
0
543
8
    case GIM_CheckConstantInt: {
544
8
      int64_t InsnID = MatchTable[CurrentIdx++];
545
8
      int64_t OpIdx = MatchTable[CurrentIdx++];
546
8
      int64_t Value = MatchTable[CurrentIdx++];
547
8
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
548
8
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
549
8
                             << InsnID << "]->getOperand(" << OpIdx
550
8
                             << "), Value=" << Value << ")\n");
551
8
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
552
8
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
553
8
      if (MO.isReg()) {
554
8
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
555
8
        LLT Ty = MRI.getType(MO.getReg());
556
8
        Value = SignExtend64(Value, Ty.getSizeInBits());
557
8
558
8
        if (!isOperandImmEqual(MO, Value, MRI)) {
559
8
          if (handleReject() == RejectAndGiveUp)
560
0
            return false;
561
0
        }
562
0
      } else if (handleReject() == RejectAndGiveUp)
563
0
        return false;
564
8
565
8
      break;
566
8
    }
567
8
568
8
    case GIM_CheckLiteralInt: {
569
0
      int64_t InsnID = MatchTable[CurrentIdx++];
570
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
571
0
      int64_t Value = MatchTable[CurrentIdx++];
572
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
573
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
574
0
                             << InsnID << "]->getOperand(" << OpIdx
575
0
                             << "), Value=" << Value << ")\n");
576
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
577
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
578
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
579
0
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
0
      }
582
0
      break;
583
0
    }
584
0
585
0
    case GIM_CheckIntrinsicID: {
586
0
      int64_t InsnID = MatchTable[CurrentIdx++];
587
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
588
0
      int64_t Value = MatchTable[CurrentIdx++];
589
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
590
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
591
0
                             << InsnID << "]->getOperand(" << OpIdx
592
0
                             << "), Value=" << Value << ")\n");
593
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
594
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
595
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
596
0
        if (handleReject() == RejectAndGiveUp)
597
0
          return false;
598
0
      break;
599
0
    }
600
0
601
16
    case GIM_CheckIsMBB: {
602
16
      int64_t InsnID = MatchTable[CurrentIdx++];
603
16
      int64_t OpIdx = MatchTable[CurrentIdx++];
604
16
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
605
16
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
606
16
                             << "]->getOperand(" << OpIdx << "))\n");
607
16
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
608
16
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
609
0
        if (handleReject() == RejectAndGiveUp)
610
0
          return false;
611
16
      }
612
16
      break;
613
16
    }
614
16
615
54
    case GIM_CheckIsSafeToFold: {
616
54
      int64_t InsnID = MatchTable[CurrentIdx++];
617
54
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
618
54
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
619
54
                             << InsnID << "])\n");
620
54
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
621
54
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
622
0
        if (handleReject() == RejectAndGiveUp)
623
0
          return false;
624
54
      }
625
54
      break;
626
54
    }
627
54
    case GIM_CheckIsSameOperand: {
628
0
      int64_t InsnID = MatchTable[CurrentIdx++];
629
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
630
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
631
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
632
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
633
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
634
0
                             << InsnID << "][" << OpIdx << "], MIs["
635
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
636
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
637
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
638
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
639
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
640
0
        if (handleReject() == RejectAndGiveUp)
641
0
          return false;
642
0
      }
643
0
      break;
644
0
    }
645
459
    case GIM_Reject:
646
459
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
647
459
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
648
459
      if (handleReject() == RejectAndGiveUp)
649
206
        return false;
650
253
      break;
651
253
652
253
    case GIR_MutateOpcode: {
653
156
      int64_t OldInsnID = MatchTable[CurrentIdx++];
654
156
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
655
156
      int64_t NewOpcode = MatchTable[CurrentIdx++];
656
156
      if (NewInsnID >= OutMIs.size())
657
156
        OutMIs.resize(NewInsnID + 1);
658
156
659
156
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
660
156
                                              State.MIs[OldInsnID]);
661
156
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
662
156
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
663
156
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
664
156
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
665
156
                             << NewOpcode << ")\n");
666
156
      break;
667
253
    }
668
253
669
253
    case GIR_BuildMI: {
670
61
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
671
61
      int64_t Opcode = MatchTable[CurrentIdx++];
672
61
      if (NewInsnID >= OutMIs.size())
673
61
        OutMIs.resize(NewInsnID + 1);
674
61
675
61
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
676
61
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
677
61
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
678
61
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
679
61
                             << NewInsnID << "], " << Opcode << ")\n");
680
61
      break;
681
253
    }
682
253
683
253
    case GIR_Copy: {
684
136
      int64_t NewInsnID = MatchTable[CurrentIdx++];
685
136
      int64_t OldInsnID = MatchTable[CurrentIdx++];
686
136
      int64_t OpIdx = MatchTable[CurrentIdx++];
687
136
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
688
136
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
689
136
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
690
136
                      dbgs()
691
136
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
692
136
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
693
136
      break;
694
253
    }
695
253
696
253
    case GIR_CopyOrAddZeroReg: {
697
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
698
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
699
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
700
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
701
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
702
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
703
0
      if (isOperandImmEqual(MO, 0, MRI))
704
0
        OutMIs[NewInsnID].addReg(ZeroReg);
705
0
      else
706
0
        OutMIs[NewInsnID].add(MO);
707
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
708
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
709
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
710
0
                             << OpIdx << ", " << ZeroReg << ")\n");
711
0
      break;
712
253
    }
713
253
714
253
    case GIR_CopySubReg: {
715
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
716
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
717
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
718
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
719
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
720
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
721
0
                               0, SubRegIdx);
722
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
723
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
724
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
725
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
726
0
      break;
727
253
    }
728
253
729
253
    case GIR_AddImplicitDef: {
730
32
      int64_t InsnID = MatchTable[CurrentIdx++];
731
32
      int64_t RegNum = MatchTable[CurrentIdx++];
732
32
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
733
32
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
734
32
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
735
32
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
736
32
                             << InsnID << "], " << RegNum << ")\n");
737
32
      break;
738
253
    }
739
253
740
253
    case GIR_AddImplicitUse: {
741
0
      int64_t InsnID = MatchTable[CurrentIdx++];
742
0
      int64_t RegNum = MatchTable[CurrentIdx++];
743
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
744
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
745
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
746
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
747
0
                             << InsnID << "], " << RegNum << ")\n");
748
0
      break;
749
253
    }
750
253
751
253
    case GIR_AddRegister: {
752
0
      int64_t InsnID = MatchTable[CurrentIdx++];
753
0
      int64_t RegNum = MatchTable[CurrentIdx++];
754
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
755
0
      OutMIs[InsnID].addReg(RegNum);
756
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
757
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
758
0
                             << InsnID << "], " << RegNum << ")\n");
759
0
      break;
760
253
    }
761
253
762
253
    case GIR_AddTempRegister: {
763
0
      int64_t InsnID = MatchTable[CurrentIdx++];
764
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
765
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
766
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
767
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
768
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
769
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
770
0
                             << InsnID << "], TempRegisters[" << TempRegID
771
0
                             << "], " << TempRegFlags << ")\n");
772
0
      break;
773
253
    }
774
253
775
253
    case GIR_AddImm: {
776
0
      int64_t InsnID = MatchTable[CurrentIdx++];
777
0
      int64_t Imm = MatchTable[CurrentIdx++];
778
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
779
0
      OutMIs[InsnID].addImm(Imm);
780
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
781
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
782
0
                             << "], " << Imm << ")\n");
783
0
      break;
784
253
    }
785
253
786
253
    case GIR_ComplexRenderer: {
787
0
      int64_t InsnID = MatchTable[CurrentIdx++];
788
0
      int64_t RendererID = MatchTable[CurrentIdx++];
789
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
790
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
791
0
        RenderOpFn(OutMIs[InsnID]);
792
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
793
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
794
0
                             << InsnID << "], " << RendererID << ")\n");
795
0
      break;
796
253
    }
797
253
    case GIR_ComplexSubOperandRenderer: {
798
0
      int64_t InsnID = MatchTable[CurrentIdx++];
799
0
      int64_t RendererID = MatchTable[CurrentIdx++];
800
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
801
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
802
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
803
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
804
0
                      dbgs() << CurrentIdx
805
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
806
0
                             << InsnID << "], " << RendererID << ", "
807
0
                             << RenderOpID << ")\n");
808
0
      break;
809
253
    }
810
253
811
253
    case GIR_CopyConstantAsSImm: {
812
54
      int64_t NewInsnID = MatchTable[CurrentIdx++];
813
54
      int64_t OldInsnID = MatchTable[CurrentIdx++];
814
54
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
815
54
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
816
54
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
817
54
        OutMIs[NewInsnID].addImm(
818
54
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
819
54
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
820
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
821
0
      else
822
0
        llvm_unreachable("Expected Imm or CImm operand");
823
54
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
824
54
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
825
54
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
826
54
      break;
827
54
    }
828
54
829
54
    // TODO: Needs a test case once we have a pattern that uses this.
830
54
    case GIR_CopyFConstantAsFPImm: {
831
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
832
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
833
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
834
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
835
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
836
0
        OutMIs[NewInsnID].addFPImm(
837
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
838
0
      else
839
0
        llvm_unreachable("Expected FPImm operand");
840
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
841
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
842
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
843
0
      break;
844
0
    }
845
0
846
0
    case GIR_CustomRenderer: {
847
0
      int64_t InsnID = MatchTable[CurrentIdx++];
848
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
849
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
850
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
851
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
852
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
853
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
854
0
                             << RendererFnID << ")\n");
855
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
856
0
                                                     *State.MIs[OldInsnID]);
857
0
      break;
858
0
    }
859
0
    case GIR_ConstrainOperandRC: {
860
0
      int64_t InsnID = MatchTable[CurrentIdx++];
861
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
862
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
863
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
864
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
865
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
866
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
867
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
868
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
869
0
                             << ")\n");
870
0
      break;
871
0
    }
872
0
873
217
    case GIR_ConstrainSelectedInstOperands: {
874
217
      int64_t InsnID = MatchTable[CurrentIdx++];
875
217
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
876
217
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
877
217
                                       RBI);
878
217
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
879
217
                      dbgs() << CurrentIdx
880
217
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
881
217
                             << InsnID << "])\n");
882
217
      break;
883
0
    }
884
0
885
0
    case GIR_MergeMemOperands: {
886
0
      int64_t InsnID = MatchTable[CurrentIdx++];
887
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
888
0
889
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
890
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
891
0
                             << InsnID << "]");
892
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
893
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
894
0
             GIU_MergeMemOperands_EndOfList) {
895
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
896
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
897
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
898
0
          OutMIs[InsnID].addMemOperand(MMO);
899
0
      }
900
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
901
0
      break;
902
0
    }
903
0
904
61
    case GIR_EraseFromParent: {
905
61
      int64_t InsnID = MatchTable[CurrentIdx++];
906
61
      assert(State.MIs[InsnID] &&
907
61
             "Attempted to erase an undefined instruction");
908
61
      State.MIs[InsnID]->eraseFromParent();
909
61
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
910
61
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
911
61
                             << InsnID << "])\n");
912
61
      break;
913
0
    }
914
0
915
0
    case GIR_MakeTempReg: {
916
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
917
0
      int64_t TypeID = MatchTable[CurrentIdx++];
918
0
919
0
      State.TempRegisters[TempRegID] =
920
0
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
921
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
922
0
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
923
0
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
924
0
      break;
925
0
    }
926
0
927
0
    case GIR_Coverage: {
928
0
      int64_t RuleID = MatchTable[CurrentIdx++];
929
0
      CoverageInfo.setCovered(RuleID);
930
0
931
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
932
0
                      dbgs()
933
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
934
0
      break;
935
0
    }
936
0
937
217
    case GIR_Done:
938
217
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
939
217
                      dbgs() << CurrentIdx << ": GIR_Done\n");
940
217
      return true;
941
0
942
0
    default:
943
0
      llvm_unreachable("Unexpected command");
944
5.52k
    }
945
5.52k
  }
946
423
}
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<112ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<112ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<112ul> const&, llvm::CodeGenCoverage