Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
Line
Count
Source (jump to first uncovered line)
1
//===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file This file declares the API for the instruction selector.
10
/// This class is responsible for selecting machine instructions.
11
/// It's implemented by the target. It's used by the InstructionSelect pass.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
16
#define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
17
18
#include "llvm/ADT/SmallVector.h"
19
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
20
#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
21
#include "llvm/CodeGen/GlobalISel/Utils.h"
22
#include "llvm/CodeGen/MachineInstrBuilder.h"
23
#include "llvm/CodeGen/MachineOperand.h"
24
#include "llvm/CodeGen/MachineRegisterInfo.h"
25
#include "llvm/CodeGen/TargetInstrInfo.h"
26
#include "llvm/CodeGen/TargetOpcodes.h"
27
#include "llvm/CodeGen/TargetRegisterInfo.h"
28
#include "llvm/IR/Constants.h"
29
#include "llvm/Support/Debug.h"
30
#include "llvm/Support/ErrorHandling.h"
31
#include "llvm/Support/raw_ostream.h"
32
#include <cassert>
33
#include <cstddef>
34
#include <cstdint>
35
36
namespace llvm {
37
38
/// GlobalISel PatFrag Predicates
39
enum {
40
  GIPFP_I64_Invalid = 0,
41
  GIPFP_APInt_Invalid = 0,
42
  GIPFP_APFloat_Invalid = 0,
43
  GIPFP_MI_Invalid = 0,
44
};
45
46
template <class TgtInstructionSelector, class PredicateBitset,
47
          class ComplexMatcherMemFn, class CustomRendererFn>
48
bool InstructionSelector::executeMatchTable(
49
    TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
50
    const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
51
        &ISelInfo,
52
    const int64_t *MatchTable, const TargetInstrInfo &TII,
53
    MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
54
    const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
55
5.94M
    CodeGenCoverage &CoverageInfo) const {
56
5.94M
57
5.94M
  uint64_t CurrentIdx = 0;
58
5.94M
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
5.94M
60
5.94M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
9.66M
  auto handleReject = [&]() -> RejectAction {
62
9.66M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
9.66M
                    dbgs() << CurrentIdx << ": Rejected\n");
64
9.66M
    if (OnFailResumeAt.empty())
65
3.13M
      return RejectAndGiveUp;
66
6.52M
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
6.52M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
6.52M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
6.52M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
6.52M
    return RejectAndResume;
71
6.52M
  };
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<22ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<22ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<22ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
9.64M
  auto handleReject = [&]() -> RejectAction {
62
9.64M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
9.64M
                    dbgs() << CurrentIdx << ": Rejected\n");
64
9.64M
    if (OnFailResumeAt.empty())
65
3.13M
      return RejectAndGiveUp;
66
6.51M
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
6.51M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
6.51M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
6.51M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
6.51M
    return RejectAndResume;
71
6.51M
  };
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<46ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<46ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<46ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
6.53k
  auto handleReject = [&]() -> RejectAction {
62
6.53k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
6.53k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
6.53k
    if (OnFailResumeAt.empty())
65
408
      return RejectAndGiveUp;
66
6.12k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
6.12k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
6.12k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
6.12k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
6.12k
    return RejectAndResume;
71
6.12k
  };
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<70ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<70ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<70ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
3.30k
  auto handleReject = [&]() -> RejectAction {
62
3.30k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
3.30k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
3.30k
    if (OnFailResumeAt.empty())
65
820
      return RejectAndGiveUp;
66
2.48k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
2.48k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
2.48k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
2.48k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
2.48k
    return RejectAndResume;
71
2.48k
  };
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<42ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
3.16k
  auto handleReject = [&]() -> RejectAction {
62
3.16k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
3.16k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
3.16k
    if (OnFailResumeAt.empty())
65
861
      return RejectAndGiveUp;
66
2.30k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
2.30k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
2.30k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
2.30k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
2.30k
    return RejectAndResume;
71
2.30k
  };
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<114ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const
Line
Count
Source
61
4.05k
  auto handleReject = [&]() -> RejectAction {
62
4.05k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
4.05k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
4.05k
    if (OnFailResumeAt.empty())
65
1.46k
      return RejectAndGiveUp;
66
2.59k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
2.59k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
2.59k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
2.59k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
2.59k
    return RejectAndResume;
71
2.59k
  };
72
5.94M
73
61.9M
  while (true) {
74
61.9M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
61.9M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
61.9M
    switch (MatcherOpcode) {
77
61.9M
    case GIM_Try: {
78
9.15M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
9.15M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
9.15M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
9.15M
      break;
82
61.9M
    }
83
61.9M
84
61.9M
    case GIM_RecordInsn: {
85
3.04M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
3.04M
      int64_t InsnID = MatchTable[CurrentIdx++];
87
3.04M
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
3.04M
89
3.04M
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
3.04M
      // any attempt to modify it.
91
3.04M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
3.04M
93
3.04M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
3.04M
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
3.04M
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
3.04M
109
3.04M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
3.04M
      if ((size_t)NewInsnID < State.MIs.size())
111
1.98M
        State.MIs[NewInsnID] = NewMI;
112
1.06M
      else {
113
1.06M
        assert((size_t)NewInsnID == State.MIs.size() &&
114
1.06M
               "Expected to store MIs in order");
115
1.06M
        State.MIs.push_back(NewMI);
116
1.06M
      }
117
3.04M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
3.04M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
3.04M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
3.04M
                             << ")\n");
121
3.04M
      break;
122
3.04M
    }
123
3.04M
124
3.04M
    case GIM_CheckFeatures: {
125
918k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
918k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
918k
                      dbgs() << CurrentIdx
128
918k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
918k
                             << ExpectedBitsetID << ")\n");
130
918k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
918k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
84.2k
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
918k
      }
135
918k
      break;
136
918k
    }
137
918k
138
3.04M
    case GIM_CheckOpcode: {
139
3.04M
      int64_t InsnID = MatchTable[CurrentIdx++];
140
3.04M
      int64_t Expected = MatchTable[CurrentIdx++];
141
3.04M
142
3.04M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
3.04M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
3.04M
145
3.04M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
3.04M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
3.04M
                             << "], ExpectedOpcode=" << Expected
148
3.04M
                             << ") // Got=" << Opcode << "\n");
149
3.04M
      if (Opcode != Expected) {
150
2.92M
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
3.04M
      }
153
3.04M
      break;
154
3.04M
    }
155
3.04M
156
5.94M
    case GIM_SwitchOpcode: {
157
5.94M
      int64_t InsnID = MatchTable[CurrentIdx++];
158
5.94M
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
5.94M
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
5.94M
      int64_t Default = MatchTable[CurrentIdx++];
161
5.94M
162
5.94M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
5.94M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
5.94M
165
5.94M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
5.94M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
5.94M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
5.94M
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
5.94M
      });
170
5.94M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
769
        CurrentIdx = Default;
172
769
        break;
173
769
      }
174
5.94M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
5.94M
      if (!CurrentIdx) {
176
1.82M
        CurrentIdx = Default;
177
1.82M
  break;
178
1.82M
      }
179
4.11M
      OnFailResumeAt.push_back(Default);
180
4.11M
      break;
181
4.11M
    }
182
4.11M
183
4.11M
    case GIM_SwitchType: {
184
3.76M
      int64_t InsnID = MatchTable[CurrentIdx++];
185
3.76M
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
3.76M
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
3.76M
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
3.76M
      int64_t Default = MatchTable[CurrentIdx++];
189
3.76M
190
3.76M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
3.76M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
3.76M
193
3.76M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
3.76M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
3.76M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
3.76M
               << UpperBound << "), Default=" << Default
197
3.76M
               << ", JumpTable...) // Got=";
198
3.76M
        if (!MO.isReg())
199
3.76M
          dbgs() << "Not a VReg\n";
200
3.76M
        else
201
3.76M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
3.76M
      });
203
3.76M
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
3.76M
      const LLT Ty = MRI.getType(MO.getReg());
208
3.76M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
3.76M
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
913k
        CurrentIdx = Default;
211
913k
        break;
212
913k
      }
213
2.85M
      const int64_t TypeID = TyI->second;
214
2.85M
      if (TypeID < LowerBound || 
UpperBound <= TypeID2.66M
) {
215
191k
        CurrentIdx = Default;
216
191k
        break;
217
191k
      }
218
2.66M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
2.66M
      if (!CurrentIdx) {
220
490
        CurrentIdx = Default;
221
490
        break;
222
490
      }
223
2.65M
      OnFailResumeAt.push_back(Default);
224
2.65M
      break;
225
2.65M
    }
226
2.65M
227
2.65M
    case GIM_CheckNumOperands: {
228
22.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
229
22.9k
      int64_t Expected = MatchTable[CurrentIdx++];
230
22.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
22.9k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
22.9k
                             << InsnID << "], Expected=" << Expected << ")\n");
233
22.9k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
22.9k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
17.2k
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
22.9k
      }
238
22.9k
      break;
239
22.9k
    }
240
63.4k
    case GIM_CheckI64ImmPredicate: {
241
63.4k
      int64_t InsnID = MatchTable[CurrentIdx++];
242
63.4k
      int64_t Predicate = MatchTable[CurrentIdx++];
243
63.4k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
63.4k
                      dbgs()
245
63.4k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
63.4k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
63.4k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
63.4k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
63.4k
             "Expected G_CONSTANT");
250
63.4k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
63.4k
      int64_t Value = 0;
252
63.4k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
63.4k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
63.4k
259
63.4k
      if (!testImmPredicate_I64(Predicate, Value))
260
58
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
63.4k
      break;
263
63.4k
    }
264
63.4k
    case GIM_CheckAPIntImmPredicate: {
265
1
      int64_t InsnID = MatchTable[CurrentIdx++];
266
1
      int64_t Predicate = MatchTable[CurrentIdx++];
267
1
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
1
                      dbgs()
269
1
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
1
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
1
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
1
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
1
             "Expected G_CONSTANT");
274
1
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
1
      APInt Value;
276
1
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
1
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
1
      else
279
1
        
llvm_unreachable0
("Expected Imm or CImm operand");
280
1
281
1
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
1
      break;
285
1
    }
286
24.3k
    case GIM_CheckAPFloatImmPredicate: {
287
24.3k
      int64_t InsnID = MatchTable[CurrentIdx++];
288
24.3k
      int64_t Predicate = MatchTable[CurrentIdx++];
289
24.3k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
24.3k
                      dbgs()
291
24.3k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
24.3k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
24.3k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
24.3k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
24.3k
             "Expected G_FCONSTANT");
296
24.3k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
24.3k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
24.3k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
24.3k
300
24.3k
      if (!testImmPredicate_APFloat(Predicate, Value))
301
20.9k
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
24.3k
      break;
304
24.3k
    }
305
24.3k
    case GIM_CheckCxxInsnPredicate: {
306
406
      int64_t InsnID = MatchTable[CurrentIdx++];
307
406
      int64_t Predicate = MatchTable[CurrentIdx++];
308
406
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
406
                      dbgs()
310
406
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
406
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
406
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
406
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
406
315
406
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
4
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
406
      break;
319
406
    }
320
915k
    case GIM_CheckAtomicOrdering: {
321
915k
      int64_t InsnID = MatchTable[CurrentIdx++];
322
915k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
915k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
915k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
915k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
915k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
915k
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
915k
331
915k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
915k
        if (MMO->getOrdering() != Ordering)
333
4.29k
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
915k
      break;
336
915k
    }
337
915k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
2.19k
    case GIM_CheckMemoryAddressSpace: {
374
2.19k
      int64_t InsnID = MatchTable[CurrentIdx++];
375
2.19k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
2.19k
      // This accepts a list of possible address spaces.
377
2.19k
      const int NumAddrSpace = MatchTable[CurrentIdx++];
378
2.19k
379
2.19k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
380
0
        if (handleReject() == RejectAndGiveUp)
381
0
          return false;
382
0
        break;
383
0
      }
384
2.19k
385
2.19k
      // Need to still jump to the end of the list of address spaces if we find
386
2.19k
      // a match earlier.
387
2.19k
      const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
388
2.19k
389
2.19k
      const MachineMemOperand *MMO
390
2.19k
        = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
391
2.19k
      const unsigned MMOAddrSpace = MMO->getAddrSpace();
392
2.19k
393
2.19k
      bool Success = false;
394
4.08k
      for (int I = 0; I != NumAddrSpace; 
++I1.89k
) {
395
2.54k
        unsigned AddrSpace = MatchTable[CurrentIdx++];
396
2.54k
        DEBUG_WITH_TYPE(
397
2.54k
          TgtInstructionSelector::getName(),
398
2.54k
          dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
399
2.54k
                 << AddrSpace << '\n');
400
2.54k
401
2.54k
        if (AddrSpace == MMOAddrSpace) {
402
649
          Success = true;
403
649
          break;
404
649
        }
405
2.54k
      }
406
2.19k
407
2.19k
      CurrentIdx = LastIdx;
408
2.19k
      if (!Success && 
handleReject() == RejectAndGiveUp1.54k
)
409
0
        return false;
410
2.19k
      break;
411
2.19k
    }
412
152k
    case GIM_CheckMemorySizeEqualTo: {
413
152k
      int64_t InsnID = MatchTable[CurrentIdx++];
414
152k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
415
152k
      uint64_t Size = MatchTable[CurrentIdx++];
416
152k
417
152k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
418
152k
                      dbgs() << CurrentIdx
419
152k
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
420
152k
                             << "]->memoperands() + " << MMOIdx
421
152k
                             << ", Size=" << Size << ")\n");
422
152k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
423
152k
424
152k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
152k
430
152k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
431
152k
432
152k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
433
152k
                      dbgs() << MMO->getSize() << " bytes vs " << Size
434
152k
                             << " bytes\n");
435
152k
      if (MMO->getSize() != Size)
436
92.2k
        if (handleReject() == RejectAndGiveUp)
437
0
          return false;
438
152k
439
152k
      break;
440
152k
    }
441
450k
    case GIM_CheckMemorySizeEqualToLLT:
442
450k
    case GIM_CheckMemorySizeLessThanLLT:
443
450k
    case GIM_CheckMemorySizeGreaterThanLLT: {
444
450k
      int64_t InsnID = MatchTable[CurrentIdx++];
445
450k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
446
450k
      int64_t OpIdx = MatchTable[CurrentIdx++];
447
450k
448
450k
      DEBUG_WITH_TYPE(
449
450k
          TgtInstructionSelector::getName(),
450
450k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
451
450k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
452
450k
                         ? "EqualTo"
453
450k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
454
450k
                               ? "GreaterThan"
455
450k
                               : "LessThan")
456
450k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
457
450k
                 << ", OpIdx=" << OpIdx << ")\n");
458
450k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
459
450k
460
450k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
461
450k
      if (!MO.isReg()) {
462
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
463
0
                        dbgs() << CurrentIdx << ": Not a register\n");
464
0
        if (handleReject() == RejectAndGiveUp)
465
0
          return false;
466
0
        break;
467
0
      }
468
450k
469
450k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
470
0
        if (handleReject() == RejectAndGiveUp)
471
0
          return false;
472
0
        break;
473
0
      }
474
450k
475
450k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
476
450k
477
450k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
478
450k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
479
450k
          
MMO->getSizeInBits() != Size447k
) {
480
534
        if (handleReject() == RejectAndGiveUp)
481
0
          return false;
482
449k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
483
449k
                 
MMO->getSizeInBits() >= Size2.58k
) {
484
1.72k
        if (handleReject() == RejectAndGiveUp)
485
0
          return false;
486
447k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
487
447k
                 
MMO->getSizeInBits() <= Size0
)
488
0
        if (handleReject() == RejectAndGiveUp)
489
0
          return false;
490
450k
491
450k
      break;
492
450k
    }
493
2.03M
    case GIM_CheckType: {
494
2.03M
      int64_t InsnID = MatchTable[CurrentIdx++];
495
2.03M
      int64_t OpIdx = MatchTable[CurrentIdx++];
496
2.03M
      int64_t TypeID = MatchTable[CurrentIdx++];
497
2.03M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498
2.03M
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
499
2.03M
                             << "]->getOperand(" << OpIdx
500
2.03M
                             << "), TypeID=" << TypeID << ")\n");
501
2.03M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
502
2.03M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
503
2.03M
      if (!MO.isReg() ||
504
2.03M
          
MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]2.03M
) {
505
76.7k
        if (handleReject() == RejectAndGiveUp)
506
0
          return false;
507
2.03M
      }
508
2.03M
      break;
509
2.03M
    }
510
2.03M
    case GIM_CheckPointerToAny: {
511
877k
      int64_t InsnID = MatchTable[CurrentIdx++];
512
877k
      int64_t OpIdx = MatchTable[CurrentIdx++];
513
877k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
514
877k
515
877k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
516
877k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
517
877k
                             << InsnID << "]->getOperand(" << OpIdx
518
877k
                             << "), SizeInBits=" << SizeInBits << ")\n");
519
877k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
520
877k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
521
877k
      const LLT Ty = MRI.getType(MO.getReg());
522
877k
523
877k
      // iPTR must be looked up in the target.
524
877k
      if (SizeInBits == 0) {
525
876k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
526
876k
        const unsigned AddrSpace = Ty.getAddressSpace();
527
876k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
528
876k
      }
529
877k
530
877k
      assert(SizeInBits != 0 && "Pointer size must be known");
531
877k
532
877k
      if (MO.isReg()) {
533
877k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
534
0
          if (handleReject() == RejectAndGiveUp)
535
0
            return false;
536
0
      } else if (handleReject() == RejectAndGiveUp)
537
0
        return false;
538
877k
539
877k
      break;
540
877k
    }
541
7.67M
    case GIM_CheckRegBankForClass: {
542
7.67M
      int64_t InsnID = MatchTable[CurrentIdx++];
543
7.67M
      int64_t OpIdx = MatchTable[CurrentIdx++];
544
7.67M
      int64_t RCEnum = MatchTable[CurrentIdx++];
545
7.67M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
546
7.67M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
547
7.67M
                             << InsnID << "]->getOperand(" << OpIdx
548
7.67M
                             << "), RCEnum=" << RCEnum << ")\n");
549
7.67M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
550
7.67M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
551
7.67M
      if (!MO.isReg() ||
552
7.67M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
553
7.67M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
554
744k
        if (handleReject() == RejectAndGiveUp)
555
0
          return false;
556
7.67M
      }
557
7.67M
      break;
558
7.67M
    }
559
7.67M
560
7.67M
    case GIM_CheckComplexPattern: {
561
1.44M
      int64_t InsnID = MatchTable[CurrentIdx++];
562
1.44M
      int64_t OpIdx = MatchTable[CurrentIdx++];
563
1.44M
      int64_t RendererID = MatchTable[CurrentIdx++];
564
1.44M
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
565
1.44M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
566
1.44M
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
567
1.44M
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
568
1.44M
                             << "]->getOperand(" << OpIdx
569
1.44M
                             << "), ComplexPredicateID=" << ComplexPredicateID
570
1.44M
                             << ")\n");
571
1.44M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572
1.44M
      // FIXME: Use std::invoke() when it's available.
573
1.44M
      ComplexRendererFns Renderer =
574
1.44M
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
575
1.44M
              State.MIs[InsnID]->getOperand(OpIdx));
576
1.44M
      if (Renderer.hasValue())
577
929k
        State.Renderers[RendererID] = Renderer.getValue();
578
512k
      else
579
512k
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
1.44M
      break;
582
1.44M
    }
583
1.44M
584
1.44M
    case GIM_CheckConstantInt: {
585
286k
      int64_t InsnID = MatchTable[CurrentIdx++];
586
286k
      int64_t OpIdx = MatchTable[CurrentIdx++];
587
286k
      int64_t Value = MatchTable[CurrentIdx++];
588
286k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
589
286k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
590
286k
                             << InsnID << "]->getOperand(" << OpIdx
591
286k
                             << "), Value=" << Value << ")\n");
592
286k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593
286k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
594
286k
      if (MO.isReg()) {
595
286k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
596
286k
        LLT Ty = MRI.getType(MO.getReg());
597
286k
        Value = SignExtend64(Value, Ty.getSizeInBits());
598
286k
599
286k
        if (!isOperandImmEqual(MO, Value, MRI)) {
600
255k
          if (handleReject() == RejectAndGiveUp)
601
0
            return false;
602
0
        }
603
0
      } else if (handleReject() == RejectAndGiveUp)
604
0
        return false;
605
286k
606
286k
      break;
607
286k
    }
608
286k
609
286k
    case GIM_CheckLiteralInt: {
610
45
      int64_t InsnID = MatchTable[CurrentIdx++];
611
45
      int64_t OpIdx = MatchTable[CurrentIdx++];
612
45
      int64_t Value = MatchTable[CurrentIdx++];
613
45
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
614
45
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
615
45
                             << InsnID << "]->getOperand(" << OpIdx
616
45
                             << "), Value=" << Value << ")\n");
617
45
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
618
45
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
619
45
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
620
23
        if (handleReject() == RejectAndGiveUp)
621
0
          return false;
622
45
      }
623
45
      break;
624
45
    }
625
45
626
242k
    case GIM_CheckIntrinsicID: {
627
242k
      int64_t InsnID = MatchTable[CurrentIdx++];
628
242k
      int64_t OpIdx = MatchTable[CurrentIdx++];
629
242k
      int64_t Value = MatchTable[CurrentIdx++];
630
242k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
631
242k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
632
242k
                             << InsnID << "]->getOperand(" << OpIdx
633
242k
                             << "), Value=" << Value << ")\n");
634
242k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
635
242k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
636
242k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value233k
)
637
235k
        if (handleReject() == RejectAndGiveUp)
638
0
          return false;
639
242k
      break;
640
242k
    }
641
242k
642
307k
    case GIM_CheckIsMBB: {
643
307k
      int64_t InsnID = MatchTable[CurrentIdx++];
644
307k
      int64_t OpIdx = MatchTable[CurrentIdx++];
645
307k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
646
307k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
647
307k
                             << "]->getOperand(" << OpIdx << "))\n");
648
307k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
649
307k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
650
0
        if (handleReject() == RejectAndGiveUp)
651
0
          return false;
652
307k
      }
653
307k
      break;
654
307k
    }
655
307k
656
307k
    case GIM_CheckIsSafeToFold: {
657
73.3k
      int64_t InsnID = MatchTable[CurrentIdx++];
658
73.3k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
659
73.3k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
660
73.3k
                             << InsnID << "])\n");
661
73.3k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
662
73.3k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
663
0
        if (handleReject() == RejectAndGiveUp)
664
0
          return false;
665
73.3k
      }
666
73.3k
      break;
667
73.3k
    }
668
73.3k
    case GIM_CheckIsSameOperand: {
669
19
      int64_t InsnID = MatchTable[CurrentIdx++];
670
19
      int64_t OpIdx = MatchTable[CurrentIdx++];
671
19
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
672
19
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
673
19
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
674
19
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
675
19
                             << InsnID << "][" << OpIdx << "], MIs["
676
19
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
677
19
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
678
19
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
679
19
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
680
19
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
681
2
        if (handleReject() == RejectAndGiveUp)
682
0
          return false;
683
19
      }
684
19
      break;
685
19
    }
686
4.69M
    case GIM_Reject:
687
4.69M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
688
4.69M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
689
4.69M
      if (handleReject() == RejectAndGiveUp)
690
3.13M
        return false;
691
1.55M
      break;
692
1.55M
693
1.55M
    case GIR_MutateOpcode: {
694
823k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
695
823k
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
696
823k
      int64_t NewOpcode = MatchTable[CurrentIdx++];
697
823k
      if (NewInsnID >= OutMIs.size())
698
823k
        OutMIs.resize(NewInsnID + 1);
699
823k
700
823k
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
701
823k
                                              State.MIs[OldInsnID]);
702
823k
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
703
823k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
704
823k
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
705
823k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
706
823k
                             << NewOpcode << ")\n");
707
823k
      break;
708
1.55M
    }
709
1.55M
710
1.99M
    case GIR_BuildMI: {
711
1.99M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
712
1.99M
      int64_t Opcode = MatchTable[CurrentIdx++];
713
1.99M
      if (NewInsnID >= OutMIs.size())
714
1.98M
        OutMIs.resize(NewInsnID + 1);
715
1.99M
716
1.99M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
717
1.99M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
718
1.99M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
719
1.99M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
720
1.99M
                             << NewInsnID << "], " << Opcode << ")\n");
721
1.99M
      break;
722
1.55M
    }
723
1.55M
724
2.20M
    case GIR_Copy: {
725
2.20M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
726
2.20M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
727
2.20M
      int64_t OpIdx = MatchTable[CurrentIdx++];
728
2.20M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
729
2.20M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
730
2.20M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
731
2.20M
                      dbgs()
732
2.20M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
733
2.20M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
734
2.20M
      break;
735
1.55M
    }
736
1.55M
737
1.55M
    case GIR_CopyOrAddZeroReg: {
738
272k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
739
272k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
740
272k
      int64_t OpIdx = MatchTable[CurrentIdx++];
741
272k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
742
272k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
743
272k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
744
272k
      if (isOperandImmEqual(MO, 0, MRI))
745
26.6k
        OutMIs[NewInsnID].addReg(ZeroReg);
746
245k
      else
747
245k
        OutMIs[NewInsnID].add(MO);
748
272k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
749
272k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
750
272k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
751
272k
                             << OpIdx << ", " << ZeroReg << ")\n");
752
272k
      break;
753
1.55M
    }
754
1.55M
755
1.55M
    case GIR_CopySubReg: {
756
93.7k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
757
93.7k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
758
93.7k
      int64_t OpIdx = MatchTable[CurrentIdx++];
759
93.7k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
760
93.7k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
761
93.7k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
762
93.7k
                               0, SubRegIdx);
763
93.7k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
764
93.7k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
765
93.7k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
766
93.7k
                             << OpIdx << ", " << SubRegIdx << ")\n");
767
93.7k
      break;
768
1.55M
    }
769
1.55M
770
1.55M
    case GIR_AddImplicitDef: {
771
73.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
772
73.1k
      int64_t RegNum = MatchTable[CurrentIdx++];
773
73.1k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
774
73.1k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
775
73.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
776
73.1k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
777
73.1k
                             << InsnID << "], " << RegNum << ")\n");
778
73.1k
      break;
779
1.55M
    }
780
1.55M
781
1.55M
    case GIR_AddImplicitUse: {
782
31
      int64_t InsnID = MatchTable[CurrentIdx++];
783
31
      int64_t RegNum = MatchTable[CurrentIdx++];
784
31
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
785
31
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
786
31
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
787
31
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
788
31
                             << InsnID << "], " << RegNum << ")\n");
789
31
      break;
790
1.55M
    }
791
1.55M
792
1.55M
    case GIR_AddRegister: {
793
163k
      int64_t InsnID = MatchTable[CurrentIdx++];
794
163k
      int64_t RegNum = MatchTable[CurrentIdx++];
795
163k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
796
163k
      OutMIs[InsnID].addReg(RegNum);
797
163k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
798
163k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
799
163k
                             << InsnID << "], " << RegNum << ")\n");
800
163k
      break;
801
1.55M
    }
802
1.55M
803
1.55M
    case GIR_AddTempRegister: {
804
8.09k
      int64_t InsnID = MatchTable[CurrentIdx++];
805
8.09k
      int64_t TempRegID = MatchTable[CurrentIdx++];
806
8.09k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
807
8.09k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
808
8.09k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
809
8.09k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
810
8.09k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
811
8.09k
                             << InsnID << "], TempRegisters[" << TempRegID
812
8.09k
                             << "], " << TempRegFlags << ")\n");
813
8.09k
      break;
814
1.55M
    }
815
1.55M
816
1.55M
    case GIR_AddImm: {
817
61.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
818
61.1k
      int64_t Imm = MatchTable[CurrentIdx++];
819
61.1k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
820
61.1k
      OutMIs[InsnID].addImm(Imm);
821
61.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
822
61.1k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
823
61.1k
                             << "], " << Imm << ")\n");
824
61.1k
      break;
825
1.55M
    }
826
1.55M
827
1.55M
    case GIR_ComplexRenderer: {
828
111k
      int64_t InsnID = MatchTable[CurrentIdx++];
829
111k
      int64_t RendererID = MatchTable[CurrentIdx++];
830
111k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
831
111k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
832
223k
        RenderOpFn(OutMIs[InsnID]);
833
111k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
834
111k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
835
111k
                             << InsnID << "], " << RendererID << ")\n");
836
111k
      break;
837
1.55M
    }
838
1.63M
    case GIR_ComplexSubOperandRenderer: {
839
1.63M
      int64_t InsnID = MatchTable[CurrentIdx++];
840
1.63M
      int64_t RendererID = MatchTable[CurrentIdx++];
841
1.63M
      int64_t RenderOpID = MatchTable[CurrentIdx++];
842
1.63M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
843
1.63M
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
844
1.63M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
845
1.63M
                      dbgs() << CurrentIdx
846
1.63M
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
847
1.63M
                             << InsnID << "], " << RendererID << ", "
848
1.63M
                             << RenderOpID << ")\n");
849
1.63M
      break;
850
1.55M
    }
851
1.55M
852
1.55M
    case GIR_CopyConstantAsSImm: {
853
790k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
854
790k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
855
790k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
856
790k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
857
790k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
858
790k
        OutMIs[NewInsnID].addImm(
859
790k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
860
790k
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
861
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
862
0
      else
863
0
        llvm_unreachable("Expected Imm or CImm operand");
864
790k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
865
790k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
866
790k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
867
790k
      break;
868
790k
    }
869
790k
870
790k
    // TODO: Needs a test case once we have a pattern that uses this.
871
790k
    case GIR_CopyFConstantAsFPImm: {
872
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
873
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
874
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
875
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
876
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
877
0
        OutMIs[NewInsnID].addFPImm(
878
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
879
0
      else
880
0
        llvm_unreachable("Expected FPImm operand");
881
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
882
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
883
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
884
0
      break;
885
0
    }
886
0
887
4.02k
    case GIR_CustomRenderer: {
888
4.02k
      int64_t InsnID = MatchTable[CurrentIdx++];
889
4.02k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
890
4.02k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
891
4.02k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
892
4.02k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
893
4.02k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
894
4.02k
                             << InsnID << "], MIs[" << OldInsnID << "], "
895
4.02k
                             << RendererFnID << ")\n");
896
4.02k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
897
4.02k
                                                     *State.MIs[OldInsnID]);
898
4.02k
      break;
899
0
    }
900
188k
    case GIR_ConstrainOperandRC: {
901
188k
      int64_t InsnID = MatchTable[CurrentIdx++];
902
188k
      int64_t OpIdx = MatchTable[CurrentIdx++];
903
188k
      int64_t RCEnum = MatchTable[CurrentIdx++];
904
188k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
905
188k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
906
188k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
907
188k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
908
188k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
909
188k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
910
188k
                             << ")\n");
911
188k
      break;
912
0
    }
913
0
914
2.72M
    case GIR_ConstrainSelectedInstOperands: {
915
2.72M
      int64_t InsnID = MatchTable[CurrentIdx++];
916
2.72M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
917
2.72M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
918
2.72M
                                       RBI);
919
2.72M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
920
2.72M
                      dbgs() << CurrentIdx
921
2.72M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
922
2.72M
                             << InsnID << "])\n");
923
2.72M
      break;
924
0
    }
925
0
926
821k
    case GIR_MergeMemOperands: {
927
821k
      int64_t InsnID = MatchTable[CurrentIdx++];
928
821k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
929
821k
930
821k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
931
821k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
932
821k
                             << InsnID << "]");
933
821k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
934
1.64M
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
935
1.64M
             GIU_MergeMemOperands_EndOfList) {
936
823k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
937
823k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
938
823k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
939
818k
          OutMIs[InsnID].addMemOperand(MMO);
940
823k
      }
941
821k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
942
821k
      break;
943
0
    }
944
0
945
1.98M
    case GIR_EraseFromParent: {
946
1.98M
      int64_t InsnID = MatchTable[CurrentIdx++];
947
1.98M
      assert(State.MIs[InsnID] &&
948
1.98M
             "Attempted to erase an undefined instruction");
949
1.98M
      State.MIs[InsnID]->eraseFromParent();
950
1.98M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
951
1.98M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
952
1.98M
                             << InsnID << "])\n");
953
1.98M
      break;
954
0
    }
955
0
956
4.04k
    case GIR_MakeTempReg: {
957
4.04k
      int64_t TempRegID = MatchTable[CurrentIdx++];
958
4.04k
      int64_t TypeID = MatchTable[CurrentIdx++];
959
4.04k
960
4.04k
      State.TempRegisters[TempRegID] =
961
4.04k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
962
4.04k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
963
4.04k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
964
4.04k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
965
4.04k
      break;
966
0
    }
967
0
968
0
    case GIR_Coverage: {
969
0
      int64_t RuleID = MatchTable[CurrentIdx++];
970
0
      CoverageInfo.setCovered(RuleID);
971
0
972
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
973
0
                      dbgs()
974
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
975
0
      break;
976
0
    }
977
0
978
2.81M
    case GIR_Done:
979
2.81M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
980
2.81M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
981
2.81M
      return true;
982
0
983
0
    default:
984
0
      llvm_unreachable("Unexpected command");
985
61.9M
    }
986
61.9M
  }
987
5.94M
}
AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<22ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<22ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<22ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
5.94M
    CodeGenCoverage &CoverageInfo) const {
56
5.94M
57
5.94M
  uint64_t CurrentIdx = 0;
58
5.94M
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
5.94M
60
5.94M
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
5.94M
  auto handleReject = [&]() -> RejectAction {
62
5.94M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
5.94M
                    dbgs() << CurrentIdx << ": Rejected\n");
64
5.94M
    if (OnFailResumeAt.empty())
65
5.94M
      return RejectAndGiveUp;
66
5.94M
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
5.94M
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
5.94M
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
5.94M
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
5.94M
    return RejectAndResume;
71
5.94M
  };
72
5.94M
73
61.8M
  while (true) {
74
61.8M
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
61.8M
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
61.8M
    switch (MatcherOpcode) {
77
61.8M
    case GIM_Try: {
78
9.13M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
9.13M
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
9.13M
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
9.13M
      break;
82
61.8M
    }
83
61.8M
84
61.8M
    case GIM_RecordInsn: {
85
3.04M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
3.04M
      int64_t InsnID = MatchTable[CurrentIdx++];
87
3.04M
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
3.04M
89
3.04M
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
3.04M
      // any attempt to modify it.
91
3.04M
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
3.04M
93
3.04M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
3.04M
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
3.04M
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
3.04M
109
3.04M
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
3.04M
      if ((size_t)NewInsnID < State.MIs.size())
111
1.98M
        State.MIs[NewInsnID] = NewMI;
112
1.05M
      else {
113
1.05M
        assert((size_t)NewInsnID == State.MIs.size() &&
114
1.05M
               "Expected to store MIs in order");
115
1.05M
        State.MIs.push_back(NewMI);
116
1.05M
      }
117
3.04M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
3.04M
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
3.04M
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
3.04M
                             << ")\n");
121
3.04M
      break;
122
3.04M
    }
123
3.04M
124
3.04M
    case GIM_CheckFeatures: {
125
906k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
906k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
906k
                      dbgs() << CurrentIdx
128
906k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
906k
                             << ExpectedBitsetID << ")\n");
130
906k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
906k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
79.1k
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
906k
      }
135
906k
      break;
136
906k
    }
137
906k
138
3.04M
    case GIM_CheckOpcode: {
139
3.04M
      int64_t InsnID = MatchTable[CurrentIdx++];
140
3.04M
      int64_t Expected = MatchTable[CurrentIdx++];
141
3.04M
142
3.04M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
3.04M
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
3.04M
145
3.04M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
3.04M
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
3.04M
                             << "], ExpectedOpcode=" << Expected
148
3.04M
                             << ") // Got=" << Opcode << "\n");
149
3.04M
      if (Opcode != Expected) {
150
2.92M
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
3.04M
      }
153
3.04M
      break;
154
3.04M
    }
155
3.04M
156
5.94M
    case GIM_SwitchOpcode: {
157
5.94M
      int64_t InsnID = MatchTable[CurrentIdx++];
158
5.94M
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
5.94M
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
5.94M
      int64_t Default = MatchTable[CurrentIdx++];
161
5.94M
162
5.94M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
5.94M
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
5.94M
165
5.94M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
5.94M
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
5.94M
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
5.94M
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
5.94M
      });
170
5.94M
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
763
        CurrentIdx = Default;
172
763
        break;
173
763
      }
174
5.94M
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
5.94M
      if (!CurrentIdx) {
176
1.82M
        CurrentIdx = Default;
177
1.82M
  break;
178
1.82M
      }
179
4.11M
      OnFailResumeAt.push_back(Default);
180
4.11M
      break;
181
4.11M
    }
182
4.11M
183
4.11M
    case GIM_SwitchType: {
184
3.76M
      int64_t InsnID = MatchTable[CurrentIdx++];
185
3.76M
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
3.76M
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
3.76M
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
3.76M
      int64_t Default = MatchTable[CurrentIdx++];
189
3.76M
190
3.76M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
3.76M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
3.76M
193
3.76M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
3.76M
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
3.76M
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
3.76M
               << UpperBound << "), Default=" << Default
197
3.76M
               << ", JumpTable...) // Got=";
198
3.76M
        if (!MO.isReg())
199
3.76M
          dbgs() << "Not a VReg\n";
200
3.76M
        else
201
3.76M
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
3.76M
      });
203
3.76M
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
3.76M
      const LLT Ty = MRI.getType(MO.getReg());
208
3.76M
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
3.76M
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
913k
        CurrentIdx = Default;
211
913k
        break;
212
913k
      }
213
2.84M
      const int64_t TypeID = TyI->second;
214
2.84M
      if (TypeID < LowerBound || 
UpperBound <= TypeID2.65M
) {
215
191k
        CurrentIdx = Default;
216
191k
        break;
217
191k
      }
218
2.65M
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
2.65M
      if (!CurrentIdx) {
220
39
        CurrentIdx = Default;
221
39
        break;
222
39
      }
223
2.65M
      OnFailResumeAt.push_back(Default);
224
2.65M
      break;
225
2.65M
    }
226
2.65M
227
2.65M
    case GIM_CheckNumOperands: {
228
22.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
229
22.9k
      int64_t Expected = MatchTable[CurrentIdx++];
230
22.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
22.9k
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
22.9k
                             << InsnID << "], Expected=" << Expected << ")\n");
233
22.9k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
22.9k
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
17.2k
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
22.9k
      }
238
22.9k
      break;
239
22.9k
    }
240
63.1k
    case GIM_CheckI64ImmPredicate: {
241
63.1k
      int64_t InsnID = MatchTable[CurrentIdx++];
242
63.1k
      int64_t Predicate = MatchTable[CurrentIdx++];
243
63.1k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
63.1k
                      dbgs()
245
63.1k
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
63.1k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
63.1k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
63.1k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
63.1k
             "Expected G_CONSTANT");
250
63.1k
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
63.1k
      int64_t Value = 0;
252
63.1k
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
63.1k
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
63.1k
259
63.1k
      if (!testImmPredicate_I64(Predicate, Value))
260
27
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
63.1k
      break;
263
63.1k
    }
264
63.1k
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
24.3k
    case GIM_CheckAPFloatImmPredicate: {
287
24.3k
      int64_t InsnID = MatchTable[CurrentIdx++];
288
24.3k
      int64_t Predicate = MatchTable[CurrentIdx++];
289
24.3k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
24.3k
                      dbgs()
291
24.3k
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
24.3k
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
24.3k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
24.3k
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
24.3k
             "Expected G_FCONSTANT");
296
24.3k
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
24.3k
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
24.3k
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
24.3k
300
24.3k
      if (!testImmPredicate_APFloat(Predicate, Value))
301
20.9k
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
24.3k
      break;
304
24.3k
    }
305
24.3k
    case GIM_CheckCxxInsnPredicate: {
306
0
      int64_t InsnID = MatchTable[CurrentIdx++];
307
0
      int64_t Predicate = MatchTable[CurrentIdx++];
308
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
0
                      dbgs()
310
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
0
315
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
0
      break;
319
0
    }
320
914k
    case GIM_CheckAtomicOrdering: {
321
914k
      int64_t InsnID = MatchTable[CurrentIdx++];
322
914k
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
914k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
914k
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
914k
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
914k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
914k
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
914k
331
914k
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
914k
        if (MMO->getOrdering() != Ordering)
333
4.24k
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
914k
      break;
336
914k
    }
337
914k
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
0
    case GIM_CheckMemoryAddressSpace: {
374
0
      int64_t InsnID = MatchTable[CurrentIdx++];
375
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
0
      // This accepts a list of possible address spaces.
377
0
      const int NumAddrSpace = MatchTable[CurrentIdx++];
378
0
379
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
380
0
        if (handleReject() == RejectAndGiveUp)
381
0
          return false;
382
0
        break;
383
0
      }
384
0
385
0
      // Need to still jump to the end of the list of address spaces if we find
386
0
      // a match earlier.
387
0
      const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
388
0
389
0
      const MachineMemOperand *MMO
390
0
        = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
391
0
      const unsigned MMOAddrSpace = MMO->getAddrSpace();
392
0
393
0
      bool Success = false;
394
0
      for (int I = 0; I != NumAddrSpace; ++I) {
395
0
        unsigned AddrSpace = MatchTable[CurrentIdx++];
396
0
        DEBUG_WITH_TYPE(
397
0
          TgtInstructionSelector::getName(),
398
0
          dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
399
0
                 << AddrSpace << '\n');
400
0
401
0
        if (AddrSpace == MMOAddrSpace) {
402
0
          Success = true;
403
0
          break;
404
0
        }
405
0
      }
406
0
407
0
      CurrentIdx = LastIdx;
408
0
      if (!Success && handleReject() == RejectAndGiveUp)
409
0
        return false;
410
0
      break;
411
0
    }
412
151k
    case GIM_CheckMemorySizeEqualTo: {
413
151k
      int64_t InsnID = MatchTable[CurrentIdx++];
414
151k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
415
151k
      uint64_t Size = MatchTable[CurrentIdx++];
416
151k
417
151k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
418
151k
                      dbgs() << CurrentIdx
419
151k
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
420
151k
                             << "]->memoperands() + " << MMOIdx
421
151k
                             << ", Size=" << Size << ")\n");
422
151k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
423
151k
424
151k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
151k
430
151k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
431
151k
432
151k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
433
151k
                      dbgs() << MMO->getSize() << " bytes vs " << Size
434
151k
                             << " bytes\n");
435
151k
      if (MMO->getSize() != Size)
436
92.1k
        if (handleReject() == RejectAndGiveUp)
437
0
          return false;
438
151k
439
151k
      break;
440
151k
    }
441
446k
    case GIM_CheckMemorySizeEqualToLLT:
442
446k
    case GIM_CheckMemorySizeLessThanLLT:
443
446k
    case GIM_CheckMemorySizeGreaterThanLLT: {
444
446k
      int64_t InsnID = MatchTable[CurrentIdx++];
445
446k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
446
446k
      int64_t OpIdx = MatchTable[CurrentIdx++];
447
446k
448
446k
      DEBUG_WITH_TYPE(
449
446k
          TgtInstructionSelector::getName(),
450
446k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
451
446k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
452
446k
                         ? "EqualTo"
453
446k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
454
446k
                               ? "GreaterThan"
455
446k
                               : "LessThan")
456
446k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
457
446k
                 << ", OpIdx=" << OpIdx << ")\n");
458
446k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
459
446k
460
446k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
461
446k
      if (!MO.isReg()) {
462
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
463
0
                        dbgs() << CurrentIdx << ": Not a register\n");
464
0
        if (handleReject() == RejectAndGiveUp)
465
0
          return false;
466
0
        break;
467
0
      }
468
446k
469
446k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
470
0
        if (handleReject() == RejectAndGiveUp)
471
0
          return false;
472
0
        break;
473
0
      }
474
446k
475
446k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
476
446k
477
446k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
478
446k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
479
446k
          
MMO->getSizeInBits() != Size446k
) {
480
28
        if (handleReject() == RejectAndGiveUp)
481
0
          return false;
482
446k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
483
446k
                 
MMO->getSizeInBits() >= Size48
) {
484
36
        if (handleReject() == RejectAndGiveUp)
485
0
          return false;
486
446k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
487
446k
                 
MMO->getSizeInBits() <= Size0
)
488
0
        if (handleReject() == RejectAndGiveUp)
489
0
          return false;
490
446k
491
446k
      break;
492
446k
    }
493
2.03M
    case GIM_CheckType: {
494
2.03M
      int64_t InsnID = MatchTable[CurrentIdx++];
495
2.03M
      int64_t OpIdx = MatchTable[CurrentIdx++];
496
2.03M
      int64_t TypeID = MatchTable[CurrentIdx++];
497
2.03M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498
2.03M
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
499
2.03M
                             << "]->getOperand(" << OpIdx
500
2.03M
                             << "), TypeID=" << TypeID << ")\n");
501
2.03M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
502
2.03M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
503
2.03M
      if (!MO.isReg() ||
504
2.03M
          
MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]2.03M
) {
505
76.5k
        if (handleReject() == RejectAndGiveUp)
506
0
          return false;
507
2.03M
      }
508
2.03M
      break;
509
2.03M
    }
510
2.03M
    case GIM_CheckPointerToAny: {
511
877k
      int64_t InsnID = MatchTable[CurrentIdx++];
512
877k
      int64_t OpIdx = MatchTable[CurrentIdx++];
513
877k
      int64_t SizeInBits = MatchTable[CurrentIdx++];
514
877k
515
877k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
516
877k
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
517
877k
                             << InsnID << "]->getOperand(" << OpIdx
518
877k
                             << "), SizeInBits=" << SizeInBits << ")\n");
519
877k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
520
877k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
521
877k
      const LLT Ty = MRI.getType(MO.getReg());
522
877k
523
877k
      // iPTR must be looked up in the target.
524
877k
      if (SizeInBits == 0) {
525
875k
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
526
875k
        const unsigned AddrSpace = Ty.getAddressSpace();
527
875k
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
528
875k
      }
529
877k
530
877k
      assert(SizeInBits != 0 && "Pointer size must be known");
531
877k
532
877k
      if (MO.isReg()) {
533
877k
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
534
0
          if (handleReject() == RejectAndGiveUp)
535
0
            return false;
536
0
      } else if (handleReject() == RejectAndGiveUp)
537
0
        return false;
538
877k
539
877k
      break;
540
877k
    }
541
7.66M
    case GIM_CheckRegBankForClass: {
542
7.66M
      int64_t InsnID = MatchTable[CurrentIdx++];
543
7.66M
      int64_t OpIdx = MatchTable[CurrentIdx++];
544
7.66M
      int64_t RCEnum = MatchTable[CurrentIdx++];
545
7.66M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
546
7.66M
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
547
7.66M
                             << InsnID << "]->getOperand(" << OpIdx
548
7.66M
                             << "), RCEnum=" << RCEnum << ")\n");
549
7.66M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
550
7.66M
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
551
7.66M
      if (!MO.isReg() ||
552
7.66M
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
553
7.66M
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
554
744k
        if (handleReject() == RejectAndGiveUp)
555
0
          return false;
556
7.66M
      }
557
7.66M
      break;
558
7.66M
    }
559
7.66M
560
7.66M
    case GIM_CheckComplexPattern: {
561
1.44M
      int64_t InsnID = MatchTable[CurrentIdx++];
562
1.44M
      int64_t OpIdx = MatchTable[CurrentIdx++];
563
1.44M
      int64_t RendererID = MatchTable[CurrentIdx++];
564
1.44M
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
565
1.44M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
566
1.44M
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
567
1.44M
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
568
1.44M
                             << "]->getOperand(" << OpIdx
569
1.44M
                             << "), ComplexPredicateID=" << ComplexPredicateID
570
1.44M
                             << ")\n");
571
1.44M
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572
1.44M
      // FIXME: Use std::invoke() when it's available.
573
1.44M
      ComplexRendererFns Renderer =
574
1.44M
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
575
1.44M
              State.MIs[InsnID]->getOperand(OpIdx));
576
1.44M
      if (Renderer.hasValue())
577
928k
        State.Renderers[RendererID] = Renderer.getValue();
578
512k
      else
579
512k
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
1.44M
      break;
582
1.44M
    }
583
1.44M
584
1.44M
    case GIM_CheckConstantInt: {
585
286k
      int64_t InsnID = MatchTable[CurrentIdx++];
586
286k
      int64_t OpIdx = MatchTable[CurrentIdx++];
587
286k
      int64_t Value = MatchTable[CurrentIdx++];
588
286k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
589
286k
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
590
286k
                             << InsnID << "]->getOperand(" << OpIdx
591
286k
                             << "), Value=" << Value << ")\n");
592
286k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593
286k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
594
286k
      if (MO.isReg()) {
595
286k
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
596
286k
        LLT Ty = MRI.getType(MO.getReg());
597
286k
        Value = SignExtend64(Value, Ty.getSizeInBits());
598
286k
599
286k
        if (!isOperandImmEqual(MO, Value, MRI)) {
600
254k
          if (handleReject() == RejectAndGiveUp)
601
0
            return false;
602
0
        }
603
0
      } else if (handleReject() == RejectAndGiveUp)
604
0
        return false;
605
286k
606
286k
      break;
607
286k
    }
608
286k
609
286k
    case GIM_CheckLiteralInt: {
610
0
      int64_t InsnID = MatchTable[CurrentIdx++];
611
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
612
0
      int64_t Value = MatchTable[CurrentIdx++];
613
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
614
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
615
0
                             << InsnID << "]->getOperand(" << OpIdx
616
0
                             << "), Value=" << Value << ")\n");
617
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
618
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
619
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
620
0
        if (handleReject() == RejectAndGiveUp)
621
0
          return false;
622
0
      }
623
0
      break;
624
0
    }
625
0
626
241k
    case GIM_CheckIntrinsicID: {
627
241k
      int64_t InsnID = MatchTable[CurrentIdx++];
628
241k
      int64_t OpIdx = MatchTable[CurrentIdx++];
629
241k
      int64_t Value = MatchTable[CurrentIdx++];
630
241k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
631
241k
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
632
241k
                             << InsnID << "]->getOperand(" << OpIdx
633
241k
                             << "), Value=" << Value << ")\n");
634
241k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
635
241k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
636
241k
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value233k
)
637
235k
        if (handleReject() == RejectAndGiveUp)
638
0
          return false;
639
241k
      break;
640
241k
    }
641
241k
642
307k
    case GIM_CheckIsMBB: {
643
307k
      int64_t InsnID = MatchTable[CurrentIdx++];
644
307k
      int64_t OpIdx = MatchTable[CurrentIdx++];
645
307k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
646
307k
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
647
307k
                             << "]->getOperand(" << OpIdx << "))\n");
648
307k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
649
307k
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
650
0
        if (handleReject() == RejectAndGiveUp)
651
0
          return false;
652
307k
      }
653
307k
      break;
654
307k
    }
655
307k
656
307k
    case GIM_CheckIsSafeToFold: {
657
73.0k
      int64_t InsnID = MatchTable[CurrentIdx++];
658
73.0k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
659
73.0k
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
660
73.0k
                             << InsnID << "])\n");
661
73.0k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
662
73.0k
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
663
0
        if (handleReject() == RejectAndGiveUp)
664
0
          return false;
665
73.0k
      }
666
73.0k
      break;
667
73.0k
    }
668
73.0k
    case GIM_CheckIsSameOperand: {
669
11
      int64_t InsnID = MatchTable[CurrentIdx++];
670
11
      int64_t OpIdx = MatchTable[CurrentIdx++];
671
11
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
672
11
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
673
11
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
674
11
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
675
11
                             << InsnID << "][" << OpIdx << "], MIs["
676
11
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
677
11
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
678
11
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
679
11
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
680
11
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
681
0
        if (handleReject() == RejectAndGiveUp)
682
0
          return false;
683
11
      }
684
11
      break;
685
11
    }
686
4.68M
    case GIM_Reject:
687
4.68M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
688
4.68M
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
689
4.68M
      if (handleReject() == RejectAndGiveUp)
690
3.13M
        return false;
691
1.54M
      break;
692
1.54M
693
1.54M
    case GIR_MutateOpcode: {
694
822k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
695
822k
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
696
822k
      int64_t NewOpcode = MatchTable[CurrentIdx++];
697
822k
      if (NewInsnID >= OutMIs.size())
698
822k
        OutMIs.resize(NewInsnID + 1);
699
822k
700
822k
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
701
822k
                                              State.MIs[OldInsnID]);
702
822k
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
703
822k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
704
822k
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
705
822k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
706
822k
                             << NewOpcode << ")\n");
707
822k
      break;
708
1.54M
    }
709
1.54M
710
1.99M
    case GIR_BuildMI: {
711
1.99M
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
712
1.99M
      int64_t Opcode = MatchTable[CurrentIdx++];
713
1.99M
      if (NewInsnID >= OutMIs.size())
714
1.98M
        OutMIs.resize(NewInsnID + 1);
715
1.99M
716
1.99M
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
717
1.99M
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
718
1.99M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
719
1.99M
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
720
1.99M
                             << NewInsnID << "], " << Opcode << ")\n");
721
1.99M
      break;
722
1.54M
    }
723
1.54M
724
2.20M
    case GIR_Copy: {
725
2.20M
      int64_t NewInsnID = MatchTable[CurrentIdx++];
726
2.20M
      int64_t OldInsnID = MatchTable[CurrentIdx++];
727
2.20M
      int64_t OpIdx = MatchTable[CurrentIdx++];
728
2.20M
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
729
2.20M
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
730
2.20M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
731
2.20M
                      dbgs()
732
2.20M
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
733
2.20M
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
734
2.20M
      break;
735
1.54M
    }
736
1.54M
737
1.54M
    case GIR_CopyOrAddZeroReg: {
738
272k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
739
272k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
740
272k
      int64_t OpIdx = MatchTable[CurrentIdx++];
741
272k
      int64_t ZeroReg = MatchTable[CurrentIdx++];
742
272k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
743
272k
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
744
272k
      if (isOperandImmEqual(MO, 0, MRI))
745
26.6k
        OutMIs[NewInsnID].addReg(ZeroReg);
746
245k
      else
747
245k
        OutMIs[NewInsnID].add(MO);
748
272k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
749
272k
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
750
272k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
751
272k
                             << OpIdx << ", " << ZeroReg << ")\n");
752
272k
      break;
753
1.54M
    }
754
1.54M
755
1.54M
    case GIR_CopySubReg: {
756
93.5k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
757
93.5k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
758
93.5k
      int64_t OpIdx = MatchTable[CurrentIdx++];
759
93.5k
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
760
93.5k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
761
93.5k
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
762
93.5k
                               0, SubRegIdx);
763
93.5k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
764
93.5k
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
765
93.5k
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
766
93.5k
                             << OpIdx << ", " << SubRegIdx << ")\n");
767
93.5k
      break;
768
1.54M
    }
769
1.54M
770
1.54M
    case GIR_AddImplicitDef: {
771
72.9k
      int64_t InsnID = MatchTable[CurrentIdx++];
772
72.9k
      int64_t RegNum = MatchTable[CurrentIdx++];
773
72.9k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
774
72.9k
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
775
72.9k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
776
72.9k
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
777
72.9k
                             << InsnID << "], " << RegNum << ")\n");
778
72.9k
      break;
779
1.54M
    }
780
1.54M
781
1.54M
    case GIR_AddImplicitUse: {
782
0
      int64_t InsnID = MatchTable[CurrentIdx++];
783
0
      int64_t RegNum = MatchTable[CurrentIdx++];
784
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
785
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
786
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
787
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
788
0
                             << InsnID << "], " << RegNum << ")\n");
789
0
      break;
790
1.54M
    }
791
1.54M
792
1.54M
    case GIR_AddRegister: {
793
163k
      int64_t InsnID = MatchTable[CurrentIdx++];
794
163k
      int64_t RegNum = MatchTable[CurrentIdx++];
795
163k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
796
163k
      OutMIs[InsnID].addReg(RegNum);
797
163k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
798
163k
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
799
163k
                             << InsnID << "], " << RegNum << ")\n");
800
163k
      break;
801
1.54M
    }
802
1.54M
803
1.54M
    case GIR_AddTempRegister: {
804
8.05k
      int64_t InsnID = MatchTable[CurrentIdx++];
805
8.05k
      int64_t TempRegID = MatchTable[CurrentIdx++];
806
8.05k
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
807
8.05k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
808
8.05k
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
809
8.05k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
810
8.05k
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
811
8.05k
                             << InsnID << "], TempRegisters[" << TempRegID
812
8.05k
                             << "], " << TempRegFlags << ")\n");
813
8.05k
      break;
814
1.54M
    }
815
1.54M
816
1.54M
    case GIR_AddImm: {
817
59.2k
      int64_t InsnID = MatchTable[CurrentIdx++];
818
59.2k
      int64_t Imm = MatchTable[CurrentIdx++];
819
59.2k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
820
59.2k
      OutMIs[InsnID].addImm(Imm);
821
59.2k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
822
59.2k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
823
59.2k
                             << "], " << Imm << ")\n");
824
59.2k
      break;
825
1.54M
    }
826
1.54M
827
1.54M
    case GIR_ComplexRenderer: {
828
111k
      int64_t InsnID = MatchTable[CurrentIdx++];
829
111k
      int64_t RendererID = MatchTable[CurrentIdx++];
830
111k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
831
111k
      for (const auto &RenderOpFn : State.Renderers[RendererID])
832
223k
        RenderOpFn(OutMIs[InsnID]);
833
111k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
834
111k
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
835
111k
                             << InsnID << "], " << RendererID << ")\n");
836
111k
      break;
837
1.54M
    }
838
1.63M
    case GIR_ComplexSubOperandRenderer: {
839
1.63M
      int64_t InsnID = MatchTable[CurrentIdx++];
840
1.63M
      int64_t RendererID = MatchTable[CurrentIdx++];
841
1.63M
      int64_t RenderOpID = MatchTable[CurrentIdx++];
842
1.63M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
843
1.63M
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
844
1.63M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
845
1.63M
                      dbgs() << CurrentIdx
846
1.63M
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
847
1.63M
                             << InsnID << "], " << RendererID << ", "
848
1.63M
                             << RenderOpID << ")\n");
849
1.63M
      break;
850
1.54M
    }
851
1.54M
852
1.54M
    case GIR_CopyConstantAsSImm: {
853
790k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
854
790k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
855
790k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
856
790k
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
857
790k
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
858
790k
        OutMIs[NewInsnID].addImm(
859
790k
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
860
790k
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
861
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
862
0
      else
863
0
        llvm_unreachable("Expected Imm or CImm operand");
864
790k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
865
790k
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
866
790k
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
867
790k
      break;
868
790k
    }
869
790k
870
790k
    // TODO: Needs a test case once we have a pattern that uses this.
871
790k
    case GIR_CopyFConstantAsFPImm: {
872
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
873
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
874
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
875
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
876
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
877
0
        OutMIs[NewInsnID].addFPImm(
878
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
879
0
      else
880
0
        llvm_unreachable("Expected FPImm operand");
881
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
882
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
883
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
884
0
      break;
885
0
    }
886
0
887
4.02k
    case GIR_CustomRenderer: {
888
4.02k
      int64_t InsnID = MatchTable[CurrentIdx++];
889
4.02k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
890
4.02k
      int64_t RendererFnID = MatchTable[CurrentIdx++];
891
4.02k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
892
4.02k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
893
4.02k
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
894
4.02k
                             << InsnID << "], MIs[" << OldInsnID << "], "
895
4.02k
                             << RendererFnID << ")\n");
896
4.02k
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
897
4.02k
                                                     *State.MIs[OldInsnID]);
898
4.02k
      break;
899
0
    }
900
187k
    case GIR_ConstrainOperandRC: {
901
187k
      int64_t InsnID = MatchTable[CurrentIdx++];
902
187k
      int64_t OpIdx = MatchTable[CurrentIdx++];
903
187k
      int64_t RCEnum = MatchTable[CurrentIdx++];
904
187k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
905
187k
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
906
187k
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
907
187k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
908
187k
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
909
187k
                             << InsnID << "], " << OpIdx << ", " << RCEnum
910
187k
                             << ")\n");
911
187k
      break;
912
0
    }
913
0
914
2.71M
    case GIR_ConstrainSelectedInstOperands: {
915
2.71M
      int64_t InsnID = MatchTable[CurrentIdx++];
916
2.71M
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
917
2.71M
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
918
2.71M
                                       RBI);
919
2.71M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
920
2.71M
                      dbgs() << CurrentIdx
921
2.71M
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
922
2.71M
                             << InsnID << "])\n");
923
2.71M
      break;
924
0
    }
925
0
926
820k
    case GIR_MergeMemOperands: {
927
820k
      int64_t InsnID = MatchTable[CurrentIdx++];
928
820k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
929
820k
930
820k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
931
820k
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
932
820k
                             << InsnID << "]");
933
820k
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
934
1.64M
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
935
1.64M
             GIU_MergeMemOperands_EndOfList) {
936
823k
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
937
823k
                        dbgs() << ", MIs[" << MergeInsnID << "]");
938
823k
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
939
818k
          OutMIs[InsnID].addMemOperand(MMO);
940
823k
      }
941
820k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
942
820k
      break;
943
0
    }
944
0
945
1.98M
    case GIR_EraseFromParent: {
946
1.98M
      int64_t InsnID = MatchTable[CurrentIdx++];
947
1.98M
      assert(State.MIs[InsnID] &&
948
1.98M
             "Attempted to erase an undefined instruction");
949
1.98M
      State.MIs[InsnID]->eraseFromParent();
950
1.98M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
951
1.98M
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
952
1.98M
                             << InsnID << "])\n");
953
1.98M
      break;
954
0
    }
955
0
956
4.02k
    case GIR_MakeTempReg: {
957
4.02k
      int64_t TempRegID = MatchTable[CurrentIdx++];
958
4.02k
      int64_t TypeID = MatchTable[CurrentIdx++];
959
4.02k
960
4.02k
      State.TempRegisters[TempRegID] =
961
4.02k
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
962
4.02k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
963
4.02k
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
964
4.02k
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
965
4.02k
      break;
966
0
    }
967
0
968
0
    case GIR_Coverage: {
969
0
      int64_t RuleID = MatchTable[CurrentIdx++];
970
0
      CoverageInfo.setCovered(RuleID);
971
0
972
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
973
0
                      dbgs()
974
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
975
0
      break;
976
0
    }
977
0
978
2.80M
    case GIR_Done:
979
2.80M
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
980
2.80M
                      dbgs() << CurrentIdx << ": GIR_Done\n");
981
2.80M
      return true;
982
0
983
0
    default:
984
0
      llvm_unreachable("Unexpected command");
985
61.8M
    }
986
61.8M
  }
987
5.94M
}
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<46ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<46ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<46ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
1.34k
    CodeGenCoverage &CoverageInfo) const {
56
1.34k
57
1.34k
  uint64_t CurrentIdx = 0;
58
1.34k
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
1.34k
60
1.34k
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
1.34k
  auto handleReject = [&]() -> RejectAction {
62
1.34k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
1.34k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
1.34k
    if (OnFailResumeAt.empty())
65
1.34k
      return RejectAndGiveUp;
66
1.34k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
1.34k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
1.34k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
1.34k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
1.34k
    return RejectAndResume;
71
1.34k
  };
72
1.34k
73
37.0k
  while (true) {
74
37.0k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
37.0k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
37.0k
    switch (MatcherOpcode) {
77
37.0k
    case GIM_Try: {
78
6.96k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
6.96k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
6.96k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
6.96k
      break;
82
37.0k
    }
83
37.0k
84
37.0k
    case GIM_RecordInsn: {
85
37
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
37
      int64_t InsnID = MatchTable[CurrentIdx++];
87
37
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
37
89
37
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
37
      // any attempt to modify it.
91
37
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
37
93
37
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
37
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
37
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
37
109
37
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
37
      if ((size_t)NewInsnID < State.MIs.size())
111
30
        State.MIs[NewInsnID] = NewMI;
112
7
      else {
113
7
        assert((size_t)NewInsnID == State.MIs.size() &&
114
7
               "Expected to store MIs in order");
115
7
        State.MIs.push_back(NewMI);
116
7
      }
117
37
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
37
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
37
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
37
                             << ")\n");
121
37
      break;
122
37
    }
123
37
124
6.30k
    case GIM_CheckFeatures: {
125
6.30k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
6.30k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
6.30k
                      dbgs() << CurrentIdx
128
6.30k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
6.30k
                             << ExpectedBitsetID << ")\n");
130
6.30k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
6.30k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
1.18k
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
6.30k
      }
135
6.30k
      break;
136
6.30k
    }
137
6.30k
138
6.30k
    case GIM_CheckOpcode: {
139
37
      int64_t InsnID = MatchTable[CurrentIdx++];
140
37
      int64_t Expected = MatchTable[CurrentIdx++];
141
37
142
37
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
37
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
37
145
37
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
37
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
37
                             << "], ExpectedOpcode=" << Expected
148
37
                             << ") // Got=" << Opcode << "\n");
149
37
      if (Opcode != Expected) {
150
36
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
37
      }
153
37
      break;
154
37
    }
155
37
156
1.34k
    case GIM_SwitchOpcode: {
157
1.34k
      int64_t InsnID = MatchTable[CurrentIdx++];
158
1.34k
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
1.34k
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
1.34k
      int64_t Default = MatchTable[CurrentIdx++];
161
1.34k
162
1.34k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
1.34k
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
1.34k
165
1.34k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
1.34k
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
1.34k
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
1.34k
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
1.34k
      });
170
1.34k
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
6
        CurrentIdx = Default;
172
6
        break;
173
6
      }
174
1.34k
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
1.34k
      if (!CurrentIdx) {
176
9
        CurrentIdx = Default;
177
9
  break;
178
9
      }
179
1.33k
      OnFailResumeAt.push_back(Default);
180
1.33k
      break;
181
1.33k
    }
182
1.33k
183
1.33k
    case GIM_SwitchType: {
184
1.23k
      int64_t InsnID = MatchTable[CurrentIdx++];
185
1.23k
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
1.23k
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
1.23k
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
1.23k
      int64_t Default = MatchTable[CurrentIdx++];
189
1.23k
190
1.23k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
1.23k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
1.23k
193
1.23k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
1.23k
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
1.23k
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
1.23k
               << UpperBound << "), Default=" << Default
197
1.23k
               << ", JumpTable...) // Got=";
198
1.23k
        if (!MO.isReg())
199
1.23k
          dbgs() << "Not a VReg\n";
200
1.23k
        else
201
1.23k
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
1.23k
      });
203
1.23k
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
1.23k
      const LLT Ty = MRI.getType(MO.getReg());
208
1.23k
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
1.23k
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
138
        CurrentIdx = Default;
211
138
        break;
212
138
      }
213
1.09k
      const int64_t TypeID = TyI->second;
214
1.09k
      if (TypeID < LowerBound || 
UpperBound <= TypeID1.04k
) {
215
82
        CurrentIdx = Default;
216
82
        break;
217
82
      }
218
1.01k
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
1.01k
      if (!CurrentIdx) {
220
74
        CurrentIdx = Default;
221
74
        break;
222
74
      }
223
942
      OnFailResumeAt.push_back(Default);
224
942
      break;
225
942
    }
226
942
227
942
    case GIM_CheckNumOperands: {
228
27
      int64_t InsnID = MatchTable[CurrentIdx++];
229
27
      int64_t Expected = MatchTable[CurrentIdx++];
230
27
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
27
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
27
                             << InsnID << "], Expected=" << Expected << ")\n");
233
27
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
27
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
18
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
27
      }
238
27
      break;
239
27
    }
240
27
    case GIM_CheckI64ImmPredicate: {
241
0
      int64_t InsnID = MatchTable[CurrentIdx++];
242
0
      int64_t Predicate = MatchTable[CurrentIdx++];
243
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
0
                      dbgs()
245
0
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
0
             "Expected G_CONSTANT");
250
0
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
0
      int64_t Value = 0;
252
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
0
259
0
      if (!testImmPredicate_I64(Predicate, Value))
260
0
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
0
      break;
263
0
    }
264
0
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
0
    case GIM_CheckAPFloatImmPredicate: {
287
0
      int64_t InsnID = MatchTable[CurrentIdx++];
288
0
      int64_t Predicate = MatchTable[CurrentIdx++];
289
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
0
                      dbgs()
291
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
0
             "Expected G_FCONSTANT");
296
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
0
300
0
      if (!testImmPredicate_APFloat(Predicate, Value))
301
0
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
0
      break;
304
0
    }
305
396
    case GIM_CheckCxxInsnPredicate: {
306
396
      int64_t InsnID = MatchTable[CurrentIdx++];
307
396
      int64_t Predicate = MatchTable[CurrentIdx++];
308
396
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
396
                      dbgs()
310
396
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
396
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
396
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
396
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
396
315
396
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
396
      break;
319
396
    }
320
949
    case GIM_CheckAtomicOrdering: {
321
949
      int64_t InsnID = MatchTable[CurrentIdx++];
322
949
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
949
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
949
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
949
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
949
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
949
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
949
331
949
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
949
        if (MMO->getOrdering() != Ordering)
333
54
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
949
      break;
336
949
    }
337
949
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
2.19k
    case GIM_CheckMemoryAddressSpace: {
374
2.19k
      int64_t InsnID = MatchTable[CurrentIdx++];
375
2.19k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
2.19k
      // This accepts a list of possible address spaces.
377
2.19k
      const int NumAddrSpace = MatchTable[CurrentIdx++];
378
2.19k
379
2.19k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
380
0
        if (handleReject() == RejectAndGiveUp)
381
0
          return false;
382
0
        break;
383
0
      }
384
2.19k
385
2.19k
      // Need to still jump to the end of the list of address spaces if we find
386
2.19k
      // a match earlier.
387
2.19k
      const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
388
2.19k
389
2.19k
      const MachineMemOperand *MMO
390
2.19k
        = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
391
2.19k
      const unsigned MMOAddrSpace = MMO->getAddrSpace();
392
2.19k
393
2.19k
      bool Success = false;
394
4.08k
      for (int I = 0; I != NumAddrSpace; 
++I1.89k
) {
395
2.54k
        unsigned AddrSpace = MatchTable[CurrentIdx++];
396
2.54k
        DEBUG_WITH_TYPE(
397
2.54k
          TgtInstructionSelector::getName(),
398
2.54k
          dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
399
2.54k
                 << AddrSpace << '\n');
400
2.54k
401
2.54k
        if (AddrSpace == MMOAddrSpace) {
402
649
          Success = true;
403
649
          break;
404
649
        }
405
2.54k
      }
406
2.19k
407
2.19k
      CurrentIdx = LastIdx;
408
2.19k
      if (!Success && 
handleReject() == RejectAndGiveUp1.54k
)
409
0
        return false;
410
2.19k
      break;
411
2.19k
    }
412
2.19k
    case GIM_CheckMemorySizeEqualTo: {
413
278
      int64_t InsnID = MatchTable[CurrentIdx++];
414
278
      int64_t MMOIdx = MatchTable[CurrentIdx++];
415
278
      uint64_t Size = MatchTable[CurrentIdx++];
416
278
417
278
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
418
278
                      dbgs() << CurrentIdx
419
278
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
420
278
                             << "]->memoperands() + " << MMOIdx
421
278
                             << ", Size=" << Size << ")\n");
422
278
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
423
278
424
278
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
278
430
278
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
431
278
432
278
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
433
278
                      dbgs() << MMO->getSize() << " bytes vs " << Size
434
278
                             << " bytes\n");
435
278
      if (MMO->getSize() != Size)
436
62
        if (handleReject() == RejectAndGiveUp)
437
0
          return false;
438
278
439
278
      break;
440
278
    }
441
3.50k
    case GIM_CheckMemorySizeEqualToLLT:
442
3.50k
    case GIM_CheckMemorySizeLessThanLLT:
443
3.50k
    case GIM_CheckMemorySizeGreaterThanLLT: {
444
3.50k
      int64_t InsnID = MatchTable[CurrentIdx++];
445
3.50k
      int64_t MMOIdx = MatchTable[CurrentIdx++];
446
3.50k
      int64_t OpIdx = MatchTable[CurrentIdx++];
447
3.50k
448
3.50k
      DEBUG_WITH_TYPE(
449
3.50k
          TgtInstructionSelector::getName(),
450
3.50k
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
451
3.50k
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
452
3.50k
                         ? "EqualTo"
453
3.50k
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
454
3.50k
                               ? "GreaterThan"
455
3.50k
                               : "LessThan")
456
3.50k
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
457
3.50k
                 << ", OpIdx=" << OpIdx << ")\n");
458
3.50k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
459
3.50k
460
3.50k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
461
3.50k
      if (!MO.isReg()) {
462
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
463
0
                        dbgs() << CurrentIdx << ": Not a register\n");
464
0
        if (handleReject() == RejectAndGiveUp)
465
0
          return false;
466
0
        break;
467
0
      }
468
3.50k
469
3.50k
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
470
0
        if (handleReject() == RejectAndGiveUp)
471
0
          return false;
472
0
        break;
473
0
      }
474
3.50k
475
3.50k
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
476
3.50k
477
3.50k
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
478
3.50k
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
479
3.50k
          
MMO->getSizeInBits() != Size964
) {
480
506
        if (handleReject() == RejectAndGiveUp)
481
0
          return false;
482
2.99k
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
483
2.99k
                 
MMO->getSizeInBits() >= Size2.54k
) {
484
1.68k
        if (handleReject() == RejectAndGiveUp)
485
0
          return false;
486
1.31k
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
487
1.31k
                 
MMO->getSizeInBits() <= Size0
)
488
0
        if (handleReject() == RejectAndGiveUp)
489
0
          return false;
490
3.50k
491
3.50k
      break;
492
3.50k
    }
493
3.50k
    case GIM_CheckType: {
494
606
      int64_t InsnID = MatchTable[CurrentIdx++];
495
606
      int64_t OpIdx = MatchTable[CurrentIdx++];
496
606
      int64_t TypeID = MatchTable[CurrentIdx++];
497
606
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498
606
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
499
606
                             << "]->getOperand(" << OpIdx
500
606
                             << "), TypeID=" << TypeID << ")\n");
501
606
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
502
606
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
503
606
      if (!MO.isReg() ||
504
606
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
505
22
        if (handleReject() == RejectAndGiveUp)
506
0
          return false;
507
606
      }
508
606
      break;
509
606
    }
510
763
    case GIM_CheckPointerToAny: {
511
763
      int64_t InsnID = MatchTable[CurrentIdx++];
512
763
      int64_t OpIdx = MatchTable[CurrentIdx++];
513
763
      int64_t SizeInBits = MatchTable[CurrentIdx++];
514
763
515
763
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
516
763
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
517
763
                             << InsnID << "]->getOperand(" << OpIdx
518
763
                             << "), SizeInBits=" << SizeInBits << ")\n");
519
763
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
520
763
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
521
763
      const LLT Ty = MRI.getType(MO.getReg());
522
763
523
763
      // iPTR must be looked up in the target.
524
763
      if (SizeInBits == 0) {
525
746
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
526
746
        const unsigned AddrSpace = Ty.getAddressSpace();
527
746
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
528
746
      }
529
763
530
763
      assert(SizeInBits != 0 && "Pointer size must be known");
531
763
532
763
      if (MO.isReg()) {
533
763
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
534
0
          if (handleReject() == RejectAndGiveUp)
535
0
            return false;
536
0
      } else if (handleReject() == RejectAndGiveUp)
537
0
        return false;
538
763
539
763
      break;
540
763
    }
541
1.10k
    case GIM_CheckRegBankForClass: {
542
1.10k
      int64_t InsnID = MatchTable[CurrentIdx++];
543
1.10k
      int64_t OpIdx = MatchTable[CurrentIdx++];
544
1.10k
      int64_t RCEnum = MatchTable[CurrentIdx++];
545
1.10k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
546
1.10k
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
547
1.10k
                             << InsnID << "]->getOperand(" << OpIdx
548
1.10k
                             << "), RCEnum=" << RCEnum << ")\n");
549
1.10k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
550
1.10k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
551
1.10k
      if (!MO.isReg() ||
552
1.10k
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
553
1.10k
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
554
237
        if (handleReject() == RejectAndGiveUp)
555
0
          return false;
556
1.10k
      }
557
1.10k
      break;
558
1.10k
    }
559
1.10k
560
1.10k
    case GIM_CheckComplexPattern: {
561
1.03k
      int64_t InsnID = MatchTable[CurrentIdx++];
562
1.03k
      int64_t OpIdx = MatchTable[CurrentIdx++];
563
1.03k
      int64_t RendererID = MatchTable[CurrentIdx++];
564
1.03k
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
565
1.03k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
566
1.03k
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
567
1.03k
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
568
1.03k
                             << "]->getOperand(" << OpIdx
569
1.03k
                             << "), ComplexPredicateID=" << ComplexPredicateID
570
1.03k
                             << ")\n");
571
1.03k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572
1.03k
      // FIXME: Use std::invoke() when it's available.
573
1.03k
      ComplexRendererFns Renderer =
574
1.03k
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
575
1.03k
              State.MIs[InsnID]->getOperand(OpIdx));
576
1.03k
      if (Renderer.hasValue())
577
768
        State.Renderers[RendererID] = Renderer.getValue();
578
270
      else
579
270
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
1.03k
      break;
582
1.03k
    }
583
1.03k
584
1.03k
    case GIM_CheckConstantInt: {
585
0
      int64_t InsnID = MatchTable[CurrentIdx++];
586
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
587
0
      int64_t Value = MatchTable[CurrentIdx++];
588
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
589
0
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
590
0
                             << InsnID << "]->getOperand(" << OpIdx
591
0
                             << "), Value=" << Value << ")\n");
592
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
594
0
      if (MO.isReg()) {
595
0
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
596
0
        LLT Ty = MRI.getType(MO.getReg());
597
0
        Value = SignExtend64(Value, Ty.getSizeInBits());
598
0
599
0
        if (!isOperandImmEqual(MO, Value, MRI)) {
600
0
          if (handleReject() == RejectAndGiveUp)
601
0
            return false;
602
0
        }
603
0
      } else if (handleReject() == RejectAndGiveUp)
604
0
        return false;
605
0
606
0
      break;
607
0
    }
608
0
609
0
    case GIM_CheckLiteralInt: {
610
0
      int64_t InsnID = MatchTable[CurrentIdx++];
611
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
612
0
      int64_t Value = MatchTable[CurrentIdx++];
613
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
614
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
615
0
                             << InsnID << "]->getOperand(" << OpIdx
616
0
                             << "), Value=" << Value << ")\n");
617
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
618
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
619
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
620
0
        if (handleReject() == RejectAndGiveUp)
621
0
          return false;
622
0
      }
623
0
      break;
624
0
    }
625
0
626
46
    case GIM_CheckIntrinsicID: {
627
46
      int64_t InsnID = MatchTable[CurrentIdx++];
628
46
      int64_t OpIdx = MatchTable[CurrentIdx++];
629
46
      int64_t Value = MatchTable[CurrentIdx++];
630
46
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
631
46
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
632
46
                             << InsnID << "]->getOperand(" << OpIdx
633
46
                             << "), Value=" << Value << ")\n");
634
46
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
635
46
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
636
46
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
637
36
        if (handleReject() == RejectAndGiveUp)
638
0
          return false;
639
46
      break;
640
46
    }
641
46
642
46
    case GIM_CheckIsMBB: {
643
28
      int64_t InsnID = MatchTable[CurrentIdx++];
644
28
      int64_t OpIdx = MatchTable[CurrentIdx++];
645
28
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
646
28
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
647
28
                             << "]->getOperand(" << OpIdx << "))\n");
648
28
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
649
28
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
650
0
        if (handleReject() == RejectAndGiveUp)
651
0
          return false;
652
28
      }
653
28
      break;
654
28
    }
655
28
656
28
    case GIM_CheckIsSafeToFold: {
657
1
      int64_t InsnID = MatchTable[CurrentIdx++];
658
1
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
659
1
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
660
1
                             << InsnID << "])\n");
661
1
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
662
1
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
663
0
        if (handleReject() == RejectAndGiveUp)
664
0
          return false;
665
1
      }
666
1
      break;
667
1
    }
668
1
    case GIM_CheckIsSameOperand: {
669
0
      int64_t InsnID = MatchTable[CurrentIdx++];
670
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
671
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
672
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
673
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
674
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
675
0
                             << InsnID << "][" << OpIdx << "], MIs["
676
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
677
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
678
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
679
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
680
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
681
0
        if (handleReject() == RejectAndGiveUp)
682
0
          return false;
683
0
      }
684
0
      break;
685
0
    }
686
872
    case GIM_Reject:
687
872
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
688
872
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
689
872
      if (handleReject() == RejectAndGiveUp)
690
408
        return false;
691
464
      break;
692
464
693
464
    case GIR_MutateOpcode: {
694
90
      int64_t OldInsnID = MatchTable[CurrentIdx++];
695
90
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
696
90
      int64_t NewOpcode = MatchTable[CurrentIdx++];
697
90
      if (NewInsnID >= OutMIs.size())
698
90
        OutMIs.resize(NewInsnID + 1);
699
90
700
90
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
701
90
                                              State.MIs[OldInsnID]);
702
90
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
703
90
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
704
90
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
705
90
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
706
90
                             << NewOpcode << ")\n");
707
90
      break;
708
464
    }
709
464
710
851
    case GIR_BuildMI: {
711
851
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
712
851
      int64_t Opcode = MatchTable[CurrentIdx++];
713
851
      if (NewInsnID >= OutMIs.size())
714
851
        OutMIs.resize(NewInsnID + 1);
715
851
716
851
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
717
851
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
718
851
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
719
851
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
720
851
                             << NewInsnID << "], " << Opcode << ")\n");
721
851
      break;
722
464
    }
723
464
724
1.11k
    case GIR_Copy: {
725
1.11k
      int64_t NewInsnID = MatchTable[CurrentIdx++];
726
1.11k
      int64_t OldInsnID = MatchTable[CurrentIdx++];
727
1.11k
      int64_t OpIdx = MatchTable[CurrentIdx++];
728
1.11k
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
729
1.11k
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
730
1.11k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
731
1.11k
                      dbgs()
732
1.11k
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
733
1.11k
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
734
1.11k
      break;
735
464
    }
736
464
737
464
    case GIR_CopyOrAddZeroReg: {
738
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
739
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
740
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
741
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
742
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
743
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
744
0
      if (isOperandImmEqual(MO, 0, MRI))
745
0
        OutMIs[NewInsnID].addReg(ZeroReg);
746
0
      else
747
0
        OutMIs[NewInsnID].add(MO);
748
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
749
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
750
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
751
0
                             << OpIdx << ", " << ZeroReg << ")\n");
752
0
      break;
753
464
    }
754
464
755
464
    case GIR_CopySubReg: {
756
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
757
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
758
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
759
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
760
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
761
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
762
0
                               0, SubRegIdx);
763
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
764
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
765
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
766
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
767
0
      break;
768
464
    }
769
464
770
464
    case GIR_AddImplicitDef: {
771
30
      int64_t InsnID = MatchTable[CurrentIdx++];
772
30
      int64_t RegNum = MatchTable[CurrentIdx++];
773
30
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
774
30
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
775
30
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
776
30
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
777
30
                             << InsnID << "], " << RegNum << ")\n");
778
30
      break;
779
464
    }
780
464
781
464
    case GIR_AddImplicitUse: {
782
31
      int64_t InsnID = MatchTable[CurrentIdx++];
783
31
      int64_t RegNum = MatchTable[CurrentIdx++];
784
31
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
785
31
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
786
31
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
787
31
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
788
31
                             << InsnID << "], " << RegNum << ")\n");
789
31
      break;
790
464
    }
791
464
792
464
    case GIR_AddRegister: {
793
0
      int64_t InsnID = MatchTable[CurrentIdx++];
794
0
      int64_t RegNum = MatchTable[CurrentIdx++];
795
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
796
0
      OutMIs[InsnID].addReg(RegNum);
797
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
798
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
799
0
                             << InsnID << "], " << RegNum << ")\n");
800
0
      break;
801
464
    }
802
464
803
464
    case GIR_AddTempRegister: {
804
0
      int64_t InsnID = MatchTable[CurrentIdx++];
805
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
806
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
807
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
808
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
809
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
810
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
811
0
                             << InsnID << "], TempRegisters[" << TempRegID
812
0
                             << "], " << TempRegFlags << ")\n");
813
0
      break;
814
464
    }
815
464
816
1.53k
    case GIR_AddImm: {
817
1.53k
      int64_t InsnID = MatchTable[CurrentIdx++];
818
1.53k
      int64_t Imm = MatchTable[CurrentIdx++];
819
1.53k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
820
1.53k
      OutMIs[InsnID].addImm(Imm);
821
1.53k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
822
1.53k
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
823
1.53k
                             << "], " << Imm << ")\n");
824
1.53k
      break;
825
464
    }
826
464
827
464
    case GIR_ComplexRenderer: {
828
0
      int64_t InsnID = MatchTable[CurrentIdx++];
829
0
      int64_t RendererID = MatchTable[CurrentIdx++];
830
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
831
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
832
0
        RenderOpFn(OutMIs[InsnID]);
833
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
834
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
835
0
                             << InsnID << "], " << RendererID << ")\n");
836
0
      break;
837
464
    }
838
2.26k
    case GIR_ComplexSubOperandRenderer: {
839
2.26k
      int64_t InsnID = MatchTable[CurrentIdx++];
840
2.26k
      int64_t RendererID = MatchTable[CurrentIdx++];
841
2.26k
      int64_t RenderOpID = MatchTable[CurrentIdx++];
842
2.26k
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
843
2.26k
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
844
2.26k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
845
2.26k
                      dbgs() << CurrentIdx
846
2.26k
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
847
2.26k
                             << InsnID << "], " << RendererID << ", "
848
2.26k
                             << RenderOpID << ")\n");
849
2.26k
      break;
850
464
    }
851
464
852
464
    case GIR_CopyConstantAsSImm: {
853
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
854
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
855
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
856
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
857
0
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
858
0
        OutMIs[NewInsnID].addImm(
859
0
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
860
0
      } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
861
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
862
0
      else
863
0
        llvm_unreachable("Expected Imm or CImm operand");
864
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
865
0
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
866
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
867
0
      break;
868
0
    }
869
0
870
0
    // TODO: Needs a test case once we have a pattern that uses this.
871
0
    case GIR_CopyFConstantAsFPImm: {
872
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
873
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
874
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
875
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
876
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
877
0
        OutMIs[NewInsnID].addFPImm(
878
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
879
0
      else
880
0
        llvm_unreachable("Expected FPImm operand");
881
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
882
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
883
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
884
0
      break;
885
0
    }
886
0
887
0
    case GIR_CustomRenderer: {
888
0
      int64_t InsnID = MatchTable[CurrentIdx++];
889
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
890
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
891
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
892
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
893
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
894
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
895
0
                             << RendererFnID << ")\n");
896
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
897
0
                                                     *State.MIs[OldInsnID]);
898
0
      break;
899
0
    }
900
0
    case GIR_ConstrainOperandRC: {
901
0
      int64_t InsnID = MatchTable[CurrentIdx++];
902
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
903
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
904
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
905
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
906
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
907
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
908
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
909
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
910
0
                             << ")\n");
911
0
      break;
912
0
    }
913
0
914
941
    case GIR_ConstrainSelectedInstOperands: {
915
941
      int64_t InsnID = MatchTable[CurrentIdx++];
916
941
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
917
941
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
918
941
                                       RBI);
919
941
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
920
941
                      dbgs() << CurrentIdx
921
941
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
922
941
                             << InsnID << "])\n");
923
941
      break;
924
0
    }
925
0
926
648
    case GIR_MergeMemOperands: {
927
648
      int64_t InsnID = MatchTable[CurrentIdx++];
928
648
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
929
648
930
648
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
931
648
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
932
648
                             << InsnID << "]");
933
648
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
934
1.29k
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
935
1.29k
             GIU_MergeMemOperands_EndOfList) {
936
648
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
937
648
                        dbgs() << ", MIs[" << MergeInsnID << "]");
938
648
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
939
647
          OutMIs[InsnID].addMemOperand(MMO);
940
648
      }
941
648
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
942
648
      break;
943
0
    }
944
0
945
851
    case GIR_EraseFromParent: {
946
851
      int64_t InsnID = MatchTable[CurrentIdx++];
947
851
      assert(State.MIs[InsnID] &&
948
851
             "Attempted to erase an undefined instruction");
949
851
      State.MIs[InsnID]->eraseFromParent();
950
851
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
951
851
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
952
851
                             << InsnID << "])\n");
953
851
      break;
954
0
    }
955
0
956
0
    case GIR_MakeTempReg: {
957
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
958
0
      int64_t TypeID = MatchTable[CurrentIdx++];
959
0
960
0
      State.TempRegisters[TempRegID] =
961
0
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
962
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
963
0
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
964
0
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
965
0
      break;
966
0
    }
967
0
968
0
    case GIR_Coverage: {
969
0
      int64_t RuleID = MatchTable[CurrentIdx++];
970
0
      CoverageInfo.setCovered(RuleID);
971
0
972
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
973
0
                      dbgs()
974
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
975
0
      break;
976
0
    }
977
0
978
941
    case GIR_Done:
979
941
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
980
941
                      dbgs() << CurrentIdx << ": GIR_Done\n");
981
941
      return true;
982
0
983
0
    default:
984
0
      llvm_unreachable("Unexpected command");
985
37.0k
    }
986
37.0k
  }
987
1.34k
}
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<70ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<70ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<70ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
1.11k
    CodeGenCoverage &CoverageInfo) const {
56
1.11k
57
1.11k
  uint64_t CurrentIdx = 0;
58
1.11k
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
1.11k
60
1.11k
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
1.11k
  auto handleReject = [&]() -> RejectAction {
62
1.11k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
1.11k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
1.11k
    if (OnFailResumeAt.empty())
65
1.11k
      return RejectAndGiveUp;
66
1.11k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
1.11k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
1.11k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
1.11k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
1.11k
    return RejectAndResume;
71
1.11k
  };
72
1.11k
73
15.4k
  while (true) {
74
15.4k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
15.4k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
15.4k
    switch (MatcherOpcode) {
77
15.4k
    case GIM_Try: {
78
2.76k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
2.76k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
2.76k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
2.76k
      break;
82
15.4k
    }
83
15.4k
84
15.4k
    case GIM_RecordInsn: {
85
827
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
827
      int64_t InsnID = MatchTable[CurrentIdx++];
87
827
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
827
89
827
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
827
      // any attempt to modify it.
91
827
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
827
93
827
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
827
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
827
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
827
109
827
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
827
      if ((size_t)NewInsnID < State.MIs.size())
111
571
        State.MIs[NewInsnID] = NewMI;
112
256
      else {
113
256
        assert((size_t)NewInsnID == State.MIs.size() &&
114
256
               "Expected to store MIs in order");
115
256
        State.MIs.push_back(NewMI);
116
256
      }
117
827
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
827
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
827
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
827
                             << ")\n");
121
827
      break;
122
827
    }
123
827
124
2.40k
    case GIM_CheckFeatures: {
125
2.40k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
2.40k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
2.40k
                      dbgs() << CurrentIdx
128
2.40k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
2.40k
                             << ExpectedBitsetID << ")\n");
130
2.40k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
2.40k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
1.42k
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
2.40k
      }
135
2.40k
      break;
136
2.40k
    }
137
2.40k
138
2.40k
    case GIM_CheckOpcode: {
139
827
      int64_t InsnID = MatchTable[CurrentIdx++];
140
827
      int64_t Expected = MatchTable[CurrentIdx++];
141
827
142
827
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
827
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
827
145
827
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
827
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
827
                             << "], ExpectedOpcode=" << Expected
148
827
                             << ") // Got=" << Opcode << "\n");
149
827
      if (Opcode != Expected) {
150
584
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
827
      }
153
827
      break;
154
827
    }
155
827
156
1.11k
    case GIM_SwitchOpcode: {
157
1.11k
      int64_t InsnID = MatchTable[CurrentIdx++];
158
1.11k
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
1.11k
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
1.11k
      int64_t Default = MatchTable[CurrentIdx++];
161
1.11k
162
1.11k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
1.11k
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
1.11k
165
1.11k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
1.11k
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
1.11k
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
1.11k
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
1.11k
      });
170
1.11k
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
0
        CurrentIdx = Default;
172
0
        break;
173
0
      }
174
1.11k
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
1.11k
      if (!CurrentIdx) {
176
579
        CurrentIdx = Default;
177
579
  break;
178
579
      }
179
534
      OnFailResumeAt.push_back(Default);
180
534
      break;
181
534
    }
182
534
183
534
    case GIM_SwitchType: {
184
368
      int64_t InsnID = MatchTable[CurrentIdx++];
185
368
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
368
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
368
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
368
      int64_t Default = MatchTable[CurrentIdx++];
189
368
190
368
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
368
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
368
193
368
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
368
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
368
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
368
               << UpperBound << "), Default=" << Default
197
368
               << ", JumpTable...) // Got=";
198
368
        if (!MO.isReg())
199
368
          dbgs() << "Not a VReg\n";
200
368
        else
201
368
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
368
      });
203
368
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
368
      const LLT Ty = MRI.getType(MO.getReg());
208
368
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
368
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
49
        CurrentIdx = Default;
211
49
        break;
212
49
      }
213
319
      const int64_t TypeID = TyI->second;
214
319
      if (TypeID < LowerBound || 
UpperBound <= TypeID219
) {
215
100
        CurrentIdx = Default;
216
100
        break;
217
100
      }
218
219
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
219
      if (!CurrentIdx) {
220
0
        CurrentIdx = Default;
221
0
        break;
222
0
      }
223
219
      OnFailResumeAt.push_back(Default);
224
219
      break;
225
219
    }
226
219
227
219
    case GIM_CheckNumOperands: {
228
0
      int64_t InsnID = MatchTable[CurrentIdx++];
229
0
      int64_t Expected = MatchTable[CurrentIdx++];
230
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
0
                             << InsnID << "], Expected=" << Expected << ")\n");
233
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
0
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
0
      }
238
0
      break;
239
0
    }
240
81
    case GIM_CheckI64ImmPredicate: {
241
81
      int64_t InsnID = MatchTable[CurrentIdx++];
242
81
      int64_t Predicate = MatchTable[CurrentIdx++];
243
81
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
81
                      dbgs()
245
81
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
81
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
81
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
81
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
81
             "Expected G_CONSTANT");
250
81
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
81
      int64_t Value = 0;
252
81
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
81
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
81
259
81
      if (!testImmPredicate_I64(Predicate, Value))
260
15
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
81
      break;
263
81
    }
264
81
    case GIM_CheckAPIntImmPredicate: {
265
1
      int64_t InsnID = MatchTable[CurrentIdx++];
266
1
      int64_t Predicate = MatchTable[CurrentIdx++];
267
1
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
1
                      dbgs()
269
1
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
1
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
1
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
1
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
1
             "Expected G_CONSTANT");
274
1
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
1
      APInt Value;
276
1
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
1
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
1
      else
279
1
        
llvm_unreachable0
("Expected Imm or CImm operand");
280
1
281
1
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
1
      break;
285
1
    }
286
1
    case GIM_CheckAPFloatImmPredicate: {
287
0
      int64_t InsnID = MatchTable[CurrentIdx++];
288
0
      int64_t Predicate = MatchTable[CurrentIdx++];
289
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
0
                      dbgs()
291
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
0
             "Expected G_FCONSTANT");
296
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
0
300
0
      if (!testImmPredicate_APFloat(Predicate, Value))
301
0
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
0
      break;
304
0
    }
305
10
    case GIM_CheckCxxInsnPredicate: {
306
10
      int64_t InsnID = MatchTable[CurrentIdx++];
307
10
      int64_t Predicate = MatchTable[CurrentIdx++];
308
10
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
10
                      dbgs()
310
10
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
10
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
10
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
10
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
10
315
10
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
4
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
10
      break;
319
10
    }
320
10
    case GIM_CheckAtomicOrdering: {
321
0
      int64_t InsnID = MatchTable[CurrentIdx++];
322
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
0
331
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
0
        if (MMO->getOrdering() != Ordering)
333
0
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
0
      break;
336
0
    }
337
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
0
    case GIM_CheckMemoryAddressSpace: {
374
0
      int64_t InsnID = MatchTable[CurrentIdx++];
375
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
0
      // This accepts a list of possible address spaces.
377
0
      const int NumAddrSpace = MatchTable[CurrentIdx++];
378
0
379
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
380
0
        if (handleReject() == RejectAndGiveUp)
381
0
          return false;
382
0
        break;
383
0
      }
384
0
385
0
      // Need to still jump to the end of the list of address spaces if we find
386
0
      // a match earlier.
387
0
      const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
388
0
389
0
      const MachineMemOperand *MMO
390
0
        = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
391
0
      const unsigned MMOAddrSpace = MMO->getAddrSpace();
392
0
393
0
      bool Success = false;
394
0
      for (int I = 0; I != NumAddrSpace; ++I) {
395
0
        unsigned AddrSpace = MatchTable[CurrentIdx++];
396
0
        DEBUG_WITH_TYPE(
397
0
          TgtInstructionSelector::getName(),
398
0
          dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
399
0
                 << AddrSpace << '\n');
400
0
401
0
        if (AddrSpace == MMOAddrSpace) {
402
0
          Success = true;
403
0
          break;
404
0
        }
405
0
      }
406
0
407
0
      CurrentIdx = LastIdx;
408
0
      if (!Success && handleReject() == RejectAndGiveUp)
409
0
        return false;
410
0
      break;
411
0
    }
412
0
    case GIM_CheckMemorySizeEqualTo: {
413
0
      int64_t InsnID = MatchTable[CurrentIdx++];
414
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
415
0
      uint64_t Size = MatchTable[CurrentIdx++];
416
0
417
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
418
0
                      dbgs() << CurrentIdx
419
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
420
0
                             << "]->memoperands() + " << MMOIdx
421
0
                             << ", Size=" << Size << ")\n");
422
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
423
0
424
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
0
430
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
431
0
432
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
433
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
434
0
                             << " bytes\n");
435
0
      if (MMO->getSize() != Size)
436
0
        if (handleReject() == RejectAndGiveUp)
437
0
          return false;
438
0
439
0
      break;
440
0
    }
441
0
    case GIM_CheckMemorySizeEqualToLLT:
442
0
    case GIM_CheckMemorySizeLessThanLLT:
443
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
444
0
      int64_t InsnID = MatchTable[CurrentIdx++];
445
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
446
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
447
0
448
0
      DEBUG_WITH_TYPE(
449
0
          TgtInstructionSelector::getName(),
450
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
451
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
452
0
                         ? "EqualTo"
453
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
454
0
                               ? "GreaterThan"
455
0
                               : "LessThan")
456
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
457
0
                 << ", OpIdx=" << OpIdx << ")\n");
458
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
459
0
460
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
461
0
      if (!MO.isReg()) {
462
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
463
0
                        dbgs() << CurrentIdx << ": Not a register\n");
464
0
        if (handleReject() == RejectAndGiveUp)
465
0
          return false;
466
0
        break;
467
0
      }
468
0
469
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
470
0
        if (handleReject() == RejectAndGiveUp)
471
0
          return false;
472
0
        break;
473
0
      }
474
0
475
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
476
0
477
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
478
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
479
0
          MMO->getSizeInBits() != Size) {
480
0
        if (handleReject() == RejectAndGiveUp)
481
0
          return false;
482
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
483
0
                 MMO->getSizeInBits() >= Size) {
484
0
        if (handleReject() == RejectAndGiveUp)
485
0
          return false;
486
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
487
0
                 MMO->getSizeInBits() <= Size)
488
0
        if (handleReject() == RejectAndGiveUp)
489
0
          return false;
490
0
491
0
      break;
492
0
    }
493
1.15k
    case GIM_CheckType: {
494
1.15k
      int64_t InsnID = MatchTable[CurrentIdx++];
495
1.15k
      int64_t OpIdx = MatchTable[CurrentIdx++];
496
1.15k
      int64_t TypeID = MatchTable[CurrentIdx++];
497
1.15k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498
1.15k
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
499
1.15k
                             << "]->getOperand(" << OpIdx
500
1.15k
                             << "), TypeID=" << TypeID << ")\n");
501
1.15k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
502
1.15k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
503
1.15k
      if (!MO.isReg() ||
504
1.15k
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
505
7
        if (handleReject() == RejectAndGiveUp)
506
0
          return false;
507
1.15k
      }
508
1.15k
      break;
509
1.15k
    }
510
1.15k
    case GIM_CheckPointerToAny: {
511
0
      int64_t InsnID = MatchTable[CurrentIdx++];
512
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
513
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
514
0
515
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
516
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
517
0
                             << InsnID << "]->getOperand(" << OpIdx
518
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
519
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
520
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
521
0
      const LLT Ty = MRI.getType(MO.getReg());
522
0
523
0
      // iPTR must be looked up in the target.
524
0
      if (SizeInBits == 0) {
525
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
526
0
        const unsigned AddrSpace = Ty.getAddressSpace();
527
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
528
0
      }
529
0
530
0
      assert(SizeInBits != 0 && "Pointer size must be known");
531
0
532
0
      if (MO.isReg()) {
533
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
534
0
          if (handleReject() == RejectAndGiveUp)
535
0
            return false;
536
0
      } else if (handleReject() == RejectAndGiveUp)
537
0
        return false;
538
0
539
0
      break;
540
0
    }
541
1.70k
    case GIM_CheckRegBankForClass: {
542
1.70k
      int64_t InsnID = MatchTable[CurrentIdx++];
543
1.70k
      int64_t OpIdx = MatchTable[CurrentIdx++];
544
1.70k
      int64_t RCEnum = MatchTable[CurrentIdx++];
545
1.70k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
546
1.70k
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
547
1.70k
                             << InsnID << "]->getOperand(" << OpIdx
548
1.70k
                             << "), RCEnum=" << RCEnum << ")\n");
549
1.70k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
550
1.70k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
551
1.70k
      if (!MO.isReg() ||
552
1.70k
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
553
1.70k
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
554
0
        if (handleReject() == RejectAndGiveUp)
555
0
          return false;
556
1.70k
      }
557
1.70k
      break;
558
1.70k
    }
559
1.70k
560
1.70k
    case GIM_CheckComplexPattern: {
561
0
      int64_t InsnID = MatchTable[CurrentIdx++];
562
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
563
0
      int64_t RendererID = MatchTable[CurrentIdx++];
564
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
565
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
566
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
567
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
568
0
                             << "]->getOperand(" << OpIdx
569
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
570
0
                             << ")\n");
571
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572
0
      // FIXME: Use std::invoke() when it's available.
573
0
      ComplexRendererFns Renderer =
574
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
575
0
              State.MIs[InsnID]->getOperand(OpIdx));
576
0
      if (Renderer.hasValue())
577
0
        State.Renderers[RendererID] = Renderer.getValue();
578
0
      else
579
0
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
0
      break;
582
0
    }
583
0
584
183
    case GIM_CheckConstantInt: {
585
183
      int64_t InsnID = MatchTable[CurrentIdx++];
586
183
      int64_t OpIdx = MatchTable[CurrentIdx++];
587
183
      int64_t Value = MatchTable[CurrentIdx++];
588
183
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
589
183
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
590
183
                             << InsnID << "]->getOperand(" << OpIdx
591
183
                             << "), Value=" << Value << ")\n");
592
183
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593
183
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
594
183
      if (MO.isReg()) {
595
183
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
596
183
        LLT Ty = MRI.getType(MO.getReg());
597
183
        Value = SignExtend64(Value, Ty.getSizeInBits());
598
183
599
183
        if (!isOperandImmEqual(MO, Value, MRI)) {
600
83
          if (handleReject() == RejectAndGiveUp)
601
0
            return false;
602
0
        }
603
0
      } else if (handleReject() == RejectAndGiveUp)
604
0
        return false;
605
183
606
183
      break;
607
183
    }
608
183
609
183
    case GIM_CheckLiteralInt: {
610
0
      int64_t InsnID = MatchTable[CurrentIdx++];
611
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
612
0
      int64_t Value = MatchTable[CurrentIdx++];
613
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
614
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
615
0
                             << InsnID << "]->getOperand(" << OpIdx
616
0
                             << "), Value=" << Value << ")\n");
617
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
618
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
619
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
620
0
        if (handleReject() == RejectAndGiveUp)
621
0
          return false;
622
0
      }
623
0
      break;
624
0
    }
625
0
626
0
    case GIM_CheckIntrinsicID: {
627
0
      int64_t InsnID = MatchTable[CurrentIdx++];
628
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
629
0
      int64_t Value = MatchTable[CurrentIdx++];
630
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
631
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
632
0
                             << InsnID << "]->getOperand(" << OpIdx
633
0
                             << "), Value=" << Value << ")\n");
634
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
635
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
636
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
637
0
        if (handleReject() == RejectAndGiveUp)
638
0
          return false;
639
0
      break;
640
0
    }
641
0
642
15
    case GIM_CheckIsMBB: {
643
15
      int64_t InsnID = MatchTable[CurrentIdx++];
644
15
      int64_t OpIdx = MatchTable[CurrentIdx++];
645
15
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
646
15
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
647
15
                             << "]->getOperand(" << OpIdx << "))\n");
648
15
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
649
15
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
650
0
        if (handleReject() == RejectAndGiveUp)
651
0
          return false;
652
15
      }
653
15
      break;
654
15
    }
655
15
656
118
    case GIM_CheckIsSafeToFold: {
657
118
      int64_t InsnID = MatchTable[CurrentIdx++];
658
118
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
659
118
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
660
118
                             << InsnID << "])\n");
661
118
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
662
118
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
663
0
        if (handleReject() == RejectAndGiveUp)
664
0
          return false;
665
118
      }
666
118
      break;
667
118
    }
668
118
    case GIM_CheckIsSameOperand: {
669
4
      int64_t InsnID = MatchTable[CurrentIdx++];
670
4
      int64_t OpIdx = MatchTable[CurrentIdx++];
671
4
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
672
4
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
673
4
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
674
4
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
675
4
                             << InsnID << "][" << OpIdx << "], MIs["
676
4
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
677
4
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
678
4
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
679
4
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
680
4
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
681
0
        if (handleReject() == RejectAndGiveUp)
682
0
          return false;
683
4
      }
684
4
      break;
685
4
    }
686
1.18k
    case GIM_Reject:
687
1.18k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
688
1.18k
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
689
1.18k
      if (handleReject() == RejectAndGiveUp)
690
820
        return false;
691
367
      break;
692
367
693
367
    case GIR_MutateOpcode: {
694
9
      int64_t OldInsnID = MatchTable[CurrentIdx++];
695
9
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
696
9
      int64_t NewOpcode = MatchTable[CurrentIdx++];
697
9
      if (NewInsnID >= OutMIs.size())
698
9
        OutMIs.resize(NewInsnID + 1);
699
9
700
9
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
701
9
                                              State.MIs[OldInsnID]);
702
9
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
703
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
704
9
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
705
9
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
706
9
                             << NewOpcode << ")\n");
707
9
      break;
708
367
    }
709
367
710
367
    case GIR_BuildMI: {
711
301
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
712
301
      int64_t Opcode = MatchTable[CurrentIdx++];
713
301
      if (NewInsnID >= OutMIs.size())
714
284
        OutMIs.resize(NewInsnID + 1);
715
301
716
301
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
717
301
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
718
301
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
719
301
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
720
301
                             << NewInsnID << "], " << Opcode << ")\n");
721
301
      break;
722
367
    }
723
367
724
695
    case GIR_Copy: {
725
695
      int64_t NewInsnID = MatchTable[CurrentIdx++];
726
695
      int64_t OldInsnID = MatchTable[CurrentIdx++];
727
695
      int64_t OpIdx = MatchTable[CurrentIdx++];
728
695
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
729
695
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
730
695
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
731
695
                      dbgs()
732
695
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
733
695
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
734
695
      break;
735
367
    }
736
367
737
367
    case GIR_CopyOrAddZeroReg: {
738
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
739
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
740
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
741
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
742
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
743
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
744
0
      if (isOperandImmEqual(MO, 0, MRI))
745
0
        OutMIs[NewInsnID].addReg(ZeroReg);
746
0
      else
747
0
        OutMIs[NewInsnID].add(MO);
748
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
749
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
750
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
751
0
                             << OpIdx << ", " << ZeroReg << ")\n");
752
0
      break;
753
367
    }
754
367
755
367
    case GIR_CopySubReg: {
756
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
757
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
758
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
759
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
760
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
761
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
762
0
                               0, SubRegIdx);
763
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
764
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
765
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
766
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
767
0
      break;
768
367
    }
769
367
770
367
    case GIR_AddImplicitDef: {
771
0
      int64_t InsnID = MatchTable[CurrentIdx++];
772
0
      int64_t RegNum = MatchTable[CurrentIdx++];
773
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
774
0
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
775
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
776
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
777
0
                             << InsnID << "], " << RegNum << ")\n");
778
0
      break;
779
367
    }
780
367
781
367
    case GIR_AddImplicitUse: {
782
0
      int64_t InsnID = MatchTable[CurrentIdx++];
783
0
      int64_t RegNum = MatchTable[CurrentIdx++];
784
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
785
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
786
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
787
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
788
0
                             << InsnID << "], " << RegNum << ")\n");
789
0
      break;
790
367
    }
791
367
792
383
    case GIR_AddRegister: {
793
383
      int64_t InsnID = MatchTable[CurrentIdx++];
794
383
      int64_t RegNum = MatchTable[CurrentIdx++];
795
383
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
796
383
      OutMIs[InsnID].addReg(RegNum);
797
383
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
798
383
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
799
383
                             << InsnID << "], " << RegNum << ")\n");
800
383
      break;
801
367
    }
802
367
803
367
    case GIR_AddTempRegister: {
804
34
      int64_t InsnID = MatchTable[CurrentIdx++];
805
34
      int64_t TempRegID = MatchTable[CurrentIdx++];
806
34
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
807
34
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
808
34
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
809
34
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
810
34
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
811
34
                             << InsnID << "], TempRegisters[" << TempRegID
812
34
                             << "], " << TempRegFlags << ")\n");
813
34
      break;
814
367
    }
815
367
816
367
    case GIR_AddImm: {
817
319
      int64_t InsnID = MatchTable[CurrentIdx++];
818
319
      int64_t Imm = MatchTable[CurrentIdx++];
819
319
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
820
319
      OutMIs[InsnID].addImm(Imm);
821
319
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
822
319
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
823
319
                             << "], " << Imm << ")\n");
824
319
      break;
825
367
    }
826
367
827
367
    case GIR_ComplexRenderer: {
828
0
      int64_t InsnID = MatchTable[CurrentIdx++];
829
0
      int64_t RendererID = MatchTable[CurrentIdx++];
830
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
831
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
832
0
        RenderOpFn(OutMIs[InsnID]);
833
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
834
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
835
0
                             << InsnID << "], " << RendererID << ")\n");
836
0
      break;
837
367
    }
838
367
    case GIR_ComplexSubOperandRenderer: {
839
0
      int64_t InsnID = MatchTable[CurrentIdx++];
840
0
      int64_t RendererID = MatchTable[CurrentIdx++];
841
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
842
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
843
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
844
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
845
0
                      dbgs() << CurrentIdx
846
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
847
0
                             << InsnID << "], " << RendererID << ", "
848
0
                             << RenderOpID << ")\n");
849
0
      break;
850
367
    }
851
367
852
367
    case GIR_CopyConstantAsSImm: {
853
70
      int64_t NewInsnID = MatchTable[CurrentIdx++];
854
70
      int64_t OldInsnID = MatchTable[CurrentIdx++];
855
70
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
856
70
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
857
70
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
858
70
        OutMIs[NewInsnID].addImm(
859
70
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
860
70
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
861
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
862
0
      else
863
0
        llvm_unreachable("Expected Imm or CImm operand");
864
70
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
865
70
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
866
70
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
867
70
      break;
868
70
    }
869
70
870
70
    // TODO: Needs a test case once we have a pattern that uses this.
871
70
    case GIR_CopyFConstantAsFPImm: {
872
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
873
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
874
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
875
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
876
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
877
0
        OutMIs[NewInsnID].addFPImm(
878
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
879
0
      else
880
0
        llvm_unreachable("Expected FPImm operand");
881
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
882
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
883
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
884
0
      break;
885
0
    }
886
0
887
4
    case GIR_CustomRenderer: {
888
4
      int64_t InsnID = MatchTable[CurrentIdx++];
889
4
      int64_t OldInsnID = MatchTable[CurrentIdx++];
890
4
      int64_t RendererFnID = MatchTable[CurrentIdx++];
891
4
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
892
4
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
893
4
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
894
4
                             << InsnID << "], MIs[" << OldInsnID << "], "
895
4
                             << RendererFnID << ")\n");
896
4
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
897
4
                                                     *State.MIs[OldInsnID]);
898
4
      break;
899
0
    }
900
9
    case GIR_ConstrainOperandRC: {
901
9
      int64_t InsnID = MatchTable[CurrentIdx++];
902
9
      int64_t OpIdx = MatchTable[CurrentIdx++];
903
9
      int64_t RCEnum = MatchTable[CurrentIdx++];
904
9
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
905
9
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
906
9
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
907
9
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
908
9
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
909
9
                             << InsnID << "], " << OpIdx << ", " << RCEnum
910
9
                             << ")\n");
911
9
      break;
912
0
    }
913
0
914
301
    case GIR_ConstrainSelectedInstOperands: {
915
301
      int64_t InsnID = MatchTable[CurrentIdx++];
916
301
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
917
301
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
918
301
                                       RBI);
919
301
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
920
301
                      dbgs() << CurrentIdx
921
301
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
922
301
                             << InsnID << "])\n");
923
301
      break;
924
0
    }
925
0
926
0
    case GIR_MergeMemOperands: {
927
0
      int64_t InsnID = MatchTable[CurrentIdx++];
928
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
929
0
930
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
931
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
932
0
                             << InsnID << "]");
933
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
934
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
935
0
             GIU_MergeMemOperands_EndOfList) {
936
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
937
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
938
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
939
0
          OutMIs[InsnID].addMemOperand(MMO);
940
0
      }
941
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
942
0
      break;
943
0
    }
944
0
945
284
    case GIR_EraseFromParent: {
946
284
      int64_t InsnID = MatchTable[CurrentIdx++];
947
284
      assert(State.MIs[InsnID] &&
948
284
             "Attempted to erase an undefined instruction");
949
284
      State.MIs[InsnID]->eraseFromParent();
950
284
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
951
284
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
952
284
                             << InsnID << "])\n");
953
284
      break;
954
0
    }
955
0
956
17
    case GIR_MakeTempReg: {
957
17
      int64_t TempRegID = MatchTable[CurrentIdx++];
958
17
      int64_t TypeID = MatchTable[CurrentIdx++];
959
17
960
17
      State.TempRegisters[TempRegID] =
961
17
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
962
17
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
963
17
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
964
17
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
965
17
      break;
966
0
    }
967
0
968
0
    case GIR_Coverage: {
969
0
      int64_t RuleID = MatchTable[CurrentIdx++];
970
0
      CoverageInfo.setCovered(RuleID);
971
0
972
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
973
0
                      dbgs()
974
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
975
0
      break;
976
0
    }
977
0
978
293
    case GIR_Done:
979
293
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
980
293
                      dbgs() << CurrentIdx << ": GIR_Done\n");
981
293
      return true;
982
0
983
0
    default:
984
0
      llvm_unreachable("Unexpected command");
985
15.4k
    }
986
15.4k
  }
987
1.11k
}
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<42ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
1.40k
    CodeGenCoverage &CoverageInfo) const {
56
1.40k
57
1.40k
  uint64_t CurrentIdx = 0;
58
1.40k
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
1.40k
60
1.40k
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
1.40k
  auto handleReject = [&]() -> RejectAction {
62
1.40k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
1.40k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
1.40k
    if (OnFailResumeAt.empty())
65
1.40k
      return RejectAndGiveUp;
66
1.40k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
1.40k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
1.40k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
1.40k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
1.40k
    return RejectAndResume;
71
1.40k
  };
72
1.40k
73
14.0k
  while (true) {
74
14.0k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
14.0k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
14.0k
    switch (MatcherOpcode) {
77
14.0k
    case GIM_Try: {
78
2.89k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
2.89k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
2.89k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
2.89k
      break;
82
14.0k
    }
83
14.0k
84
14.0k
    case GIM_RecordInsn: {
85
120
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
120
      int64_t InsnID = MatchTable[CurrentIdx++];
87
120
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
120
89
120
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
120
      // any attempt to modify it.
91
120
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
120
93
120
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
120
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
120
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
120
109
120
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
120
      if ((size_t)NewInsnID < State.MIs.size())
111
4
        State.MIs[NewInsnID] = NewMI;
112
116
      else {
113
116
        assert((size_t)NewInsnID == State.MIs.size() &&
114
116
               "Expected to store MIs in order");
115
116
        State.MIs.push_back(NewMI);
116
116
      }
117
120
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
120
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
120
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
120
                             << ")\n");
121
120
      break;
122
120
    }
123
120
124
2.19k
    case GIM_CheckFeatures: {
125
2.19k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
2.19k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
2.19k
                      dbgs() << CurrentIdx
128
2.19k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
2.19k
                             << ExpectedBitsetID << ")\n");
130
2.19k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
2.19k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
1.60k
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
2.19k
      }
135
2.19k
      break;
136
2.19k
    }
137
2.19k
138
2.19k
    case GIM_CheckOpcode: {
139
120
      int64_t InsnID = MatchTable[CurrentIdx++];
140
120
      int64_t Expected = MatchTable[CurrentIdx++];
141
120
142
120
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
120
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
120
145
120
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
120
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
120
                             << "], ExpectedOpcode=" << Expected
148
120
                             << ") // Got=" << Opcode << "\n");
149
120
      if (Opcode != Expected) {
150
30
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
120
      }
153
120
      break;
154
120
    }
155
120
156
1.40k
    case GIM_SwitchOpcode: {
157
1.40k
      int64_t InsnID = MatchTable[CurrentIdx++];
158
1.40k
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
1.40k
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
1.40k
      int64_t Default = MatchTable[CurrentIdx++];
161
1.40k
162
1.40k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
1.40k
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
1.40k
165
1.40k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
1.40k
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
1.40k
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
1.40k
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
1.40k
      });
170
1.40k
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
0
        CurrentIdx = Default;
172
0
        break;
173
0
      }
174
1.40k
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
1.40k
      if (!CurrentIdx) {
176
463
        CurrentIdx = Default;
177
463
  break;
178
463
      }
179
942
      OnFailResumeAt.push_back(Default);
180
942
      break;
181
942
    }
182
942
183
942
    case GIM_SwitchType: {
184
516
      int64_t InsnID = MatchTable[CurrentIdx++];
185
516
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
516
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
516
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
516
      int64_t Default = MatchTable[CurrentIdx++];
189
516
190
516
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
516
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
516
193
516
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
516
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
516
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
516
               << UpperBound << "), Default=" << Default
197
516
               << ", JumpTable...) // Got=";
198
516
        if (!MO.isReg())
199
516
          dbgs() << "Not a VReg\n";
200
516
        else
201
516
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
516
      });
203
516
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
516
      const LLT Ty = MRI.getType(MO.getReg());
208
516
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
516
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
3
        CurrentIdx = Default;
211
3
        break;
212
3
      }
213
513
      const int64_t TypeID = TyI->second;
214
513
      if (TypeID < LowerBound || 
UpperBound <= TypeID497
) {
215
16
        CurrentIdx = Default;
216
16
        break;
217
16
      }
218
497
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
497
      if (!CurrentIdx) {
220
0
        CurrentIdx = Default;
221
0
        break;
222
0
      }
223
497
      OnFailResumeAt.push_back(Default);
224
497
      break;
225
497
    }
226
497
227
497
    case GIM_CheckNumOperands: {
228
0
      int64_t InsnID = MatchTable[CurrentIdx++];
229
0
      int64_t Expected = MatchTable[CurrentIdx++];
230
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
0
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
0
                             << InsnID << "], Expected=" << Expected << ")\n");
233
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
0
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
0
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
0
      }
238
0
      break;
239
0
    }
240
90
    case GIM_CheckI64ImmPredicate: {
241
90
      int64_t InsnID = MatchTable[CurrentIdx++];
242
90
      int64_t Predicate = MatchTable[CurrentIdx++];
243
90
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
90
                      dbgs()
245
90
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
90
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
90
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
90
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
90
             "Expected G_CONSTANT");
250
90
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
90
      int64_t Value = 0;
252
90
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
90
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
90
259
90
      if (!testImmPredicate_I64(Predicate, Value))
260
0
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
90
      break;
263
90
    }
264
90
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
0
    case GIM_CheckAPFloatImmPredicate: {
287
0
      int64_t InsnID = MatchTable[CurrentIdx++];
288
0
      int64_t Predicate = MatchTable[CurrentIdx++];
289
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
0
                      dbgs()
291
0
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
0
             "Expected G_FCONSTANT");
296
0
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
0
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
0
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
0
300
0
      if (!testImmPredicate_APFloat(Predicate, Value))
301
0
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
0
      break;
304
0
    }
305
0
    case GIM_CheckCxxInsnPredicate: {
306
0
      int64_t InsnID = MatchTable[CurrentIdx++];
307
0
      int64_t Predicate = MatchTable[CurrentIdx++];
308
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
0
                      dbgs()
310
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
0
315
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
0
      break;
319
0
    }
320
0
    case GIM_CheckAtomicOrdering: {
321
0
      int64_t InsnID = MatchTable[CurrentIdx++];
322
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
0
331
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
0
        if (MMO->getOrdering() != Ordering)
333
0
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
0
      break;
336
0
    }
337
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
0
    case GIM_CheckMemoryAddressSpace: {
374
0
      int64_t InsnID = MatchTable[CurrentIdx++];
375
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
0
      // This accepts a list of possible address spaces.
377
0
      const int NumAddrSpace = MatchTable[CurrentIdx++];
378
0
379
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
380
0
        if (handleReject() == RejectAndGiveUp)
381
0
          return false;
382
0
        break;
383
0
      }
384
0
385
0
      // Need to still jump to the end of the list of address spaces if we find
386
0
      // a match earlier.
387
0
      const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
388
0
389
0
      const MachineMemOperand *MMO
390
0
        = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
391
0
      const unsigned MMOAddrSpace = MMO->getAddrSpace();
392
0
393
0
      bool Success = false;
394
0
      for (int I = 0; I != NumAddrSpace; ++I) {
395
0
        unsigned AddrSpace = MatchTable[CurrentIdx++];
396
0
        DEBUG_WITH_TYPE(
397
0
          TgtInstructionSelector::getName(),
398
0
          dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
399
0
                 << AddrSpace << '\n');
400
0
401
0
        if (AddrSpace == MMOAddrSpace) {
402
0
          Success = true;
403
0
          break;
404
0
        }
405
0
      }
406
0
407
0
      CurrentIdx = LastIdx;
408
0
      if (!Success && handleReject() == RejectAndGiveUp)
409
0
        return false;
410
0
      break;
411
0
    }
412
0
    case GIM_CheckMemorySizeEqualTo: {
413
0
      int64_t InsnID = MatchTable[CurrentIdx++];
414
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
415
0
      uint64_t Size = MatchTable[CurrentIdx++];
416
0
417
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
418
0
                      dbgs() << CurrentIdx
419
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
420
0
                             << "]->memoperands() + " << MMOIdx
421
0
                             << ", Size=" << Size << ")\n");
422
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
423
0
424
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
0
430
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
431
0
432
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
433
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
434
0
                             << " bytes\n");
435
0
      if (MMO->getSize() != Size)
436
0
        if (handleReject() == RejectAndGiveUp)
437
0
          return false;
438
0
439
0
      break;
440
0
    }
441
0
    case GIM_CheckMemorySizeEqualToLLT:
442
0
    case GIM_CheckMemorySizeLessThanLLT:
443
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
444
0
      int64_t InsnID = MatchTable[CurrentIdx++];
445
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
446
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
447
0
448
0
      DEBUG_WITH_TYPE(
449
0
          TgtInstructionSelector::getName(),
450
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
451
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
452
0
                         ? "EqualTo"
453
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
454
0
                               ? "GreaterThan"
455
0
                               : "LessThan")
456
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
457
0
                 << ", OpIdx=" << OpIdx << ")\n");
458
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
459
0
460
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
461
0
      if (!MO.isReg()) {
462
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
463
0
                        dbgs() << CurrentIdx << ": Not a register\n");
464
0
        if (handleReject() == RejectAndGiveUp)
465
0
          return false;
466
0
        break;
467
0
      }
468
0
469
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
470
0
        if (handleReject() == RejectAndGiveUp)
471
0
          return false;
472
0
        break;
473
0
      }
474
0
475
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
476
0
477
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
478
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
479
0
          MMO->getSizeInBits() != Size) {
480
0
        if (handleReject() == RejectAndGiveUp)
481
0
          return false;
482
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
483
0
                 MMO->getSizeInBits() >= Size) {
484
0
        if (handleReject() == RejectAndGiveUp)
485
0
          return false;
486
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
487
0
                 MMO->getSizeInBits() <= Size)
488
0
        if (handleReject() == RejectAndGiveUp)
489
0
          return false;
490
0
491
0
      break;
492
0
    }
493
1.21k
    case GIM_CheckType: {
494
1.21k
      int64_t InsnID = MatchTable[CurrentIdx++];
495
1.21k
      int64_t OpIdx = MatchTable[CurrentIdx++];
496
1.21k
      int64_t TypeID = MatchTable[CurrentIdx++];
497
1.21k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498
1.21k
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
499
1.21k
                             << "]->getOperand(" << OpIdx
500
1.21k
                             << "), TypeID=" << TypeID << ")\n");
501
1.21k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
502
1.21k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
503
1.21k
      if (!MO.isReg() ||
504
1.21k
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
505
0
        if (handleReject() == RejectAndGiveUp)
506
0
          return false;
507
1.21k
      }
508
1.21k
      break;
509
1.21k
    }
510
1.21k
    case GIM_CheckPointerToAny: {
511
0
      int64_t InsnID = MatchTable[CurrentIdx++];
512
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
513
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
514
0
515
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
516
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
517
0
                             << InsnID << "]->getOperand(" << OpIdx
518
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
519
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
520
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
521
0
      const LLT Ty = MRI.getType(MO.getReg());
522
0
523
0
      // iPTR must be looked up in the target.
524
0
      if (SizeInBits == 0) {
525
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
526
0
        const unsigned AddrSpace = Ty.getAddressSpace();
527
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
528
0
      }
529
0
530
0
      assert(SizeInBits != 0 && "Pointer size must be known");
531
0
532
0
      if (MO.isReg()) {
533
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
534
0
          if (handleReject() == RejectAndGiveUp)
535
0
            return false;
536
0
      } else if (handleReject() == RejectAndGiveUp)
537
0
        return false;
538
0
539
0
      break;
540
0
    }
541
1.46k
    case GIM_CheckRegBankForClass: {
542
1.46k
      int64_t InsnID = MatchTable[CurrentIdx++];
543
1.46k
      int64_t OpIdx = MatchTable[CurrentIdx++];
544
1.46k
      int64_t RCEnum = MatchTable[CurrentIdx++];
545
1.46k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
546
1.46k
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
547
1.46k
                             << InsnID << "]->getOperand(" << OpIdx
548
1.46k
                             << "), RCEnum=" << RCEnum << ")\n");
549
1.46k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
550
1.46k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
551
1.46k
      if (!MO.isReg() ||
552
1.46k
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
553
1.46k
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
554
7
        if (handleReject() == RejectAndGiveUp)
555
0
          return false;
556
1.46k
      }
557
1.46k
      break;
558
1.46k
    }
559
1.46k
560
1.46k
    case GIM_CheckComplexPattern: {
561
0
      int64_t InsnID = MatchTable[CurrentIdx++];
562
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
563
0
      int64_t RendererID = MatchTable[CurrentIdx++];
564
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
565
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
566
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
567
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
568
0
                             << "]->getOperand(" << OpIdx
569
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
570
0
                             << ")\n");
571
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572
0
      // FIXME: Use std::invoke() when it's available.
573
0
      ComplexRendererFns Renderer =
574
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
575
0
              State.MIs[InsnID]->getOperand(OpIdx));
576
0
      if (Renderer.hasValue())
577
0
        State.Renderers[RendererID] = Renderer.getValue();
578
0
      else
579
0
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
0
      break;
582
0
    }
583
0
584
12
    case GIM_CheckConstantInt: {
585
12
      int64_t InsnID = MatchTable[CurrentIdx++];
586
12
      int64_t OpIdx = MatchTable[CurrentIdx++];
587
12
      int64_t Value = MatchTable[CurrentIdx++];
588
12
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
589
12
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
590
12
                             << InsnID << "]->getOperand(" << OpIdx
591
12
                             << "), Value=" << Value << ")\n");
592
12
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593
12
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
594
12
      if (MO.isReg()) {
595
12
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
596
12
        LLT Ty = MRI.getType(MO.getReg());
597
12
        Value = SignExtend64(Value, Ty.getSizeInBits());
598
12
599
12
        if (!isOperandImmEqual(MO, Value, MRI)) {
600
11
          if (handleReject() == RejectAndGiveUp)
601
0
            return false;
602
0
        }
603
0
      } else if (handleReject() == RejectAndGiveUp)
604
0
        return false;
605
12
606
12
      break;
607
12
    }
608
12
609
12
    case GIM_CheckLiteralInt: {
610
0
      int64_t InsnID = MatchTable[CurrentIdx++];
611
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
612
0
      int64_t Value = MatchTable[CurrentIdx++];
613
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
614
0
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
615
0
                             << InsnID << "]->getOperand(" << OpIdx
616
0
                             << "), Value=" << Value << ")\n");
617
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
618
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
619
0
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
620
0
        if (handleReject() == RejectAndGiveUp)
621
0
          return false;
622
0
      }
623
0
      break;
624
0
    }
625
0
626
0
    case GIM_CheckIntrinsicID: {
627
0
      int64_t InsnID = MatchTable[CurrentIdx++];
628
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
629
0
      int64_t Value = MatchTable[CurrentIdx++];
630
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
631
0
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
632
0
                             << InsnID << "]->getOperand(" << OpIdx
633
0
                             << "), Value=" << Value << ")\n");
634
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
635
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
636
0
      if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
637
0
        if (handleReject() == RejectAndGiveUp)
638
0
          return false;
639
0
      break;
640
0
    }
641
0
642
80
    case GIM_CheckIsMBB: {
643
80
      int64_t InsnID = MatchTable[CurrentIdx++];
644
80
      int64_t OpIdx = MatchTable[CurrentIdx++];
645
80
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
646
80
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
647
80
                             << "]->getOperand(" << OpIdx << "))\n");
648
80
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
649
80
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
650
0
        if (handleReject() == RejectAndGiveUp)
651
0
          return false;
652
80
      }
653
80
      break;
654
80
    }
655
80
656
90
    case GIM_CheckIsSafeToFold: {
657
90
      int64_t InsnID = MatchTable[CurrentIdx++];
658
90
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
659
90
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
660
90
                             << InsnID << "])\n");
661
90
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
662
90
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
663
0
        if (handleReject() == RejectAndGiveUp)
664
0
          return false;
665
90
      }
666
90
      break;
667
90
    }
668
90
    case GIM_CheckIsSameOperand: {
669
0
      int64_t InsnID = MatchTable[CurrentIdx++];
670
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
671
0
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
672
0
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
673
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
674
0
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
675
0
                             << InsnID << "][" << OpIdx << "], MIs["
676
0
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
677
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
678
0
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
679
0
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
680
0
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
681
0
        if (handleReject() == RejectAndGiveUp)
682
0
          return false;
683
0
      }
684
0
      break;
685
0
    }
686
1.51k
    case GIM_Reject:
687
1.51k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
688
1.51k
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
689
1.51k
      if (handleReject() == RejectAndGiveUp)
690
861
        return false;
691
651
      break;
692
651
693
651
    case GIR_MutateOpcode: {
694
405
      int64_t OldInsnID = MatchTable[CurrentIdx++];
695
405
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
696
405
      int64_t NewOpcode = MatchTable[CurrentIdx++];
697
405
      if (NewInsnID >= OutMIs.size())
698
405
        OutMIs.resize(NewInsnID + 1);
699
405
700
405
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
701
405
                                              State.MIs[OldInsnID]);
702
405
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
703
405
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
704
405
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
705
405
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
706
405
                             << NewOpcode << ")\n");
707
405
      break;
708
651
    }
709
651
710
651
    case GIR_BuildMI: {
711
139
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
712
139
      int64_t Opcode = MatchTable[CurrentIdx++];
713
139
      if (NewInsnID >= OutMIs.size())
714
139
        OutMIs.resize(NewInsnID + 1);
715
139
716
139
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
717
139
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
718
139
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
719
139
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
720
139
                             << NewInsnID << "], " << Opcode << ")\n");
721
139
      break;
722
651
    }
723
651
724
651
    case GIR_Copy: {
725
374
      int64_t NewInsnID = MatchTable[CurrentIdx++];
726
374
      int64_t OldInsnID = MatchTable[CurrentIdx++];
727
374
      int64_t OpIdx = MatchTable[CurrentIdx++];
728
374
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
729
374
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
730
374
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
731
374
                      dbgs()
732
374
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
733
374
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
734
374
      break;
735
651
    }
736
651
737
651
    case GIR_CopyOrAddZeroReg: {
738
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
739
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
740
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
741
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
742
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
743
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
744
0
      if (isOperandImmEqual(MO, 0, MRI))
745
0
        OutMIs[NewInsnID].addReg(ZeroReg);
746
0
      else
747
0
        OutMIs[NewInsnID].add(MO);
748
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
749
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
750
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
751
0
                             << OpIdx << ", " << ZeroReg << ")\n");
752
0
      break;
753
651
    }
754
651
755
651
    case GIR_CopySubReg: {
756
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
757
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
758
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
759
0
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
760
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
761
0
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
762
0
                               0, SubRegIdx);
763
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
764
0
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
765
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
766
0
                             << OpIdx << ", " << SubRegIdx << ")\n");
767
0
      break;
768
651
    }
769
651
770
651
    case GIR_AddImplicitDef: {
771
80
      int64_t InsnID = MatchTable[CurrentIdx++];
772
80
      int64_t RegNum = MatchTable[CurrentIdx++];
773
80
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
774
80
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
775
80
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
776
80
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
777
80
                             << InsnID << "], " << RegNum << ")\n");
778
80
      break;
779
651
    }
780
651
781
651
    case GIR_AddImplicitUse: {
782
0
      int64_t InsnID = MatchTable[CurrentIdx++];
783
0
      int64_t RegNum = MatchTable[CurrentIdx++];
784
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
785
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
786
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
787
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
788
0
                             << InsnID << "], " << RegNum << ")\n");
789
0
      break;
790
651
    }
791
651
792
651
    case GIR_AddRegister: {
793
1
      int64_t InsnID = MatchTable[CurrentIdx++];
794
1
      int64_t RegNum = MatchTable[CurrentIdx++];
795
1
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
796
1
      OutMIs[InsnID].addReg(RegNum);
797
1
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
798
1
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
799
1
                             << InsnID << "], " << RegNum << ")\n");
800
1
      break;
801
651
    }
802
651
803
651
    case GIR_AddTempRegister: {
804
0
      int64_t InsnID = MatchTable[CurrentIdx++];
805
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
806
0
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
807
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
808
0
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
809
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
810
0
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
811
0
                             << InsnID << "], TempRegisters[" << TempRegID
812
0
                             << "], " << TempRegFlags << ")\n");
813
0
      break;
814
651
    }
815
651
816
651
    case GIR_AddImm: {
817
0
      int64_t InsnID = MatchTable[CurrentIdx++];
818
0
      int64_t Imm = MatchTable[CurrentIdx++];
819
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
820
0
      OutMIs[InsnID].addImm(Imm);
821
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
822
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
823
0
                             << "], " << Imm << ")\n");
824
0
      break;
825
651
    }
826
651
827
651
    case GIR_ComplexRenderer: {
828
0
      int64_t InsnID = MatchTable[CurrentIdx++];
829
0
      int64_t RendererID = MatchTable[CurrentIdx++];
830
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
831
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
832
0
        RenderOpFn(OutMIs[InsnID]);
833
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
834
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
835
0
                             << InsnID << "], " << RendererID << ")\n");
836
0
      break;
837
651
    }
838
651
    case GIR_ComplexSubOperandRenderer: {
839
0
      int64_t InsnID = MatchTable[CurrentIdx++];
840
0
      int64_t RendererID = MatchTable[CurrentIdx++];
841
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
842
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
843
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
844
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
845
0
                      dbgs() << CurrentIdx
846
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
847
0
                             << InsnID << "], " << RendererID << ", "
848
0
                             << RenderOpID << ")\n");
849
0
      break;
850
651
    }
851
651
852
651
    case GIR_CopyConstantAsSImm: {
853
90
      int64_t NewInsnID = MatchTable[CurrentIdx++];
854
90
      int64_t OldInsnID = MatchTable[CurrentIdx++];
855
90
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
856
90
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
857
90
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
858
90
        OutMIs[NewInsnID].addImm(
859
90
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
860
90
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
861
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
862
0
      else
863
0
        llvm_unreachable("Expected Imm or CImm operand");
864
90
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
865
90
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
866
90
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
867
90
      break;
868
90
    }
869
90
870
90
    // TODO: Needs a test case once we have a pattern that uses this.
871
90
    case GIR_CopyFConstantAsFPImm: {
872
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
873
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
874
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
875
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
876
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
877
0
        OutMIs[NewInsnID].addFPImm(
878
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
879
0
      else
880
0
        llvm_unreachable("Expected FPImm operand");
881
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
882
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
883
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
884
0
      break;
885
0
    }
886
0
887
0
    case GIR_CustomRenderer: {
888
0
      int64_t InsnID = MatchTable[CurrentIdx++];
889
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
890
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
891
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
892
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
893
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
894
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
895
0
                             << RendererFnID << ")\n");
896
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
897
0
                                                     *State.MIs[OldInsnID]);
898
0
      break;
899
0
    }
900
0
    case GIR_ConstrainOperandRC: {
901
0
      int64_t InsnID = MatchTable[CurrentIdx++];
902
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
903
0
      int64_t RCEnum = MatchTable[CurrentIdx++];
904
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
905
0
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
906
0
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
907
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
908
0
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
909
0
                             << InsnID << "], " << OpIdx << ", " << RCEnum
910
0
                             << ")\n");
911
0
      break;
912
0
    }
913
0
914
544
    case GIR_ConstrainSelectedInstOperands: {
915
544
      int64_t InsnID = MatchTable[CurrentIdx++];
916
544
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
917
544
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
918
544
                                       RBI);
919
544
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
920
544
                      dbgs() << CurrentIdx
921
544
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
922
544
                             << InsnID << "])\n");
923
544
      break;
924
0
    }
925
0
926
0
    case GIR_MergeMemOperands: {
927
0
      int64_t InsnID = MatchTable[CurrentIdx++];
928
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
929
0
930
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
931
0
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
932
0
                             << InsnID << "]");
933
0
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
934
0
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
935
0
             GIU_MergeMemOperands_EndOfList) {
936
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
937
0
                        dbgs() << ", MIs[" << MergeInsnID << "]");
938
0
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
939
0
          OutMIs[InsnID].addMemOperand(MMO);
940
0
      }
941
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
942
0
      break;
943
0
    }
944
0
945
139
    case GIR_EraseFromParent: {
946
139
      int64_t InsnID = MatchTable[CurrentIdx++];
947
139
      assert(State.MIs[InsnID] &&
948
139
             "Attempted to erase an undefined instruction");
949
139
      State.MIs[InsnID]->eraseFromParent();
950
139
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
951
139
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
952
139
                             << InsnID << "])\n");
953
139
      break;
954
0
    }
955
0
956
0
    case GIR_MakeTempReg: {
957
0
      int64_t TempRegID = MatchTable[CurrentIdx++];
958
0
      int64_t TypeID = MatchTable[CurrentIdx++];
959
0
960
0
      State.TempRegisters[TempRegID] =
961
0
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
962
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
963
0
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
964
0
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
965
0
      break;
966
0
    }
967
0
968
0
    case GIR_Coverage: {
969
0
      int64_t RuleID = MatchTable[CurrentIdx++];
970
0
      CoverageInfo.setCovered(RuleID);
971
0
972
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
973
0
                      dbgs()
974
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
975
0
      break;
976
0
    }
977
0
978
544
    case GIR_Done:
979
544
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
980
544
                      dbgs() << CurrentIdx << ": GIR_Done\n");
981
544
      return true;
982
0
983
0
    default:
984
0
      llvm_unreachable("Unexpected command");
985
14.0k
    }
986
14.0k
  }
987
1.40k
}
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<114ul> const&, llvm::CodeGenCoverage&) const
Line
Count
Source
55
2.27k
    CodeGenCoverage &CoverageInfo) const {
56
2.27k
57
2.27k
  uint64_t CurrentIdx = 0;
58
2.27k
  SmallVector<uint64_t, 4> OnFailResumeAt;
59
2.27k
60
2.27k
  enum RejectAction { RejectAndGiveUp, RejectAndResume };
61
2.27k
  auto handleReject = [&]() -> RejectAction {
62
2.27k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
63
2.27k
                    dbgs() << CurrentIdx << ": Rejected\n");
64
2.27k
    if (OnFailResumeAt.empty())
65
2.27k
      return RejectAndGiveUp;
66
2.27k
    CurrentIdx = OnFailResumeAt.pop_back_val();
67
2.27k
    DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
68
2.27k
                    dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
69
2.27k
                           << OnFailResumeAt.size() << " try-blocks remain)\n");
70
2.27k
    return RejectAndResume;
71
2.27k
  };
72
2.27k
73
19.3k
  while (true) {
74
19.3k
    assert(CurrentIdx != ~0u && "Invalid MatchTable index");
75
19.3k
    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
76
19.3k
    switch (MatcherOpcode) {
77
19.3k
    case GIM_Try: {
78
3.08k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
79
3.08k
                      dbgs() << CurrentIdx << ": Begin try-block\n");
80
3.08k
      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
81
3.08k
      break;
82
19.3k
    }
83
19.3k
84
19.3k
    case GIM_RecordInsn: {
85
370
      int64_t NewInsnID = MatchTable[CurrentIdx++];
86
370
      int64_t InsnID = MatchTable[CurrentIdx++];
87
370
      int64_t OpIdx = MatchTable[CurrentIdx++];
88
370
89
370
      // As an optimisation we require that MIs[0] is always the root. Refuse
90
370
      // any attempt to modify it.
91
370
      assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
92
370
93
370
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
94
370
      if (!MO.isReg()) {
95
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
96
0
                        dbgs() << CurrentIdx << ": Not a register\n");
97
0
        if (handleReject() == RejectAndGiveUp)
98
0
          return false;
99
0
        break;
100
0
      }
101
370
      if (TRI.isPhysicalRegister(MO.getReg())) {
102
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
103
0
                        dbgs() << CurrentIdx << ": Is a physical register\n");
104
0
        if (handleReject() == RejectAndGiveUp)
105
0
          return false;
106
0
        break;
107
0
      }
108
370
109
370
      MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
110
370
      if ((size_t)NewInsnID < State.MIs.size())
111
110
        State.MIs[NewInsnID] = NewMI;
112
260
      else {
113
260
        assert((size_t)NewInsnID == State.MIs.size() &&
114
260
               "Expected to store MIs in order");
115
260
        State.MIs.push_back(NewMI);
116
260
      }
117
370
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
118
370
                      dbgs() << CurrentIdx << ": MIs[" << NewInsnID
119
370
                             << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
120
370
                             << ")\n");
121
370
      break;
122
370
    }
123
370
124
1.47k
    case GIM_CheckFeatures: {
125
1.47k
      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
126
1.47k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
127
1.47k
                      dbgs() << CurrentIdx
128
1.47k
                             << ": GIM_CheckFeatures(ExpectedBitsetID="
129
1.47k
                             << ExpectedBitsetID << ")\n");
130
1.47k
      if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
131
1.47k
          ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
132
887
        if (handleReject() == RejectAndGiveUp)
133
0
          return false;
134
1.47k
      }
135
1.47k
      break;
136
1.47k
    }
137
1.47k
138
1.47k
    case GIM_CheckOpcode: {
139
370
      int64_t InsnID = MatchTable[CurrentIdx++];
140
370
      int64_t Expected = MatchTable[CurrentIdx++];
141
370
142
370
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
143
370
      unsigned Opcode = State.MIs[InsnID]->getOpcode();
144
370
145
370
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
146
370
                      dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
147
370
                             << "], ExpectedOpcode=" << Expected
148
370
                             << ") // Got=" << Opcode << "\n");
149
370
      if (Opcode != Expected) {
150
296
        if (handleReject() == RejectAndGiveUp)
151
0
          return false;
152
370
      }
153
370
      break;
154
370
    }
155
370
156
2.27k
    case GIM_SwitchOpcode: {
157
2.27k
      int64_t InsnID = MatchTable[CurrentIdx++];
158
2.27k
      int64_t LowerBound = MatchTable[CurrentIdx++];
159
2.27k
      int64_t UpperBound = MatchTable[CurrentIdx++];
160
2.27k
      int64_t Default = MatchTable[CurrentIdx++];
161
2.27k
162
2.27k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
163
2.27k
      const int64_t Opcode = State.MIs[InsnID]->getOpcode();
164
2.27k
165
2.27k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
166
2.27k
        dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
167
2.27k
               << LowerBound << ", " << UpperBound << "), Default=" << Default
168
2.27k
               << ", JumpTable...) // Got=" << Opcode << "\n";
169
2.27k
      });
170
2.27k
      if (Opcode < LowerBound || UpperBound <= Opcode) {
171
0
        CurrentIdx = Default;
172
0
        break;
173
0
      }
174
2.27k
      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
175
2.27k
      if (!CurrentIdx) {
176
875
        CurrentIdx = Default;
177
875
  break;
178
875
      }
179
1.39k
      OnFailResumeAt.push_back(Default);
180
1.39k
      break;
181
1.39k
    }
182
1.39k
183
1.39k
    case GIM_SwitchType: {
184
1.37k
      int64_t InsnID = MatchTable[CurrentIdx++];
185
1.37k
      int64_t OpIdx = MatchTable[CurrentIdx++];
186
1.37k
      int64_t LowerBound = MatchTable[CurrentIdx++];
187
1.37k
      int64_t UpperBound = MatchTable[CurrentIdx++];
188
1.37k
      int64_t Default = MatchTable[CurrentIdx++];
189
1.37k
190
1.37k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
191
1.37k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
192
1.37k
193
1.37k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
194
1.37k
        dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
195
1.37k
               << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
196
1.37k
               << UpperBound << "), Default=" << Default
197
1.37k
               << ", JumpTable...) // Got=";
198
1.37k
        if (!MO.isReg())
199
1.37k
          dbgs() << "Not a VReg\n";
200
1.37k
        else
201
1.37k
          dbgs() << MRI.getType(MO.getReg()) << "\n";
202
1.37k
      });
203
1.37k
      if (!MO.isReg()) {
204
0
        CurrentIdx = Default;
205
0
        break;
206
0
      }
207
1.37k
      const LLT Ty = MRI.getType(MO.getReg());
208
1.37k
      const auto TyI = ISelInfo.TypeIDMap.find(Ty);
209
1.37k
      if (TyI == ISelInfo.TypeIDMap.end()) {
210
1
        CurrentIdx = Default;
211
1
        break;
212
1
      }
213
1.37k
      const int64_t TypeID = TyI->second;
214
1.37k
      if (TypeID < LowerBound || 
UpperBound <= TypeID1.32k
) {
215
48
        CurrentIdx = Default;
216
48
        break;
217
48
      }
218
1.32k
      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
219
1.32k
      if (!CurrentIdx) {
220
377
        CurrentIdx = Default;
221
377
        break;
222
377
      }
223
946
      OnFailResumeAt.push_back(Default);
224
946
      break;
225
946
    }
226
946
227
946
    case GIM_CheckNumOperands: {
228
23
      int64_t InsnID = MatchTable[CurrentIdx++];
229
23
      int64_t Expected = MatchTable[CurrentIdx++];
230
23
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
231
23
                      dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
232
23
                             << InsnID << "], Expected=" << Expected << ")\n");
233
23
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
234
23
      if (State.MIs[InsnID]->getNumOperands() != Expected) {
235
6
        if (handleReject() == RejectAndGiveUp)
236
0
          return false;
237
23
      }
238
23
      break;
239
23
    }
240
74
    case GIM_CheckI64ImmPredicate: {
241
74
      int64_t InsnID = MatchTable[CurrentIdx++];
242
74
      int64_t Predicate = MatchTable[CurrentIdx++];
243
74
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
244
74
                      dbgs()
245
74
                          << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
246
74
                          << InsnID << "], Predicate=" << Predicate << ")\n");
247
74
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
248
74
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
249
74
             "Expected G_CONSTANT");
250
74
      assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
251
74
      int64_t Value = 0;
252
74
      if (State.MIs[InsnID]->getOperand(1).isCImm())
253
74
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
254
0
      else if (State.MIs[InsnID]->getOperand(1).isImm())
255
0
        Value = State.MIs[InsnID]->getOperand(1).getImm();
256
0
      else
257
0
        llvm_unreachable("Expected Imm or CImm operand");
258
74
259
74
      if (!testImmPredicate_I64(Predicate, Value))
260
16
        if (handleReject() == RejectAndGiveUp)
261
0
          return false;
262
74
      break;
263
74
    }
264
74
    case GIM_CheckAPIntImmPredicate: {
265
0
      int64_t InsnID = MatchTable[CurrentIdx++];
266
0
      int64_t Predicate = MatchTable[CurrentIdx++];
267
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
268
0
                      dbgs()
269
0
                          << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
270
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
271
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
272
0
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
273
0
             "Expected G_CONSTANT");
274
0
      assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
275
0
      APInt Value;
276
0
      if (State.MIs[InsnID]->getOperand(1).isCImm())
277
0
        Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
278
0
      else
279
0
        llvm_unreachable("Expected Imm or CImm operand");
280
0
281
0
      if (!testImmPredicate_APInt(Predicate, Value))
282
0
        if (handleReject() == RejectAndGiveUp)
283
0
          return false;
284
0
      break;
285
0
    }
286
5
    case GIM_CheckAPFloatImmPredicate: {
287
5
      int64_t InsnID = MatchTable[CurrentIdx++];
288
5
      int64_t Predicate = MatchTable[CurrentIdx++];
289
5
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
290
5
                      dbgs()
291
5
                          << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
292
5
                          << InsnID << "], Predicate=" << Predicate << ")\n");
293
5
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
294
5
      assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
295
5
             "Expected G_FCONSTANT");
296
5
      assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
297
5
      assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
298
5
      APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
299
5
300
5
      if (!testImmPredicate_APFloat(Predicate, Value))
301
5
        if (handleReject() == RejectAndGiveUp)
302
0
          return false;
303
5
      break;
304
5
    }
305
5
    case GIM_CheckCxxInsnPredicate: {
306
0
      int64_t InsnID = MatchTable[CurrentIdx++];
307
0
      int64_t Predicate = MatchTable[CurrentIdx++];
308
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
309
0
                      dbgs()
310
0
                          << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
311
0
                          << InsnID << "], Predicate=" << Predicate << ")\n");
312
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
313
0
      assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
314
0
315
0
      if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
316
0
        if (handleReject() == RejectAndGiveUp)
317
0
          return false;
318
0
      break;
319
0
    }
320
0
    case GIM_CheckAtomicOrdering: {
321
0
      int64_t InsnID = MatchTable[CurrentIdx++];
322
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
323
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
324
0
                      dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
325
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
326
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
327
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
328
0
        if (handleReject() == RejectAndGiveUp)
329
0
          return false;
330
0
331
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
332
0
        if (MMO->getOrdering() != Ordering)
333
0
          if (handleReject() == RejectAndGiveUp)
334
0
            return false;
335
0
      break;
336
0
    }
337
0
    case GIM_CheckAtomicOrderingOrStrongerThan: {
338
0
      int64_t InsnID = MatchTable[CurrentIdx++];
339
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
340
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
341
0
                      dbgs() << CurrentIdx
342
0
                             << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
343
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
344
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
345
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
346
0
        if (handleReject() == RejectAndGiveUp)
347
0
          return false;
348
0
349
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
350
0
        if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
351
0
          if (handleReject() == RejectAndGiveUp)
352
0
            return false;
353
0
      break;
354
0
    }
355
0
    case GIM_CheckAtomicOrderingWeakerThan: {
356
0
      int64_t InsnID = MatchTable[CurrentIdx++];
357
0
      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
358
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
359
0
                      dbgs() << CurrentIdx
360
0
                             << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
361
0
                             << InsnID << "], " << (uint64_t)Ordering << ")\n");
362
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
363
0
      if (!State.MIs[InsnID]->hasOneMemOperand())
364
0
        if (handleReject() == RejectAndGiveUp)
365
0
          return false;
366
0
367
0
      for (const auto &MMO : State.MIs[InsnID]->memoperands())
368
0
        if (!isStrongerThan(Ordering, MMO->getOrdering()))
369
0
          if (handleReject() == RejectAndGiveUp)
370
0
            return false;
371
0
      break;
372
0
    }
373
0
    case GIM_CheckMemoryAddressSpace: {
374
0
      int64_t InsnID = MatchTable[CurrentIdx++];
375
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
376
0
      // This accepts a list of possible address spaces.
377
0
      const int NumAddrSpace = MatchTable[CurrentIdx++];
378
0
379
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
380
0
        if (handleReject() == RejectAndGiveUp)
381
0
          return false;
382
0
        break;
383
0
      }
384
0
385
0
      // Need to still jump to the end of the list of address spaces if we find
386
0
      // a match earlier.
387
0
      const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
388
0
389
0
      const MachineMemOperand *MMO
390
0
        = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
391
0
      const unsigned MMOAddrSpace = MMO->getAddrSpace();
392
0
393
0
      bool Success = false;
394
0
      for (int I = 0; I != NumAddrSpace; ++I) {
395
0
        unsigned AddrSpace = MatchTable[CurrentIdx++];
396
0
        DEBUG_WITH_TYPE(
397
0
          TgtInstructionSelector::getName(),
398
0
          dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
399
0
                 << AddrSpace << '\n');
400
0
401
0
        if (AddrSpace == MMOAddrSpace) {
402
0
          Success = true;
403
0
          break;
404
0
        }
405
0
      }
406
0
407
0
      CurrentIdx = LastIdx;
408
0
      if (!Success && handleReject() == RejectAndGiveUp)
409
0
        return false;
410
0
      break;
411
0
    }
412
0
    case GIM_CheckMemorySizeEqualTo: {
413
0
      int64_t InsnID = MatchTable[CurrentIdx++];
414
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
415
0
      uint64_t Size = MatchTable[CurrentIdx++];
416
0
417
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
418
0
                      dbgs() << CurrentIdx
419
0
                             << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
420
0
                             << "]->memoperands() + " << MMOIdx
421
0
                             << ", Size=" << Size << ")\n");
422
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
423
0
424
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
425
0
        if (handleReject() == RejectAndGiveUp)
426
0
          return false;
427
0
        break;
428
0
      }
429
0
430
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
431
0
432
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
433
0
                      dbgs() << MMO->getSize() << " bytes vs " << Size
434
0
                             << " bytes\n");
435
0
      if (MMO->getSize() != Size)
436
0
        if (handleReject() == RejectAndGiveUp)
437
0
          return false;
438
0
439
0
      break;
440
0
    }
441
0
    case GIM_CheckMemorySizeEqualToLLT:
442
0
    case GIM_CheckMemorySizeLessThanLLT:
443
0
    case GIM_CheckMemorySizeGreaterThanLLT: {
444
0
      int64_t InsnID = MatchTable[CurrentIdx++];
445
0
      int64_t MMOIdx = MatchTable[CurrentIdx++];
446
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
447
0
448
0
      DEBUG_WITH_TYPE(
449
0
          TgtInstructionSelector::getName(),
450
0
          dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
451
0
                 << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
452
0
                         ? "EqualTo"
453
0
                         : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
454
0
                               ? "GreaterThan"
455
0
                               : "LessThan")
456
0
                 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
457
0
                 << ", OpIdx=" << OpIdx << ")\n");
458
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
459
0
460
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
461
0
      if (!MO.isReg()) {
462
0
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
463
0
                        dbgs() << CurrentIdx << ": Not a register\n");
464
0
        if (handleReject() == RejectAndGiveUp)
465
0
          return false;
466
0
        break;
467
0
      }
468
0
469
0
      if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
470
0
        if (handleReject() == RejectAndGiveUp)
471
0
          return false;
472
0
        break;
473
0
      }
474
0
475
0
      MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
476
0
477
0
      unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
478
0
      if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
479
0
          MMO->getSizeInBits() != Size) {
480
0
        if (handleReject() == RejectAndGiveUp)
481
0
          return false;
482
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
483
0
                 MMO->getSizeInBits() >= Size) {
484
0
        if (handleReject() == RejectAndGiveUp)
485
0
          return false;
486
0
      } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
487
0
                 MMO->getSizeInBits() <= Size)
488
0
        if (handleReject() == RejectAndGiveUp)
489
0
          return false;
490
0
491
0
      break;
492
0
    }
493
1.62k
    case GIM_CheckType: {
494
1.62k
      int64_t InsnID = MatchTable[CurrentIdx++];
495
1.62k
      int64_t OpIdx = MatchTable[CurrentIdx++];
496
1.62k
      int64_t TypeID = MatchTable[CurrentIdx++];
497
1.62k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
498
1.62k
                      dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
499
1.62k
                             << "]->getOperand(" << OpIdx
500
1.62k
                             << "), TypeID=" << TypeID << ")\n");
501
1.62k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
502
1.62k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
503
1.62k
      if (!MO.isReg() ||
504
1.62k
          MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
505
227
        if (handleReject() == RejectAndGiveUp)
506
0
          return false;
507
1.62k
      }
508
1.62k
      break;
509
1.62k
    }
510
1.62k
    case GIM_CheckPointerToAny: {
511
0
      int64_t InsnID = MatchTable[CurrentIdx++];
512
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
513
0
      int64_t SizeInBits = MatchTable[CurrentIdx++];
514
0
515
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
516
0
                      dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
517
0
                             << InsnID << "]->getOperand(" << OpIdx
518
0
                             << "), SizeInBits=" << SizeInBits << ")\n");
519
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
520
0
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
521
0
      const LLT Ty = MRI.getType(MO.getReg());
522
0
523
0
      // iPTR must be looked up in the target.
524
0
      if (SizeInBits == 0) {
525
0
        MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
526
0
        const unsigned AddrSpace = Ty.getAddressSpace();
527
0
        SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
528
0
      }
529
0
530
0
      assert(SizeInBits != 0 && "Pointer size must be known");
531
0
532
0
      if (MO.isReg()) {
533
0
        if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
534
0
          if (handleReject() == RejectAndGiveUp)
535
0
            return false;
536
0
      } else if (handleReject() == RejectAndGiveUp)
537
0
        return false;
538
0
539
0
      break;
540
0
    }
541
2.17k
    case GIM_CheckRegBankForClass: {
542
2.17k
      int64_t InsnID = MatchTable[CurrentIdx++];
543
2.17k
      int64_t OpIdx = MatchTable[CurrentIdx++];
544
2.17k
      int64_t RCEnum = MatchTable[CurrentIdx++];
545
2.17k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
546
2.17k
                      dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
547
2.17k
                             << InsnID << "]->getOperand(" << OpIdx
548
2.17k
                             << "), RCEnum=" << RCEnum << ")\n");
549
2.17k
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
550
2.17k
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
551
2.17k
      if (!MO.isReg() ||
552
2.17k
          &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
553
2.17k
              RBI.getRegBank(MO.getReg(), MRI, TRI)) {
554
0
        if (handleReject() == RejectAndGiveUp)
555
0
          return false;
556
2.17k
      }
557
2.17k
      break;
558
2.17k
    }
559
2.17k
560
2.17k
    case GIM_CheckComplexPattern: {
561
0
      int64_t InsnID = MatchTable[CurrentIdx++];
562
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
563
0
      int64_t RendererID = MatchTable[CurrentIdx++];
564
0
      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
565
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
566
0
                      dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
567
0
                             << "] = GIM_CheckComplexPattern(MIs[" << InsnID
568
0
                             << "]->getOperand(" << OpIdx
569
0
                             << "), ComplexPredicateID=" << ComplexPredicateID
570
0
                             << ")\n");
571
0
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
572
0
      // FIXME: Use std::invoke() when it's available.
573
0
      ComplexRendererFns Renderer =
574
0
          (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
575
0
              State.MIs[InsnID]->getOperand(OpIdx));
576
0
      if (Renderer.hasValue())
577
0
        State.Renderers[RendererID] = Renderer.getValue();
578
0
      else
579
0
        if (handleReject() == RejectAndGiveUp)
580
0
          return false;
581
0
      break;
582
0
    }
583
0
584
321
    case GIM_CheckConstantInt: {
585
321
      int64_t InsnID = MatchTable[CurrentIdx++];
586
321
      int64_t OpIdx = MatchTable[CurrentIdx++];
587
321
      int64_t Value = MatchTable[CurrentIdx++];
588
321
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
589
321
                      dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
590
321
                             << InsnID << "]->getOperand(" << OpIdx
591
321
                             << "), Value=" << Value << ")\n");
592
321
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
593
321
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
594
321
      if (MO.isReg()) {
595
321
        // isOperandImmEqual() will sign-extend to 64-bits, so should we.
596
321
        LLT Ty = MRI.getType(MO.getReg());
597
321
        Value = SignExtend64(Value, Ty.getSizeInBits());
598
321
599
321
        if (!isOperandImmEqual(MO, Value, MRI)) {
600
285
          if (handleReject() == RejectAndGiveUp)
601
0
            return false;
602
0
        }
603
0
      } else if (handleReject() == RejectAndGiveUp)
604
0
        return false;
605
321
606
321
      break;
607
321
    }
608
321
609
321
    case GIM_CheckLiteralInt: {
610
45
      int64_t InsnID = MatchTable[CurrentIdx++];
611
45
      int64_t OpIdx = MatchTable[CurrentIdx++];
612
45
      int64_t Value = MatchTable[CurrentIdx++];
613
45
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
614
45
                      dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
615
45
                             << InsnID << "]->getOperand(" << OpIdx
616
45
                             << "), Value=" << Value << ")\n");
617
45
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
618
45
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
619
45
      if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
620
23
        if (handleReject() == RejectAndGiveUp)
621
0
          return false;
622
45
      }
623
45
      break;
624
45
    }
625
45
626
45
    case GIM_CheckIntrinsicID: {
627
24
      int64_t InsnID = MatchTable[CurrentIdx++];
628
24
      int64_t OpIdx = MatchTable[CurrentIdx++];
629
24
      int64_t Value = MatchTable[CurrentIdx++];
630
24
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
631
24
                      dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
632
24
                             << InsnID << "]->getOperand(" << OpIdx
633
24
                             << "), Value=" << Value << ")\n");
634
24
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
635
24
      MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
636
24
      if (!MO.isIntrinsicID() || 
MO.getIntrinsicID() != Value17
)
637
21
        if (handleReject() == RejectAndGiveUp)
638
0
          return false;
639
24
      break;
640
24
    }
641
24
642
24
    case GIM_CheckIsMBB: {
643
19
      int64_t InsnID = MatchTable[CurrentIdx++];
644
19
      int64_t OpIdx = MatchTable[CurrentIdx++];
645
19
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
646
19
                      dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
647
19
                             << "]->getOperand(" << OpIdx << "))\n");
648
19
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
649
19
      if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
650
0
        if (handleReject() == RejectAndGiveUp)
651
0
          return false;
652
19
      }
653
19
      break;
654
19
    }
655
19
656
69
    case GIM_CheckIsSafeToFold: {
657
69
      int64_t InsnID = MatchTable[CurrentIdx++];
658
69
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
659
69
                      dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
660
69
                             << InsnID << "])\n");
661
69
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
662
69
      if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
663
0
        if (handleReject() == RejectAndGiveUp)
664
0
          return false;
665
69
      }
666
69
      break;
667
69
    }
668
69
    case GIM_CheckIsSameOperand: {
669
4
      int64_t InsnID = MatchTable[CurrentIdx++];
670
4
      int64_t OpIdx = MatchTable[CurrentIdx++];
671
4
      int64_t OtherInsnID = MatchTable[CurrentIdx++];
672
4
      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
673
4
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
674
4
                      dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
675
4
                             << InsnID << "][" << OpIdx << "], MIs["
676
4
                             << OtherInsnID << "][" << OtherOpIdx << "])\n");
677
4
      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
678
4
      assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
679
4
      if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
680
4
              State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
681
2
        if (handleReject() == RejectAndGiveUp)
682
0
          return false;
683
4
      }
684
4
      break;
685
4
    }
686
2.29k
    case GIM_Reject:
687
2.29k
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
688
2.29k
                      dbgs() << CurrentIdx << ": GIM_Reject\n");
689
2.29k
      if (handleReject() == RejectAndGiveUp)
690
1.46k
        return false;
691
824
      break;
692
824
693
824
    case GIR_MutateOpcode: {
694
473
      int64_t OldInsnID = MatchTable[CurrentIdx++];
695
473
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
696
473
      int64_t NewOpcode = MatchTable[CurrentIdx++];
697
473
      if (NewInsnID >= OutMIs.size())
698
473
        OutMIs.resize(NewInsnID + 1);
699
473
700
473
      OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
701
473
                                              State.MIs[OldInsnID]);
702
473
      OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
703
473
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
704
473
                      dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
705
473
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
706
473
                             << NewOpcode << ")\n");
707
473
      break;
708
824
    }
709
824
710
824
    case GIR_BuildMI: {
711
334
      uint64_t NewInsnID = MatchTable[CurrentIdx++];
712
334
      int64_t Opcode = MatchTable[CurrentIdx++];
713
334
      if (NewInsnID >= OutMIs.size())
714
330
        OutMIs.resize(NewInsnID + 1);
715
334
716
334
      OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
717
334
                                  State.MIs[0]->getDebugLoc(), TII.get(Opcode));
718
334
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
719
334
                      dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
720
334
                             << NewInsnID << "], " << Opcode << ")\n");
721
334
      break;
722
824
    }
723
824
724
824
    case GIR_Copy: {
725
435
      int64_t NewInsnID = MatchTable[CurrentIdx++];
726
435
      int64_t OldInsnID = MatchTable[CurrentIdx++];
727
435
      int64_t OpIdx = MatchTable[CurrentIdx++];
728
435
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
729
435
      OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
730
435
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
731
435
                      dbgs()
732
435
                          << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
733
435
                          << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
734
435
      break;
735
824
    }
736
824
737
824
    case GIR_CopyOrAddZeroReg: {
738
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
739
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
740
0
      int64_t OpIdx = MatchTable[CurrentIdx++];
741
0
      int64_t ZeroReg = MatchTable[CurrentIdx++];
742
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
743
0
      MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
744
0
      if (isOperandImmEqual(MO, 0, MRI))
745
0
        OutMIs[NewInsnID].addReg(ZeroReg);
746
0
      else
747
0
        OutMIs[NewInsnID].add(MO);
748
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
749
0
                      dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
750
0
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
751
0
                             << OpIdx << ", " << ZeroReg << ")\n");
752
0
      break;
753
824
    }
754
824
755
824
    case GIR_CopySubReg: {
756
170
      int64_t NewInsnID = MatchTable[CurrentIdx++];
757
170
      int64_t OldInsnID = MatchTable[CurrentIdx++];
758
170
      int64_t OpIdx = MatchTable[CurrentIdx++];
759
170
      int64_t SubRegIdx = MatchTable[CurrentIdx++];
760
170
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
761
170
      OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
762
170
                               0, SubRegIdx);
763
170
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
764
170
                      dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
765
170
                             << NewInsnID << "], MIs[" << OldInsnID << "], "
766
170
                             << OpIdx << ", " << SubRegIdx << ")\n");
767
170
      break;
768
824
    }
769
824
770
824
    case GIR_AddImplicitDef: {
771
116
      int64_t InsnID = MatchTable[CurrentIdx++];
772
116
      int64_t RegNum = MatchTable[CurrentIdx++];
773
116
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
774
116
      OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
775
116
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
776
116
                      dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
777
116
                             << InsnID << "], " << RegNum << ")\n");
778
116
      break;
779
824
    }
780
824
781
824
    case GIR_AddImplicitUse: {
782
0
      int64_t InsnID = MatchTable[CurrentIdx++];
783
0
      int64_t RegNum = MatchTable[CurrentIdx++];
784
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
785
0
      OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
786
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
787
0
                      dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
788
0
                             << InsnID << "], " << RegNum << ")\n");
789
0
      break;
790
824
    }
791
824
792
824
    case GIR_AddRegister: {
793
0
      int64_t InsnID = MatchTable[CurrentIdx++];
794
0
      int64_t RegNum = MatchTable[CurrentIdx++];
795
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
796
0
      OutMIs[InsnID].addReg(RegNum);
797
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
798
0
                      dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
799
0
                             << InsnID << "], " << RegNum << ")\n");
800
0
      break;
801
824
    }
802
824
803
824
    case GIR_AddTempRegister: {
804
8
      int64_t InsnID = MatchTable[CurrentIdx++];
805
8
      int64_t TempRegID = MatchTable[CurrentIdx++];
806
8
      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
807
8
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
808
8
      OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
809
8
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
810
8
                      dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
811
8
                             << InsnID << "], TempRegisters[" << TempRegID
812
8
                             << "], " << TempRegFlags << ")\n");
813
8
      break;
814
824
    }
815
824
816
824
    case GIR_AddImm: {
817
0
      int64_t InsnID = MatchTable[CurrentIdx++];
818
0
      int64_t Imm = MatchTable[CurrentIdx++];
819
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
820
0
      OutMIs[InsnID].addImm(Imm);
821
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
822
0
                      dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
823
0
                             << "], " << Imm << ")\n");
824
0
      break;
825
824
    }
826
824
827
824
    case GIR_ComplexRenderer: {
828
0
      int64_t InsnID = MatchTable[CurrentIdx++];
829
0
      int64_t RendererID = MatchTable[CurrentIdx++];
830
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
831
0
      for (const auto &RenderOpFn : State.Renderers[RendererID])
832
0
        RenderOpFn(OutMIs[InsnID]);
833
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
834
0
                      dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
835
0
                             << InsnID << "], " << RendererID << ")\n");
836
0
      break;
837
824
    }
838
824
    case GIR_ComplexSubOperandRenderer: {
839
0
      int64_t InsnID = MatchTable[CurrentIdx++];
840
0
      int64_t RendererID = MatchTable[CurrentIdx++];
841
0
      int64_t RenderOpID = MatchTable[CurrentIdx++];
842
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
843
0
      State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
844
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
845
0
                      dbgs() << CurrentIdx
846
0
                             << ": GIR_ComplexSubOperandRenderer(OutMIs["
847
0
                             << InsnID << "], " << RendererID << ", "
848
0
                             << RenderOpID << ")\n");
849
0
      break;
850
824
    }
851
824
852
824
    case GIR_CopyConstantAsSImm: {
853
107
      int64_t NewInsnID = MatchTable[CurrentIdx++];
854
107
      int64_t OldInsnID = MatchTable[CurrentIdx++];
855
107
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
856
107
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
857
107
      if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
858
107
        OutMIs[NewInsnID].addImm(
859
107
            State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
860
107
      } else 
if (0
State.MIs[OldInsnID]->getOperand(1).isImm()0
)
861
0
        OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
862
0
      else
863
0
        llvm_unreachable("Expected Imm or CImm operand");
864
107
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
865
107
                      dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
866
107
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
867
107
      break;
868
107
    }
869
107
870
107
    // TODO: Needs a test case once we have a pattern that uses this.
871
107
    case GIR_CopyFConstantAsFPImm: {
872
0
      int64_t NewInsnID = MatchTable[CurrentIdx++];
873
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
874
0
      assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
875
0
      assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
876
0
      if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
877
0
        OutMIs[NewInsnID].addFPImm(
878
0
            State.MIs[OldInsnID]->getOperand(1).getFPImm());
879
0
      else
880
0
        llvm_unreachable("Expected FPImm operand");
881
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
882
0
                      dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
883
0
                             << NewInsnID << "], MIs[" << OldInsnID << "])\n");
884
0
      break;
885
0
    }
886
0
887
0
    case GIR_CustomRenderer: {
888
0
      int64_t InsnID = MatchTable[CurrentIdx++];
889
0
      int64_t OldInsnID = MatchTable[CurrentIdx++];
890
0
      int64_t RendererFnID = MatchTable[CurrentIdx++];
891
0
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
892
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
893
0
                      dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
894
0
                             << InsnID << "], MIs[" << OldInsnID << "], "
895
0
                             << RendererFnID << ")\n");
896
0
      (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
897
0
                                                     *State.MIs[OldInsnID]);
898
0
      break;
899
0
    }
900
332
    case GIR_ConstrainOperandRC: {
901
332
      int64_t InsnID = MatchTable[CurrentIdx++];
902
332
      int64_t OpIdx = MatchTable[CurrentIdx++];
903
332
      int64_t RCEnum = MatchTable[CurrentIdx++];
904
332
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
905
332
      constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
906
332
                                    *TRI.getRegClass(RCEnum), TII, TRI, RBI);
907
332
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
908
332
                      dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
909
332
                             << InsnID << "], " << OpIdx << ", " << RCEnum
910
332
                             << ")\n");
911
332
      break;
912
0
    }
913
0
914
641
    case GIR_ConstrainSelectedInstOperands: {
915
641
      int64_t InsnID = MatchTable[CurrentIdx++];
916
641
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
917
641
      constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
918
641
                                       RBI);
919
641
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
920
641
                      dbgs() << CurrentIdx
921
641
                             << ": GIR_ConstrainSelectedInstOperands(OutMIs["
922
641
                             << InsnID << "])\n");
923
641
      break;
924
0
    }
925
0
926
1
    case GIR_MergeMemOperands: {
927
1
      int64_t InsnID = MatchTable[CurrentIdx++];
928
1
      assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
929
1
930
1
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
931
1
                      dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
932
1
                             << InsnID << "]");
933
1
      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
934
2
      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
935
2
             GIU_MergeMemOperands_EndOfList) {
936
1
        DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
937
1
                        dbgs() << ", MIs[" << MergeInsnID << "]");
938
1
        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
939
0
          OutMIs[InsnID].addMemOperand(MMO);
940
1
      }
941
1
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
942
1
      break;
943
0
    }
944
0
945
330
    case GIR_EraseFromParent: {
946
330
      int64_t InsnID = MatchTable[CurrentIdx++];
947
330
      assert(State.MIs[InsnID] &&
948
330
             "Attempted to erase an undefined instruction");
949
330
      State.MIs[InsnID]->eraseFromParent();
950
330
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
951
330
                      dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
952
330
                             << InsnID << "])\n");
953
330
      break;
954
0
    }
955
0
956
4
    case GIR_MakeTempReg: {
957
4
      int64_t TempRegID = MatchTable[CurrentIdx++];
958
4
      int64_t TypeID = MatchTable[CurrentIdx++];
959
4
960
4
      State.TempRegisters[TempRegID] =
961
4
          MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
962
4
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
963
4
                      dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
964
4
                             << "] = GIR_MakeTempReg(" << TypeID << ")\n");
965
4
      break;
966
0
    }
967
0
968
0
    case GIR_Coverage: {
969
0
      int64_t RuleID = MatchTable[CurrentIdx++];
970
0
      CoverageInfo.setCovered(RuleID);
971
0
972
0
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
973
0
                      dbgs()
974
0
                          << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
975
0
      break;
976
0
    }
977
0
978
803
    case GIR_Done:
979
803
      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
980
803
                      dbgs() << CurrentIdx << ": GIR_Done\n");
981
803
      return true;
982
0
983
0
    default:
984
0
      llvm_unreachable("Unexpected command");
985
19.3k
    }
986
19.3k
  }
987
2.27k
}
988
989
} // end namespace llvm
990
991
#endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H