/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
Line | Count | Source (jump to first uncovered line) |
1 | | //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | /// \file This file declares the API for the instruction selector. |
10 | | /// This class is responsible for selecting machine instructions. |
11 | | /// It's implemented by the target. It's used by the InstructionSelect pass. |
12 | | // |
13 | | //===----------------------------------------------------------------------===// |
14 | | |
15 | | #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H |
16 | | #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H |
17 | | |
18 | | #include "llvm/ADT/SmallVector.h" |
19 | | #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" |
20 | | #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" |
21 | | #include "llvm/CodeGen/GlobalISel/Utils.h" |
22 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
23 | | #include "llvm/CodeGen/MachineOperand.h" |
24 | | #include "llvm/CodeGen/MachineRegisterInfo.h" |
25 | | #include "llvm/CodeGen/TargetInstrInfo.h" |
26 | | #include "llvm/CodeGen/TargetOpcodes.h" |
27 | | #include "llvm/CodeGen/TargetRegisterInfo.h" |
28 | | #include "llvm/IR/Constants.h" |
29 | | #include "llvm/Support/Debug.h" |
30 | | #include "llvm/Support/ErrorHandling.h" |
31 | | #include "llvm/Support/raw_ostream.h" |
32 | | #include <cassert> |
33 | | #include <cstddef> |
34 | | #include <cstdint> |
35 | | |
36 | | namespace llvm { |
37 | | |
38 | | /// GlobalISel PatFrag Predicates |
39 | | enum { |
40 | | GIPFP_I64_Invalid = 0, |
41 | | GIPFP_APInt_Invalid = 0, |
42 | | GIPFP_APFloat_Invalid = 0, |
43 | | GIPFP_MI_Invalid = 0, |
44 | | }; |
45 | | |
46 | | template <class TgtInstructionSelector, class PredicateBitset, |
47 | | class ComplexMatcherMemFn, class CustomRendererFn> |
48 | | bool InstructionSelector::executeMatchTable( |
49 | | TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State, |
50 | | const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn> |
51 | | &ISelInfo, |
52 | | const int64_t *MatchTable, const TargetInstrInfo &TII, |
53 | | MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, |
54 | | const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures, |
55 | 5.94M | CodeGenCoverage &CoverageInfo) const { |
56 | 5.94M | |
57 | 5.94M | uint64_t CurrentIdx = 0; |
58 | 5.94M | SmallVector<uint64_t, 4> OnFailResumeAt; |
59 | 5.94M | |
60 | 5.94M | enum RejectAction { RejectAndGiveUp, RejectAndResume }; |
61 | 9.66M | auto handleReject = [&]() -> RejectAction { |
62 | 9.66M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
63 | 9.66M | dbgs() << CurrentIdx << ": Rejected\n"); |
64 | 9.66M | if (OnFailResumeAt.empty()) |
65 | 3.13M | return RejectAndGiveUp; |
66 | 6.52M | CurrentIdx = OnFailResumeAt.pop_back_val(); |
67 | 6.52M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
68 | 6.52M | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" |
69 | 6.52M | << OnFailResumeAt.size() << " try-blocks remain)\n"); |
70 | 6.52M | return RejectAndResume; |
71 | 6.52M | }; AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<22ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<22ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<22ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const Line | Count | Source | 61 | 9.64M | auto handleReject = [&]() -> RejectAction { | 62 | 9.64M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 9.64M | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 9.64M | if (OnFailResumeAt.empty()) | 65 | 3.13M | return RejectAndGiveUp; | 66 | 6.51M | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 6.51M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 6.51M | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 6.51M | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 6.51M | return RejectAndResume; | 71 | 6.51M | }; |
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<46ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<46ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<46ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const Line | Count | Source | 61 | 6.53k | auto handleReject = [&]() -> RejectAction { | 62 | 6.53k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 6.53k | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 6.53k | if (OnFailResumeAt.empty()) | 65 | 408 | return RejectAndGiveUp; | 66 | 6.12k | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 6.12k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 6.12k | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 6.12k | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 6.12k | return RejectAndResume; | 71 | 6.12k | }; |
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<70ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<70ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<70ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const Line | Count | Source | 61 | 3.30k | auto handleReject = [&]() -> RejectAction { | 62 | 3.30k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 3.30k | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 3.30k | if (OnFailResumeAt.empty()) | 65 | 820 | return RejectAndGiveUp; | 66 | 2.48k | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 2.48k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 2.48k | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 2.48k | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 2.48k | return RejectAndResume; | 71 | 2.48k | }; |
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<42ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const Line | Count | Source | 61 | 3.16k | auto handleReject = [&]() -> RejectAction { | 62 | 3.16k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 3.16k | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 3.16k | if (OnFailResumeAt.empty()) | 65 | 861 | return RejectAndGiveUp; | 66 | 2.30k | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 2.30k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 2.30k | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 2.30k | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 2.30k | return RejectAndResume; | 71 | 2.30k | }; |
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<114ul> const&, llvm::CodeGenCoverage&) const::'lambda'()::operator()() const Line | Count | Source | 61 | 4.05k | auto handleReject = [&]() -> RejectAction { | 62 | 4.05k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 4.05k | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 4.05k | if (OnFailResumeAt.empty()) | 65 | 1.46k | return RejectAndGiveUp; | 66 | 2.59k | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 2.59k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 2.59k | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 2.59k | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 2.59k | return RejectAndResume; | 71 | 2.59k | }; |
|
72 | 5.94M | |
73 | 61.9M | while (true) { |
74 | 61.9M | assert(CurrentIdx != ~0u && "Invalid MatchTable index"); |
75 | 61.9M | int64_t MatcherOpcode = MatchTable[CurrentIdx++]; |
76 | 61.9M | switch (MatcherOpcode) { |
77 | 61.9M | case GIM_Try: { |
78 | 9.15M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
79 | 9.15M | dbgs() << CurrentIdx << ": Begin try-block\n"); |
80 | 9.15M | OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); |
81 | 9.15M | break; |
82 | 61.9M | } |
83 | 61.9M | |
84 | 61.9M | case GIM_RecordInsn: { |
85 | 3.04M | int64_t NewInsnID = MatchTable[CurrentIdx++]; |
86 | 3.04M | int64_t InsnID = MatchTable[CurrentIdx++]; |
87 | 3.04M | int64_t OpIdx = MatchTable[CurrentIdx++]; |
88 | 3.04M | |
89 | 3.04M | // As an optimisation we require that MIs[0] is always the root. Refuse |
90 | 3.04M | // any attempt to modify it. |
91 | 3.04M | assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); |
92 | 3.04M | |
93 | 3.04M | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
94 | 3.04M | if (!MO.isReg()) { |
95 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
96 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); |
97 | 0 | if (handleReject() == RejectAndGiveUp) |
98 | 0 | return false; |
99 | 0 | break; |
100 | 0 | } |
101 | 3.04M | if (TRI.isPhysicalRegister(MO.getReg())) { |
102 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
103 | 0 | dbgs() << CurrentIdx << ": Is a physical register\n"); |
104 | 0 | if (handleReject() == RejectAndGiveUp) |
105 | 0 | return false; |
106 | 0 | break; |
107 | 0 | } |
108 | 3.04M | |
109 | 3.04M | MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); |
110 | 3.04M | if ((size_t)NewInsnID < State.MIs.size()) |
111 | 1.98M | State.MIs[NewInsnID] = NewMI; |
112 | 1.06M | else { |
113 | 1.06M | assert((size_t)NewInsnID == State.MIs.size() && |
114 | 1.06M | "Expected to store MIs in order"); |
115 | 1.06M | State.MIs.push_back(NewMI); |
116 | 1.06M | } |
117 | 3.04M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
118 | 3.04M | dbgs() << CurrentIdx << ": MIs[" << NewInsnID |
119 | 3.04M | << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx |
120 | 3.04M | << ")\n"); |
121 | 3.04M | break; |
122 | 3.04M | } |
123 | 3.04M | |
124 | 3.04M | case GIM_CheckFeatures: { |
125 | 918k | int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; |
126 | 918k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
127 | 918k | dbgs() << CurrentIdx |
128 | 918k | << ": GIM_CheckFeatures(ExpectedBitsetID=" |
129 | 918k | << ExpectedBitsetID << ")\n"); |
130 | 918k | if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != |
131 | 918k | ISelInfo.FeatureBitsets[ExpectedBitsetID]) { |
132 | 84.2k | if (handleReject() == RejectAndGiveUp) |
133 | 0 | return false; |
134 | 918k | } |
135 | 918k | break; |
136 | 918k | } |
137 | 918k | |
138 | 3.04M | case GIM_CheckOpcode: { |
139 | 3.04M | int64_t InsnID = MatchTable[CurrentIdx++]; |
140 | 3.04M | int64_t Expected = MatchTable[CurrentIdx++]; |
141 | 3.04M | |
142 | 3.04M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
143 | 3.04M | unsigned Opcode = State.MIs[InsnID]->getOpcode(); |
144 | 3.04M | |
145 | 3.04M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
146 | 3.04M | dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID |
147 | 3.04M | << "], ExpectedOpcode=" << Expected |
148 | 3.04M | << ") // Got=" << Opcode << "\n"); |
149 | 3.04M | if (Opcode != Expected) { |
150 | 2.92M | if (handleReject() == RejectAndGiveUp) |
151 | 0 | return false; |
152 | 3.04M | } |
153 | 3.04M | break; |
154 | 3.04M | } |
155 | 3.04M | |
156 | 5.94M | case GIM_SwitchOpcode: { |
157 | 5.94M | int64_t InsnID = MatchTable[CurrentIdx++]; |
158 | 5.94M | int64_t LowerBound = MatchTable[CurrentIdx++]; |
159 | 5.94M | int64_t UpperBound = MatchTable[CurrentIdx++]; |
160 | 5.94M | int64_t Default = MatchTable[CurrentIdx++]; |
161 | 5.94M | |
162 | 5.94M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
163 | 5.94M | const int64_t Opcode = State.MIs[InsnID]->getOpcode(); |
164 | 5.94M | |
165 | 5.94M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { |
166 | 5.94M | dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" |
167 | 5.94M | << LowerBound << ", " << UpperBound << "), Default=" << Default |
168 | 5.94M | << ", JumpTable...) // Got=" << Opcode << "\n"; |
169 | 5.94M | }); |
170 | 5.94M | if (Opcode < LowerBound || UpperBound <= Opcode) { |
171 | 769 | CurrentIdx = Default; |
172 | 769 | break; |
173 | 769 | } |
174 | 5.94M | CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; |
175 | 5.94M | if (!CurrentIdx) { |
176 | 1.82M | CurrentIdx = Default; |
177 | 1.82M | break; |
178 | 1.82M | } |
179 | 4.11M | OnFailResumeAt.push_back(Default); |
180 | 4.11M | break; |
181 | 4.11M | } |
182 | 4.11M | |
183 | 4.11M | case GIM_SwitchType: { |
184 | 3.76M | int64_t InsnID = MatchTable[CurrentIdx++]; |
185 | 3.76M | int64_t OpIdx = MatchTable[CurrentIdx++]; |
186 | 3.76M | int64_t LowerBound = MatchTable[CurrentIdx++]; |
187 | 3.76M | int64_t UpperBound = MatchTable[CurrentIdx++]; |
188 | 3.76M | int64_t Default = MatchTable[CurrentIdx++]; |
189 | 3.76M | |
190 | 3.76M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
191 | 3.76M | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
192 | 3.76M | |
193 | 3.76M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { |
194 | 3.76M | dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID |
195 | 3.76M | << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " |
196 | 3.76M | << UpperBound << "), Default=" << Default |
197 | 3.76M | << ", JumpTable...) // Got="; |
198 | 3.76M | if (!MO.isReg()) |
199 | 3.76M | dbgs() << "Not a VReg\n"; |
200 | 3.76M | else |
201 | 3.76M | dbgs() << MRI.getType(MO.getReg()) << "\n"; |
202 | 3.76M | }); |
203 | 3.76M | if (!MO.isReg()) { |
204 | 0 | CurrentIdx = Default; |
205 | 0 | break; |
206 | 0 | } |
207 | 3.76M | const LLT Ty = MRI.getType(MO.getReg()); |
208 | 3.76M | const auto TyI = ISelInfo.TypeIDMap.find(Ty); |
209 | 3.76M | if (TyI == ISelInfo.TypeIDMap.end()) { |
210 | 913k | CurrentIdx = Default; |
211 | 913k | break; |
212 | 913k | } |
213 | 2.85M | const int64_t TypeID = TyI->second; |
214 | 2.85M | if (TypeID < LowerBound || UpperBound <= TypeID2.66M ) { |
215 | 191k | CurrentIdx = Default; |
216 | 191k | break; |
217 | 191k | } |
218 | 2.66M | CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; |
219 | 2.66M | if (!CurrentIdx) { |
220 | 490 | CurrentIdx = Default; |
221 | 490 | break; |
222 | 490 | } |
223 | 2.65M | OnFailResumeAt.push_back(Default); |
224 | 2.65M | break; |
225 | 2.65M | } |
226 | 2.65M | |
227 | 2.65M | case GIM_CheckNumOperands: { |
228 | 22.9k | int64_t InsnID = MatchTable[CurrentIdx++]; |
229 | 22.9k | int64_t Expected = MatchTable[CurrentIdx++]; |
230 | 22.9k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
231 | 22.9k | dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" |
232 | 22.9k | << InsnID << "], Expected=" << Expected << ")\n"); |
233 | 22.9k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
234 | 22.9k | if (State.MIs[InsnID]->getNumOperands() != Expected) { |
235 | 17.2k | if (handleReject() == RejectAndGiveUp) |
236 | 0 | return false; |
237 | 22.9k | } |
238 | 22.9k | break; |
239 | 22.9k | } |
240 | 63.4k | case GIM_CheckI64ImmPredicate: { |
241 | 63.4k | int64_t InsnID = MatchTable[CurrentIdx++]; |
242 | 63.4k | int64_t Predicate = MatchTable[CurrentIdx++]; |
243 | 63.4k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
244 | 63.4k | dbgs() |
245 | 63.4k | << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" |
246 | 63.4k | << InsnID << "], Predicate=" << Predicate << ")\n"); |
247 | 63.4k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
248 | 63.4k | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && |
249 | 63.4k | "Expected G_CONSTANT"); |
250 | 63.4k | assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); |
251 | 63.4k | int64_t Value = 0; |
252 | 63.4k | if (State.MIs[InsnID]->getOperand(1).isCImm()) |
253 | 63.4k | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); |
254 | 0 | else if (State.MIs[InsnID]->getOperand(1).isImm()) |
255 | 0 | Value = State.MIs[InsnID]->getOperand(1).getImm(); |
256 | 0 | else |
257 | 0 | llvm_unreachable("Expected Imm or CImm operand"); |
258 | 63.4k | |
259 | 63.4k | if (!testImmPredicate_I64(Predicate, Value)) |
260 | 58 | if (handleReject() == RejectAndGiveUp) |
261 | 0 | return false; |
262 | 63.4k | break; |
263 | 63.4k | } |
264 | 63.4k | case GIM_CheckAPIntImmPredicate: { |
265 | 1 | int64_t InsnID = MatchTable[CurrentIdx++]; |
266 | 1 | int64_t Predicate = MatchTable[CurrentIdx++]; |
267 | 1 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
268 | 1 | dbgs() |
269 | 1 | << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" |
270 | 1 | << InsnID << "], Predicate=" << Predicate << ")\n"); |
271 | 1 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
272 | 1 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && |
273 | 1 | "Expected G_CONSTANT"); |
274 | 1 | assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); |
275 | 1 | APInt Value; |
276 | 1 | if (State.MIs[InsnID]->getOperand(1).isCImm()) |
277 | 1 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); |
278 | 1 | else |
279 | 1 | llvm_unreachable0 ("Expected Imm or CImm operand"); |
280 | 1 | |
281 | 1 | if (!testImmPredicate_APInt(Predicate, Value)) |
282 | 0 | if (handleReject() == RejectAndGiveUp) |
283 | 0 | return false; |
284 | 1 | break; |
285 | 1 | } |
286 | 24.3k | case GIM_CheckAPFloatImmPredicate: { |
287 | 24.3k | int64_t InsnID = MatchTable[CurrentIdx++]; |
288 | 24.3k | int64_t Predicate = MatchTable[CurrentIdx++]; |
289 | 24.3k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
290 | 24.3k | dbgs() |
291 | 24.3k | << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" |
292 | 24.3k | << InsnID << "], Predicate=" << Predicate << ")\n"); |
293 | 24.3k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
294 | 24.3k | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && |
295 | 24.3k | "Expected G_FCONSTANT"); |
296 | 24.3k | assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); |
297 | 24.3k | assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); |
298 | 24.3k | APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); |
299 | 24.3k | |
300 | 24.3k | if (!testImmPredicate_APFloat(Predicate, Value)) |
301 | 20.9k | if (handleReject() == RejectAndGiveUp) |
302 | 0 | return false; |
303 | 24.3k | break; |
304 | 24.3k | } |
305 | 24.3k | case GIM_CheckCxxInsnPredicate: { |
306 | 406 | int64_t InsnID = MatchTable[CurrentIdx++]; |
307 | 406 | int64_t Predicate = MatchTable[CurrentIdx++]; |
308 | 406 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
309 | 406 | dbgs() |
310 | 406 | << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" |
311 | 406 | << InsnID << "], Predicate=" << Predicate << ")\n"); |
312 | 406 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
313 | 406 | assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); |
314 | 406 | |
315 | 406 | if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID])) |
316 | 4 | if (handleReject() == RejectAndGiveUp) |
317 | 0 | return false; |
318 | 406 | break; |
319 | 406 | } |
320 | 915k | case GIM_CheckAtomicOrdering: { |
321 | 915k | int64_t InsnID = MatchTable[CurrentIdx++]; |
322 | 915k | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; |
323 | 915k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
324 | 915k | dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" |
325 | 915k | << InsnID << "], " << (uint64_t)Ordering << ")\n"); |
326 | 915k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
327 | 915k | if (!State.MIs[InsnID]->hasOneMemOperand()) |
328 | 0 | if (handleReject() == RejectAndGiveUp) |
329 | 0 | return false; |
330 | 915k | |
331 | 915k | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
332 | 915k | if (MMO->getOrdering() != Ordering) |
333 | 4.29k | if (handleReject() == RejectAndGiveUp) |
334 | 0 | return false; |
335 | 915k | break; |
336 | 915k | } |
337 | 915k | case GIM_CheckAtomicOrderingOrStrongerThan: { |
338 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; |
339 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; |
340 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
341 | 0 | dbgs() << CurrentIdx |
342 | 0 | << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" |
343 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); |
344 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
345 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) |
346 | 0 | if (handleReject() == RejectAndGiveUp) |
347 | 0 | return false; |
348 | 0 | |
349 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
350 | 0 | if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) |
351 | 0 | if (handleReject() == RejectAndGiveUp) |
352 | 0 | return false; |
353 | 0 | break; |
354 | 0 | } |
355 | 0 | case GIM_CheckAtomicOrderingWeakerThan: { |
356 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; |
357 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; |
358 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
359 | 0 | dbgs() << CurrentIdx |
360 | 0 | << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" |
361 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); |
362 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
363 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) |
364 | 0 | if (handleReject() == RejectAndGiveUp) |
365 | 0 | return false; |
366 | 0 | |
367 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) |
368 | 0 | if (!isStrongerThan(Ordering, MMO->getOrdering())) |
369 | 0 | if (handleReject() == RejectAndGiveUp) |
370 | 0 | return false; |
371 | 0 | break; |
372 | 0 | } |
373 | 2.19k | case GIM_CheckMemoryAddressSpace: { |
374 | 2.19k | int64_t InsnID = MatchTable[CurrentIdx++]; |
375 | 2.19k | int64_t MMOIdx = MatchTable[CurrentIdx++]; |
376 | 2.19k | // This accepts a list of possible address spaces. |
377 | 2.19k | const int NumAddrSpace = MatchTable[CurrentIdx++]; |
378 | 2.19k | |
379 | 2.19k | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
380 | 0 | if (handleReject() == RejectAndGiveUp) |
381 | 0 | return false; |
382 | 0 | break; |
383 | 0 | } |
384 | 2.19k | |
385 | 2.19k | // Need to still jump to the end of the list of address spaces if we find |
386 | 2.19k | // a match earlier. |
387 | 2.19k | const uint64_t LastIdx = CurrentIdx + NumAddrSpace; |
388 | 2.19k | |
389 | 2.19k | const MachineMemOperand *MMO |
390 | 2.19k | = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
391 | 2.19k | const unsigned MMOAddrSpace = MMO->getAddrSpace(); |
392 | 2.19k | |
393 | 2.19k | bool Success = false; |
394 | 4.08k | for (int I = 0; I != NumAddrSpace; ++I1.89k ) { |
395 | 2.54k | unsigned AddrSpace = MatchTable[CurrentIdx++]; |
396 | 2.54k | DEBUG_WITH_TYPE( |
397 | 2.54k | TgtInstructionSelector::getName(), |
398 | 2.54k | dbgs() << "addrspace(" << MMOAddrSpace << ") vs " |
399 | 2.54k | << AddrSpace << '\n'); |
400 | 2.54k | |
401 | 2.54k | if (AddrSpace == MMOAddrSpace) { |
402 | 649 | Success = true; |
403 | 649 | break; |
404 | 649 | } |
405 | 2.54k | } |
406 | 2.19k | |
407 | 2.19k | CurrentIdx = LastIdx; |
408 | 2.19k | if (!Success && handleReject() == RejectAndGiveUp1.54k ) |
409 | 0 | return false; |
410 | 2.19k | break; |
411 | 2.19k | } |
412 | 152k | case GIM_CheckMemorySizeEqualTo: { |
413 | 152k | int64_t InsnID = MatchTable[CurrentIdx++]; |
414 | 152k | int64_t MMOIdx = MatchTable[CurrentIdx++]; |
415 | 152k | uint64_t Size = MatchTable[CurrentIdx++]; |
416 | 152k | |
417 | 152k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
418 | 152k | dbgs() << CurrentIdx |
419 | 152k | << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID |
420 | 152k | << "]->memoperands() + " << MMOIdx |
421 | 152k | << ", Size=" << Size << ")\n"); |
422 | 152k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
423 | 152k | |
424 | 152k | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
425 | 0 | if (handleReject() == RejectAndGiveUp) |
426 | 0 | return false; |
427 | 0 | break; |
428 | 0 | } |
429 | 152k | |
430 | 152k | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
431 | 152k | |
432 | 152k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
433 | 152k | dbgs() << MMO->getSize() << " bytes vs " << Size |
434 | 152k | << " bytes\n"); |
435 | 152k | if (MMO->getSize() != Size) |
436 | 92.2k | if (handleReject() == RejectAndGiveUp) |
437 | 0 | return false; |
438 | 152k | |
439 | 152k | break; |
440 | 152k | } |
441 | 450k | case GIM_CheckMemorySizeEqualToLLT: |
442 | 450k | case GIM_CheckMemorySizeLessThanLLT: |
443 | 450k | case GIM_CheckMemorySizeGreaterThanLLT: { |
444 | 450k | int64_t InsnID = MatchTable[CurrentIdx++]; |
445 | 450k | int64_t MMOIdx = MatchTable[CurrentIdx++]; |
446 | 450k | int64_t OpIdx = MatchTable[CurrentIdx++]; |
447 | 450k | |
448 | 450k | DEBUG_WITH_TYPE( |
449 | 450k | TgtInstructionSelector::getName(), |
450 | 450k | dbgs() << CurrentIdx << ": GIM_CheckMemorySize" |
451 | 450k | << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT |
452 | 450k | ? "EqualTo" |
453 | 450k | : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT |
454 | 450k | ? "GreaterThan" |
455 | 450k | : "LessThan") |
456 | 450k | << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx |
457 | 450k | << ", OpIdx=" << OpIdx << ")\n"); |
458 | 450k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
459 | 450k | |
460 | 450k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
461 | 450k | if (!MO.isReg()) { |
462 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
463 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); |
464 | 0 | if (handleReject() == RejectAndGiveUp) |
465 | 0 | return false; |
466 | 0 | break; |
467 | 0 | } |
468 | 450k | |
469 | 450k | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { |
470 | 0 | if (handleReject() == RejectAndGiveUp) |
471 | 0 | return false; |
472 | 0 | break; |
473 | 0 | } |
474 | 450k | |
475 | 450k | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); |
476 | 450k | |
477 | 450k | unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); |
478 | 450k | if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && |
479 | 450k | MMO->getSizeInBits() != Size447k ) { |
480 | 534 | if (handleReject() == RejectAndGiveUp) |
481 | 0 | return false; |
482 | 449k | } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && |
483 | 449k | MMO->getSizeInBits() >= Size2.58k ) { |
484 | 1.72k | if (handleReject() == RejectAndGiveUp) |
485 | 0 | return false; |
486 | 447k | } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && |
487 | 447k | MMO->getSizeInBits() <= Size0 ) |
488 | 0 | if (handleReject() == RejectAndGiveUp) |
489 | 0 | return false; |
490 | 450k | |
491 | 450k | break; |
492 | 450k | } |
493 | 2.03M | case GIM_CheckType: { |
494 | 2.03M | int64_t InsnID = MatchTable[CurrentIdx++]; |
495 | 2.03M | int64_t OpIdx = MatchTable[CurrentIdx++]; |
496 | 2.03M | int64_t TypeID = MatchTable[CurrentIdx++]; |
497 | 2.03M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
498 | 2.03M | dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID |
499 | 2.03M | << "]->getOperand(" << OpIdx |
500 | 2.03M | << "), TypeID=" << TypeID << ")\n"); |
501 | 2.03M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
502 | 2.03M | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
503 | 2.03M | if (!MO.isReg() || |
504 | 2.03M | MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]2.03M ) { |
505 | 76.7k | if (handleReject() == RejectAndGiveUp) |
506 | 0 | return false; |
507 | 2.03M | } |
508 | 2.03M | break; |
509 | 2.03M | } |
510 | 2.03M | case GIM_CheckPointerToAny: { |
511 | 877k | int64_t InsnID = MatchTable[CurrentIdx++]; |
512 | 877k | int64_t OpIdx = MatchTable[CurrentIdx++]; |
513 | 877k | int64_t SizeInBits = MatchTable[CurrentIdx++]; |
514 | 877k | |
515 | 877k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
516 | 877k | dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" |
517 | 877k | << InsnID << "]->getOperand(" << OpIdx |
518 | 877k | << "), SizeInBits=" << SizeInBits << ")\n"); |
519 | 877k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
520 | 877k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
521 | 877k | const LLT Ty = MRI.getType(MO.getReg()); |
522 | 877k | |
523 | 877k | // iPTR must be looked up in the target. |
524 | 877k | if (SizeInBits == 0) { |
525 | 876k | MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); |
526 | 876k | const unsigned AddrSpace = Ty.getAddressSpace(); |
527 | 876k | SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); |
528 | 876k | } |
529 | 877k | |
530 | 877k | assert(SizeInBits != 0 && "Pointer size must be known"); |
531 | 877k | |
532 | 877k | if (MO.isReg()) { |
533 | 877k | if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) |
534 | 0 | if (handleReject() == RejectAndGiveUp) |
535 | 0 | return false; |
536 | 0 | } else if (handleReject() == RejectAndGiveUp) |
537 | 0 | return false; |
538 | 877k | |
539 | 877k | break; |
540 | 877k | } |
541 | 7.67M | case GIM_CheckRegBankForClass: { |
542 | 7.67M | int64_t InsnID = MatchTable[CurrentIdx++]; |
543 | 7.67M | int64_t OpIdx = MatchTable[CurrentIdx++]; |
544 | 7.67M | int64_t RCEnum = MatchTable[CurrentIdx++]; |
545 | 7.67M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
546 | 7.67M | dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" |
547 | 7.67M | << InsnID << "]->getOperand(" << OpIdx |
548 | 7.67M | << "), RCEnum=" << RCEnum << ")\n"); |
549 | 7.67M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
550 | 7.67M | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
551 | 7.67M | if (!MO.isReg() || |
552 | 7.67M | &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != |
553 | 7.67M | RBI.getRegBank(MO.getReg(), MRI, TRI)) { |
554 | 744k | if (handleReject() == RejectAndGiveUp) |
555 | 0 | return false; |
556 | 7.67M | } |
557 | 7.67M | break; |
558 | 7.67M | } |
559 | 7.67M | |
560 | 7.67M | case GIM_CheckComplexPattern: { |
561 | 1.44M | int64_t InsnID = MatchTable[CurrentIdx++]; |
562 | 1.44M | int64_t OpIdx = MatchTable[CurrentIdx++]; |
563 | 1.44M | int64_t RendererID = MatchTable[CurrentIdx++]; |
564 | 1.44M | int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; |
565 | 1.44M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
566 | 1.44M | dbgs() << CurrentIdx << ": State.Renderers[" << RendererID |
567 | 1.44M | << "] = GIM_CheckComplexPattern(MIs[" << InsnID |
568 | 1.44M | << "]->getOperand(" << OpIdx |
569 | 1.44M | << "), ComplexPredicateID=" << ComplexPredicateID |
570 | 1.44M | << ")\n"); |
571 | 1.44M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
572 | 1.44M | // FIXME: Use std::invoke() when it's available. |
573 | 1.44M | ComplexRendererFns Renderer = |
574 | 1.44M | (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( |
575 | 1.44M | State.MIs[InsnID]->getOperand(OpIdx)); |
576 | 1.44M | if (Renderer.hasValue()) |
577 | 929k | State.Renderers[RendererID] = Renderer.getValue(); |
578 | 512k | else |
579 | 512k | if (handleReject() == RejectAndGiveUp) |
580 | 0 | return false; |
581 | 1.44M | break; |
582 | 1.44M | } |
583 | 1.44M | |
584 | 1.44M | case GIM_CheckConstantInt: { |
585 | 286k | int64_t InsnID = MatchTable[CurrentIdx++]; |
586 | 286k | int64_t OpIdx = MatchTable[CurrentIdx++]; |
587 | 286k | int64_t Value = MatchTable[CurrentIdx++]; |
588 | 286k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
589 | 286k | dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" |
590 | 286k | << InsnID << "]->getOperand(" << OpIdx |
591 | 286k | << "), Value=" << Value << ")\n"); |
592 | 286k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
593 | 286k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
594 | 286k | if (MO.isReg()) { |
595 | 286k | // isOperandImmEqual() will sign-extend to 64-bits, so should we. |
596 | 286k | LLT Ty = MRI.getType(MO.getReg()); |
597 | 286k | Value = SignExtend64(Value, Ty.getSizeInBits()); |
598 | 286k | |
599 | 286k | if (!isOperandImmEqual(MO, Value, MRI)) { |
600 | 255k | if (handleReject() == RejectAndGiveUp) |
601 | 0 | return false; |
602 | 0 | } |
603 | 0 | } else if (handleReject() == RejectAndGiveUp) |
604 | 0 | return false; |
605 | 286k | |
606 | 286k | break; |
607 | 286k | } |
608 | 286k | |
609 | 286k | case GIM_CheckLiteralInt: { |
610 | 45 | int64_t InsnID = MatchTable[CurrentIdx++]; |
611 | 45 | int64_t OpIdx = MatchTable[CurrentIdx++]; |
612 | 45 | int64_t Value = MatchTable[CurrentIdx++]; |
613 | 45 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
614 | 45 | dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" |
615 | 45 | << InsnID << "]->getOperand(" << OpIdx |
616 | 45 | << "), Value=" << Value << ")\n"); |
617 | 45 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
618 | 45 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
619 | 45 | if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { |
620 | 23 | if (handleReject() == RejectAndGiveUp) |
621 | 0 | return false; |
622 | 45 | } |
623 | 45 | break; |
624 | 45 | } |
625 | 45 | |
626 | 242k | case GIM_CheckIntrinsicID: { |
627 | 242k | int64_t InsnID = MatchTable[CurrentIdx++]; |
628 | 242k | int64_t OpIdx = MatchTable[CurrentIdx++]; |
629 | 242k | int64_t Value = MatchTable[CurrentIdx++]; |
630 | 242k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
631 | 242k | dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" |
632 | 242k | << InsnID << "]->getOperand(" << OpIdx |
633 | 242k | << "), Value=" << Value << ")\n"); |
634 | 242k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
635 | 242k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); |
636 | 242k | if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value233k ) |
637 | 235k | if (handleReject() == RejectAndGiveUp) |
638 | 0 | return false; |
639 | 242k | break; |
640 | 242k | } |
641 | 242k | |
642 | 307k | case GIM_CheckIsMBB: { |
643 | 307k | int64_t InsnID = MatchTable[CurrentIdx++]; |
644 | 307k | int64_t OpIdx = MatchTable[CurrentIdx++]; |
645 | 307k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
646 | 307k | dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID |
647 | 307k | << "]->getOperand(" << OpIdx << "))\n"); |
648 | 307k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
649 | 307k | if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { |
650 | 0 | if (handleReject() == RejectAndGiveUp) |
651 | 0 | return false; |
652 | 307k | } |
653 | 307k | break; |
654 | 307k | } |
655 | 307k | |
656 | 307k | case GIM_CheckIsSafeToFold: { |
657 | 73.3k | int64_t InsnID = MatchTable[CurrentIdx++]; |
658 | 73.3k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
659 | 73.3k | dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" |
660 | 73.3k | << InsnID << "])\n"); |
661 | 73.3k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
662 | 73.3k | if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { |
663 | 0 | if (handleReject() == RejectAndGiveUp) |
664 | 0 | return false; |
665 | 73.3k | } |
666 | 73.3k | break; |
667 | 73.3k | } |
668 | 73.3k | case GIM_CheckIsSameOperand: { |
669 | 19 | int64_t InsnID = MatchTable[CurrentIdx++]; |
670 | 19 | int64_t OpIdx = MatchTable[CurrentIdx++]; |
671 | 19 | int64_t OtherInsnID = MatchTable[CurrentIdx++]; |
672 | 19 | int64_t OtherOpIdx = MatchTable[CurrentIdx++]; |
673 | 19 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
674 | 19 | dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" |
675 | 19 | << InsnID << "][" << OpIdx << "], MIs[" |
676 | 19 | << OtherInsnID << "][" << OtherOpIdx << "])\n"); |
677 | 19 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); |
678 | 19 | assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); |
679 | 19 | if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( |
680 | 19 | State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { |
681 | 2 | if (handleReject() == RejectAndGiveUp) |
682 | 0 | return false; |
683 | 19 | } |
684 | 19 | break; |
685 | 19 | } |
686 | 4.69M | case GIM_Reject: |
687 | 4.69M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
688 | 4.69M | dbgs() << CurrentIdx << ": GIM_Reject\n"); |
689 | 4.69M | if (handleReject() == RejectAndGiveUp) |
690 | 3.13M | return false; |
691 | 1.55M | break; |
692 | 1.55M | |
693 | 1.55M | case GIR_MutateOpcode: { |
694 | 823k | int64_t OldInsnID = MatchTable[CurrentIdx++]; |
695 | 823k | uint64_t NewInsnID = MatchTable[CurrentIdx++]; |
696 | 823k | int64_t NewOpcode = MatchTable[CurrentIdx++]; |
697 | 823k | if (NewInsnID >= OutMIs.size()) |
698 | 823k | OutMIs.resize(NewInsnID + 1); |
699 | 823k | |
700 | 823k | OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), |
701 | 823k | State.MIs[OldInsnID]); |
702 | 823k | OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); |
703 | 823k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
704 | 823k | dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" |
705 | 823k | << NewInsnID << "], MIs[" << OldInsnID << "], " |
706 | 823k | << NewOpcode << ")\n"); |
707 | 823k | break; |
708 | 1.55M | } |
709 | 1.55M | |
710 | 1.99M | case GIR_BuildMI: { |
711 | 1.99M | uint64_t NewInsnID = MatchTable[CurrentIdx++]; |
712 | 1.99M | int64_t Opcode = MatchTable[CurrentIdx++]; |
713 | 1.99M | if (NewInsnID >= OutMIs.size()) |
714 | 1.98M | OutMIs.resize(NewInsnID + 1); |
715 | 1.99M | |
716 | 1.99M | OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], |
717 | 1.99M | State.MIs[0]->getDebugLoc(), TII.get(Opcode)); |
718 | 1.99M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
719 | 1.99M | dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" |
720 | 1.99M | << NewInsnID << "], " << Opcode << ")\n"); |
721 | 1.99M | break; |
722 | 1.55M | } |
723 | 1.55M | |
724 | 2.20M | case GIR_Copy: { |
725 | 2.20M | int64_t NewInsnID = MatchTable[CurrentIdx++]; |
726 | 2.20M | int64_t OldInsnID = MatchTable[CurrentIdx++]; |
727 | 2.20M | int64_t OpIdx = MatchTable[CurrentIdx++]; |
728 | 2.20M | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
729 | 2.20M | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); |
730 | 2.20M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
731 | 2.20M | dbgs() |
732 | 2.20M | << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID |
733 | 2.20M | << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); |
734 | 2.20M | break; |
735 | 1.55M | } |
736 | 1.55M | |
737 | 1.55M | case GIR_CopyOrAddZeroReg: { |
738 | 272k | int64_t NewInsnID = MatchTable[CurrentIdx++]; |
739 | 272k | int64_t OldInsnID = MatchTable[CurrentIdx++]; |
740 | 272k | int64_t OpIdx = MatchTable[CurrentIdx++]; |
741 | 272k | int64_t ZeroReg = MatchTable[CurrentIdx++]; |
742 | 272k | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
743 | 272k | MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); |
744 | 272k | if (isOperandImmEqual(MO, 0, MRI)) |
745 | 26.6k | OutMIs[NewInsnID].addReg(ZeroReg); |
746 | 245k | else |
747 | 245k | OutMIs[NewInsnID].add(MO); |
748 | 272k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
749 | 272k | dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" |
750 | 272k | << NewInsnID << "], MIs[" << OldInsnID << "], " |
751 | 272k | << OpIdx << ", " << ZeroReg << ")\n"); |
752 | 272k | break; |
753 | 1.55M | } |
754 | 1.55M | |
755 | 1.55M | case GIR_CopySubReg: { |
756 | 93.7k | int64_t NewInsnID = MatchTable[CurrentIdx++]; |
757 | 93.7k | int64_t OldInsnID = MatchTable[CurrentIdx++]; |
758 | 93.7k | int64_t OpIdx = MatchTable[CurrentIdx++]; |
759 | 93.7k | int64_t SubRegIdx = MatchTable[CurrentIdx++]; |
760 | 93.7k | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
761 | 93.7k | OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), |
762 | 93.7k | 0, SubRegIdx); |
763 | 93.7k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
764 | 93.7k | dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" |
765 | 93.7k | << NewInsnID << "], MIs[" << OldInsnID << "], " |
766 | 93.7k | << OpIdx << ", " << SubRegIdx << ")\n"); |
767 | 93.7k | break; |
768 | 1.55M | } |
769 | 1.55M | |
770 | 1.55M | case GIR_AddImplicitDef: { |
771 | 73.1k | int64_t InsnID = MatchTable[CurrentIdx++]; |
772 | 73.1k | int64_t RegNum = MatchTable[CurrentIdx++]; |
773 | 73.1k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
774 | 73.1k | OutMIs[InsnID].addDef(RegNum, RegState::Implicit); |
775 | 73.1k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
776 | 73.1k | dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" |
777 | 73.1k | << InsnID << "], " << RegNum << ")\n"); |
778 | 73.1k | break; |
779 | 1.55M | } |
780 | 1.55M | |
781 | 1.55M | case GIR_AddImplicitUse: { |
782 | 31 | int64_t InsnID = MatchTable[CurrentIdx++]; |
783 | 31 | int64_t RegNum = MatchTable[CurrentIdx++]; |
784 | 31 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
785 | 31 | OutMIs[InsnID].addUse(RegNum, RegState::Implicit); |
786 | 31 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
787 | 31 | dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" |
788 | 31 | << InsnID << "], " << RegNum << ")\n"); |
789 | 31 | break; |
790 | 1.55M | } |
791 | 1.55M | |
792 | 1.55M | case GIR_AddRegister: { |
793 | 163k | int64_t InsnID = MatchTable[CurrentIdx++]; |
794 | 163k | int64_t RegNum = MatchTable[CurrentIdx++]; |
795 | 163k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
796 | 163k | OutMIs[InsnID].addReg(RegNum); |
797 | 163k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
798 | 163k | dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" |
799 | 163k | << InsnID << "], " << RegNum << ")\n"); |
800 | 163k | break; |
801 | 1.55M | } |
802 | 1.55M | |
803 | 1.55M | case GIR_AddTempRegister: { |
804 | 8.09k | int64_t InsnID = MatchTable[CurrentIdx++]; |
805 | 8.09k | int64_t TempRegID = MatchTable[CurrentIdx++]; |
806 | 8.09k | uint64_t TempRegFlags = MatchTable[CurrentIdx++]; |
807 | 8.09k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
808 | 8.09k | OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags); |
809 | 8.09k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
810 | 8.09k | dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" |
811 | 8.09k | << InsnID << "], TempRegisters[" << TempRegID |
812 | 8.09k | << "], " << TempRegFlags << ")\n"); |
813 | 8.09k | break; |
814 | 1.55M | } |
815 | 1.55M | |
816 | 1.55M | case GIR_AddImm: { |
817 | 61.1k | int64_t InsnID = MatchTable[CurrentIdx++]; |
818 | 61.1k | int64_t Imm = MatchTable[CurrentIdx++]; |
819 | 61.1k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
820 | 61.1k | OutMIs[InsnID].addImm(Imm); |
821 | 61.1k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
822 | 61.1k | dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID |
823 | 61.1k | << "], " << Imm << ")\n"); |
824 | 61.1k | break; |
825 | 1.55M | } |
826 | 1.55M | |
827 | 1.55M | case GIR_ComplexRenderer: { |
828 | 111k | int64_t InsnID = MatchTable[CurrentIdx++]; |
829 | 111k | int64_t RendererID = MatchTable[CurrentIdx++]; |
830 | 111k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
831 | 111k | for (const auto &RenderOpFn : State.Renderers[RendererID]) |
832 | 223k | RenderOpFn(OutMIs[InsnID]); |
833 | 111k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
834 | 111k | dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" |
835 | 111k | << InsnID << "], " << RendererID << ")\n"); |
836 | 111k | break; |
837 | 1.55M | } |
838 | 1.63M | case GIR_ComplexSubOperandRenderer: { |
839 | 1.63M | int64_t InsnID = MatchTable[CurrentIdx++]; |
840 | 1.63M | int64_t RendererID = MatchTable[CurrentIdx++]; |
841 | 1.63M | int64_t RenderOpID = MatchTable[CurrentIdx++]; |
842 | 1.63M | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
843 | 1.63M | State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); |
844 | 1.63M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
845 | 1.63M | dbgs() << CurrentIdx |
846 | 1.63M | << ": GIR_ComplexSubOperandRenderer(OutMIs[" |
847 | 1.63M | << InsnID << "], " << RendererID << ", " |
848 | 1.63M | << RenderOpID << ")\n"); |
849 | 1.63M | break; |
850 | 1.55M | } |
851 | 1.55M | |
852 | 1.55M | case GIR_CopyConstantAsSImm: { |
853 | 790k | int64_t NewInsnID = MatchTable[CurrentIdx++]; |
854 | 790k | int64_t OldInsnID = MatchTable[CurrentIdx++]; |
855 | 790k | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
856 | 790k | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); |
857 | 790k | if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { |
858 | 790k | OutMIs[NewInsnID].addImm( |
859 | 790k | State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); |
860 | 790k | } else if (0 State.MIs[OldInsnID]->getOperand(1).isImm()0 ) |
861 | 0 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); |
862 | 0 | else |
863 | 0 | llvm_unreachable("Expected Imm or CImm operand"); |
864 | 790k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
865 | 790k | dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" |
866 | 790k | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); |
867 | 790k | break; |
868 | 790k | } |
869 | 790k | |
870 | 790k | // TODO: Needs a test case once we have a pattern that uses this. |
871 | 790k | case GIR_CopyFConstantAsFPImm: { |
872 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; |
873 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; |
874 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); |
875 | 0 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); |
876 | 0 | if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) |
877 | 0 | OutMIs[NewInsnID].addFPImm( |
878 | 0 | State.MIs[OldInsnID]->getOperand(1).getFPImm()); |
879 | 0 | else |
880 | 0 | llvm_unreachable("Expected FPImm operand"); |
881 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
882 | 0 | dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" |
883 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); |
884 | 0 | break; |
885 | 0 | } |
886 | 0 |
|
887 | 4.02k | case GIR_CustomRenderer: { |
888 | 4.02k | int64_t InsnID = MatchTable[CurrentIdx++]; |
889 | 4.02k | int64_t OldInsnID = MatchTable[CurrentIdx++]; |
890 | 4.02k | int64_t RendererFnID = MatchTable[CurrentIdx++]; |
891 | 4.02k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
892 | 4.02k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
893 | 4.02k | dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" |
894 | 4.02k | << InsnID << "], MIs[" << OldInsnID << "], " |
895 | 4.02k | << RendererFnID << ")\n"); |
896 | 4.02k | (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], |
897 | 4.02k | *State.MIs[OldInsnID]); |
898 | 4.02k | break; |
899 | 0 | } |
900 | 188k | case GIR_ConstrainOperandRC: { |
901 | 188k | int64_t InsnID = MatchTable[CurrentIdx++]; |
902 | 188k | int64_t OpIdx = MatchTable[CurrentIdx++]; |
903 | 188k | int64_t RCEnum = MatchTable[CurrentIdx++]; |
904 | 188k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
905 | 188k | constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, |
906 | 188k | *TRI.getRegClass(RCEnum), TII, TRI, RBI); |
907 | 188k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
908 | 188k | dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" |
909 | 188k | << InsnID << "], " << OpIdx << ", " << RCEnum |
910 | 188k | << ")\n"); |
911 | 188k | break; |
912 | 0 | } |
913 | 0 |
|
914 | 2.72M | case GIR_ConstrainSelectedInstOperands: { |
915 | 2.72M | int64_t InsnID = MatchTable[CurrentIdx++]; |
916 | 2.72M | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
917 | 2.72M | constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, |
918 | 2.72M | RBI); |
919 | 2.72M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
920 | 2.72M | dbgs() << CurrentIdx |
921 | 2.72M | << ": GIR_ConstrainSelectedInstOperands(OutMIs[" |
922 | 2.72M | << InsnID << "])\n"); |
923 | 2.72M | break; |
924 | 0 | } |
925 | 0 |
|
926 | 821k | case GIR_MergeMemOperands: { |
927 | 821k | int64_t InsnID = MatchTable[CurrentIdx++]; |
928 | 821k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); |
929 | 821k | |
930 | 821k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
931 | 821k | dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" |
932 | 821k | << InsnID << "]"); |
933 | 821k | int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; |
934 | 1.64M | while ((MergeInsnID = MatchTable[CurrentIdx++]) != |
935 | 1.64M | GIU_MergeMemOperands_EndOfList) { |
936 | 823k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
937 | 823k | dbgs() << ", MIs[" << MergeInsnID << "]"); |
938 | 823k | for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) |
939 | 818k | OutMIs[InsnID].addMemOperand(MMO); |
940 | 823k | } |
941 | 821k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); |
942 | 821k | break; |
943 | 0 | } |
944 | 0 |
|
945 | 1.98M | case GIR_EraseFromParent: { |
946 | 1.98M | int64_t InsnID = MatchTable[CurrentIdx++]; |
947 | 1.98M | assert(State.MIs[InsnID] && |
948 | 1.98M | "Attempted to erase an undefined instruction"); |
949 | 1.98M | State.MIs[InsnID]->eraseFromParent(); |
950 | 1.98M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
951 | 1.98M | dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" |
952 | 1.98M | << InsnID << "])\n"); |
953 | 1.98M | break; |
954 | 0 | } |
955 | 0 |
|
956 | 4.04k | case GIR_MakeTempReg: { |
957 | 4.04k | int64_t TempRegID = MatchTable[CurrentIdx++]; |
958 | 4.04k | int64_t TypeID = MatchTable[CurrentIdx++]; |
959 | 4.04k | |
960 | 4.04k | State.TempRegisters[TempRegID] = |
961 | 4.04k | MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); |
962 | 4.04k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
963 | 4.04k | dbgs() << CurrentIdx << ": TempRegs[" << TempRegID |
964 | 4.04k | << "] = GIR_MakeTempReg(" << TypeID << ")\n"); |
965 | 4.04k | break; |
966 | 0 | } |
967 | 0 |
|
968 | 0 | case GIR_Coverage: { |
969 | 0 | int64_t RuleID = MatchTable[CurrentIdx++]; |
970 | 0 | CoverageInfo.setCovered(RuleID); |
971 | 0 |
|
972 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
973 | 0 | dbgs() |
974 | 0 | << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); |
975 | 0 | break; |
976 | 0 | } |
977 | 0 |
|
978 | 2.81M | case GIR_Done: |
979 | 2.81M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), |
980 | 2.81M | dbgs() << CurrentIdx << ": GIR_Done\n"); |
981 | 2.81M | return true; |
982 | 0 |
|
983 | 0 | default: |
984 | 0 | llvm_unreachable("Unexpected command"); |
985 | 61.9M | } |
986 | 61.9M | } |
987 | 5.94M | } AArch64InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::AArch64InstructionSelector const, llvm::PredicateBitsetImpl<22ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::AArch64InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<22ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::AArch64InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<22ul> const&, llvm::CodeGenCoverage&) const Line | Count | Source | 55 | 5.94M | CodeGenCoverage &CoverageInfo) const { | 56 | 5.94M | | 57 | 5.94M | uint64_t CurrentIdx = 0; | 58 | 5.94M | SmallVector<uint64_t, 4> OnFailResumeAt; | 59 | 5.94M | | 60 | 5.94M | enum RejectAction { RejectAndGiveUp, RejectAndResume }; | 61 | 5.94M | auto handleReject = [&]() -> RejectAction { | 62 | 5.94M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 5.94M | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 5.94M | if (OnFailResumeAt.empty()) | 65 | 5.94M | return RejectAndGiveUp; | 66 | 5.94M | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 5.94M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 5.94M | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 5.94M | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 5.94M | return RejectAndResume; | 71 | 5.94M | }; | 72 | 5.94M | | 73 | 61.8M | while (true) { | 74 | 61.8M | assert(CurrentIdx != ~0u && "Invalid MatchTable index"); | 75 | 61.8M | int64_t MatcherOpcode = MatchTable[CurrentIdx++]; | 76 | 61.8M | switch (MatcherOpcode) { | 77 | 61.8M | case GIM_Try: { | 78 | 9.13M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 79 | 9.13M | dbgs() << CurrentIdx << ": Begin try-block\n"); | 80 | 9.13M | OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); | 81 | 9.13M | break; | 82 | 61.8M | } | 83 | 61.8M | | 84 | 61.8M | case GIM_RecordInsn: { | 85 | 3.04M | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 86 | 3.04M | int64_t InsnID = MatchTable[CurrentIdx++]; | 87 | 3.04M | int64_t OpIdx = MatchTable[CurrentIdx++]; | 88 | 3.04M | | 89 | 3.04M | // As an optimisation we require that MIs[0] is always the root. Refuse | 90 | 3.04M | // any attempt to modify it. | 91 | 3.04M | assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); | 92 | 3.04M | | 93 | 3.04M | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 94 | 3.04M | if (!MO.isReg()) { | 95 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 96 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 97 | 0 | if (handleReject() == RejectAndGiveUp) | 98 | 0 | return false; | 99 | 0 | break; | 100 | 0 | } | 101 | 3.04M | if (TRI.isPhysicalRegister(MO.getReg())) { | 102 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 103 | 0 | dbgs() << CurrentIdx << ": Is a physical register\n"); | 104 | 0 | if (handleReject() == RejectAndGiveUp) | 105 | 0 | return false; | 106 | 0 | break; | 107 | 0 | } | 108 | 3.04M | | 109 | 3.04M | MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); | 110 | 3.04M | if ((size_t)NewInsnID < State.MIs.size()) | 111 | 1.98M | State.MIs[NewInsnID] = NewMI; | 112 | 1.05M | else { | 113 | 1.05M | assert((size_t)NewInsnID == State.MIs.size() && | 114 | 1.05M | "Expected to store MIs in order"); | 115 | 1.05M | State.MIs.push_back(NewMI); | 116 | 1.05M | } | 117 | 3.04M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 118 | 3.04M | dbgs() << CurrentIdx << ": MIs[" << NewInsnID | 119 | 3.04M | << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx | 120 | 3.04M | << ")\n"); | 121 | 3.04M | break; | 122 | 3.04M | } | 123 | 3.04M | | 124 | 3.04M | case GIM_CheckFeatures: { | 125 | 906k | int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; | 126 | 906k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 127 | 906k | dbgs() << CurrentIdx | 128 | 906k | << ": GIM_CheckFeatures(ExpectedBitsetID=" | 129 | 906k | << ExpectedBitsetID << ")\n"); | 130 | 906k | if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != | 131 | 906k | ISelInfo.FeatureBitsets[ExpectedBitsetID]) { | 132 | 79.1k | if (handleReject() == RejectAndGiveUp) | 133 | 0 | return false; | 134 | 906k | } | 135 | 906k | break; | 136 | 906k | } | 137 | 906k | | 138 | 3.04M | case GIM_CheckOpcode: { | 139 | 3.04M | int64_t InsnID = MatchTable[CurrentIdx++]; | 140 | 3.04M | int64_t Expected = MatchTable[CurrentIdx++]; | 141 | 3.04M | | 142 | 3.04M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 143 | 3.04M | unsigned Opcode = State.MIs[InsnID]->getOpcode(); | 144 | 3.04M | | 145 | 3.04M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 146 | 3.04M | dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID | 147 | 3.04M | << "], ExpectedOpcode=" << Expected | 148 | 3.04M | << ") // Got=" << Opcode << "\n"); | 149 | 3.04M | if (Opcode != Expected) { | 150 | 2.92M | if (handleReject() == RejectAndGiveUp) | 151 | 0 | return false; | 152 | 3.04M | } | 153 | 3.04M | break; | 154 | 3.04M | } | 155 | 3.04M | | 156 | 5.94M | case GIM_SwitchOpcode: { | 157 | 5.94M | int64_t InsnID = MatchTable[CurrentIdx++]; | 158 | 5.94M | int64_t LowerBound = MatchTable[CurrentIdx++]; | 159 | 5.94M | int64_t UpperBound = MatchTable[CurrentIdx++]; | 160 | 5.94M | int64_t Default = MatchTable[CurrentIdx++]; | 161 | 5.94M | | 162 | 5.94M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 163 | 5.94M | const int64_t Opcode = State.MIs[InsnID]->getOpcode(); | 164 | 5.94M | | 165 | 5.94M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 166 | 5.94M | dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" | 167 | 5.94M | << LowerBound << ", " << UpperBound << "), Default=" << Default | 168 | 5.94M | << ", JumpTable...) // Got=" << Opcode << "\n"; | 169 | 5.94M | }); | 170 | 5.94M | if (Opcode < LowerBound || UpperBound <= Opcode) { | 171 | 763 | CurrentIdx = Default; | 172 | 763 | break; | 173 | 763 | } | 174 | 5.94M | CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; | 175 | 5.94M | if (!CurrentIdx) { | 176 | 1.82M | CurrentIdx = Default; | 177 | 1.82M | break; | 178 | 1.82M | } | 179 | 4.11M | OnFailResumeAt.push_back(Default); | 180 | 4.11M | break; | 181 | 4.11M | } | 182 | 4.11M | | 183 | 4.11M | case GIM_SwitchType: { | 184 | 3.76M | int64_t InsnID = MatchTable[CurrentIdx++]; | 185 | 3.76M | int64_t OpIdx = MatchTable[CurrentIdx++]; | 186 | 3.76M | int64_t LowerBound = MatchTable[CurrentIdx++]; | 187 | 3.76M | int64_t UpperBound = MatchTable[CurrentIdx++]; | 188 | 3.76M | int64_t Default = MatchTable[CurrentIdx++]; | 189 | 3.76M | | 190 | 3.76M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 191 | 3.76M | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 192 | 3.76M | | 193 | 3.76M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 194 | 3.76M | dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID | 195 | 3.76M | << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " | 196 | 3.76M | << UpperBound << "), Default=" << Default | 197 | 3.76M | << ", JumpTable...) // Got="; | 198 | 3.76M | if (!MO.isReg()) | 199 | 3.76M | dbgs() << "Not a VReg\n"; | 200 | 3.76M | else | 201 | 3.76M | dbgs() << MRI.getType(MO.getReg()) << "\n"; | 202 | 3.76M | }); | 203 | 3.76M | if (!MO.isReg()) { | 204 | 0 | CurrentIdx = Default; | 205 | 0 | break; | 206 | 0 | } | 207 | 3.76M | const LLT Ty = MRI.getType(MO.getReg()); | 208 | 3.76M | const auto TyI = ISelInfo.TypeIDMap.find(Ty); | 209 | 3.76M | if (TyI == ISelInfo.TypeIDMap.end()) { | 210 | 913k | CurrentIdx = Default; | 211 | 913k | break; | 212 | 913k | } | 213 | 2.84M | const int64_t TypeID = TyI->second; | 214 | 2.84M | if (TypeID < LowerBound || UpperBound <= TypeID2.65M ) { | 215 | 191k | CurrentIdx = Default; | 216 | 191k | break; | 217 | 191k | } | 218 | 2.65M | CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; | 219 | 2.65M | if (!CurrentIdx) { | 220 | 39 | CurrentIdx = Default; | 221 | 39 | break; | 222 | 39 | } | 223 | 2.65M | OnFailResumeAt.push_back(Default); | 224 | 2.65M | break; | 225 | 2.65M | } | 226 | 2.65M | | 227 | 2.65M | case GIM_CheckNumOperands: { | 228 | 22.9k | int64_t InsnID = MatchTable[CurrentIdx++]; | 229 | 22.9k | int64_t Expected = MatchTable[CurrentIdx++]; | 230 | 22.9k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 231 | 22.9k | dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" | 232 | 22.9k | << InsnID << "], Expected=" << Expected << ")\n"); | 233 | 22.9k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 234 | 22.9k | if (State.MIs[InsnID]->getNumOperands() != Expected) { | 235 | 17.2k | if (handleReject() == RejectAndGiveUp) | 236 | 0 | return false; | 237 | 22.9k | } | 238 | 22.9k | break; | 239 | 22.9k | } | 240 | 63.1k | case GIM_CheckI64ImmPredicate: { | 241 | 63.1k | int64_t InsnID = MatchTable[CurrentIdx++]; | 242 | 63.1k | int64_t Predicate = MatchTable[CurrentIdx++]; | 243 | 63.1k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 244 | 63.1k | dbgs() | 245 | 63.1k | << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" | 246 | 63.1k | << InsnID << "], Predicate=" << Predicate << ")\n"); | 247 | 63.1k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 248 | 63.1k | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 249 | 63.1k | "Expected G_CONSTANT"); | 250 | 63.1k | assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); | 251 | 63.1k | int64_t Value = 0; | 252 | 63.1k | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 253 | 63.1k | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); | 254 | 0 | else if (State.MIs[InsnID]->getOperand(1).isImm()) | 255 | 0 | Value = State.MIs[InsnID]->getOperand(1).getImm(); | 256 | 0 | else | 257 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 258 | 63.1k | | 259 | 63.1k | if (!testImmPredicate_I64(Predicate, Value)) | 260 | 27 | if (handleReject() == RejectAndGiveUp) | 261 | 0 | return false; | 262 | 63.1k | break; | 263 | 63.1k | } | 264 | 63.1k | case GIM_CheckAPIntImmPredicate: { | 265 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 266 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 267 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 268 | 0 | dbgs() | 269 | 0 | << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" | 270 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 271 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 272 | 0 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 273 | 0 | "Expected G_CONSTANT"); | 274 | 0 | assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); | 275 | 0 | APInt Value; | 276 | 0 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 277 | 0 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); | 278 | 0 | else | 279 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 280 | 0 |
| 281 | 0 | if (!testImmPredicate_APInt(Predicate, Value)) | 282 | 0 | if (handleReject() == RejectAndGiveUp) | 283 | 0 | return false; | 284 | 0 | break; | 285 | 0 | } | 286 | 24.3k | case GIM_CheckAPFloatImmPredicate: { | 287 | 24.3k | int64_t InsnID = MatchTable[CurrentIdx++]; | 288 | 24.3k | int64_t Predicate = MatchTable[CurrentIdx++]; | 289 | 24.3k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 290 | 24.3k | dbgs() | 291 | 24.3k | << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" | 292 | 24.3k | << InsnID << "], Predicate=" << Predicate << ")\n"); | 293 | 24.3k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 294 | 24.3k | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && | 295 | 24.3k | "Expected G_FCONSTANT"); | 296 | 24.3k | assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); | 297 | 24.3k | assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); | 298 | 24.3k | APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); | 299 | 24.3k | | 300 | 24.3k | if (!testImmPredicate_APFloat(Predicate, Value)) | 301 | 20.9k | if (handleReject() == RejectAndGiveUp) | 302 | 0 | return false; | 303 | 24.3k | break; | 304 | 24.3k | } | 305 | 24.3k | case GIM_CheckCxxInsnPredicate: { | 306 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 307 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 308 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 309 | 0 | dbgs() | 310 | 0 | << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" | 311 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 312 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 313 | 0 | assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); | 314 | 0 |
| 315 | 0 | if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID])) | 316 | 0 | if (handleReject() == RejectAndGiveUp) | 317 | 0 | return false; | 318 | 0 | break; | 319 | 0 | } | 320 | 914k | case GIM_CheckAtomicOrdering: { | 321 | 914k | int64_t InsnID = MatchTable[CurrentIdx++]; | 322 | 914k | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 323 | 914k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 324 | 914k | dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" | 325 | 914k | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 326 | 914k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 327 | 914k | if (!State.MIs[InsnID]->hasOneMemOperand()) | 328 | 0 | if (handleReject() == RejectAndGiveUp) | 329 | 0 | return false; | 330 | 914k | | 331 | 914k | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 332 | 914k | if (MMO->getOrdering() != Ordering) | 333 | 4.24k | if (handleReject() == RejectAndGiveUp) | 334 | 0 | return false; | 335 | 914k | break; | 336 | 914k | } | 337 | 914k | case GIM_CheckAtomicOrderingOrStrongerThan: { | 338 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 339 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 340 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 341 | 0 | dbgs() << CurrentIdx | 342 | 0 | << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" | 343 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 344 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 345 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 346 | 0 | if (handleReject() == RejectAndGiveUp) | 347 | 0 | return false; | 348 | 0 | | 349 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 350 | 0 | if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) | 351 | 0 | if (handleReject() == RejectAndGiveUp) | 352 | 0 | return false; | 353 | 0 | break; | 354 | 0 | } | 355 | 0 | case GIM_CheckAtomicOrderingWeakerThan: { | 356 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 357 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 358 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 359 | 0 | dbgs() << CurrentIdx | 360 | 0 | << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" | 361 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 362 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 363 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 364 | 0 | if (handleReject() == RejectAndGiveUp) | 365 | 0 | return false; | 366 | 0 | | 367 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 368 | 0 | if (!isStrongerThan(Ordering, MMO->getOrdering())) | 369 | 0 | if (handleReject() == RejectAndGiveUp) | 370 | 0 | return false; | 371 | 0 | break; | 372 | 0 | } | 373 | 0 | case GIM_CheckMemoryAddressSpace: { | 374 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 375 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 376 | 0 | // This accepts a list of possible address spaces. | 377 | 0 | const int NumAddrSpace = MatchTable[CurrentIdx++]; | 378 | 0 |
| 379 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 380 | 0 | if (handleReject() == RejectAndGiveUp) | 381 | 0 | return false; | 382 | 0 | break; | 383 | 0 | } | 384 | 0 | | 385 | 0 | // Need to still jump to the end of the list of address spaces if we find | 386 | 0 | // a match earlier. | 387 | 0 | const uint64_t LastIdx = CurrentIdx + NumAddrSpace; | 388 | 0 |
| 389 | 0 | const MachineMemOperand *MMO | 390 | 0 | = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 391 | 0 | const unsigned MMOAddrSpace = MMO->getAddrSpace(); | 392 | 0 |
| 393 | 0 | bool Success = false; | 394 | 0 | for (int I = 0; I != NumAddrSpace; ++I) { | 395 | 0 | unsigned AddrSpace = MatchTable[CurrentIdx++]; | 396 | 0 | DEBUG_WITH_TYPE( | 397 | 0 | TgtInstructionSelector::getName(), | 398 | 0 | dbgs() << "addrspace(" << MMOAddrSpace << ") vs " | 399 | 0 | << AddrSpace << '\n'); | 400 | 0 |
| 401 | 0 | if (AddrSpace == MMOAddrSpace) { | 402 | 0 | Success = true; | 403 | 0 | break; | 404 | 0 | } | 405 | 0 | } | 406 | 0 |
| 407 | 0 | CurrentIdx = LastIdx; | 408 | 0 | if (!Success && handleReject() == RejectAndGiveUp) | 409 | 0 | return false; | 410 | 0 | break; | 411 | 0 | } | 412 | 151k | case GIM_CheckMemorySizeEqualTo: { | 413 | 151k | int64_t InsnID = MatchTable[CurrentIdx++]; | 414 | 151k | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 415 | 151k | uint64_t Size = MatchTable[CurrentIdx++]; | 416 | 151k | | 417 | 151k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 418 | 151k | dbgs() << CurrentIdx | 419 | 151k | << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID | 420 | 151k | << "]->memoperands() + " << MMOIdx | 421 | 151k | << ", Size=" << Size << ")\n"); | 422 | 151k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 423 | 151k | | 424 | 151k | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 425 | 0 | if (handleReject() == RejectAndGiveUp) | 426 | 0 | return false; | 427 | 0 | break; | 428 | 0 | } | 429 | 151k | | 430 | 151k | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 431 | 151k | | 432 | 151k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 433 | 151k | dbgs() << MMO->getSize() << " bytes vs " << Size | 434 | 151k | << " bytes\n"); | 435 | 151k | if (MMO->getSize() != Size) | 436 | 92.1k | if (handleReject() == RejectAndGiveUp) | 437 | 0 | return false; | 438 | 151k | | 439 | 151k | break; | 440 | 151k | } | 441 | 446k | case GIM_CheckMemorySizeEqualToLLT: | 442 | 446k | case GIM_CheckMemorySizeLessThanLLT: | 443 | 446k | case GIM_CheckMemorySizeGreaterThanLLT: { | 444 | 446k | int64_t InsnID = MatchTable[CurrentIdx++]; | 445 | 446k | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 446 | 446k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 447 | 446k | | 448 | 446k | DEBUG_WITH_TYPE( | 449 | 446k | TgtInstructionSelector::getName(), | 450 | 446k | dbgs() << CurrentIdx << ": GIM_CheckMemorySize" | 451 | 446k | << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT | 452 | 446k | ? "EqualTo" | 453 | 446k | : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT | 454 | 446k | ? "GreaterThan" | 455 | 446k | : "LessThan") | 456 | 446k | << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx | 457 | 446k | << ", OpIdx=" << OpIdx << ")\n"); | 458 | 446k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 459 | 446k | | 460 | 446k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 461 | 446k | if (!MO.isReg()) { | 462 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 463 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 464 | 0 | if (handleReject() == RejectAndGiveUp) | 465 | 0 | return false; | 466 | 0 | break; | 467 | 0 | } | 468 | 446k | | 469 | 446k | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 470 | 0 | if (handleReject() == RejectAndGiveUp) | 471 | 0 | return false; | 472 | 0 | break; | 473 | 0 | } | 474 | 446k | | 475 | 446k | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 476 | 446k | | 477 | 446k | unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); | 478 | 446k | if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && | 479 | 446k | MMO->getSizeInBits() != Size446k ) { | 480 | 28 | if (handleReject() == RejectAndGiveUp) | 481 | 0 | return false; | 482 | 446k | } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && | 483 | 446k | MMO->getSizeInBits() >= Size48 ) { | 484 | 36 | if (handleReject() == RejectAndGiveUp) | 485 | 0 | return false; | 486 | 446k | } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && | 487 | 446k | MMO->getSizeInBits() <= Size0 ) | 488 | 0 | if (handleReject() == RejectAndGiveUp) | 489 | 0 | return false; | 490 | 446k | | 491 | 446k | break; | 492 | 446k | } | 493 | 2.03M | case GIM_CheckType: { | 494 | 2.03M | int64_t InsnID = MatchTable[CurrentIdx++]; | 495 | 2.03M | int64_t OpIdx = MatchTable[CurrentIdx++]; | 496 | 2.03M | int64_t TypeID = MatchTable[CurrentIdx++]; | 497 | 2.03M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 498 | 2.03M | dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID | 499 | 2.03M | << "]->getOperand(" << OpIdx | 500 | 2.03M | << "), TypeID=" << TypeID << ")\n"); | 501 | 2.03M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 502 | 2.03M | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 503 | 2.03M | if (!MO.isReg() || | 504 | 2.03M | MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]2.03M ) { | 505 | 76.5k | if (handleReject() == RejectAndGiveUp) | 506 | 0 | return false; | 507 | 2.03M | } | 508 | 2.03M | break; | 509 | 2.03M | } | 510 | 2.03M | case GIM_CheckPointerToAny: { | 511 | 877k | int64_t InsnID = MatchTable[CurrentIdx++]; | 512 | 877k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 513 | 877k | int64_t SizeInBits = MatchTable[CurrentIdx++]; | 514 | 877k | | 515 | 877k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 516 | 877k | dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" | 517 | 877k | << InsnID << "]->getOperand(" << OpIdx | 518 | 877k | << "), SizeInBits=" << SizeInBits << ")\n"); | 519 | 877k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 520 | 877k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 521 | 877k | const LLT Ty = MRI.getType(MO.getReg()); | 522 | 877k | | 523 | 877k | // iPTR must be looked up in the target. | 524 | 877k | if (SizeInBits == 0) { | 525 | 875k | MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); | 526 | 875k | const unsigned AddrSpace = Ty.getAddressSpace(); | 527 | 875k | SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); | 528 | 875k | } | 529 | 877k | | 530 | 877k | assert(SizeInBits != 0 && "Pointer size must be known"); | 531 | 877k | | 532 | 877k | if (MO.isReg()) { | 533 | 877k | if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) | 534 | 0 | if (handleReject() == RejectAndGiveUp) | 535 | 0 | return false; | 536 | 0 | } else if (handleReject() == RejectAndGiveUp) | 537 | 0 | return false; | 538 | 877k | | 539 | 877k | break; | 540 | 877k | } | 541 | 7.66M | case GIM_CheckRegBankForClass: { | 542 | 7.66M | int64_t InsnID = MatchTable[CurrentIdx++]; | 543 | 7.66M | int64_t OpIdx = MatchTable[CurrentIdx++]; | 544 | 7.66M | int64_t RCEnum = MatchTable[CurrentIdx++]; | 545 | 7.66M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 546 | 7.66M | dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" | 547 | 7.66M | << InsnID << "]->getOperand(" << OpIdx | 548 | 7.66M | << "), RCEnum=" << RCEnum << ")\n"); | 549 | 7.66M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 550 | 7.66M | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 551 | 7.66M | if (!MO.isReg() || | 552 | 7.66M | &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != | 553 | 7.66M | RBI.getRegBank(MO.getReg(), MRI, TRI)) { | 554 | 744k | if (handleReject() == RejectAndGiveUp) | 555 | 0 | return false; | 556 | 7.66M | } | 557 | 7.66M | break; | 558 | 7.66M | } | 559 | 7.66M | | 560 | 7.66M | case GIM_CheckComplexPattern: { | 561 | 1.44M | int64_t InsnID = MatchTable[CurrentIdx++]; | 562 | 1.44M | int64_t OpIdx = MatchTable[CurrentIdx++]; | 563 | 1.44M | int64_t RendererID = MatchTable[CurrentIdx++]; | 564 | 1.44M | int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; | 565 | 1.44M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 566 | 1.44M | dbgs() << CurrentIdx << ": State.Renderers[" << RendererID | 567 | 1.44M | << "] = GIM_CheckComplexPattern(MIs[" << InsnID | 568 | 1.44M | << "]->getOperand(" << OpIdx | 569 | 1.44M | << "), ComplexPredicateID=" << ComplexPredicateID | 570 | 1.44M | << ")\n"); | 571 | 1.44M | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 572 | 1.44M | // FIXME: Use std::invoke() when it's available. | 573 | 1.44M | ComplexRendererFns Renderer = | 574 | 1.44M | (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( | 575 | 1.44M | State.MIs[InsnID]->getOperand(OpIdx)); | 576 | 1.44M | if (Renderer.hasValue()) | 577 | 928k | State.Renderers[RendererID] = Renderer.getValue(); | 578 | 512k | else | 579 | 512k | if (handleReject() == RejectAndGiveUp) | 580 | 0 | return false; | 581 | 1.44M | break; | 582 | 1.44M | } | 583 | 1.44M | | 584 | 1.44M | case GIM_CheckConstantInt: { | 585 | 286k | int64_t InsnID = MatchTable[CurrentIdx++]; | 586 | 286k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 587 | 286k | int64_t Value = MatchTable[CurrentIdx++]; | 588 | 286k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 589 | 286k | dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" | 590 | 286k | << InsnID << "]->getOperand(" << OpIdx | 591 | 286k | << "), Value=" << Value << ")\n"); | 592 | 286k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 593 | 286k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 594 | 286k | if (MO.isReg()) { | 595 | 286k | // isOperandImmEqual() will sign-extend to 64-bits, so should we. | 596 | 286k | LLT Ty = MRI.getType(MO.getReg()); | 597 | 286k | Value = SignExtend64(Value, Ty.getSizeInBits()); | 598 | 286k | | 599 | 286k | if (!isOperandImmEqual(MO, Value, MRI)) { | 600 | 254k | if (handleReject() == RejectAndGiveUp) | 601 | 0 | return false; | 602 | 0 | } | 603 | 0 | } else if (handleReject() == RejectAndGiveUp) | 604 | 0 | return false; | 605 | 286k | | 606 | 286k | break; | 607 | 286k | } | 608 | 286k | | 609 | 286k | case GIM_CheckLiteralInt: { | 610 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 611 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 612 | 0 | int64_t Value = MatchTable[CurrentIdx++]; | 613 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 614 | 0 | dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" | 615 | 0 | << InsnID << "]->getOperand(" << OpIdx | 616 | 0 | << "), Value=" << Value << ")\n"); | 617 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 618 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 619 | 0 | if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { | 620 | 0 | if (handleReject() == RejectAndGiveUp) | 621 | 0 | return false; | 622 | 0 | } | 623 | 0 | break; | 624 | 0 | } | 625 | 0 |
| 626 | 241k | case GIM_CheckIntrinsicID: { | 627 | 241k | int64_t InsnID = MatchTable[CurrentIdx++]; | 628 | 241k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 629 | 241k | int64_t Value = MatchTable[CurrentIdx++]; | 630 | 241k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 631 | 241k | dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" | 632 | 241k | << InsnID << "]->getOperand(" << OpIdx | 633 | 241k | << "), Value=" << Value << ")\n"); | 634 | 241k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 635 | 241k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 636 | 241k | if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value233k ) | 637 | 235k | if (handleReject() == RejectAndGiveUp) | 638 | 0 | return false; | 639 | 241k | break; | 640 | 241k | } | 641 | 241k | | 642 | 307k | case GIM_CheckIsMBB: { | 643 | 307k | int64_t InsnID = MatchTable[CurrentIdx++]; | 644 | 307k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 645 | 307k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 646 | 307k | dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID | 647 | 307k | << "]->getOperand(" << OpIdx << "))\n"); | 648 | 307k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 649 | 307k | if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { | 650 | 0 | if (handleReject() == RejectAndGiveUp) | 651 | 0 | return false; | 652 | 307k | } | 653 | 307k | break; | 654 | 307k | } | 655 | 307k | | 656 | 307k | case GIM_CheckIsSafeToFold: { | 657 | 73.0k | int64_t InsnID = MatchTable[CurrentIdx++]; | 658 | 73.0k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 659 | 73.0k | dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" | 660 | 73.0k | << InsnID << "])\n"); | 661 | 73.0k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 662 | 73.0k | if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { | 663 | 0 | if (handleReject() == RejectAndGiveUp) | 664 | 0 | return false; | 665 | 73.0k | } | 666 | 73.0k | break; | 667 | 73.0k | } | 668 | 73.0k | case GIM_CheckIsSameOperand: { | 669 | 11 | int64_t InsnID = MatchTable[CurrentIdx++]; | 670 | 11 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 671 | 11 | int64_t OtherInsnID = MatchTable[CurrentIdx++]; | 672 | 11 | int64_t OtherOpIdx = MatchTable[CurrentIdx++]; | 673 | 11 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 674 | 11 | dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" | 675 | 11 | << InsnID << "][" << OpIdx << "], MIs[" | 676 | 11 | << OtherInsnID << "][" << OtherOpIdx << "])\n"); | 677 | 11 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 678 | 11 | assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); | 679 | 11 | if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( | 680 | 11 | State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { | 681 | 0 | if (handleReject() == RejectAndGiveUp) | 682 | 0 | return false; | 683 | 11 | } | 684 | 11 | break; | 685 | 11 | } | 686 | 4.68M | case GIM_Reject: | 687 | 4.68M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 688 | 4.68M | dbgs() << CurrentIdx << ": GIM_Reject\n"); | 689 | 4.68M | if (handleReject() == RejectAndGiveUp) | 690 | 3.13M | return false; | 691 | 1.54M | break; | 692 | 1.54M | | 693 | 1.54M | case GIR_MutateOpcode: { | 694 | 822k | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 695 | 822k | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 696 | 822k | int64_t NewOpcode = MatchTable[CurrentIdx++]; | 697 | 822k | if (NewInsnID >= OutMIs.size()) | 698 | 822k | OutMIs.resize(NewInsnID + 1); | 699 | 822k | | 700 | 822k | OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), | 701 | 822k | State.MIs[OldInsnID]); | 702 | 822k | OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); | 703 | 822k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 704 | 822k | dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" | 705 | 822k | << NewInsnID << "], MIs[" << OldInsnID << "], " | 706 | 822k | << NewOpcode << ")\n"); | 707 | 822k | break; | 708 | 1.54M | } | 709 | 1.54M | | 710 | 1.99M | case GIR_BuildMI: { | 711 | 1.99M | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 712 | 1.99M | int64_t Opcode = MatchTable[CurrentIdx++]; | 713 | 1.99M | if (NewInsnID >= OutMIs.size()) | 714 | 1.98M | OutMIs.resize(NewInsnID + 1); | 715 | 1.99M | | 716 | 1.99M | OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], | 717 | 1.99M | State.MIs[0]->getDebugLoc(), TII.get(Opcode)); | 718 | 1.99M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 719 | 1.99M | dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" | 720 | 1.99M | << NewInsnID << "], " << Opcode << ")\n"); | 721 | 1.99M | break; | 722 | 1.54M | } | 723 | 1.54M | | 724 | 2.20M | case GIR_Copy: { | 725 | 2.20M | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 726 | 2.20M | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 727 | 2.20M | int64_t OpIdx = MatchTable[CurrentIdx++]; | 728 | 2.20M | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 729 | 2.20M | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); | 730 | 2.20M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 731 | 2.20M | dbgs() | 732 | 2.20M | << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID | 733 | 2.20M | << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); | 734 | 2.20M | break; | 735 | 1.54M | } | 736 | 1.54M | | 737 | 1.54M | case GIR_CopyOrAddZeroReg: { | 738 | 272k | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 739 | 272k | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 740 | 272k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 741 | 272k | int64_t ZeroReg = MatchTable[CurrentIdx++]; | 742 | 272k | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 743 | 272k | MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); | 744 | 272k | if (isOperandImmEqual(MO, 0, MRI)) | 745 | 26.6k | OutMIs[NewInsnID].addReg(ZeroReg); | 746 | 245k | else | 747 | 245k | OutMIs[NewInsnID].add(MO); | 748 | 272k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 749 | 272k | dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" | 750 | 272k | << NewInsnID << "], MIs[" << OldInsnID << "], " | 751 | 272k | << OpIdx << ", " << ZeroReg << ")\n"); | 752 | 272k | break; | 753 | 1.54M | } | 754 | 1.54M | | 755 | 1.54M | case GIR_CopySubReg: { | 756 | 93.5k | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 757 | 93.5k | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 758 | 93.5k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 759 | 93.5k | int64_t SubRegIdx = MatchTable[CurrentIdx++]; | 760 | 93.5k | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 761 | 93.5k | OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), | 762 | 93.5k | 0, SubRegIdx); | 763 | 93.5k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 764 | 93.5k | dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" | 765 | 93.5k | << NewInsnID << "], MIs[" << OldInsnID << "], " | 766 | 93.5k | << OpIdx << ", " << SubRegIdx << ")\n"); | 767 | 93.5k | break; | 768 | 1.54M | } | 769 | 1.54M | | 770 | 1.54M | case GIR_AddImplicitDef: { | 771 | 72.9k | int64_t InsnID = MatchTable[CurrentIdx++]; | 772 | 72.9k | int64_t RegNum = MatchTable[CurrentIdx++]; | 773 | 72.9k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 774 | 72.9k | OutMIs[InsnID].addDef(RegNum, RegState::Implicit); | 775 | 72.9k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 776 | 72.9k | dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" | 777 | 72.9k | << InsnID << "], " << RegNum << ")\n"); | 778 | 72.9k | break; | 779 | 1.54M | } | 780 | 1.54M | | 781 | 1.54M | case GIR_AddImplicitUse: { | 782 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 783 | 0 | int64_t RegNum = MatchTable[CurrentIdx++]; | 784 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 785 | 0 | OutMIs[InsnID].addUse(RegNum, RegState::Implicit); | 786 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 787 | 0 | dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" | 788 | 0 | << InsnID << "], " << RegNum << ")\n"); | 789 | 0 | break; | 790 | 1.54M | } | 791 | 1.54M | | 792 | 1.54M | case GIR_AddRegister: { | 793 | 163k | int64_t InsnID = MatchTable[CurrentIdx++]; | 794 | 163k | int64_t RegNum = MatchTable[CurrentIdx++]; | 795 | 163k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 796 | 163k | OutMIs[InsnID].addReg(RegNum); | 797 | 163k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 798 | 163k | dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" | 799 | 163k | << InsnID << "], " << RegNum << ")\n"); | 800 | 163k | break; | 801 | 1.54M | } | 802 | 1.54M | | 803 | 1.54M | case GIR_AddTempRegister: { | 804 | 8.05k | int64_t InsnID = MatchTable[CurrentIdx++]; | 805 | 8.05k | int64_t TempRegID = MatchTable[CurrentIdx++]; | 806 | 8.05k | uint64_t TempRegFlags = MatchTable[CurrentIdx++]; | 807 | 8.05k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 808 | 8.05k | OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags); | 809 | 8.05k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 810 | 8.05k | dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" | 811 | 8.05k | << InsnID << "], TempRegisters[" << TempRegID | 812 | 8.05k | << "], " << TempRegFlags << ")\n"); | 813 | 8.05k | break; | 814 | 1.54M | } | 815 | 1.54M | | 816 | 1.54M | case GIR_AddImm: { | 817 | 59.2k | int64_t InsnID = MatchTable[CurrentIdx++]; | 818 | 59.2k | int64_t Imm = MatchTable[CurrentIdx++]; | 819 | 59.2k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 820 | 59.2k | OutMIs[InsnID].addImm(Imm); | 821 | 59.2k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 822 | 59.2k | dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID | 823 | 59.2k | << "], " << Imm << ")\n"); | 824 | 59.2k | break; | 825 | 1.54M | } | 826 | 1.54M | | 827 | 1.54M | case GIR_ComplexRenderer: { | 828 | 111k | int64_t InsnID = MatchTable[CurrentIdx++]; | 829 | 111k | int64_t RendererID = MatchTable[CurrentIdx++]; | 830 | 111k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 831 | 111k | for (const auto &RenderOpFn : State.Renderers[RendererID]) | 832 | 223k | RenderOpFn(OutMIs[InsnID]); | 833 | 111k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 834 | 111k | dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" | 835 | 111k | << InsnID << "], " << RendererID << ")\n"); | 836 | 111k | break; | 837 | 1.54M | } | 838 | 1.63M | case GIR_ComplexSubOperandRenderer: { | 839 | 1.63M | int64_t InsnID = MatchTable[CurrentIdx++]; | 840 | 1.63M | int64_t RendererID = MatchTable[CurrentIdx++]; | 841 | 1.63M | int64_t RenderOpID = MatchTable[CurrentIdx++]; | 842 | 1.63M | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 843 | 1.63M | State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); | 844 | 1.63M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 845 | 1.63M | dbgs() << CurrentIdx | 846 | 1.63M | << ": GIR_ComplexSubOperandRenderer(OutMIs[" | 847 | 1.63M | << InsnID << "], " << RendererID << ", " | 848 | 1.63M | << RenderOpID << ")\n"); | 849 | 1.63M | break; | 850 | 1.54M | } | 851 | 1.54M | | 852 | 1.54M | case GIR_CopyConstantAsSImm: { | 853 | 790k | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 854 | 790k | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 855 | 790k | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 856 | 790k | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); | 857 | 790k | if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { | 858 | 790k | OutMIs[NewInsnID].addImm( | 859 | 790k | State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); | 860 | 790k | } else if (0 State.MIs[OldInsnID]->getOperand(1).isImm()0 ) | 861 | 0 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); | 862 | 0 | else | 863 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 864 | 790k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 865 | 790k | dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" | 866 | 790k | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 867 | 790k | break; | 868 | 790k | } | 869 | 790k | | 870 | 790k | // TODO: Needs a test case once we have a pattern that uses this. | 871 | 790k | case GIR_CopyFConstantAsFPImm: { | 872 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 873 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 874 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 875 | 0 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); | 876 | 0 | if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) | 877 | 0 | OutMIs[NewInsnID].addFPImm( | 878 | 0 | State.MIs[OldInsnID]->getOperand(1).getFPImm()); | 879 | 0 | else | 880 | 0 | llvm_unreachable("Expected FPImm operand"); | 881 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 882 | 0 | dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" | 883 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 884 | 0 | break; | 885 | 0 | } | 886 | 0 |
| 887 | 4.02k | case GIR_CustomRenderer: { | 888 | 4.02k | int64_t InsnID = MatchTable[CurrentIdx++]; | 889 | 4.02k | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 890 | 4.02k | int64_t RendererFnID = MatchTable[CurrentIdx++]; | 891 | 4.02k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 892 | 4.02k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 893 | 4.02k | dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" | 894 | 4.02k | << InsnID << "], MIs[" << OldInsnID << "], " | 895 | 4.02k | << RendererFnID << ")\n"); | 896 | 4.02k | (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], | 897 | 4.02k | *State.MIs[OldInsnID]); | 898 | 4.02k | break; | 899 | 0 | } | 900 | 187k | case GIR_ConstrainOperandRC: { | 901 | 187k | int64_t InsnID = MatchTable[CurrentIdx++]; | 902 | 187k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 903 | 187k | int64_t RCEnum = MatchTable[CurrentIdx++]; | 904 | 187k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 905 | 187k | constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, | 906 | 187k | *TRI.getRegClass(RCEnum), TII, TRI, RBI); | 907 | 187k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 908 | 187k | dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" | 909 | 187k | << InsnID << "], " << OpIdx << ", " << RCEnum | 910 | 187k | << ")\n"); | 911 | 187k | break; | 912 | 0 | } | 913 | 0 |
| 914 | 2.71M | case GIR_ConstrainSelectedInstOperands: { | 915 | 2.71M | int64_t InsnID = MatchTable[CurrentIdx++]; | 916 | 2.71M | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 917 | 2.71M | constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, | 918 | 2.71M | RBI); | 919 | 2.71M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 920 | 2.71M | dbgs() << CurrentIdx | 921 | 2.71M | << ": GIR_ConstrainSelectedInstOperands(OutMIs[" | 922 | 2.71M | << InsnID << "])\n"); | 923 | 2.71M | break; | 924 | 0 | } | 925 | 0 |
| 926 | 820k | case GIR_MergeMemOperands: { | 927 | 820k | int64_t InsnID = MatchTable[CurrentIdx++]; | 928 | 820k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 929 | 820k | | 930 | 820k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 931 | 820k | dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" | 932 | 820k | << InsnID << "]"); | 933 | 820k | int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; | 934 | 1.64M | while ((MergeInsnID = MatchTable[CurrentIdx++]) != | 935 | 1.64M | GIU_MergeMemOperands_EndOfList) { | 936 | 823k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 937 | 823k | dbgs() << ", MIs[" << MergeInsnID << "]"); | 938 | 823k | for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) | 939 | 818k | OutMIs[InsnID].addMemOperand(MMO); | 940 | 823k | } | 941 | 820k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); | 942 | 820k | break; | 943 | 0 | } | 944 | 0 |
| 945 | 1.98M | case GIR_EraseFromParent: { | 946 | 1.98M | int64_t InsnID = MatchTable[CurrentIdx++]; | 947 | 1.98M | assert(State.MIs[InsnID] && | 948 | 1.98M | "Attempted to erase an undefined instruction"); | 949 | 1.98M | State.MIs[InsnID]->eraseFromParent(); | 950 | 1.98M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 951 | 1.98M | dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" | 952 | 1.98M | << InsnID << "])\n"); | 953 | 1.98M | break; | 954 | 0 | } | 955 | 0 |
| 956 | 4.02k | case GIR_MakeTempReg: { | 957 | 4.02k | int64_t TempRegID = MatchTable[CurrentIdx++]; | 958 | 4.02k | int64_t TypeID = MatchTable[CurrentIdx++]; | 959 | 4.02k | | 960 | 4.02k | State.TempRegisters[TempRegID] = | 961 | 4.02k | MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); | 962 | 4.02k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 963 | 4.02k | dbgs() << CurrentIdx << ": TempRegs[" << TempRegID | 964 | 4.02k | << "] = GIR_MakeTempReg(" << TypeID << ")\n"); | 965 | 4.02k | break; | 966 | 0 | } | 967 | 0 |
| 968 | 0 | case GIR_Coverage: { | 969 | 0 | int64_t RuleID = MatchTable[CurrentIdx++]; | 970 | 0 | CoverageInfo.setCovered(RuleID); | 971 | 0 |
| 972 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 973 | 0 | dbgs() | 974 | 0 | << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); | 975 | 0 | break; | 976 | 0 | } | 977 | 0 |
| 978 | 2.80M | case GIR_Done: | 979 | 2.80M | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 980 | 2.80M | dbgs() << CurrentIdx << ": GIR_Done\n"); | 981 | 2.80M | return true; | 982 | 0 |
| 983 | 0 | default: | 984 | 0 | llvm_unreachable("Unexpected command"); | 985 | 61.8M | } | 986 | 61.8M | } | 987 | 5.94M | } |
bool llvm::InstructionSelector::executeMatchTable<llvm::AMDGPUInstructionSelector const, llvm::PredicateBitsetImpl<46ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>(llvm::AMDGPUInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<46ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > (llvm::AMDGPUInstructionSelector::*)(llvm::MachineOperand&) const, void (llvm::AMDGPUInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<46ul> const&, llvm::CodeGenCoverage&) const Line | Count | Source | 55 | 1.34k | CodeGenCoverage &CoverageInfo) const { | 56 | 1.34k | | 57 | 1.34k | uint64_t CurrentIdx = 0; | 58 | 1.34k | SmallVector<uint64_t, 4> OnFailResumeAt; | 59 | 1.34k | | 60 | 1.34k | enum RejectAction { RejectAndGiveUp, RejectAndResume }; | 61 | 1.34k | auto handleReject = [&]() -> RejectAction { | 62 | 1.34k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 1.34k | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 1.34k | if (OnFailResumeAt.empty()) | 65 | 1.34k | return RejectAndGiveUp; | 66 | 1.34k | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 1.34k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 1.34k | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 1.34k | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 1.34k | return RejectAndResume; | 71 | 1.34k | }; | 72 | 1.34k | | 73 | 37.0k | while (true) { | 74 | 37.0k | assert(CurrentIdx != ~0u && "Invalid MatchTable index"); | 75 | 37.0k | int64_t MatcherOpcode = MatchTable[CurrentIdx++]; | 76 | 37.0k | switch (MatcherOpcode) { | 77 | 37.0k | case GIM_Try: { | 78 | 6.96k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 79 | 6.96k | dbgs() << CurrentIdx << ": Begin try-block\n"); | 80 | 6.96k | OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); | 81 | 6.96k | break; | 82 | 37.0k | } | 83 | 37.0k | | 84 | 37.0k | case GIM_RecordInsn: { | 85 | 37 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 86 | 37 | int64_t InsnID = MatchTable[CurrentIdx++]; | 87 | 37 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 88 | 37 | | 89 | 37 | // As an optimisation we require that MIs[0] is always the root. Refuse | 90 | 37 | // any attempt to modify it. | 91 | 37 | assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); | 92 | 37 | | 93 | 37 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 94 | 37 | if (!MO.isReg()) { | 95 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 96 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 97 | 0 | if (handleReject() == RejectAndGiveUp) | 98 | 0 | return false; | 99 | 0 | break; | 100 | 0 | } | 101 | 37 | if (TRI.isPhysicalRegister(MO.getReg())) { | 102 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 103 | 0 | dbgs() << CurrentIdx << ": Is a physical register\n"); | 104 | 0 | if (handleReject() == RejectAndGiveUp) | 105 | 0 | return false; | 106 | 0 | break; | 107 | 0 | } | 108 | 37 | | 109 | 37 | MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); | 110 | 37 | if ((size_t)NewInsnID < State.MIs.size()) | 111 | 30 | State.MIs[NewInsnID] = NewMI; | 112 | 7 | else { | 113 | 7 | assert((size_t)NewInsnID == State.MIs.size() && | 114 | 7 | "Expected to store MIs in order"); | 115 | 7 | State.MIs.push_back(NewMI); | 116 | 7 | } | 117 | 37 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 118 | 37 | dbgs() << CurrentIdx << ": MIs[" << NewInsnID | 119 | 37 | << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx | 120 | 37 | << ")\n"); | 121 | 37 | break; | 122 | 37 | } | 123 | 37 | | 124 | 6.30k | case GIM_CheckFeatures: { | 125 | 6.30k | int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; | 126 | 6.30k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 127 | 6.30k | dbgs() << CurrentIdx | 128 | 6.30k | << ": GIM_CheckFeatures(ExpectedBitsetID=" | 129 | 6.30k | << ExpectedBitsetID << ")\n"); | 130 | 6.30k | if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != | 131 | 6.30k | ISelInfo.FeatureBitsets[ExpectedBitsetID]) { | 132 | 1.18k | if (handleReject() == RejectAndGiveUp) | 133 | 0 | return false; | 134 | 6.30k | } | 135 | 6.30k | break; | 136 | 6.30k | } | 137 | 6.30k | | 138 | 6.30k | case GIM_CheckOpcode: { | 139 | 37 | int64_t InsnID = MatchTable[CurrentIdx++]; | 140 | 37 | int64_t Expected = MatchTable[CurrentIdx++]; | 141 | 37 | | 142 | 37 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 143 | 37 | unsigned Opcode = State.MIs[InsnID]->getOpcode(); | 144 | 37 | | 145 | 37 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 146 | 37 | dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID | 147 | 37 | << "], ExpectedOpcode=" << Expected | 148 | 37 | << ") // Got=" << Opcode << "\n"); | 149 | 37 | if (Opcode != Expected) { | 150 | 36 | if (handleReject() == RejectAndGiveUp) | 151 | 0 | return false; | 152 | 37 | } | 153 | 37 | break; | 154 | 37 | } | 155 | 37 | | 156 | 1.34k | case GIM_SwitchOpcode: { | 157 | 1.34k | int64_t InsnID = MatchTable[CurrentIdx++]; | 158 | 1.34k | int64_t LowerBound = MatchTable[CurrentIdx++]; | 159 | 1.34k | int64_t UpperBound = MatchTable[CurrentIdx++]; | 160 | 1.34k | int64_t Default = MatchTable[CurrentIdx++]; | 161 | 1.34k | | 162 | 1.34k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 163 | 1.34k | const int64_t Opcode = State.MIs[InsnID]->getOpcode(); | 164 | 1.34k | | 165 | 1.34k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 166 | 1.34k | dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" | 167 | 1.34k | << LowerBound << ", " << UpperBound << "), Default=" << Default | 168 | 1.34k | << ", JumpTable...) // Got=" << Opcode << "\n"; | 169 | 1.34k | }); | 170 | 1.34k | if (Opcode < LowerBound || UpperBound <= Opcode) { | 171 | 6 | CurrentIdx = Default; | 172 | 6 | break; | 173 | 6 | } | 174 | 1.34k | CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; | 175 | 1.34k | if (!CurrentIdx) { | 176 | 9 | CurrentIdx = Default; | 177 | 9 | break; | 178 | 9 | } | 179 | 1.33k | OnFailResumeAt.push_back(Default); | 180 | 1.33k | break; | 181 | 1.33k | } | 182 | 1.33k | | 183 | 1.33k | case GIM_SwitchType: { | 184 | 1.23k | int64_t InsnID = MatchTable[CurrentIdx++]; | 185 | 1.23k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 186 | 1.23k | int64_t LowerBound = MatchTable[CurrentIdx++]; | 187 | 1.23k | int64_t UpperBound = MatchTable[CurrentIdx++]; | 188 | 1.23k | int64_t Default = MatchTable[CurrentIdx++]; | 189 | 1.23k | | 190 | 1.23k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 191 | 1.23k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 192 | 1.23k | | 193 | 1.23k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 194 | 1.23k | dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID | 195 | 1.23k | << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " | 196 | 1.23k | << UpperBound << "), Default=" << Default | 197 | 1.23k | << ", JumpTable...) // Got="; | 198 | 1.23k | if (!MO.isReg()) | 199 | 1.23k | dbgs() << "Not a VReg\n"; | 200 | 1.23k | else | 201 | 1.23k | dbgs() << MRI.getType(MO.getReg()) << "\n"; | 202 | 1.23k | }); | 203 | 1.23k | if (!MO.isReg()) { | 204 | 0 | CurrentIdx = Default; | 205 | 0 | break; | 206 | 0 | } | 207 | 1.23k | const LLT Ty = MRI.getType(MO.getReg()); | 208 | 1.23k | const auto TyI = ISelInfo.TypeIDMap.find(Ty); | 209 | 1.23k | if (TyI == ISelInfo.TypeIDMap.end()) { | 210 | 138 | CurrentIdx = Default; | 211 | 138 | break; | 212 | 138 | } | 213 | 1.09k | const int64_t TypeID = TyI->second; | 214 | 1.09k | if (TypeID < LowerBound || UpperBound <= TypeID1.04k ) { | 215 | 82 | CurrentIdx = Default; | 216 | 82 | break; | 217 | 82 | } | 218 | 1.01k | CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; | 219 | 1.01k | if (!CurrentIdx) { | 220 | 74 | CurrentIdx = Default; | 221 | 74 | break; | 222 | 74 | } | 223 | 942 | OnFailResumeAt.push_back(Default); | 224 | 942 | break; | 225 | 942 | } | 226 | 942 | | 227 | 942 | case GIM_CheckNumOperands: { | 228 | 27 | int64_t InsnID = MatchTable[CurrentIdx++]; | 229 | 27 | int64_t Expected = MatchTable[CurrentIdx++]; | 230 | 27 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 231 | 27 | dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" | 232 | 27 | << InsnID << "], Expected=" << Expected << ")\n"); | 233 | 27 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 234 | 27 | if (State.MIs[InsnID]->getNumOperands() != Expected) { | 235 | 18 | if (handleReject() == RejectAndGiveUp) | 236 | 0 | return false; | 237 | 27 | } | 238 | 27 | break; | 239 | 27 | } | 240 | 27 | case GIM_CheckI64ImmPredicate: { | 241 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 242 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 243 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 244 | 0 | dbgs() | 245 | 0 | << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" | 246 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 247 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 248 | 0 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 249 | 0 | "Expected G_CONSTANT"); | 250 | 0 | assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); | 251 | 0 | int64_t Value = 0; | 252 | 0 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 253 | 0 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); | 254 | 0 | else if (State.MIs[InsnID]->getOperand(1).isImm()) | 255 | 0 | Value = State.MIs[InsnID]->getOperand(1).getImm(); | 256 | 0 | else | 257 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 258 | 0 |
| 259 | 0 | if (!testImmPredicate_I64(Predicate, Value)) | 260 | 0 | if (handleReject() == RejectAndGiveUp) | 261 | 0 | return false; | 262 | 0 | break; | 263 | 0 | } | 264 | 0 | case GIM_CheckAPIntImmPredicate: { | 265 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 266 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 267 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 268 | 0 | dbgs() | 269 | 0 | << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" | 270 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 271 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 272 | 0 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 273 | 0 | "Expected G_CONSTANT"); | 274 | 0 | assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); | 275 | 0 | APInt Value; | 276 | 0 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 277 | 0 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); | 278 | 0 | else | 279 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 280 | 0 |
| 281 | 0 | if (!testImmPredicate_APInt(Predicate, Value)) | 282 | 0 | if (handleReject() == RejectAndGiveUp) | 283 | 0 | return false; | 284 | 0 | break; | 285 | 0 | } | 286 | 0 | case GIM_CheckAPFloatImmPredicate: { | 287 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 288 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 289 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 290 | 0 | dbgs() | 291 | 0 | << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" | 292 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 293 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 294 | 0 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && | 295 | 0 | "Expected G_FCONSTANT"); | 296 | 0 | assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); | 297 | 0 | assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); | 298 | 0 | APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); | 299 | 0 |
| 300 | 0 | if (!testImmPredicate_APFloat(Predicate, Value)) | 301 | 0 | if (handleReject() == RejectAndGiveUp) | 302 | 0 | return false; | 303 | 0 | break; | 304 | 0 | } | 305 | 396 | case GIM_CheckCxxInsnPredicate: { | 306 | 396 | int64_t InsnID = MatchTable[CurrentIdx++]; | 307 | 396 | int64_t Predicate = MatchTable[CurrentIdx++]; | 308 | 396 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 309 | 396 | dbgs() | 310 | 396 | << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" | 311 | 396 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 312 | 396 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 313 | 396 | assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); | 314 | 396 | | 315 | 396 | if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID])) | 316 | 0 | if (handleReject() == RejectAndGiveUp) | 317 | 0 | return false; | 318 | 396 | break; | 319 | 396 | } | 320 | 949 | case GIM_CheckAtomicOrdering: { | 321 | 949 | int64_t InsnID = MatchTable[CurrentIdx++]; | 322 | 949 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 323 | 949 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 324 | 949 | dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" | 325 | 949 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 326 | 949 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 327 | 949 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 328 | 0 | if (handleReject() == RejectAndGiveUp) | 329 | 0 | return false; | 330 | 949 | | 331 | 949 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 332 | 949 | if (MMO->getOrdering() != Ordering) | 333 | 54 | if (handleReject() == RejectAndGiveUp) | 334 | 0 | return false; | 335 | 949 | break; | 336 | 949 | } | 337 | 949 | case GIM_CheckAtomicOrderingOrStrongerThan: { | 338 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 339 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 340 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 341 | 0 | dbgs() << CurrentIdx | 342 | 0 | << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" | 343 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 344 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 345 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 346 | 0 | if (handleReject() == RejectAndGiveUp) | 347 | 0 | return false; | 348 | 0 | | 349 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 350 | 0 | if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) | 351 | 0 | if (handleReject() == RejectAndGiveUp) | 352 | 0 | return false; | 353 | 0 | break; | 354 | 0 | } | 355 | 0 | case GIM_CheckAtomicOrderingWeakerThan: { | 356 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 357 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 358 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 359 | 0 | dbgs() << CurrentIdx | 360 | 0 | << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" | 361 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 362 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 363 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 364 | 0 | if (handleReject() == RejectAndGiveUp) | 365 | 0 | return false; | 366 | 0 | | 367 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 368 | 0 | if (!isStrongerThan(Ordering, MMO->getOrdering())) | 369 | 0 | if (handleReject() == RejectAndGiveUp) | 370 | 0 | return false; | 371 | 0 | break; | 372 | 0 | } | 373 | 2.19k | case GIM_CheckMemoryAddressSpace: { | 374 | 2.19k | int64_t InsnID = MatchTable[CurrentIdx++]; | 375 | 2.19k | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 376 | 2.19k | // This accepts a list of possible address spaces. | 377 | 2.19k | const int NumAddrSpace = MatchTable[CurrentIdx++]; | 378 | 2.19k | | 379 | 2.19k | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 380 | 0 | if (handleReject() == RejectAndGiveUp) | 381 | 0 | return false; | 382 | 0 | break; | 383 | 0 | } | 384 | 2.19k | | 385 | 2.19k | // Need to still jump to the end of the list of address spaces if we find | 386 | 2.19k | // a match earlier. | 387 | 2.19k | const uint64_t LastIdx = CurrentIdx + NumAddrSpace; | 388 | 2.19k | | 389 | 2.19k | const MachineMemOperand *MMO | 390 | 2.19k | = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 391 | 2.19k | const unsigned MMOAddrSpace = MMO->getAddrSpace(); | 392 | 2.19k | | 393 | 2.19k | bool Success = false; | 394 | 4.08k | for (int I = 0; I != NumAddrSpace; ++I1.89k ) { | 395 | 2.54k | unsigned AddrSpace = MatchTable[CurrentIdx++]; | 396 | 2.54k | DEBUG_WITH_TYPE( | 397 | 2.54k | TgtInstructionSelector::getName(), | 398 | 2.54k | dbgs() << "addrspace(" << MMOAddrSpace << ") vs " | 399 | 2.54k | << AddrSpace << '\n'); | 400 | 2.54k | | 401 | 2.54k | if (AddrSpace == MMOAddrSpace) { | 402 | 649 | Success = true; | 403 | 649 | break; | 404 | 649 | } | 405 | 2.54k | } | 406 | 2.19k | | 407 | 2.19k | CurrentIdx = LastIdx; | 408 | 2.19k | if (!Success && handleReject() == RejectAndGiveUp1.54k ) | 409 | 0 | return false; | 410 | 2.19k | break; | 411 | 2.19k | } | 412 | 2.19k | case GIM_CheckMemorySizeEqualTo: { | 413 | 278 | int64_t InsnID = MatchTable[CurrentIdx++]; | 414 | 278 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 415 | 278 | uint64_t Size = MatchTable[CurrentIdx++]; | 416 | 278 | | 417 | 278 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 418 | 278 | dbgs() << CurrentIdx | 419 | 278 | << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID | 420 | 278 | << "]->memoperands() + " << MMOIdx | 421 | 278 | << ", Size=" << Size << ")\n"); | 422 | 278 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 423 | 278 | | 424 | 278 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 425 | 0 | if (handleReject() == RejectAndGiveUp) | 426 | 0 | return false; | 427 | 0 | break; | 428 | 0 | } | 429 | 278 | | 430 | 278 | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 431 | 278 | | 432 | 278 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 433 | 278 | dbgs() << MMO->getSize() << " bytes vs " << Size | 434 | 278 | << " bytes\n"); | 435 | 278 | if (MMO->getSize() != Size) | 436 | 62 | if (handleReject() == RejectAndGiveUp) | 437 | 0 | return false; | 438 | 278 | | 439 | 278 | break; | 440 | 278 | } | 441 | 3.50k | case GIM_CheckMemorySizeEqualToLLT: | 442 | 3.50k | case GIM_CheckMemorySizeLessThanLLT: | 443 | 3.50k | case GIM_CheckMemorySizeGreaterThanLLT: { | 444 | 3.50k | int64_t InsnID = MatchTable[CurrentIdx++]; | 445 | 3.50k | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 446 | 3.50k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 447 | 3.50k | | 448 | 3.50k | DEBUG_WITH_TYPE( | 449 | 3.50k | TgtInstructionSelector::getName(), | 450 | 3.50k | dbgs() << CurrentIdx << ": GIM_CheckMemorySize" | 451 | 3.50k | << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT | 452 | 3.50k | ? "EqualTo" | 453 | 3.50k | : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT | 454 | 3.50k | ? "GreaterThan" | 455 | 3.50k | : "LessThan") | 456 | 3.50k | << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx | 457 | 3.50k | << ", OpIdx=" << OpIdx << ")\n"); | 458 | 3.50k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 459 | 3.50k | | 460 | 3.50k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 461 | 3.50k | if (!MO.isReg()) { | 462 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 463 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 464 | 0 | if (handleReject() == RejectAndGiveUp) | 465 | 0 | return false; | 466 | 0 | break; | 467 | 0 | } | 468 | 3.50k | | 469 | 3.50k | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 470 | 0 | if (handleReject() == RejectAndGiveUp) | 471 | 0 | return false; | 472 | 0 | break; | 473 | 0 | } | 474 | 3.50k | | 475 | 3.50k | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 476 | 3.50k | | 477 | 3.50k | unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); | 478 | 3.50k | if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && | 479 | 3.50k | MMO->getSizeInBits() != Size964 ) { | 480 | 506 | if (handleReject() == RejectAndGiveUp) | 481 | 0 | return false; | 482 | 2.99k | } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && | 483 | 2.99k | MMO->getSizeInBits() >= Size2.54k ) { | 484 | 1.68k | if (handleReject() == RejectAndGiveUp) | 485 | 0 | return false; | 486 | 1.31k | } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && | 487 | 1.31k | MMO->getSizeInBits() <= Size0 ) | 488 | 0 | if (handleReject() == RejectAndGiveUp) | 489 | 0 | return false; | 490 | 3.50k | | 491 | 3.50k | break; | 492 | 3.50k | } | 493 | 3.50k | case GIM_CheckType: { | 494 | 606 | int64_t InsnID = MatchTable[CurrentIdx++]; | 495 | 606 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 496 | 606 | int64_t TypeID = MatchTable[CurrentIdx++]; | 497 | 606 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 498 | 606 | dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID | 499 | 606 | << "]->getOperand(" << OpIdx | 500 | 606 | << "), TypeID=" << TypeID << ")\n"); | 501 | 606 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 502 | 606 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 503 | 606 | if (!MO.isReg() || | 504 | 606 | MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) { | 505 | 22 | if (handleReject() == RejectAndGiveUp) | 506 | 0 | return false; | 507 | 606 | } | 508 | 606 | break; | 509 | 606 | } | 510 | 763 | case GIM_CheckPointerToAny: { | 511 | 763 | int64_t InsnID = MatchTable[CurrentIdx++]; | 512 | 763 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 513 | 763 | int64_t SizeInBits = MatchTable[CurrentIdx++]; | 514 | 763 | | 515 | 763 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 516 | 763 | dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" | 517 | 763 | << InsnID << "]->getOperand(" << OpIdx | 518 | 763 | << "), SizeInBits=" << SizeInBits << ")\n"); | 519 | 763 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 520 | 763 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 521 | 763 | const LLT Ty = MRI.getType(MO.getReg()); | 522 | 763 | | 523 | 763 | // iPTR must be looked up in the target. | 524 | 763 | if (SizeInBits == 0) { | 525 | 746 | MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); | 526 | 746 | const unsigned AddrSpace = Ty.getAddressSpace(); | 527 | 746 | SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); | 528 | 746 | } | 529 | 763 | | 530 | 763 | assert(SizeInBits != 0 && "Pointer size must be known"); | 531 | 763 | | 532 | 763 | if (MO.isReg()) { | 533 | 763 | if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) | 534 | 0 | if (handleReject() == RejectAndGiveUp) | 535 | 0 | return false; | 536 | 0 | } else if (handleReject() == RejectAndGiveUp) | 537 | 0 | return false; | 538 | 763 | | 539 | 763 | break; | 540 | 763 | } | 541 | 1.10k | case GIM_CheckRegBankForClass: { | 542 | 1.10k | int64_t InsnID = MatchTable[CurrentIdx++]; | 543 | 1.10k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 544 | 1.10k | int64_t RCEnum = MatchTable[CurrentIdx++]; | 545 | 1.10k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 546 | 1.10k | dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" | 547 | 1.10k | << InsnID << "]->getOperand(" << OpIdx | 548 | 1.10k | << "), RCEnum=" << RCEnum << ")\n"); | 549 | 1.10k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 550 | 1.10k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 551 | 1.10k | if (!MO.isReg() || | 552 | 1.10k | &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != | 553 | 1.10k | RBI.getRegBank(MO.getReg(), MRI, TRI)) { | 554 | 237 | if (handleReject() == RejectAndGiveUp) | 555 | 0 | return false; | 556 | 1.10k | } | 557 | 1.10k | break; | 558 | 1.10k | } | 559 | 1.10k | | 560 | 1.10k | case GIM_CheckComplexPattern: { | 561 | 1.03k | int64_t InsnID = MatchTable[CurrentIdx++]; | 562 | 1.03k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 563 | 1.03k | int64_t RendererID = MatchTable[CurrentIdx++]; | 564 | 1.03k | int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; | 565 | 1.03k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 566 | 1.03k | dbgs() << CurrentIdx << ": State.Renderers[" << RendererID | 567 | 1.03k | << "] = GIM_CheckComplexPattern(MIs[" << InsnID | 568 | 1.03k | << "]->getOperand(" << OpIdx | 569 | 1.03k | << "), ComplexPredicateID=" << ComplexPredicateID | 570 | 1.03k | << ")\n"); | 571 | 1.03k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 572 | 1.03k | // FIXME: Use std::invoke() when it's available. | 573 | 1.03k | ComplexRendererFns Renderer = | 574 | 1.03k | (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( | 575 | 1.03k | State.MIs[InsnID]->getOperand(OpIdx)); | 576 | 1.03k | if (Renderer.hasValue()) | 577 | 768 | State.Renderers[RendererID] = Renderer.getValue(); | 578 | 270 | else | 579 | 270 | if (handleReject() == RejectAndGiveUp) | 580 | 0 | return false; | 581 | 1.03k | break; | 582 | 1.03k | } | 583 | 1.03k | | 584 | 1.03k | case GIM_CheckConstantInt: { | 585 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 586 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 587 | 0 | int64_t Value = MatchTable[CurrentIdx++]; | 588 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 589 | 0 | dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" | 590 | 0 | << InsnID << "]->getOperand(" << OpIdx | 591 | 0 | << "), Value=" << Value << ")\n"); | 592 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 593 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 594 | 0 | if (MO.isReg()) { | 595 | 0 | // isOperandImmEqual() will sign-extend to 64-bits, so should we. | 596 | 0 | LLT Ty = MRI.getType(MO.getReg()); | 597 | 0 | Value = SignExtend64(Value, Ty.getSizeInBits()); | 598 | 0 |
| 599 | 0 | if (!isOperandImmEqual(MO, Value, MRI)) { | 600 | 0 | if (handleReject() == RejectAndGiveUp) | 601 | 0 | return false; | 602 | 0 | } | 603 | 0 | } else if (handleReject() == RejectAndGiveUp) | 604 | 0 | return false; | 605 | 0 | | 606 | 0 | break; | 607 | 0 | } | 608 | 0 |
| 609 | 0 | case GIM_CheckLiteralInt: { | 610 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 611 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 612 | 0 | int64_t Value = MatchTable[CurrentIdx++]; | 613 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 614 | 0 | dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" | 615 | 0 | << InsnID << "]->getOperand(" << OpIdx | 616 | 0 | << "), Value=" << Value << ")\n"); | 617 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 618 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 619 | 0 | if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { | 620 | 0 | if (handleReject() == RejectAndGiveUp) | 621 | 0 | return false; | 622 | 0 | } | 623 | 0 | break; | 624 | 0 | } | 625 | 0 |
| 626 | 46 | case GIM_CheckIntrinsicID: { | 627 | 46 | int64_t InsnID = MatchTable[CurrentIdx++]; | 628 | 46 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 629 | 46 | int64_t Value = MatchTable[CurrentIdx++]; | 630 | 46 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 631 | 46 | dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" | 632 | 46 | << InsnID << "]->getOperand(" << OpIdx | 633 | 46 | << "), Value=" << Value << ")\n"); | 634 | 46 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 635 | 46 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 636 | 46 | if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) | 637 | 36 | if (handleReject() == RejectAndGiveUp) | 638 | 0 | return false; | 639 | 46 | break; | 640 | 46 | } | 641 | 46 | | 642 | 46 | case GIM_CheckIsMBB: { | 643 | 28 | int64_t InsnID = MatchTable[CurrentIdx++]; | 644 | 28 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 645 | 28 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 646 | 28 | dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID | 647 | 28 | << "]->getOperand(" << OpIdx << "))\n"); | 648 | 28 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 649 | 28 | if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { | 650 | 0 | if (handleReject() == RejectAndGiveUp) | 651 | 0 | return false; | 652 | 28 | } | 653 | 28 | break; | 654 | 28 | } | 655 | 28 | | 656 | 28 | case GIM_CheckIsSafeToFold: { | 657 | 1 | int64_t InsnID = MatchTable[CurrentIdx++]; | 658 | 1 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 659 | 1 | dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" | 660 | 1 | << InsnID << "])\n"); | 661 | 1 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 662 | 1 | if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { | 663 | 0 | if (handleReject() == RejectAndGiveUp) | 664 | 0 | return false; | 665 | 1 | } | 666 | 1 | break; | 667 | 1 | } | 668 | 1 | case GIM_CheckIsSameOperand: { | 669 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 670 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 671 | 0 | int64_t OtherInsnID = MatchTable[CurrentIdx++]; | 672 | 0 | int64_t OtherOpIdx = MatchTable[CurrentIdx++]; | 673 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 674 | 0 | dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" | 675 | 0 | << InsnID << "][" << OpIdx << "], MIs[" | 676 | 0 | << OtherInsnID << "][" << OtherOpIdx << "])\n"); | 677 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 678 | 0 | assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); | 679 | 0 | if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( | 680 | 0 | State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { | 681 | 0 | if (handleReject() == RejectAndGiveUp) | 682 | 0 | return false; | 683 | 0 | } | 684 | 0 | break; | 685 | 0 | } | 686 | 872 | case GIM_Reject: | 687 | 872 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 688 | 872 | dbgs() << CurrentIdx << ": GIM_Reject\n"); | 689 | 872 | if (handleReject() == RejectAndGiveUp) | 690 | 408 | return false; | 691 | 464 | break; | 692 | 464 | | 693 | 464 | case GIR_MutateOpcode: { | 694 | 90 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 695 | 90 | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 696 | 90 | int64_t NewOpcode = MatchTable[CurrentIdx++]; | 697 | 90 | if (NewInsnID >= OutMIs.size()) | 698 | 90 | OutMIs.resize(NewInsnID + 1); | 699 | 90 | | 700 | 90 | OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), | 701 | 90 | State.MIs[OldInsnID]); | 702 | 90 | OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); | 703 | 90 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 704 | 90 | dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" | 705 | 90 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 706 | 90 | << NewOpcode << ")\n"); | 707 | 90 | break; | 708 | 464 | } | 709 | 464 | | 710 | 851 | case GIR_BuildMI: { | 711 | 851 | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 712 | 851 | int64_t Opcode = MatchTable[CurrentIdx++]; | 713 | 851 | if (NewInsnID >= OutMIs.size()) | 714 | 851 | OutMIs.resize(NewInsnID + 1); | 715 | 851 | | 716 | 851 | OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], | 717 | 851 | State.MIs[0]->getDebugLoc(), TII.get(Opcode)); | 718 | 851 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 719 | 851 | dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" | 720 | 851 | << NewInsnID << "], " << Opcode << ")\n"); | 721 | 851 | break; | 722 | 464 | } | 723 | 464 | | 724 | 1.11k | case GIR_Copy: { | 725 | 1.11k | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 726 | 1.11k | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 727 | 1.11k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 728 | 1.11k | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 729 | 1.11k | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); | 730 | 1.11k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 731 | 1.11k | dbgs() | 732 | 1.11k | << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID | 733 | 1.11k | << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); | 734 | 1.11k | break; | 735 | 464 | } | 736 | 464 | | 737 | 464 | case GIR_CopyOrAddZeroReg: { | 738 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 739 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 740 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 741 | 0 | int64_t ZeroReg = MatchTable[CurrentIdx++]; | 742 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 743 | 0 | MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); | 744 | 0 | if (isOperandImmEqual(MO, 0, MRI)) | 745 | 0 | OutMIs[NewInsnID].addReg(ZeroReg); | 746 | 0 | else | 747 | 0 | OutMIs[NewInsnID].add(MO); | 748 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 749 | 0 | dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" | 750 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 751 | 0 | << OpIdx << ", " << ZeroReg << ")\n"); | 752 | 0 | break; | 753 | 464 | } | 754 | 464 | | 755 | 464 | case GIR_CopySubReg: { | 756 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 757 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 758 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 759 | 0 | int64_t SubRegIdx = MatchTable[CurrentIdx++]; | 760 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 761 | 0 | OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), | 762 | 0 | 0, SubRegIdx); | 763 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 764 | 0 | dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" | 765 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 766 | 0 | << OpIdx << ", " << SubRegIdx << ")\n"); | 767 | 0 | break; | 768 | 464 | } | 769 | 464 | | 770 | 464 | case GIR_AddImplicitDef: { | 771 | 30 | int64_t InsnID = MatchTable[CurrentIdx++]; | 772 | 30 | int64_t RegNum = MatchTable[CurrentIdx++]; | 773 | 30 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 774 | 30 | OutMIs[InsnID].addDef(RegNum, RegState::Implicit); | 775 | 30 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 776 | 30 | dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" | 777 | 30 | << InsnID << "], " << RegNum << ")\n"); | 778 | 30 | break; | 779 | 464 | } | 780 | 464 | | 781 | 464 | case GIR_AddImplicitUse: { | 782 | 31 | int64_t InsnID = MatchTable[CurrentIdx++]; | 783 | 31 | int64_t RegNum = MatchTable[CurrentIdx++]; | 784 | 31 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 785 | 31 | OutMIs[InsnID].addUse(RegNum, RegState::Implicit); | 786 | 31 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 787 | 31 | dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" | 788 | 31 | << InsnID << "], " << RegNum << ")\n"); | 789 | 31 | break; | 790 | 464 | } | 791 | 464 | | 792 | 464 | case GIR_AddRegister: { | 793 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 794 | 0 | int64_t RegNum = MatchTable[CurrentIdx++]; | 795 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 796 | 0 | OutMIs[InsnID].addReg(RegNum); | 797 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 798 | 0 | dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" | 799 | 0 | << InsnID << "], " << RegNum << ")\n"); | 800 | 0 | break; | 801 | 464 | } | 802 | 464 | | 803 | 464 | case GIR_AddTempRegister: { | 804 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 805 | 0 | int64_t TempRegID = MatchTable[CurrentIdx++]; | 806 | 0 | uint64_t TempRegFlags = MatchTable[CurrentIdx++]; | 807 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 808 | 0 | OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags); | 809 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 810 | 0 | dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" | 811 | 0 | << InsnID << "], TempRegisters[" << TempRegID | 812 | 0 | << "], " << TempRegFlags << ")\n"); | 813 | 0 | break; | 814 | 464 | } | 815 | 464 | | 816 | 1.53k | case GIR_AddImm: { | 817 | 1.53k | int64_t InsnID = MatchTable[CurrentIdx++]; | 818 | 1.53k | int64_t Imm = MatchTable[CurrentIdx++]; | 819 | 1.53k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 820 | 1.53k | OutMIs[InsnID].addImm(Imm); | 821 | 1.53k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 822 | 1.53k | dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID | 823 | 1.53k | << "], " << Imm << ")\n"); | 824 | 1.53k | break; | 825 | 464 | } | 826 | 464 | | 827 | 464 | case GIR_ComplexRenderer: { | 828 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 829 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 830 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 831 | 0 | for (const auto &RenderOpFn : State.Renderers[RendererID]) | 832 | 0 | RenderOpFn(OutMIs[InsnID]); | 833 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 834 | 0 | dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" | 835 | 0 | << InsnID << "], " << RendererID << ")\n"); | 836 | 0 | break; | 837 | 464 | } | 838 | 2.26k | case GIR_ComplexSubOperandRenderer: { | 839 | 2.26k | int64_t InsnID = MatchTable[CurrentIdx++]; | 840 | 2.26k | int64_t RendererID = MatchTable[CurrentIdx++]; | 841 | 2.26k | int64_t RenderOpID = MatchTable[CurrentIdx++]; | 842 | 2.26k | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 843 | 2.26k | State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); | 844 | 2.26k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 845 | 2.26k | dbgs() << CurrentIdx | 846 | 2.26k | << ": GIR_ComplexSubOperandRenderer(OutMIs[" | 847 | 2.26k | << InsnID << "], " << RendererID << ", " | 848 | 2.26k | << RenderOpID << ")\n"); | 849 | 2.26k | break; | 850 | 464 | } | 851 | 464 | | 852 | 464 | case GIR_CopyConstantAsSImm: { | 853 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 854 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 855 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 856 | 0 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); | 857 | 0 | if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { | 858 | 0 | OutMIs[NewInsnID].addImm( | 859 | 0 | State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); | 860 | 0 | } else if (State.MIs[OldInsnID]->getOperand(1).isImm()) | 861 | 0 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); | 862 | 0 | else | 863 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 864 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 865 | 0 | dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" | 866 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 867 | 0 | break; | 868 | 0 | } | 869 | 0 |
| 870 | 0 | // TODO: Needs a test case once we have a pattern that uses this. | 871 | 0 | case GIR_CopyFConstantAsFPImm: { | 872 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 873 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 874 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 875 | 0 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); | 876 | 0 | if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) | 877 | 0 | OutMIs[NewInsnID].addFPImm( | 878 | 0 | State.MIs[OldInsnID]->getOperand(1).getFPImm()); | 879 | 0 | else | 880 | 0 | llvm_unreachable("Expected FPImm operand"); | 881 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 882 | 0 | dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" | 883 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 884 | 0 | break; | 885 | 0 | } | 886 | 0 |
| 887 | 0 | case GIR_CustomRenderer: { | 888 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 889 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 890 | 0 | int64_t RendererFnID = MatchTable[CurrentIdx++]; | 891 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 892 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 893 | 0 | dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" | 894 | 0 | << InsnID << "], MIs[" << OldInsnID << "], " | 895 | 0 | << RendererFnID << ")\n"); | 896 | 0 | (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], | 897 | 0 | *State.MIs[OldInsnID]); | 898 | 0 | break; | 899 | 0 | } | 900 | 0 | case GIR_ConstrainOperandRC: { | 901 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 902 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 903 | 0 | int64_t RCEnum = MatchTable[CurrentIdx++]; | 904 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 905 | 0 | constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, | 906 | 0 | *TRI.getRegClass(RCEnum), TII, TRI, RBI); | 907 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 908 | 0 | dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" | 909 | 0 | << InsnID << "], " << OpIdx << ", " << RCEnum | 910 | 0 | << ")\n"); | 911 | 0 | break; | 912 | 0 | } | 913 | 0 |
| 914 | 941 | case GIR_ConstrainSelectedInstOperands: { | 915 | 941 | int64_t InsnID = MatchTable[CurrentIdx++]; | 916 | 941 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 917 | 941 | constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, | 918 | 941 | RBI); | 919 | 941 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 920 | 941 | dbgs() << CurrentIdx | 921 | 941 | << ": GIR_ConstrainSelectedInstOperands(OutMIs[" | 922 | 941 | << InsnID << "])\n"); | 923 | 941 | break; | 924 | 0 | } | 925 | 0 |
| 926 | 648 | case GIR_MergeMemOperands: { | 927 | 648 | int64_t InsnID = MatchTable[CurrentIdx++]; | 928 | 648 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 929 | 648 | | 930 | 648 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 931 | 648 | dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" | 932 | 648 | << InsnID << "]"); | 933 | 648 | int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; | 934 | 1.29k | while ((MergeInsnID = MatchTable[CurrentIdx++]) != | 935 | 1.29k | GIU_MergeMemOperands_EndOfList) { | 936 | 648 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 937 | 648 | dbgs() << ", MIs[" << MergeInsnID << "]"); | 938 | 648 | for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) | 939 | 647 | OutMIs[InsnID].addMemOperand(MMO); | 940 | 648 | } | 941 | 648 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); | 942 | 648 | break; | 943 | 0 | } | 944 | 0 |
| 945 | 851 | case GIR_EraseFromParent: { | 946 | 851 | int64_t InsnID = MatchTable[CurrentIdx++]; | 947 | 851 | assert(State.MIs[InsnID] && | 948 | 851 | "Attempted to erase an undefined instruction"); | 949 | 851 | State.MIs[InsnID]->eraseFromParent(); | 950 | 851 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 951 | 851 | dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" | 952 | 851 | << InsnID << "])\n"); | 953 | 851 | break; | 954 | 0 | } | 955 | 0 |
| 956 | 0 | case GIR_MakeTempReg: { | 957 | 0 | int64_t TempRegID = MatchTable[CurrentIdx++]; | 958 | 0 | int64_t TypeID = MatchTable[CurrentIdx++]; | 959 | 0 |
| 960 | 0 | State.TempRegisters[TempRegID] = | 961 | 0 | MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); | 962 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 963 | 0 | dbgs() << CurrentIdx << ": TempRegs[" << TempRegID | 964 | 0 | << "] = GIR_MakeTempReg(" << TypeID << ")\n"); | 965 | 0 | break; | 966 | 0 | } | 967 | 0 |
| 968 | 0 | case GIR_Coverage: { | 969 | 0 | int64_t RuleID = MatchTable[CurrentIdx++]; | 970 | 0 | CoverageInfo.setCovered(RuleID); | 971 | 0 |
| 972 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 973 | 0 | dbgs() | 974 | 0 | << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); | 975 | 0 | break; | 976 | 0 | } | 977 | 0 |
| 978 | 941 | case GIR_Done: | 979 | 941 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 980 | 941 | dbgs() << CurrentIdx << ": GIR_Done\n"); | 981 | 941 | return true; | 982 | 0 |
| 983 | 0 | default: | 984 | 0 | llvm_unreachable("Unexpected command"); | 985 | 37.0k | } | 986 | 37.0k | } | 987 | 1.34k | } |
ARMInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::ARMInstructionSelector const, llvm::PredicateBitsetImpl<70ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::ARMInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<70ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::ARMInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<70ul> const&, llvm::CodeGenCoverage&) const Line | Count | Source | 55 | 1.11k | CodeGenCoverage &CoverageInfo) const { | 56 | 1.11k | | 57 | 1.11k | uint64_t CurrentIdx = 0; | 58 | 1.11k | SmallVector<uint64_t, 4> OnFailResumeAt; | 59 | 1.11k | | 60 | 1.11k | enum RejectAction { RejectAndGiveUp, RejectAndResume }; | 61 | 1.11k | auto handleReject = [&]() -> RejectAction { | 62 | 1.11k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 1.11k | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 1.11k | if (OnFailResumeAt.empty()) | 65 | 1.11k | return RejectAndGiveUp; | 66 | 1.11k | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 1.11k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 1.11k | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 1.11k | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 1.11k | return RejectAndResume; | 71 | 1.11k | }; | 72 | 1.11k | | 73 | 15.4k | while (true) { | 74 | 15.4k | assert(CurrentIdx != ~0u && "Invalid MatchTable index"); | 75 | 15.4k | int64_t MatcherOpcode = MatchTable[CurrentIdx++]; | 76 | 15.4k | switch (MatcherOpcode) { | 77 | 15.4k | case GIM_Try: { | 78 | 2.76k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 79 | 2.76k | dbgs() << CurrentIdx << ": Begin try-block\n"); | 80 | 2.76k | OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); | 81 | 2.76k | break; | 82 | 15.4k | } | 83 | 15.4k | | 84 | 15.4k | case GIM_RecordInsn: { | 85 | 827 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 86 | 827 | int64_t InsnID = MatchTable[CurrentIdx++]; | 87 | 827 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 88 | 827 | | 89 | 827 | // As an optimisation we require that MIs[0] is always the root. Refuse | 90 | 827 | // any attempt to modify it. | 91 | 827 | assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); | 92 | 827 | | 93 | 827 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 94 | 827 | if (!MO.isReg()) { | 95 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 96 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 97 | 0 | if (handleReject() == RejectAndGiveUp) | 98 | 0 | return false; | 99 | 0 | break; | 100 | 0 | } | 101 | 827 | if (TRI.isPhysicalRegister(MO.getReg())) { | 102 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 103 | 0 | dbgs() << CurrentIdx << ": Is a physical register\n"); | 104 | 0 | if (handleReject() == RejectAndGiveUp) | 105 | 0 | return false; | 106 | 0 | break; | 107 | 0 | } | 108 | 827 | | 109 | 827 | MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); | 110 | 827 | if ((size_t)NewInsnID < State.MIs.size()) | 111 | 571 | State.MIs[NewInsnID] = NewMI; | 112 | 256 | else { | 113 | 256 | assert((size_t)NewInsnID == State.MIs.size() && | 114 | 256 | "Expected to store MIs in order"); | 115 | 256 | State.MIs.push_back(NewMI); | 116 | 256 | } | 117 | 827 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 118 | 827 | dbgs() << CurrentIdx << ": MIs[" << NewInsnID | 119 | 827 | << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx | 120 | 827 | << ")\n"); | 121 | 827 | break; | 122 | 827 | } | 123 | 827 | | 124 | 2.40k | case GIM_CheckFeatures: { | 125 | 2.40k | int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; | 126 | 2.40k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 127 | 2.40k | dbgs() << CurrentIdx | 128 | 2.40k | << ": GIM_CheckFeatures(ExpectedBitsetID=" | 129 | 2.40k | << ExpectedBitsetID << ")\n"); | 130 | 2.40k | if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != | 131 | 2.40k | ISelInfo.FeatureBitsets[ExpectedBitsetID]) { | 132 | 1.42k | if (handleReject() == RejectAndGiveUp) | 133 | 0 | return false; | 134 | 2.40k | } | 135 | 2.40k | break; | 136 | 2.40k | } | 137 | 2.40k | | 138 | 2.40k | case GIM_CheckOpcode: { | 139 | 827 | int64_t InsnID = MatchTable[CurrentIdx++]; | 140 | 827 | int64_t Expected = MatchTable[CurrentIdx++]; | 141 | 827 | | 142 | 827 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 143 | 827 | unsigned Opcode = State.MIs[InsnID]->getOpcode(); | 144 | 827 | | 145 | 827 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 146 | 827 | dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID | 147 | 827 | << "], ExpectedOpcode=" << Expected | 148 | 827 | << ") // Got=" << Opcode << "\n"); | 149 | 827 | if (Opcode != Expected) { | 150 | 584 | if (handleReject() == RejectAndGiveUp) | 151 | 0 | return false; | 152 | 827 | } | 153 | 827 | break; | 154 | 827 | } | 155 | 827 | | 156 | 1.11k | case GIM_SwitchOpcode: { | 157 | 1.11k | int64_t InsnID = MatchTable[CurrentIdx++]; | 158 | 1.11k | int64_t LowerBound = MatchTable[CurrentIdx++]; | 159 | 1.11k | int64_t UpperBound = MatchTable[CurrentIdx++]; | 160 | 1.11k | int64_t Default = MatchTable[CurrentIdx++]; | 161 | 1.11k | | 162 | 1.11k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 163 | 1.11k | const int64_t Opcode = State.MIs[InsnID]->getOpcode(); | 164 | 1.11k | | 165 | 1.11k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 166 | 1.11k | dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" | 167 | 1.11k | << LowerBound << ", " << UpperBound << "), Default=" << Default | 168 | 1.11k | << ", JumpTable...) // Got=" << Opcode << "\n"; | 169 | 1.11k | }); | 170 | 1.11k | if (Opcode < LowerBound || UpperBound <= Opcode) { | 171 | 0 | CurrentIdx = Default; | 172 | 0 | break; | 173 | 0 | } | 174 | 1.11k | CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; | 175 | 1.11k | if (!CurrentIdx) { | 176 | 579 | CurrentIdx = Default; | 177 | 579 | break; | 178 | 579 | } | 179 | 534 | OnFailResumeAt.push_back(Default); | 180 | 534 | break; | 181 | 534 | } | 182 | 534 | | 183 | 534 | case GIM_SwitchType: { | 184 | 368 | int64_t InsnID = MatchTable[CurrentIdx++]; | 185 | 368 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 186 | 368 | int64_t LowerBound = MatchTable[CurrentIdx++]; | 187 | 368 | int64_t UpperBound = MatchTable[CurrentIdx++]; | 188 | 368 | int64_t Default = MatchTable[CurrentIdx++]; | 189 | 368 | | 190 | 368 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 191 | 368 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 192 | 368 | | 193 | 368 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 194 | 368 | dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID | 195 | 368 | << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " | 196 | 368 | << UpperBound << "), Default=" << Default | 197 | 368 | << ", JumpTable...) // Got="; | 198 | 368 | if (!MO.isReg()) | 199 | 368 | dbgs() << "Not a VReg\n"; | 200 | 368 | else | 201 | 368 | dbgs() << MRI.getType(MO.getReg()) << "\n"; | 202 | 368 | }); | 203 | 368 | if (!MO.isReg()) { | 204 | 0 | CurrentIdx = Default; | 205 | 0 | break; | 206 | 0 | } | 207 | 368 | const LLT Ty = MRI.getType(MO.getReg()); | 208 | 368 | const auto TyI = ISelInfo.TypeIDMap.find(Ty); | 209 | 368 | if (TyI == ISelInfo.TypeIDMap.end()) { | 210 | 49 | CurrentIdx = Default; | 211 | 49 | break; | 212 | 49 | } | 213 | 319 | const int64_t TypeID = TyI->second; | 214 | 319 | if (TypeID < LowerBound || UpperBound <= TypeID219 ) { | 215 | 100 | CurrentIdx = Default; | 216 | 100 | break; | 217 | 100 | } | 218 | 219 | CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; | 219 | 219 | if (!CurrentIdx) { | 220 | 0 | CurrentIdx = Default; | 221 | 0 | break; | 222 | 0 | } | 223 | 219 | OnFailResumeAt.push_back(Default); | 224 | 219 | break; | 225 | 219 | } | 226 | 219 | | 227 | 219 | case GIM_CheckNumOperands: { | 228 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 229 | 0 | int64_t Expected = MatchTable[CurrentIdx++]; | 230 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 231 | 0 | dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" | 232 | 0 | << InsnID << "], Expected=" << Expected << ")\n"); | 233 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 234 | 0 | if (State.MIs[InsnID]->getNumOperands() != Expected) { | 235 | 0 | if (handleReject() == RejectAndGiveUp) | 236 | 0 | return false; | 237 | 0 | } | 238 | 0 | break; | 239 | 0 | } | 240 | 81 | case GIM_CheckI64ImmPredicate: { | 241 | 81 | int64_t InsnID = MatchTable[CurrentIdx++]; | 242 | 81 | int64_t Predicate = MatchTable[CurrentIdx++]; | 243 | 81 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 244 | 81 | dbgs() | 245 | 81 | << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" | 246 | 81 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 247 | 81 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 248 | 81 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 249 | 81 | "Expected G_CONSTANT"); | 250 | 81 | assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); | 251 | 81 | int64_t Value = 0; | 252 | 81 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 253 | 81 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); | 254 | 0 | else if (State.MIs[InsnID]->getOperand(1).isImm()) | 255 | 0 | Value = State.MIs[InsnID]->getOperand(1).getImm(); | 256 | 0 | else | 257 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 258 | 81 | | 259 | 81 | if (!testImmPredicate_I64(Predicate, Value)) | 260 | 15 | if (handleReject() == RejectAndGiveUp) | 261 | 0 | return false; | 262 | 81 | break; | 263 | 81 | } | 264 | 81 | case GIM_CheckAPIntImmPredicate: { | 265 | 1 | int64_t InsnID = MatchTable[CurrentIdx++]; | 266 | 1 | int64_t Predicate = MatchTable[CurrentIdx++]; | 267 | 1 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 268 | 1 | dbgs() | 269 | 1 | << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" | 270 | 1 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 271 | 1 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 272 | 1 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 273 | 1 | "Expected G_CONSTANT"); | 274 | 1 | assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); | 275 | 1 | APInt Value; | 276 | 1 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 277 | 1 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); | 278 | 1 | else | 279 | 1 | llvm_unreachable0 ("Expected Imm or CImm operand"); | 280 | 1 | | 281 | 1 | if (!testImmPredicate_APInt(Predicate, Value)) | 282 | 0 | if (handleReject() == RejectAndGiveUp) | 283 | 0 | return false; | 284 | 1 | break; | 285 | 1 | } | 286 | 1 | case GIM_CheckAPFloatImmPredicate: { | 287 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 288 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 289 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 290 | 0 | dbgs() | 291 | 0 | << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" | 292 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 293 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 294 | 0 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && | 295 | 0 | "Expected G_FCONSTANT"); | 296 | 0 | assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); | 297 | 0 | assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); | 298 | 0 | APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); | 299 | 0 |
| 300 | 0 | if (!testImmPredicate_APFloat(Predicate, Value)) | 301 | 0 | if (handleReject() == RejectAndGiveUp) | 302 | 0 | return false; | 303 | 0 | break; | 304 | 0 | } | 305 | 10 | case GIM_CheckCxxInsnPredicate: { | 306 | 10 | int64_t InsnID = MatchTable[CurrentIdx++]; | 307 | 10 | int64_t Predicate = MatchTable[CurrentIdx++]; | 308 | 10 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 309 | 10 | dbgs() | 310 | 10 | << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" | 311 | 10 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 312 | 10 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 313 | 10 | assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); | 314 | 10 | | 315 | 10 | if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID])) | 316 | 4 | if (handleReject() == RejectAndGiveUp) | 317 | 0 | return false; | 318 | 10 | break; | 319 | 10 | } | 320 | 10 | case GIM_CheckAtomicOrdering: { | 321 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 322 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 323 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 324 | 0 | dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" | 325 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 326 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 327 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 328 | 0 | if (handleReject() == RejectAndGiveUp) | 329 | 0 | return false; | 330 | 0 | | 331 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 332 | 0 | if (MMO->getOrdering() != Ordering) | 333 | 0 | if (handleReject() == RejectAndGiveUp) | 334 | 0 | return false; | 335 | 0 | break; | 336 | 0 | } | 337 | 0 | case GIM_CheckAtomicOrderingOrStrongerThan: { | 338 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 339 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 340 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 341 | 0 | dbgs() << CurrentIdx | 342 | 0 | << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" | 343 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 344 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 345 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 346 | 0 | if (handleReject() == RejectAndGiveUp) | 347 | 0 | return false; | 348 | 0 | | 349 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 350 | 0 | if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) | 351 | 0 | if (handleReject() == RejectAndGiveUp) | 352 | 0 | return false; | 353 | 0 | break; | 354 | 0 | } | 355 | 0 | case GIM_CheckAtomicOrderingWeakerThan: { | 356 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 357 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 358 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 359 | 0 | dbgs() << CurrentIdx | 360 | 0 | << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" | 361 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 362 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 363 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 364 | 0 | if (handleReject() == RejectAndGiveUp) | 365 | 0 | return false; | 366 | 0 | | 367 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 368 | 0 | if (!isStrongerThan(Ordering, MMO->getOrdering())) | 369 | 0 | if (handleReject() == RejectAndGiveUp) | 370 | 0 | return false; | 371 | 0 | break; | 372 | 0 | } | 373 | 0 | case GIM_CheckMemoryAddressSpace: { | 374 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 375 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 376 | 0 | // This accepts a list of possible address spaces. | 377 | 0 | const int NumAddrSpace = MatchTable[CurrentIdx++]; | 378 | 0 |
| 379 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 380 | 0 | if (handleReject() == RejectAndGiveUp) | 381 | 0 | return false; | 382 | 0 | break; | 383 | 0 | } | 384 | 0 | | 385 | 0 | // Need to still jump to the end of the list of address spaces if we find | 386 | 0 | // a match earlier. | 387 | 0 | const uint64_t LastIdx = CurrentIdx + NumAddrSpace; | 388 | 0 |
| 389 | 0 | const MachineMemOperand *MMO | 390 | 0 | = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 391 | 0 | const unsigned MMOAddrSpace = MMO->getAddrSpace(); | 392 | 0 |
| 393 | 0 | bool Success = false; | 394 | 0 | for (int I = 0; I != NumAddrSpace; ++I) { | 395 | 0 | unsigned AddrSpace = MatchTable[CurrentIdx++]; | 396 | 0 | DEBUG_WITH_TYPE( | 397 | 0 | TgtInstructionSelector::getName(), | 398 | 0 | dbgs() << "addrspace(" << MMOAddrSpace << ") vs " | 399 | 0 | << AddrSpace << '\n'); | 400 | 0 |
| 401 | 0 | if (AddrSpace == MMOAddrSpace) { | 402 | 0 | Success = true; | 403 | 0 | break; | 404 | 0 | } | 405 | 0 | } | 406 | 0 |
| 407 | 0 | CurrentIdx = LastIdx; | 408 | 0 | if (!Success && handleReject() == RejectAndGiveUp) | 409 | 0 | return false; | 410 | 0 | break; | 411 | 0 | } | 412 | 0 | case GIM_CheckMemorySizeEqualTo: { | 413 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 414 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 415 | 0 | uint64_t Size = MatchTable[CurrentIdx++]; | 416 | 0 |
| 417 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 418 | 0 | dbgs() << CurrentIdx | 419 | 0 | << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID | 420 | 0 | << "]->memoperands() + " << MMOIdx | 421 | 0 | << ", Size=" << Size << ")\n"); | 422 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 423 | 0 |
| 424 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 425 | 0 | if (handleReject() == RejectAndGiveUp) | 426 | 0 | return false; | 427 | 0 | break; | 428 | 0 | } | 429 | 0 | | 430 | 0 | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 431 | 0 |
| 432 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 433 | 0 | dbgs() << MMO->getSize() << " bytes vs " << Size | 434 | 0 | << " bytes\n"); | 435 | 0 | if (MMO->getSize() != Size) | 436 | 0 | if (handleReject() == RejectAndGiveUp) | 437 | 0 | return false; | 438 | 0 | | 439 | 0 | break; | 440 | 0 | } | 441 | 0 | case GIM_CheckMemorySizeEqualToLLT: | 442 | 0 | case GIM_CheckMemorySizeLessThanLLT: | 443 | 0 | case GIM_CheckMemorySizeGreaterThanLLT: { | 444 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 445 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 446 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 447 | 0 |
| 448 | 0 | DEBUG_WITH_TYPE( | 449 | 0 | TgtInstructionSelector::getName(), | 450 | 0 | dbgs() << CurrentIdx << ": GIM_CheckMemorySize" | 451 | 0 | << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT | 452 | 0 | ? "EqualTo" | 453 | 0 | : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT | 454 | 0 | ? "GreaterThan" | 455 | 0 | : "LessThan") | 456 | 0 | << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx | 457 | 0 | << ", OpIdx=" << OpIdx << ")\n"); | 458 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 459 | 0 |
| 460 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 461 | 0 | if (!MO.isReg()) { | 462 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 463 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 464 | 0 | if (handleReject() == RejectAndGiveUp) | 465 | 0 | return false; | 466 | 0 | break; | 467 | 0 | } | 468 | 0 | | 469 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 470 | 0 | if (handleReject() == RejectAndGiveUp) | 471 | 0 | return false; | 472 | 0 | break; | 473 | 0 | } | 474 | 0 | | 475 | 0 | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 476 | 0 |
| 477 | 0 | unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); | 478 | 0 | if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && | 479 | 0 | MMO->getSizeInBits() != Size) { | 480 | 0 | if (handleReject() == RejectAndGiveUp) | 481 | 0 | return false; | 482 | 0 | } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && | 483 | 0 | MMO->getSizeInBits() >= Size) { | 484 | 0 | if (handleReject() == RejectAndGiveUp) | 485 | 0 | return false; | 486 | 0 | } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && | 487 | 0 | MMO->getSizeInBits() <= Size) | 488 | 0 | if (handleReject() == RejectAndGiveUp) | 489 | 0 | return false; | 490 | 0 | | 491 | 0 | break; | 492 | 0 | } | 493 | 1.15k | case GIM_CheckType: { | 494 | 1.15k | int64_t InsnID = MatchTable[CurrentIdx++]; | 495 | 1.15k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 496 | 1.15k | int64_t TypeID = MatchTable[CurrentIdx++]; | 497 | 1.15k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 498 | 1.15k | dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID | 499 | 1.15k | << "]->getOperand(" << OpIdx | 500 | 1.15k | << "), TypeID=" << TypeID << ")\n"); | 501 | 1.15k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 502 | 1.15k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 503 | 1.15k | if (!MO.isReg() || | 504 | 1.15k | MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) { | 505 | 7 | if (handleReject() == RejectAndGiveUp) | 506 | 0 | return false; | 507 | 1.15k | } | 508 | 1.15k | break; | 509 | 1.15k | } | 510 | 1.15k | case GIM_CheckPointerToAny: { | 511 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 512 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 513 | 0 | int64_t SizeInBits = MatchTable[CurrentIdx++]; | 514 | 0 |
| 515 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 516 | 0 | dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" | 517 | 0 | << InsnID << "]->getOperand(" << OpIdx | 518 | 0 | << "), SizeInBits=" << SizeInBits << ")\n"); | 519 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 520 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 521 | 0 | const LLT Ty = MRI.getType(MO.getReg()); | 522 | 0 |
| 523 | 0 | // iPTR must be looked up in the target. | 524 | 0 | if (SizeInBits == 0) { | 525 | 0 | MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); | 526 | 0 | const unsigned AddrSpace = Ty.getAddressSpace(); | 527 | 0 | SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); | 528 | 0 | } | 529 | 0 |
| 530 | 0 | assert(SizeInBits != 0 && "Pointer size must be known"); | 531 | 0 |
| 532 | 0 | if (MO.isReg()) { | 533 | 0 | if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) | 534 | 0 | if (handleReject() == RejectAndGiveUp) | 535 | 0 | return false; | 536 | 0 | } else if (handleReject() == RejectAndGiveUp) | 537 | 0 | return false; | 538 | 0 | | 539 | 0 | break; | 540 | 0 | } | 541 | 1.70k | case GIM_CheckRegBankForClass: { | 542 | 1.70k | int64_t InsnID = MatchTable[CurrentIdx++]; | 543 | 1.70k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 544 | 1.70k | int64_t RCEnum = MatchTable[CurrentIdx++]; | 545 | 1.70k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 546 | 1.70k | dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" | 547 | 1.70k | << InsnID << "]->getOperand(" << OpIdx | 548 | 1.70k | << "), RCEnum=" << RCEnum << ")\n"); | 549 | 1.70k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 550 | 1.70k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 551 | 1.70k | if (!MO.isReg() || | 552 | 1.70k | &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != | 553 | 1.70k | RBI.getRegBank(MO.getReg(), MRI, TRI)) { | 554 | 0 | if (handleReject() == RejectAndGiveUp) | 555 | 0 | return false; | 556 | 1.70k | } | 557 | 1.70k | break; | 558 | 1.70k | } | 559 | 1.70k | | 560 | 1.70k | case GIM_CheckComplexPattern: { | 561 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 562 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 563 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 564 | 0 | int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; | 565 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 566 | 0 | dbgs() << CurrentIdx << ": State.Renderers[" << RendererID | 567 | 0 | << "] = GIM_CheckComplexPattern(MIs[" << InsnID | 568 | 0 | << "]->getOperand(" << OpIdx | 569 | 0 | << "), ComplexPredicateID=" << ComplexPredicateID | 570 | 0 | << ")\n"); | 571 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 572 | 0 | // FIXME: Use std::invoke() when it's available. | 573 | 0 | ComplexRendererFns Renderer = | 574 | 0 | (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( | 575 | 0 | State.MIs[InsnID]->getOperand(OpIdx)); | 576 | 0 | if (Renderer.hasValue()) | 577 | 0 | State.Renderers[RendererID] = Renderer.getValue(); | 578 | 0 | else | 579 | 0 | if (handleReject() == RejectAndGiveUp) | 580 | 0 | return false; | 581 | 0 | break; | 582 | 0 | } | 583 | 0 |
| 584 | 183 | case GIM_CheckConstantInt: { | 585 | 183 | int64_t InsnID = MatchTable[CurrentIdx++]; | 586 | 183 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 587 | 183 | int64_t Value = MatchTable[CurrentIdx++]; | 588 | 183 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 589 | 183 | dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" | 590 | 183 | << InsnID << "]->getOperand(" << OpIdx | 591 | 183 | << "), Value=" << Value << ")\n"); | 592 | 183 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 593 | 183 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 594 | 183 | if (MO.isReg()) { | 595 | 183 | // isOperandImmEqual() will sign-extend to 64-bits, so should we. | 596 | 183 | LLT Ty = MRI.getType(MO.getReg()); | 597 | 183 | Value = SignExtend64(Value, Ty.getSizeInBits()); | 598 | 183 | | 599 | 183 | if (!isOperandImmEqual(MO, Value, MRI)) { | 600 | 83 | if (handleReject() == RejectAndGiveUp) | 601 | 0 | return false; | 602 | 0 | } | 603 | 0 | } else if (handleReject() == RejectAndGiveUp) | 604 | 0 | return false; | 605 | 183 | | 606 | 183 | break; | 607 | 183 | } | 608 | 183 | | 609 | 183 | case GIM_CheckLiteralInt: { | 610 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 611 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 612 | 0 | int64_t Value = MatchTable[CurrentIdx++]; | 613 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 614 | 0 | dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" | 615 | 0 | << InsnID << "]->getOperand(" << OpIdx | 616 | 0 | << "), Value=" << Value << ")\n"); | 617 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 618 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 619 | 0 | if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { | 620 | 0 | if (handleReject() == RejectAndGiveUp) | 621 | 0 | return false; | 622 | 0 | } | 623 | 0 | break; | 624 | 0 | } | 625 | 0 |
| 626 | 0 | case GIM_CheckIntrinsicID: { | 627 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 628 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 629 | 0 | int64_t Value = MatchTable[CurrentIdx++]; | 630 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 631 | 0 | dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" | 632 | 0 | << InsnID << "]->getOperand(" << OpIdx | 633 | 0 | << "), Value=" << Value << ")\n"); | 634 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 635 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 636 | 0 | if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) | 637 | 0 | if (handleReject() == RejectAndGiveUp) | 638 | 0 | return false; | 639 | 0 | break; | 640 | 0 | } | 641 | 0 |
| 642 | 15 | case GIM_CheckIsMBB: { | 643 | 15 | int64_t InsnID = MatchTable[CurrentIdx++]; | 644 | 15 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 645 | 15 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 646 | 15 | dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID | 647 | 15 | << "]->getOperand(" << OpIdx << "))\n"); | 648 | 15 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 649 | 15 | if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { | 650 | 0 | if (handleReject() == RejectAndGiveUp) | 651 | 0 | return false; | 652 | 15 | } | 653 | 15 | break; | 654 | 15 | } | 655 | 15 | | 656 | 118 | case GIM_CheckIsSafeToFold: { | 657 | 118 | int64_t InsnID = MatchTable[CurrentIdx++]; | 658 | 118 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 659 | 118 | dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" | 660 | 118 | << InsnID << "])\n"); | 661 | 118 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 662 | 118 | if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { | 663 | 0 | if (handleReject() == RejectAndGiveUp) | 664 | 0 | return false; | 665 | 118 | } | 666 | 118 | break; | 667 | 118 | } | 668 | 118 | case GIM_CheckIsSameOperand: { | 669 | 4 | int64_t InsnID = MatchTable[CurrentIdx++]; | 670 | 4 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 671 | 4 | int64_t OtherInsnID = MatchTable[CurrentIdx++]; | 672 | 4 | int64_t OtherOpIdx = MatchTable[CurrentIdx++]; | 673 | 4 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 674 | 4 | dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" | 675 | 4 | << InsnID << "][" << OpIdx << "], MIs[" | 676 | 4 | << OtherInsnID << "][" << OtherOpIdx << "])\n"); | 677 | 4 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 678 | 4 | assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); | 679 | 4 | if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( | 680 | 4 | State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { | 681 | 0 | if (handleReject() == RejectAndGiveUp) | 682 | 0 | return false; | 683 | 4 | } | 684 | 4 | break; | 685 | 4 | } | 686 | 1.18k | case GIM_Reject: | 687 | 1.18k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 688 | 1.18k | dbgs() << CurrentIdx << ": GIM_Reject\n"); | 689 | 1.18k | if (handleReject() == RejectAndGiveUp) | 690 | 820 | return false; | 691 | 367 | break; | 692 | 367 | | 693 | 367 | case GIR_MutateOpcode: { | 694 | 9 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 695 | 9 | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 696 | 9 | int64_t NewOpcode = MatchTable[CurrentIdx++]; | 697 | 9 | if (NewInsnID >= OutMIs.size()) | 698 | 9 | OutMIs.resize(NewInsnID + 1); | 699 | 9 | | 700 | 9 | OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), | 701 | 9 | State.MIs[OldInsnID]); | 702 | 9 | OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); | 703 | 9 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 704 | 9 | dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" | 705 | 9 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 706 | 9 | << NewOpcode << ")\n"); | 707 | 9 | break; | 708 | 367 | } | 709 | 367 | | 710 | 367 | case GIR_BuildMI: { | 711 | 301 | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 712 | 301 | int64_t Opcode = MatchTable[CurrentIdx++]; | 713 | 301 | if (NewInsnID >= OutMIs.size()) | 714 | 284 | OutMIs.resize(NewInsnID + 1); | 715 | 301 | | 716 | 301 | OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], | 717 | 301 | State.MIs[0]->getDebugLoc(), TII.get(Opcode)); | 718 | 301 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 719 | 301 | dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" | 720 | 301 | << NewInsnID << "], " << Opcode << ")\n"); | 721 | 301 | break; | 722 | 367 | } | 723 | 367 | | 724 | 695 | case GIR_Copy: { | 725 | 695 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 726 | 695 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 727 | 695 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 728 | 695 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 729 | 695 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); | 730 | 695 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 731 | 695 | dbgs() | 732 | 695 | << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID | 733 | 695 | << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); | 734 | 695 | break; | 735 | 367 | } | 736 | 367 | | 737 | 367 | case GIR_CopyOrAddZeroReg: { | 738 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 739 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 740 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 741 | 0 | int64_t ZeroReg = MatchTable[CurrentIdx++]; | 742 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 743 | 0 | MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); | 744 | 0 | if (isOperandImmEqual(MO, 0, MRI)) | 745 | 0 | OutMIs[NewInsnID].addReg(ZeroReg); | 746 | 0 | else | 747 | 0 | OutMIs[NewInsnID].add(MO); | 748 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 749 | 0 | dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" | 750 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 751 | 0 | << OpIdx << ", " << ZeroReg << ")\n"); | 752 | 0 | break; | 753 | 367 | } | 754 | 367 | | 755 | 367 | case GIR_CopySubReg: { | 756 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 757 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 758 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 759 | 0 | int64_t SubRegIdx = MatchTable[CurrentIdx++]; | 760 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 761 | 0 | OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), | 762 | 0 | 0, SubRegIdx); | 763 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 764 | 0 | dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" | 765 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 766 | 0 | << OpIdx << ", " << SubRegIdx << ")\n"); | 767 | 0 | break; | 768 | 367 | } | 769 | 367 | | 770 | 367 | case GIR_AddImplicitDef: { | 771 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 772 | 0 | int64_t RegNum = MatchTable[CurrentIdx++]; | 773 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 774 | 0 | OutMIs[InsnID].addDef(RegNum, RegState::Implicit); | 775 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 776 | 0 | dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" | 777 | 0 | << InsnID << "], " << RegNum << ")\n"); | 778 | 0 | break; | 779 | 367 | } | 780 | 367 | | 781 | 367 | case GIR_AddImplicitUse: { | 782 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 783 | 0 | int64_t RegNum = MatchTable[CurrentIdx++]; | 784 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 785 | 0 | OutMIs[InsnID].addUse(RegNum, RegState::Implicit); | 786 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 787 | 0 | dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" | 788 | 0 | << InsnID << "], " << RegNum << ")\n"); | 789 | 0 | break; | 790 | 367 | } | 791 | 367 | | 792 | 383 | case GIR_AddRegister: { | 793 | 383 | int64_t InsnID = MatchTable[CurrentIdx++]; | 794 | 383 | int64_t RegNum = MatchTable[CurrentIdx++]; | 795 | 383 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 796 | 383 | OutMIs[InsnID].addReg(RegNum); | 797 | 383 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 798 | 383 | dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" | 799 | 383 | << InsnID << "], " << RegNum << ")\n"); | 800 | 383 | break; | 801 | 367 | } | 802 | 367 | | 803 | 367 | case GIR_AddTempRegister: { | 804 | 34 | int64_t InsnID = MatchTable[CurrentIdx++]; | 805 | 34 | int64_t TempRegID = MatchTable[CurrentIdx++]; | 806 | 34 | uint64_t TempRegFlags = MatchTable[CurrentIdx++]; | 807 | 34 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 808 | 34 | OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags); | 809 | 34 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 810 | 34 | dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" | 811 | 34 | << InsnID << "], TempRegisters[" << TempRegID | 812 | 34 | << "], " << TempRegFlags << ")\n"); | 813 | 34 | break; | 814 | 367 | } | 815 | 367 | | 816 | 367 | case GIR_AddImm: { | 817 | 319 | int64_t InsnID = MatchTable[CurrentIdx++]; | 818 | 319 | int64_t Imm = MatchTable[CurrentIdx++]; | 819 | 319 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 820 | 319 | OutMIs[InsnID].addImm(Imm); | 821 | 319 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 822 | 319 | dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID | 823 | 319 | << "], " << Imm << ")\n"); | 824 | 319 | break; | 825 | 367 | } | 826 | 367 | | 827 | 367 | case GIR_ComplexRenderer: { | 828 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 829 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 830 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 831 | 0 | for (const auto &RenderOpFn : State.Renderers[RendererID]) | 832 | 0 | RenderOpFn(OutMIs[InsnID]); | 833 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 834 | 0 | dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" | 835 | 0 | << InsnID << "], " << RendererID << ")\n"); | 836 | 0 | break; | 837 | 367 | } | 838 | 367 | case GIR_ComplexSubOperandRenderer: { | 839 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 840 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 841 | 0 | int64_t RenderOpID = MatchTable[CurrentIdx++]; | 842 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 843 | 0 | State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); | 844 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 845 | 0 | dbgs() << CurrentIdx | 846 | 0 | << ": GIR_ComplexSubOperandRenderer(OutMIs[" | 847 | 0 | << InsnID << "], " << RendererID << ", " | 848 | 0 | << RenderOpID << ")\n"); | 849 | 0 | break; | 850 | 367 | } | 851 | 367 | | 852 | 367 | case GIR_CopyConstantAsSImm: { | 853 | 70 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 854 | 70 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 855 | 70 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 856 | 70 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); | 857 | 70 | if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { | 858 | 70 | OutMIs[NewInsnID].addImm( | 859 | 70 | State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); | 860 | 70 | } else if (0 State.MIs[OldInsnID]->getOperand(1).isImm()0 ) | 861 | 0 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); | 862 | 0 | else | 863 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 864 | 70 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 865 | 70 | dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" | 866 | 70 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 867 | 70 | break; | 868 | 70 | } | 869 | 70 | | 870 | 70 | // TODO: Needs a test case once we have a pattern that uses this. | 871 | 70 | case GIR_CopyFConstantAsFPImm: { | 872 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 873 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 874 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 875 | 0 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); | 876 | 0 | if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) | 877 | 0 | OutMIs[NewInsnID].addFPImm( | 878 | 0 | State.MIs[OldInsnID]->getOperand(1).getFPImm()); | 879 | 0 | else | 880 | 0 | llvm_unreachable("Expected FPImm operand"); | 881 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 882 | 0 | dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" | 883 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 884 | 0 | break; | 885 | 0 | } | 886 | 0 |
| 887 | 4 | case GIR_CustomRenderer: { | 888 | 4 | int64_t InsnID = MatchTable[CurrentIdx++]; | 889 | 4 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 890 | 4 | int64_t RendererFnID = MatchTable[CurrentIdx++]; | 891 | 4 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 892 | 4 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 893 | 4 | dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" | 894 | 4 | << InsnID << "], MIs[" << OldInsnID << "], " | 895 | 4 | << RendererFnID << ")\n"); | 896 | 4 | (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], | 897 | 4 | *State.MIs[OldInsnID]); | 898 | 4 | break; | 899 | 0 | } | 900 | 9 | case GIR_ConstrainOperandRC: { | 901 | 9 | int64_t InsnID = MatchTable[CurrentIdx++]; | 902 | 9 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 903 | 9 | int64_t RCEnum = MatchTable[CurrentIdx++]; | 904 | 9 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 905 | 9 | constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, | 906 | 9 | *TRI.getRegClass(RCEnum), TII, TRI, RBI); | 907 | 9 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 908 | 9 | dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" | 909 | 9 | << InsnID << "], " << OpIdx << ", " << RCEnum | 910 | 9 | << ")\n"); | 911 | 9 | break; | 912 | 0 | } | 913 | 0 |
| 914 | 301 | case GIR_ConstrainSelectedInstOperands: { | 915 | 301 | int64_t InsnID = MatchTable[CurrentIdx++]; | 916 | 301 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 917 | 301 | constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, | 918 | 301 | RBI); | 919 | 301 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 920 | 301 | dbgs() << CurrentIdx | 921 | 301 | << ": GIR_ConstrainSelectedInstOperands(OutMIs[" | 922 | 301 | << InsnID << "])\n"); | 923 | 301 | break; | 924 | 0 | } | 925 | 0 |
| 926 | 0 | case GIR_MergeMemOperands: { | 927 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 928 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 929 | 0 |
| 930 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 931 | 0 | dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" | 932 | 0 | << InsnID << "]"); | 933 | 0 | int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; | 934 | 0 | while ((MergeInsnID = MatchTable[CurrentIdx++]) != | 935 | 0 | GIU_MergeMemOperands_EndOfList) { | 936 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 937 | 0 | dbgs() << ", MIs[" << MergeInsnID << "]"); | 938 | 0 | for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) | 939 | 0 | OutMIs[InsnID].addMemOperand(MMO); | 940 | 0 | } | 941 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); | 942 | 0 | break; | 943 | 0 | } | 944 | 0 |
| 945 | 284 | case GIR_EraseFromParent: { | 946 | 284 | int64_t InsnID = MatchTable[CurrentIdx++]; | 947 | 284 | assert(State.MIs[InsnID] && | 948 | 284 | "Attempted to erase an undefined instruction"); | 949 | 284 | State.MIs[InsnID]->eraseFromParent(); | 950 | 284 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 951 | 284 | dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" | 952 | 284 | << InsnID << "])\n"); | 953 | 284 | break; | 954 | 0 | } | 955 | 0 |
| 956 | 17 | case GIR_MakeTempReg: { | 957 | 17 | int64_t TempRegID = MatchTable[CurrentIdx++]; | 958 | 17 | int64_t TypeID = MatchTable[CurrentIdx++]; | 959 | 17 | | 960 | 17 | State.TempRegisters[TempRegID] = | 961 | 17 | MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); | 962 | 17 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 963 | 17 | dbgs() << CurrentIdx << ": TempRegs[" << TempRegID | 964 | 17 | << "] = GIR_MakeTempReg(" << TypeID << ")\n"); | 965 | 17 | break; | 966 | 0 | } | 967 | 0 |
| 968 | 0 | case GIR_Coverage: { | 969 | 0 | int64_t RuleID = MatchTable[CurrentIdx++]; | 970 | 0 | CoverageInfo.setCovered(RuleID); | 971 | 0 |
| 972 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 973 | 0 | dbgs() | 974 | 0 | << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); | 975 | 0 | break; | 976 | 0 | } | 977 | 0 |
| 978 | 293 | case GIR_Done: | 979 | 293 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 980 | 293 | dbgs() << CurrentIdx << ": GIR_Done\n"); | 981 | 293 | return true; | 982 | 0 |
| 983 | 0 | default: | 984 | 0 | llvm_unreachable("Unexpected command"); | 985 | 15.4k | } | 986 | 15.4k | } | 987 | 1.11k | } |
MipsInstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::MipsInstructionSelector const, llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::MipsInstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<42ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::MipsInstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<42ul> const&, llvm::CodeGenCoverage&) const Line | Count | Source | 55 | 1.40k | CodeGenCoverage &CoverageInfo) const { | 56 | 1.40k | | 57 | 1.40k | uint64_t CurrentIdx = 0; | 58 | 1.40k | SmallVector<uint64_t, 4> OnFailResumeAt; | 59 | 1.40k | | 60 | 1.40k | enum RejectAction { RejectAndGiveUp, RejectAndResume }; | 61 | 1.40k | auto handleReject = [&]() -> RejectAction { | 62 | 1.40k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 1.40k | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 1.40k | if (OnFailResumeAt.empty()) | 65 | 1.40k | return RejectAndGiveUp; | 66 | 1.40k | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 1.40k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 1.40k | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 1.40k | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 1.40k | return RejectAndResume; | 71 | 1.40k | }; | 72 | 1.40k | | 73 | 14.0k | while (true) { | 74 | 14.0k | assert(CurrentIdx != ~0u && "Invalid MatchTable index"); | 75 | 14.0k | int64_t MatcherOpcode = MatchTable[CurrentIdx++]; | 76 | 14.0k | switch (MatcherOpcode) { | 77 | 14.0k | case GIM_Try: { | 78 | 2.89k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 79 | 2.89k | dbgs() << CurrentIdx << ": Begin try-block\n"); | 80 | 2.89k | OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); | 81 | 2.89k | break; | 82 | 14.0k | } | 83 | 14.0k | | 84 | 14.0k | case GIM_RecordInsn: { | 85 | 120 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 86 | 120 | int64_t InsnID = MatchTable[CurrentIdx++]; | 87 | 120 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 88 | 120 | | 89 | 120 | // As an optimisation we require that MIs[0] is always the root. Refuse | 90 | 120 | // any attempt to modify it. | 91 | 120 | assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); | 92 | 120 | | 93 | 120 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 94 | 120 | if (!MO.isReg()) { | 95 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 96 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 97 | 0 | if (handleReject() == RejectAndGiveUp) | 98 | 0 | return false; | 99 | 0 | break; | 100 | 0 | } | 101 | 120 | if (TRI.isPhysicalRegister(MO.getReg())) { | 102 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 103 | 0 | dbgs() << CurrentIdx << ": Is a physical register\n"); | 104 | 0 | if (handleReject() == RejectAndGiveUp) | 105 | 0 | return false; | 106 | 0 | break; | 107 | 0 | } | 108 | 120 | | 109 | 120 | MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); | 110 | 120 | if ((size_t)NewInsnID < State.MIs.size()) | 111 | 4 | State.MIs[NewInsnID] = NewMI; | 112 | 116 | else { | 113 | 116 | assert((size_t)NewInsnID == State.MIs.size() && | 114 | 116 | "Expected to store MIs in order"); | 115 | 116 | State.MIs.push_back(NewMI); | 116 | 116 | } | 117 | 120 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 118 | 120 | dbgs() << CurrentIdx << ": MIs[" << NewInsnID | 119 | 120 | << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx | 120 | 120 | << ")\n"); | 121 | 120 | break; | 122 | 120 | } | 123 | 120 | | 124 | 2.19k | case GIM_CheckFeatures: { | 125 | 2.19k | int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; | 126 | 2.19k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 127 | 2.19k | dbgs() << CurrentIdx | 128 | 2.19k | << ": GIM_CheckFeatures(ExpectedBitsetID=" | 129 | 2.19k | << ExpectedBitsetID << ")\n"); | 130 | 2.19k | if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != | 131 | 2.19k | ISelInfo.FeatureBitsets[ExpectedBitsetID]) { | 132 | 1.60k | if (handleReject() == RejectAndGiveUp) | 133 | 0 | return false; | 134 | 2.19k | } | 135 | 2.19k | break; | 136 | 2.19k | } | 137 | 2.19k | | 138 | 2.19k | case GIM_CheckOpcode: { | 139 | 120 | int64_t InsnID = MatchTable[CurrentIdx++]; | 140 | 120 | int64_t Expected = MatchTable[CurrentIdx++]; | 141 | 120 | | 142 | 120 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 143 | 120 | unsigned Opcode = State.MIs[InsnID]->getOpcode(); | 144 | 120 | | 145 | 120 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 146 | 120 | dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID | 147 | 120 | << "], ExpectedOpcode=" << Expected | 148 | 120 | << ") // Got=" << Opcode << "\n"); | 149 | 120 | if (Opcode != Expected) { | 150 | 30 | if (handleReject() == RejectAndGiveUp) | 151 | 0 | return false; | 152 | 120 | } | 153 | 120 | break; | 154 | 120 | } | 155 | 120 | | 156 | 1.40k | case GIM_SwitchOpcode: { | 157 | 1.40k | int64_t InsnID = MatchTable[CurrentIdx++]; | 158 | 1.40k | int64_t LowerBound = MatchTable[CurrentIdx++]; | 159 | 1.40k | int64_t UpperBound = MatchTable[CurrentIdx++]; | 160 | 1.40k | int64_t Default = MatchTable[CurrentIdx++]; | 161 | 1.40k | | 162 | 1.40k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 163 | 1.40k | const int64_t Opcode = State.MIs[InsnID]->getOpcode(); | 164 | 1.40k | | 165 | 1.40k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 166 | 1.40k | dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" | 167 | 1.40k | << LowerBound << ", " << UpperBound << "), Default=" << Default | 168 | 1.40k | << ", JumpTable...) // Got=" << Opcode << "\n"; | 169 | 1.40k | }); | 170 | 1.40k | if (Opcode < LowerBound || UpperBound <= Opcode) { | 171 | 0 | CurrentIdx = Default; | 172 | 0 | break; | 173 | 0 | } | 174 | 1.40k | CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; | 175 | 1.40k | if (!CurrentIdx) { | 176 | 463 | CurrentIdx = Default; | 177 | 463 | break; | 178 | 463 | } | 179 | 942 | OnFailResumeAt.push_back(Default); | 180 | 942 | break; | 181 | 942 | } | 182 | 942 | | 183 | 942 | case GIM_SwitchType: { | 184 | 516 | int64_t InsnID = MatchTable[CurrentIdx++]; | 185 | 516 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 186 | 516 | int64_t LowerBound = MatchTable[CurrentIdx++]; | 187 | 516 | int64_t UpperBound = MatchTable[CurrentIdx++]; | 188 | 516 | int64_t Default = MatchTable[CurrentIdx++]; | 189 | 516 | | 190 | 516 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 191 | 516 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 192 | 516 | | 193 | 516 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 194 | 516 | dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID | 195 | 516 | << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " | 196 | 516 | << UpperBound << "), Default=" << Default | 197 | 516 | << ", JumpTable...) // Got="; | 198 | 516 | if (!MO.isReg()) | 199 | 516 | dbgs() << "Not a VReg\n"; | 200 | 516 | else | 201 | 516 | dbgs() << MRI.getType(MO.getReg()) << "\n"; | 202 | 516 | }); | 203 | 516 | if (!MO.isReg()) { | 204 | 0 | CurrentIdx = Default; | 205 | 0 | break; | 206 | 0 | } | 207 | 516 | const LLT Ty = MRI.getType(MO.getReg()); | 208 | 516 | const auto TyI = ISelInfo.TypeIDMap.find(Ty); | 209 | 516 | if (TyI == ISelInfo.TypeIDMap.end()) { | 210 | 3 | CurrentIdx = Default; | 211 | 3 | break; | 212 | 3 | } | 213 | 513 | const int64_t TypeID = TyI->second; | 214 | 513 | if (TypeID < LowerBound || UpperBound <= TypeID497 ) { | 215 | 16 | CurrentIdx = Default; | 216 | 16 | break; | 217 | 16 | } | 218 | 497 | CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; | 219 | 497 | if (!CurrentIdx) { | 220 | 0 | CurrentIdx = Default; | 221 | 0 | break; | 222 | 0 | } | 223 | 497 | OnFailResumeAt.push_back(Default); | 224 | 497 | break; | 225 | 497 | } | 226 | 497 | | 227 | 497 | case GIM_CheckNumOperands: { | 228 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 229 | 0 | int64_t Expected = MatchTable[CurrentIdx++]; | 230 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 231 | 0 | dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" | 232 | 0 | << InsnID << "], Expected=" << Expected << ")\n"); | 233 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 234 | 0 | if (State.MIs[InsnID]->getNumOperands() != Expected) { | 235 | 0 | if (handleReject() == RejectAndGiveUp) | 236 | 0 | return false; | 237 | 0 | } | 238 | 0 | break; | 239 | 0 | } | 240 | 90 | case GIM_CheckI64ImmPredicate: { | 241 | 90 | int64_t InsnID = MatchTable[CurrentIdx++]; | 242 | 90 | int64_t Predicate = MatchTable[CurrentIdx++]; | 243 | 90 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 244 | 90 | dbgs() | 245 | 90 | << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" | 246 | 90 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 247 | 90 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 248 | 90 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 249 | 90 | "Expected G_CONSTANT"); | 250 | 90 | assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); | 251 | 90 | int64_t Value = 0; | 252 | 90 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 253 | 90 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); | 254 | 0 | else if (State.MIs[InsnID]->getOperand(1).isImm()) | 255 | 0 | Value = State.MIs[InsnID]->getOperand(1).getImm(); | 256 | 0 | else | 257 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 258 | 90 | | 259 | 90 | if (!testImmPredicate_I64(Predicate, Value)) | 260 | 0 | if (handleReject() == RejectAndGiveUp) | 261 | 0 | return false; | 262 | 90 | break; | 263 | 90 | } | 264 | 90 | case GIM_CheckAPIntImmPredicate: { | 265 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 266 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 267 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 268 | 0 | dbgs() | 269 | 0 | << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" | 270 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 271 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 272 | 0 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 273 | 0 | "Expected G_CONSTANT"); | 274 | 0 | assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); | 275 | 0 | APInt Value; | 276 | 0 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 277 | 0 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); | 278 | 0 | else | 279 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 280 | 0 |
| 281 | 0 | if (!testImmPredicate_APInt(Predicate, Value)) | 282 | 0 | if (handleReject() == RejectAndGiveUp) | 283 | 0 | return false; | 284 | 0 | break; | 285 | 0 | } | 286 | 0 | case GIM_CheckAPFloatImmPredicate: { | 287 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 288 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 289 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 290 | 0 | dbgs() | 291 | 0 | << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" | 292 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 293 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 294 | 0 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && | 295 | 0 | "Expected G_FCONSTANT"); | 296 | 0 | assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); | 297 | 0 | assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); | 298 | 0 | APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); | 299 | 0 |
| 300 | 0 | if (!testImmPredicate_APFloat(Predicate, Value)) | 301 | 0 | if (handleReject() == RejectAndGiveUp) | 302 | 0 | return false; | 303 | 0 | break; | 304 | 0 | } | 305 | 0 | case GIM_CheckCxxInsnPredicate: { | 306 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 307 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 308 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 309 | 0 | dbgs() | 310 | 0 | << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" | 311 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 312 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 313 | 0 | assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); | 314 | 0 |
| 315 | 0 | if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID])) | 316 | 0 | if (handleReject() == RejectAndGiveUp) | 317 | 0 | return false; | 318 | 0 | break; | 319 | 0 | } | 320 | 0 | case GIM_CheckAtomicOrdering: { | 321 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 322 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 323 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 324 | 0 | dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" | 325 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 326 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 327 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 328 | 0 | if (handleReject() == RejectAndGiveUp) | 329 | 0 | return false; | 330 | 0 | | 331 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 332 | 0 | if (MMO->getOrdering() != Ordering) | 333 | 0 | if (handleReject() == RejectAndGiveUp) | 334 | 0 | return false; | 335 | 0 | break; | 336 | 0 | } | 337 | 0 | case GIM_CheckAtomicOrderingOrStrongerThan: { | 338 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 339 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 340 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 341 | 0 | dbgs() << CurrentIdx | 342 | 0 | << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" | 343 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 344 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 345 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 346 | 0 | if (handleReject() == RejectAndGiveUp) | 347 | 0 | return false; | 348 | 0 | | 349 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 350 | 0 | if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) | 351 | 0 | if (handleReject() == RejectAndGiveUp) | 352 | 0 | return false; | 353 | 0 | break; | 354 | 0 | } | 355 | 0 | case GIM_CheckAtomicOrderingWeakerThan: { | 356 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 357 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 358 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 359 | 0 | dbgs() << CurrentIdx | 360 | 0 | << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" | 361 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 362 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 363 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 364 | 0 | if (handleReject() == RejectAndGiveUp) | 365 | 0 | return false; | 366 | 0 | | 367 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 368 | 0 | if (!isStrongerThan(Ordering, MMO->getOrdering())) | 369 | 0 | if (handleReject() == RejectAndGiveUp) | 370 | 0 | return false; | 371 | 0 | break; | 372 | 0 | } | 373 | 0 | case GIM_CheckMemoryAddressSpace: { | 374 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 375 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 376 | 0 | // This accepts a list of possible address spaces. | 377 | 0 | const int NumAddrSpace = MatchTable[CurrentIdx++]; | 378 | 0 |
| 379 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 380 | 0 | if (handleReject() == RejectAndGiveUp) | 381 | 0 | return false; | 382 | 0 | break; | 383 | 0 | } | 384 | 0 | | 385 | 0 | // Need to still jump to the end of the list of address spaces if we find | 386 | 0 | // a match earlier. | 387 | 0 | const uint64_t LastIdx = CurrentIdx + NumAddrSpace; | 388 | 0 |
| 389 | 0 | const MachineMemOperand *MMO | 390 | 0 | = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 391 | 0 | const unsigned MMOAddrSpace = MMO->getAddrSpace(); | 392 | 0 |
| 393 | 0 | bool Success = false; | 394 | 0 | for (int I = 0; I != NumAddrSpace; ++I) { | 395 | 0 | unsigned AddrSpace = MatchTable[CurrentIdx++]; | 396 | 0 | DEBUG_WITH_TYPE( | 397 | 0 | TgtInstructionSelector::getName(), | 398 | 0 | dbgs() << "addrspace(" << MMOAddrSpace << ") vs " | 399 | 0 | << AddrSpace << '\n'); | 400 | 0 |
| 401 | 0 | if (AddrSpace == MMOAddrSpace) { | 402 | 0 | Success = true; | 403 | 0 | break; | 404 | 0 | } | 405 | 0 | } | 406 | 0 |
| 407 | 0 | CurrentIdx = LastIdx; | 408 | 0 | if (!Success && handleReject() == RejectAndGiveUp) | 409 | 0 | return false; | 410 | 0 | break; | 411 | 0 | } | 412 | 0 | case GIM_CheckMemorySizeEqualTo: { | 413 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 414 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 415 | 0 | uint64_t Size = MatchTable[CurrentIdx++]; | 416 | 0 |
| 417 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 418 | 0 | dbgs() << CurrentIdx | 419 | 0 | << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID | 420 | 0 | << "]->memoperands() + " << MMOIdx | 421 | 0 | << ", Size=" << Size << ")\n"); | 422 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 423 | 0 |
| 424 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 425 | 0 | if (handleReject() == RejectAndGiveUp) | 426 | 0 | return false; | 427 | 0 | break; | 428 | 0 | } | 429 | 0 | | 430 | 0 | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 431 | 0 |
| 432 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 433 | 0 | dbgs() << MMO->getSize() << " bytes vs " << Size | 434 | 0 | << " bytes\n"); | 435 | 0 | if (MMO->getSize() != Size) | 436 | 0 | if (handleReject() == RejectAndGiveUp) | 437 | 0 | return false; | 438 | 0 | | 439 | 0 | break; | 440 | 0 | } | 441 | 0 | case GIM_CheckMemorySizeEqualToLLT: | 442 | 0 | case GIM_CheckMemorySizeLessThanLLT: | 443 | 0 | case GIM_CheckMemorySizeGreaterThanLLT: { | 444 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 445 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 446 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 447 | 0 |
| 448 | 0 | DEBUG_WITH_TYPE( | 449 | 0 | TgtInstructionSelector::getName(), | 450 | 0 | dbgs() << CurrentIdx << ": GIM_CheckMemorySize" | 451 | 0 | << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT | 452 | 0 | ? "EqualTo" | 453 | 0 | : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT | 454 | 0 | ? "GreaterThan" | 455 | 0 | : "LessThan") | 456 | 0 | << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx | 457 | 0 | << ", OpIdx=" << OpIdx << ")\n"); | 458 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 459 | 0 |
| 460 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 461 | 0 | if (!MO.isReg()) { | 462 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 463 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 464 | 0 | if (handleReject() == RejectAndGiveUp) | 465 | 0 | return false; | 466 | 0 | break; | 467 | 0 | } | 468 | 0 | | 469 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 470 | 0 | if (handleReject() == RejectAndGiveUp) | 471 | 0 | return false; | 472 | 0 | break; | 473 | 0 | } | 474 | 0 | | 475 | 0 | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 476 | 0 |
| 477 | 0 | unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); | 478 | 0 | if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && | 479 | 0 | MMO->getSizeInBits() != Size) { | 480 | 0 | if (handleReject() == RejectAndGiveUp) | 481 | 0 | return false; | 482 | 0 | } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && | 483 | 0 | MMO->getSizeInBits() >= Size) { | 484 | 0 | if (handleReject() == RejectAndGiveUp) | 485 | 0 | return false; | 486 | 0 | } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && | 487 | 0 | MMO->getSizeInBits() <= Size) | 488 | 0 | if (handleReject() == RejectAndGiveUp) | 489 | 0 | return false; | 490 | 0 | | 491 | 0 | break; | 492 | 0 | } | 493 | 1.21k | case GIM_CheckType: { | 494 | 1.21k | int64_t InsnID = MatchTable[CurrentIdx++]; | 495 | 1.21k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 496 | 1.21k | int64_t TypeID = MatchTable[CurrentIdx++]; | 497 | 1.21k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 498 | 1.21k | dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID | 499 | 1.21k | << "]->getOperand(" << OpIdx | 500 | 1.21k | << "), TypeID=" << TypeID << ")\n"); | 501 | 1.21k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 502 | 1.21k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 503 | 1.21k | if (!MO.isReg() || | 504 | 1.21k | MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) { | 505 | 0 | if (handleReject() == RejectAndGiveUp) | 506 | 0 | return false; | 507 | 1.21k | } | 508 | 1.21k | break; | 509 | 1.21k | } | 510 | 1.21k | case GIM_CheckPointerToAny: { | 511 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 512 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 513 | 0 | int64_t SizeInBits = MatchTable[CurrentIdx++]; | 514 | 0 |
| 515 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 516 | 0 | dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" | 517 | 0 | << InsnID << "]->getOperand(" << OpIdx | 518 | 0 | << "), SizeInBits=" << SizeInBits << ")\n"); | 519 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 520 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 521 | 0 | const LLT Ty = MRI.getType(MO.getReg()); | 522 | 0 |
| 523 | 0 | // iPTR must be looked up in the target. | 524 | 0 | if (SizeInBits == 0) { | 525 | 0 | MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); | 526 | 0 | const unsigned AddrSpace = Ty.getAddressSpace(); | 527 | 0 | SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); | 528 | 0 | } | 529 | 0 |
| 530 | 0 | assert(SizeInBits != 0 && "Pointer size must be known"); | 531 | 0 |
| 532 | 0 | if (MO.isReg()) { | 533 | 0 | if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) | 534 | 0 | if (handleReject() == RejectAndGiveUp) | 535 | 0 | return false; | 536 | 0 | } else if (handleReject() == RejectAndGiveUp) | 537 | 0 | return false; | 538 | 0 | | 539 | 0 | break; | 540 | 0 | } | 541 | 1.46k | case GIM_CheckRegBankForClass: { | 542 | 1.46k | int64_t InsnID = MatchTable[CurrentIdx++]; | 543 | 1.46k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 544 | 1.46k | int64_t RCEnum = MatchTable[CurrentIdx++]; | 545 | 1.46k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 546 | 1.46k | dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" | 547 | 1.46k | << InsnID << "]->getOperand(" << OpIdx | 548 | 1.46k | << "), RCEnum=" << RCEnum << ")\n"); | 549 | 1.46k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 550 | 1.46k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 551 | 1.46k | if (!MO.isReg() || | 552 | 1.46k | &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != | 553 | 1.46k | RBI.getRegBank(MO.getReg(), MRI, TRI)) { | 554 | 7 | if (handleReject() == RejectAndGiveUp) | 555 | 0 | return false; | 556 | 1.46k | } | 557 | 1.46k | break; | 558 | 1.46k | } | 559 | 1.46k | | 560 | 1.46k | case GIM_CheckComplexPattern: { | 561 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 562 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 563 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 564 | 0 | int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; | 565 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 566 | 0 | dbgs() << CurrentIdx << ": State.Renderers[" << RendererID | 567 | 0 | << "] = GIM_CheckComplexPattern(MIs[" << InsnID | 568 | 0 | << "]->getOperand(" << OpIdx | 569 | 0 | << "), ComplexPredicateID=" << ComplexPredicateID | 570 | 0 | << ")\n"); | 571 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 572 | 0 | // FIXME: Use std::invoke() when it's available. | 573 | 0 | ComplexRendererFns Renderer = | 574 | 0 | (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( | 575 | 0 | State.MIs[InsnID]->getOperand(OpIdx)); | 576 | 0 | if (Renderer.hasValue()) | 577 | 0 | State.Renderers[RendererID] = Renderer.getValue(); | 578 | 0 | else | 579 | 0 | if (handleReject() == RejectAndGiveUp) | 580 | 0 | return false; | 581 | 0 | break; | 582 | 0 | } | 583 | 0 |
| 584 | 12 | case GIM_CheckConstantInt: { | 585 | 12 | int64_t InsnID = MatchTable[CurrentIdx++]; | 586 | 12 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 587 | 12 | int64_t Value = MatchTable[CurrentIdx++]; | 588 | 12 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 589 | 12 | dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" | 590 | 12 | << InsnID << "]->getOperand(" << OpIdx | 591 | 12 | << "), Value=" << Value << ")\n"); | 592 | 12 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 593 | 12 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 594 | 12 | if (MO.isReg()) { | 595 | 12 | // isOperandImmEqual() will sign-extend to 64-bits, so should we. | 596 | 12 | LLT Ty = MRI.getType(MO.getReg()); | 597 | 12 | Value = SignExtend64(Value, Ty.getSizeInBits()); | 598 | 12 | | 599 | 12 | if (!isOperandImmEqual(MO, Value, MRI)) { | 600 | 11 | if (handleReject() == RejectAndGiveUp) | 601 | 0 | return false; | 602 | 0 | } | 603 | 0 | } else if (handleReject() == RejectAndGiveUp) | 604 | 0 | return false; | 605 | 12 | | 606 | 12 | break; | 607 | 12 | } | 608 | 12 | | 609 | 12 | case GIM_CheckLiteralInt: { | 610 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 611 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 612 | 0 | int64_t Value = MatchTable[CurrentIdx++]; | 613 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 614 | 0 | dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" | 615 | 0 | << InsnID << "]->getOperand(" << OpIdx | 616 | 0 | << "), Value=" << Value << ")\n"); | 617 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 618 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 619 | 0 | if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { | 620 | 0 | if (handleReject() == RejectAndGiveUp) | 621 | 0 | return false; | 622 | 0 | } | 623 | 0 | break; | 624 | 0 | } | 625 | 0 |
| 626 | 0 | case GIM_CheckIntrinsicID: { | 627 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 628 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 629 | 0 | int64_t Value = MatchTable[CurrentIdx++]; | 630 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 631 | 0 | dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" | 632 | 0 | << InsnID << "]->getOperand(" << OpIdx | 633 | 0 | << "), Value=" << Value << ")\n"); | 634 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 635 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 636 | 0 | if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value) | 637 | 0 | if (handleReject() == RejectAndGiveUp) | 638 | 0 | return false; | 639 | 0 | break; | 640 | 0 | } | 641 | 0 |
| 642 | 80 | case GIM_CheckIsMBB: { | 643 | 80 | int64_t InsnID = MatchTable[CurrentIdx++]; | 644 | 80 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 645 | 80 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 646 | 80 | dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID | 647 | 80 | << "]->getOperand(" << OpIdx << "))\n"); | 648 | 80 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 649 | 80 | if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { | 650 | 0 | if (handleReject() == RejectAndGiveUp) | 651 | 0 | return false; | 652 | 80 | } | 653 | 80 | break; | 654 | 80 | } | 655 | 80 | | 656 | 90 | case GIM_CheckIsSafeToFold: { | 657 | 90 | int64_t InsnID = MatchTable[CurrentIdx++]; | 658 | 90 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 659 | 90 | dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" | 660 | 90 | << InsnID << "])\n"); | 661 | 90 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 662 | 90 | if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { | 663 | 0 | if (handleReject() == RejectAndGiveUp) | 664 | 0 | return false; | 665 | 90 | } | 666 | 90 | break; | 667 | 90 | } | 668 | 90 | case GIM_CheckIsSameOperand: { | 669 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 670 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 671 | 0 | int64_t OtherInsnID = MatchTable[CurrentIdx++]; | 672 | 0 | int64_t OtherOpIdx = MatchTable[CurrentIdx++]; | 673 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 674 | 0 | dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" | 675 | 0 | << InsnID << "][" << OpIdx << "], MIs[" | 676 | 0 | << OtherInsnID << "][" << OtherOpIdx << "])\n"); | 677 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 678 | 0 | assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); | 679 | 0 | if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( | 680 | 0 | State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { | 681 | 0 | if (handleReject() == RejectAndGiveUp) | 682 | 0 | return false; | 683 | 0 | } | 684 | 0 | break; | 685 | 0 | } | 686 | 1.51k | case GIM_Reject: | 687 | 1.51k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 688 | 1.51k | dbgs() << CurrentIdx << ": GIM_Reject\n"); | 689 | 1.51k | if (handleReject() == RejectAndGiveUp) | 690 | 861 | return false; | 691 | 651 | break; | 692 | 651 | | 693 | 651 | case GIR_MutateOpcode: { | 694 | 405 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 695 | 405 | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 696 | 405 | int64_t NewOpcode = MatchTable[CurrentIdx++]; | 697 | 405 | if (NewInsnID >= OutMIs.size()) | 698 | 405 | OutMIs.resize(NewInsnID + 1); | 699 | 405 | | 700 | 405 | OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), | 701 | 405 | State.MIs[OldInsnID]); | 702 | 405 | OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); | 703 | 405 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 704 | 405 | dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" | 705 | 405 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 706 | 405 | << NewOpcode << ")\n"); | 707 | 405 | break; | 708 | 651 | } | 709 | 651 | | 710 | 651 | case GIR_BuildMI: { | 711 | 139 | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 712 | 139 | int64_t Opcode = MatchTable[CurrentIdx++]; | 713 | 139 | if (NewInsnID >= OutMIs.size()) | 714 | 139 | OutMIs.resize(NewInsnID + 1); | 715 | 139 | | 716 | 139 | OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], | 717 | 139 | State.MIs[0]->getDebugLoc(), TII.get(Opcode)); | 718 | 139 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 719 | 139 | dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" | 720 | 139 | << NewInsnID << "], " << Opcode << ")\n"); | 721 | 139 | break; | 722 | 651 | } | 723 | 651 | | 724 | 651 | case GIR_Copy: { | 725 | 374 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 726 | 374 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 727 | 374 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 728 | 374 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 729 | 374 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); | 730 | 374 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 731 | 374 | dbgs() | 732 | 374 | << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID | 733 | 374 | << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); | 734 | 374 | break; | 735 | 651 | } | 736 | 651 | | 737 | 651 | case GIR_CopyOrAddZeroReg: { | 738 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 739 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 740 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 741 | 0 | int64_t ZeroReg = MatchTable[CurrentIdx++]; | 742 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 743 | 0 | MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); | 744 | 0 | if (isOperandImmEqual(MO, 0, MRI)) | 745 | 0 | OutMIs[NewInsnID].addReg(ZeroReg); | 746 | 0 | else | 747 | 0 | OutMIs[NewInsnID].add(MO); | 748 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 749 | 0 | dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" | 750 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 751 | 0 | << OpIdx << ", " << ZeroReg << ")\n"); | 752 | 0 | break; | 753 | 651 | } | 754 | 651 | | 755 | 651 | case GIR_CopySubReg: { | 756 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 757 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 758 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 759 | 0 | int64_t SubRegIdx = MatchTable[CurrentIdx++]; | 760 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 761 | 0 | OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), | 762 | 0 | 0, SubRegIdx); | 763 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 764 | 0 | dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" | 765 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 766 | 0 | << OpIdx << ", " << SubRegIdx << ")\n"); | 767 | 0 | break; | 768 | 651 | } | 769 | 651 | | 770 | 651 | case GIR_AddImplicitDef: { | 771 | 80 | int64_t InsnID = MatchTable[CurrentIdx++]; | 772 | 80 | int64_t RegNum = MatchTable[CurrentIdx++]; | 773 | 80 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 774 | 80 | OutMIs[InsnID].addDef(RegNum, RegState::Implicit); | 775 | 80 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 776 | 80 | dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" | 777 | 80 | << InsnID << "], " << RegNum << ")\n"); | 778 | 80 | break; | 779 | 651 | } | 780 | 651 | | 781 | 651 | case GIR_AddImplicitUse: { | 782 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 783 | 0 | int64_t RegNum = MatchTable[CurrentIdx++]; | 784 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 785 | 0 | OutMIs[InsnID].addUse(RegNum, RegState::Implicit); | 786 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 787 | 0 | dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" | 788 | 0 | << InsnID << "], " << RegNum << ")\n"); | 789 | 0 | break; | 790 | 651 | } | 791 | 651 | | 792 | 651 | case GIR_AddRegister: { | 793 | 1 | int64_t InsnID = MatchTable[CurrentIdx++]; | 794 | 1 | int64_t RegNum = MatchTable[CurrentIdx++]; | 795 | 1 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 796 | 1 | OutMIs[InsnID].addReg(RegNum); | 797 | 1 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 798 | 1 | dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" | 799 | 1 | << InsnID << "], " << RegNum << ")\n"); | 800 | 1 | break; | 801 | 651 | } | 802 | 651 | | 803 | 651 | case GIR_AddTempRegister: { | 804 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 805 | 0 | int64_t TempRegID = MatchTable[CurrentIdx++]; | 806 | 0 | uint64_t TempRegFlags = MatchTable[CurrentIdx++]; | 807 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 808 | 0 | OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags); | 809 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 810 | 0 | dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" | 811 | 0 | << InsnID << "], TempRegisters[" << TempRegID | 812 | 0 | << "], " << TempRegFlags << ")\n"); | 813 | 0 | break; | 814 | 651 | } | 815 | 651 | | 816 | 651 | case GIR_AddImm: { | 817 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 818 | 0 | int64_t Imm = MatchTable[CurrentIdx++]; | 819 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 820 | 0 | OutMIs[InsnID].addImm(Imm); | 821 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 822 | 0 | dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID | 823 | 0 | << "], " << Imm << ")\n"); | 824 | 0 | break; | 825 | 651 | } | 826 | 651 | | 827 | 651 | case GIR_ComplexRenderer: { | 828 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 829 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 830 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 831 | 0 | for (const auto &RenderOpFn : State.Renderers[RendererID]) | 832 | 0 | RenderOpFn(OutMIs[InsnID]); | 833 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 834 | 0 | dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" | 835 | 0 | << InsnID << "], " << RendererID << ")\n"); | 836 | 0 | break; | 837 | 651 | } | 838 | 651 | case GIR_ComplexSubOperandRenderer: { | 839 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 840 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 841 | 0 | int64_t RenderOpID = MatchTable[CurrentIdx++]; | 842 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 843 | 0 | State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); | 844 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 845 | 0 | dbgs() << CurrentIdx | 846 | 0 | << ": GIR_ComplexSubOperandRenderer(OutMIs[" | 847 | 0 | << InsnID << "], " << RendererID << ", " | 848 | 0 | << RenderOpID << ")\n"); | 849 | 0 | break; | 850 | 651 | } | 851 | 651 | | 852 | 651 | case GIR_CopyConstantAsSImm: { | 853 | 90 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 854 | 90 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 855 | 90 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 856 | 90 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); | 857 | 90 | if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { | 858 | 90 | OutMIs[NewInsnID].addImm( | 859 | 90 | State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); | 860 | 90 | } else if (0 State.MIs[OldInsnID]->getOperand(1).isImm()0 ) | 861 | 0 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); | 862 | 0 | else | 863 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 864 | 90 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 865 | 90 | dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" | 866 | 90 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 867 | 90 | break; | 868 | 90 | } | 869 | 90 | | 870 | 90 | // TODO: Needs a test case once we have a pattern that uses this. | 871 | 90 | case GIR_CopyFConstantAsFPImm: { | 872 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 873 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 874 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 875 | 0 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); | 876 | 0 | if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) | 877 | 0 | OutMIs[NewInsnID].addFPImm( | 878 | 0 | State.MIs[OldInsnID]->getOperand(1).getFPImm()); | 879 | 0 | else | 880 | 0 | llvm_unreachable("Expected FPImm operand"); | 881 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 882 | 0 | dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" | 883 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 884 | 0 | break; | 885 | 0 | } | 886 | 0 |
| 887 | 0 | case GIR_CustomRenderer: { | 888 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 889 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 890 | 0 | int64_t RendererFnID = MatchTable[CurrentIdx++]; | 891 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 892 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 893 | 0 | dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" | 894 | 0 | << InsnID << "], MIs[" << OldInsnID << "], " | 895 | 0 | << RendererFnID << ")\n"); | 896 | 0 | (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], | 897 | 0 | *State.MIs[OldInsnID]); | 898 | 0 | break; | 899 | 0 | } | 900 | 0 | case GIR_ConstrainOperandRC: { | 901 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 902 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 903 | 0 | int64_t RCEnum = MatchTable[CurrentIdx++]; | 904 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 905 | 0 | constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, | 906 | 0 | *TRI.getRegClass(RCEnum), TII, TRI, RBI); | 907 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 908 | 0 | dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" | 909 | 0 | << InsnID << "], " << OpIdx << ", " << RCEnum | 910 | 0 | << ")\n"); | 911 | 0 | break; | 912 | 0 | } | 913 | 0 |
| 914 | 544 | case GIR_ConstrainSelectedInstOperands: { | 915 | 544 | int64_t InsnID = MatchTable[CurrentIdx++]; | 916 | 544 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 917 | 544 | constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, | 918 | 544 | RBI); | 919 | 544 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 920 | 544 | dbgs() << CurrentIdx | 921 | 544 | << ": GIR_ConstrainSelectedInstOperands(OutMIs[" | 922 | 544 | << InsnID << "])\n"); | 923 | 544 | break; | 924 | 0 | } | 925 | 0 |
| 926 | 0 | case GIR_MergeMemOperands: { | 927 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 928 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 929 | 0 |
| 930 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 931 | 0 | dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" | 932 | 0 | << InsnID << "]"); | 933 | 0 | int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; | 934 | 0 | while ((MergeInsnID = MatchTable[CurrentIdx++]) != | 935 | 0 | GIU_MergeMemOperands_EndOfList) { | 936 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 937 | 0 | dbgs() << ", MIs[" << MergeInsnID << "]"); | 938 | 0 | for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) | 939 | 0 | OutMIs[InsnID].addMemOperand(MMO); | 940 | 0 | } | 941 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); | 942 | 0 | break; | 943 | 0 | } | 944 | 0 |
| 945 | 139 | case GIR_EraseFromParent: { | 946 | 139 | int64_t InsnID = MatchTable[CurrentIdx++]; | 947 | 139 | assert(State.MIs[InsnID] && | 948 | 139 | "Attempted to erase an undefined instruction"); | 949 | 139 | State.MIs[InsnID]->eraseFromParent(); | 950 | 139 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 951 | 139 | dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" | 952 | 139 | << InsnID << "])\n"); | 953 | 139 | break; | 954 | 0 | } | 955 | 0 |
| 956 | 0 | case GIR_MakeTempReg: { | 957 | 0 | int64_t TempRegID = MatchTable[CurrentIdx++]; | 958 | 0 | int64_t TypeID = MatchTable[CurrentIdx++]; | 959 | 0 |
| 960 | 0 | State.TempRegisters[TempRegID] = | 961 | 0 | MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); | 962 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 963 | 0 | dbgs() << CurrentIdx << ": TempRegs[" << TempRegID | 964 | 0 | << "] = GIR_MakeTempReg(" << TypeID << ")\n"); | 965 | 0 | break; | 966 | 0 | } | 967 | 0 |
| 968 | 0 | case GIR_Coverage: { | 969 | 0 | int64_t RuleID = MatchTable[CurrentIdx++]; | 970 | 0 | CoverageInfo.setCovered(RuleID); | 971 | 0 |
| 972 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 973 | 0 | dbgs() | 974 | 0 | << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); | 975 | 0 | break; | 976 | 0 | } | 977 | 0 |
| 978 | 544 | case GIR_Done: | 979 | 544 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 980 | 544 | dbgs() << CurrentIdx << ": GIR_Done\n"); | 981 | 544 | return true; | 982 | 0 |
| 983 | 0 | default: | 984 | 0 | llvm_unreachable("Unexpected command"); | 985 | 14.0k | } | 986 | 14.0k | } | 987 | 1.40k | } |
X86InstructionSelector.cpp:bool llvm::InstructionSelector::executeMatchTable<(anonymous namespace)::X86InstructionSelector const, llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const>((anonymous namespace)::X86InstructionSelector const&, llvm::SmallVector<llvm::MachineInstrBuilder, 4u>&, llvm::InstructionSelector::MatcherState&, llvm::InstructionSelector::ISelInfoTy<llvm::PredicateBitsetImpl<114ul>, llvm::Optional<llvm::SmallVector<std::__1::function<void (llvm::MachineInstrBuilder&)>, 4u> > ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineOperand&) const, void ((anonymous namespace)::X86InstructionSelector::*)(llvm::MachineInstrBuilder&, llvm::MachineInstr const&) const> const&, long long const*, llvm::TargetInstrInfo const&, llvm::MachineRegisterInfo&, llvm::TargetRegisterInfo const&, llvm::RegisterBankInfo const&, llvm::PredicateBitsetImpl<114ul> const&, llvm::CodeGenCoverage&) const Line | Count | Source | 55 | 2.27k | CodeGenCoverage &CoverageInfo) const { | 56 | 2.27k | | 57 | 2.27k | uint64_t CurrentIdx = 0; | 58 | 2.27k | SmallVector<uint64_t, 4> OnFailResumeAt; | 59 | 2.27k | | 60 | 2.27k | enum RejectAction { RejectAndGiveUp, RejectAndResume }; | 61 | 2.27k | auto handleReject = [&]() -> RejectAction { | 62 | 2.27k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 63 | 2.27k | dbgs() << CurrentIdx << ": Rejected\n"); | 64 | 2.27k | if (OnFailResumeAt.empty()) | 65 | 2.27k | return RejectAndGiveUp; | 66 | 2.27k | CurrentIdx = OnFailResumeAt.pop_back_val(); | 67 | 2.27k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 68 | 2.27k | dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " (" | 69 | 2.27k | << OnFailResumeAt.size() << " try-blocks remain)\n"); | 70 | 2.27k | return RejectAndResume; | 71 | 2.27k | }; | 72 | 2.27k | | 73 | 19.3k | while (true) { | 74 | 19.3k | assert(CurrentIdx != ~0u && "Invalid MatchTable index"); | 75 | 19.3k | int64_t MatcherOpcode = MatchTable[CurrentIdx++]; | 76 | 19.3k | switch (MatcherOpcode) { | 77 | 19.3k | case GIM_Try: { | 78 | 3.08k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 79 | 3.08k | dbgs() << CurrentIdx << ": Begin try-block\n"); | 80 | 3.08k | OnFailResumeAt.push_back(MatchTable[CurrentIdx++]); | 81 | 3.08k | break; | 82 | 19.3k | } | 83 | 19.3k | | 84 | 19.3k | case GIM_RecordInsn: { | 85 | 370 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 86 | 370 | int64_t InsnID = MatchTable[CurrentIdx++]; | 87 | 370 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 88 | 370 | | 89 | 370 | // As an optimisation we require that MIs[0] is always the root. Refuse | 90 | 370 | // any attempt to modify it. | 91 | 370 | assert(NewInsnID != 0 && "Refusing to modify MIs[0]"); | 92 | 370 | | 93 | 370 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 94 | 370 | if (!MO.isReg()) { | 95 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 96 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 97 | 0 | if (handleReject() == RejectAndGiveUp) | 98 | 0 | return false; | 99 | 0 | break; | 100 | 0 | } | 101 | 370 | if (TRI.isPhysicalRegister(MO.getReg())) { | 102 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 103 | 0 | dbgs() << CurrentIdx << ": Is a physical register\n"); | 104 | 0 | if (handleReject() == RejectAndGiveUp) | 105 | 0 | return false; | 106 | 0 | break; | 107 | 0 | } | 108 | 370 | | 109 | 370 | MachineInstr *NewMI = MRI.getVRegDef(MO.getReg()); | 110 | 370 | if ((size_t)NewInsnID < State.MIs.size()) | 111 | 110 | State.MIs[NewInsnID] = NewMI; | 112 | 260 | else { | 113 | 260 | assert((size_t)NewInsnID == State.MIs.size() && | 114 | 260 | "Expected to store MIs in order"); | 115 | 260 | State.MIs.push_back(NewMI); | 116 | 260 | } | 117 | 370 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 118 | 370 | dbgs() << CurrentIdx << ": MIs[" << NewInsnID | 119 | 370 | << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx | 120 | 370 | << ")\n"); | 121 | 370 | break; | 122 | 370 | } | 123 | 370 | | 124 | 1.47k | case GIM_CheckFeatures: { | 125 | 1.47k | int64_t ExpectedBitsetID = MatchTable[CurrentIdx++]; | 126 | 1.47k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 127 | 1.47k | dbgs() << CurrentIdx | 128 | 1.47k | << ": GIM_CheckFeatures(ExpectedBitsetID=" | 129 | 1.47k | << ExpectedBitsetID << ")\n"); | 130 | 1.47k | if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) != | 131 | 1.47k | ISelInfo.FeatureBitsets[ExpectedBitsetID]) { | 132 | 887 | if (handleReject() == RejectAndGiveUp) | 133 | 0 | return false; | 134 | 1.47k | } | 135 | 1.47k | break; | 136 | 1.47k | } | 137 | 1.47k | | 138 | 1.47k | case GIM_CheckOpcode: { | 139 | 370 | int64_t InsnID = MatchTable[CurrentIdx++]; | 140 | 370 | int64_t Expected = MatchTable[CurrentIdx++]; | 141 | 370 | | 142 | 370 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 143 | 370 | unsigned Opcode = State.MIs[InsnID]->getOpcode(); | 144 | 370 | | 145 | 370 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 146 | 370 | dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID | 147 | 370 | << "], ExpectedOpcode=" << Expected | 148 | 370 | << ") // Got=" << Opcode << "\n"); | 149 | 370 | if (Opcode != Expected) { | 150 | 296 | if (handleReject() == RejectAndGiveUp) | 151 | 0 | return false; | 152 | 370 | } | 153 | 370 | break; | 154 | 370 | } | 155 | 370 | | 156 | 2.27k | case GIM_SwitchOpcode: { | 157 | 2.27k | int64_t InsnID = MatchTable[CurrentIdx++]; | 158 | 2.27k | int64_t LowerBound = MatchTable[CurrentIdx++]; | 159 | 2.27k | int64_t UpperBound = MatchTable[CurrentIdx++]; | 160 | 2.27k | int64_t Default = MatchTable[CurrentIdx++]; | 161 | 2.27k | | 162 | 2.27k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 163 | 2.27k | const int64_t Opcode = State.MIs[InsnID]->getOpcode(); | 164 | 2.27k | | 165 | 2.27k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 166 | 2.27k | dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], [" | 167 | 2.27k | << LowerBound << ", " << UpperBound << "), Default=" << Default | 168 | 2.27k | << ", JumpTable...) // Got=" << Opcode << "\n"; | 169 | 2.27k | }); | 170 | 2.27k | if (Opcode < LowerBound || UpperBound <= Opcode) { | 171 | 0 | CurrentIdx = Default; | 172 | 0 | break; | 173 | 0 | } | 174 | 2.27k | CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)]; | 175 | 2.27k | if (!CurrentIdx) { | 176 | 875 | CurrentIdx = Default; | 177 | 875 | break; | 178 | 875 | } | 179 | 1.39k | OnFailResumeAt.push_back(Default); | 180 | 1.39k | break; | 181 | 1.39k | } | 182 | 1.39k | | 183 | 1.39k | case GIM_SwitchType: { | 184 | 1.37k | int64_t InsnID = MatchTable[CurrentIdx++]; | 185 | 1.37k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 186 | 1.37k | int64_t LowerBound = MatchTable[CurrentIdx++]; | 187 | 1.37k | int64_t UpperBound = MatchTable[CurrentIdx++]; | 188 | 1.37k | int64_t Default = MatchTable[CurrentIdx++]; | 189 | 1.37k | | 190 | 1.37k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 191 | 1.37k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 192 | 1.37k | | 193 | 1.37k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), { | 194 | 1.37k | dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID | 195 | 1.37k | << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", " | 196 | 1.37k | << UpperBound << "), Default=" << Default | 197 | 1.37k | << ", JumpTable...) // Got="; | 198 | 1.37k | if (!MO.isReg()) | 199 | 1.37k | dbgs() << "Not a VReg\n"; | 200 | 1.37k | else | 201 | 1.37k | dbgs() << MRI.getType(MO.getReg()) << "\n"; | 202 | 1.37k | }); | 203 | 1.37k | if (!MO.isReg()) { | 204 | 0 | CurrentIdx = Default; | 205 | 0 | break; | 206 | 0 | } | 207 | 1.37k | const LLT Ty = MRI.getType(MO.getReg()); | 208 | 1.37k | const auto TyI = ISelInfo.TypeIDMap.find(Ty); | 209 | 1.37k | if (TyI == ISelInfo.TypeIDMap.end()) { | 210 | 1 | CurrentIdx = Default; | 211 | 1 | break; | 212 | 1 | } | 213 | 1.37k | const int64_t TypeID = TyI->second; | 214 | 1.37k | if (TypeID < LowerBound || UpperBound <= TypeID1.32k ) { | 215 | 48 | CurrentIdx = Default; | 216 | 48 | break; | 217 | 48 | } | 218 | 1.32k | CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)]; | 219 | 1.32k | if (!CurrentIdx) { | 220 | 377 | CurrentIdx = Default; | 221 | 377 | break; | 222 | 377 | } | 223 | 946 | OnFailResumeAt.push_back(Default); | 224 | 946 | break; | 225 | 946 | } | 226 | 946 | | 227 | 946 | case GIM_CheckNumOperands: { | 228 | 23 | int64_t InsnID = MatchTable[CurrentIdx++]; | 229 | 23 | int64_t Expected = MatchTable[CurrentIdx++]; | 230 | 23 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 231 | 23 | dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs[" | 232 | 23 | << InsnID << "], Expected=" << Expected << ")\n"); | 233 | 23 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 234 | 23 | if (State.MIs[InsnID]->getNumOperands() != Expected) { | 235 | 6 | if (handleReject() == RejectAndGiveUp) | 236 | 0 | return false; | 237 | 23 | } | 238 | 23 | break; | 239 | 23 | } | 240 | 74 | case GIM_CheckI64ImmPredicate: { | 241 | 74 | int64_t InsnID = MatchTable[CurrentIdx++]; | 242 | 74 | int64_t Predicate = MatchTable[CurrentIdx++]; | 243 | 74 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 244 | 74 | dbgs() | 245 | 74 | << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs[" | 246 | 74 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 247 | 74 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 248 | 74 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 249 | 74 | "Expected G_CONSTANT"); | 250 | 74 | assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate"); | 251 | 74 | int64_t Value = 0; | 252 | 74 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 253 | 74 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue(); | 254 | 0 | else if (State.MIs[InsnID]->getOperand(1).isImm()) | 255 | 0 | Value = State.MIs[InsnID]->getOperand(1).getImm(); | 256 | 0 | else | 257 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 258 | 74 | | 259 | 74 | if (!testImmPredicate_I64(Predicate, Value)) | 260 | 16 | if (handleReject() == RejectAndGiveUp) | 261 | 0 | return false; | 262 | 74 | break; | 263 | 74 | } | 264 | 74 | case GIM_CheckAPIntImmPredicate: { | 265 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 266 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 267 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 268 | 0 | dbgs() | 269 | 0 | << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs[" | 270 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 271 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 272 | 0 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT && | 273 | 0 | "Expected G_CONSTANT"); | 274 | 0 | assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate"); | 275 | 0 | APInt Value; | 276 | 0 | if (State.MIs[InsnID]->getOperand(1).isCImm()) | 277 | 0 | Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue(); | 278 | 0 | else | 279 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 280 | 0 |
| 281 | 0 | if (!testImmPredicate_APInt(Predicate, Value)) | 282 | 0 | if (handleReject() == RejectAndGiveUp) | 283 | 0 | return false; | 284 | 0 | break; | 285 | 0 | } | 286 | 5 | case GIM_CheckAPFloatImmPredicate: { | 287 | 5 | int64_t InsnID = MatchTable[CurrentIdx++]; | 288 | 5 | int64_t Predicate = MatchTable[CurrentIdx++]; | 289 | 5 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 290 | 5 | dbgs() | 291 | 5 | << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs[" | 292 | 5 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 293 | 5 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 294 | 5 | assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && | 295 | 5 | "Expected G_FCONSTANT"); | 296 | 5 | assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand"); | 297 | 5 | assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate"); | 298 | 5 | APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF(); | 299 | 5 | | 300 | 5 | if (!testImmPredicate_APFloat(Predicate, Value)) | 301 | 5 | if (handleReject() == RejectAndGiveUp) | 302 | 0 | return false; | 303 | 5 | break; | 304 | 5 | } | 305 | 5 | case GIM_CheckCxxInsnPredicate: { | 306 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 307 | 0 | int64_t Predicate = MatchTable[CurrentIdx++]; | 308 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 309 | 0 | dbgs() | 310 | 0 | << CurrentIdx << ": GIM_CheckCxxPredicate(MIs[" | 311 | 0 | << InsnID << "], Predicate=" << Predicate << ")\n"); | 312 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 313 | 0 | assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate"); | 314 | 0 |
| 315 | 0 | if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID])) | 316 | 0 | if (handleReject() == RejectAndGiveUp) | 317 | 0 | return false; | 318 | 0 | break; | 319 | 0 | } | 320 | 0 | case GIM_CheckAtomicOrdering: { | 321 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 322 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 323 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 324 | 0 | dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs[" | 325 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 326 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 327 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 328 | 0 | if (handleReject() == RejectAndGiveUp) | 329 | 0 | return false; | 330 | 0 | | 331 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 332 | 0 | if (MMO->getOrdering() != Ordering) | 333 | 0 | if (handleReject() == RejectAndGiveUp) | 334 | 0 | return false; | 335 | 0 | break; | 336 | 0 | } | 337 | 0 | case GIM_CheckAtomicOrderingOrStrongerThan: { | 338 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 339 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 340 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 341 | 0 | dbgs() << CurrentIdx | 342 | 0 | << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs[" | 343 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 344 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 345 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 346 | 0 | if (handleReject() == RejectAndGiveUp) | 347 | 0 | return false; | 348 | 0 | | 349 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 350 | 0 | if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering)) | 351 | 0 | if (handleReject() == RejectAndGiveUp) | 352 | 0 | return false; | 353 | 0 | break; | 354 | 0 | } | 355 | 0 | case GIM_CheckAtomicOrderingWeakerThan: { | 356 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 357 | 0 | AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++]; | 358 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 359 | 0 | dbgs() << CurrentIdx | 360 | 0 | << ": GIM_CheckAtomicOrderingWeakerThan(MIs[" | 361 | 0 | << InsnID << "], " << (uint64_t)Ordering << ")\n"); | 362 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 363 | 0 | if (!State.MIs[InsnID]->hasOneMemOperand()) | 364 | 0 | if (handleReject() == RejectAndGiveUp) | 365 | 0 | return false; | 366 | 0 | | 367 | 0 | for (const auto &MMO : State.MIs[InsnID]->memoperands()) | 368 | 0 | if (!isStrongerThan(Ordering, MMO->getOrdering())) | 369 | 0 | if (handleReject() == RejectAndGiveUp) | 370 | 0 | return false; | 371 | 0 | break; | 372 | 0 | } | 373 | 0 | case GIM_CheckMemoryAddressSpace: { | 374 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 375 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 376 | 0 | // This accepts a list of possible address spaces. | 377 | 0 | const int NumAddrSpace = MatchTable[CurrentIdx++]; | 378 | 0 |
| 379 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 380 | 0 | if (handleReject() == RejectAndGiveUp) | 381 | 0 | return false; | 382 | 0 | break; | 383 | 0 | } | 384 | 0 | | 385 | 0 | // Need to still jump to the end of the list of address spaces if we find | 386 | 0 | // a match earlier. | 387 | 0 | const uint64_t LastIdx = CurrentIdx + NumAddrSpace; | 388 | 0 |
| 389 | 0 | const MachineMemOperand *MMO | 390 | 0 | = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 391 | 0 | const unsigned MMOAddrSpace = MMO->getAddrSpace(); | 392 | 0 |
| 393 | 0 | bool Success = false; | 394 | 0 | for (int I = 0; I != NumAddrSpace; ++I) { | 395 | 0 | unsigned AddrSpace = MatchTable[CurrentIdx++]; | 396 | 0 | DEBUG_WITH_TYPE( | 397 | 0 | TgtInstructionSelector::getName(), | 398 | 0 | dbgs() << "addrspace(" << MMOAddrSpace << ") vs " | 399 | 0 | << AddrSpace << '\n'); | 400 | 0 |
| 401 | 0 | if (AddrSpace == MMOAddrSpace) { | 402 | 0 | Success = true; | 403 | 0 | break; | 404 | 0 | } | 405 | 0 | } | 406 | 0 |
| 407 | 0 | CurrentIdx = LastIdx; | 408 | 0 | if (!Success && handleReject() == RejectAndGiveUp) | 409 | 0 | return false; | 410 | 0 | break; | 411 | 0 | } | 412 | 0 | case GIM_CheckMemorySizeEqualTo: { | 413 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 414 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 415 | 0 | uint64_t Size = MatchTable[CurrentIdx++]; | 416 | 0 |
| 417 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 418 | 0 | dbgs() << CurrentIdx | 419 | 0 | << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID | 420 | 0 | << "]->memoperands() + " << MMOIdx | 421 | 0 | << ", Size=" << Size << ")\n"); | 422 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 423 | 0 |
| 424 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 425 | 0 | if (handleReject() == RejectAndGiveUp) | 426 | 0 | return false; | 427 | 0 | break; | 428 | 0 | } | 429 | 0 | | 430 | 0 | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 431 | 0 |
| 432 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 433 | 0 | dbgs() << MMO->getSize() << " bytes vs " << Size | 434 | 0 | << " bytes\n"); | 435 | 0 | if (MMO->getSize() != Size) | 436 | 0 | if (handleReject() == RejectAndGiveUp) | 437 | 0 | return false; | 438 | 0 | | 439 | 0 | break; | 440 | 0 | } | 441 | 0 | case GIM_CheckMemorySizeEqualToLLT: | 442 | 0 | case GIM_CheckMemorySizeLessThanLLT: | 443 | 0 | case GIM_CheckMemorySizeGreaterThanLLT: { | 444 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 445 | 0 | int64_t MMOIdx = MatchTable[CurrentIdx++]; | 446 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 447 | 0 |
| 448 | 0 | DEBUG_WITH_TYPE( | 449 | 0 | TgtInstructionSelector::getName(), | 450 | 0 | dbgs() << CurrentIdx << ": GIM_CheckMemorySize" | 451 | 0 | << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT | 452 | 0 | ? "EqualTo" | 453 | 0 | : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT | 454 | 0 | ? "GreaterThan" | 455 | 0 | : "LessThan") | 456 | 0 | << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx | 457 | 0 | << ", OpIdx=" << OpIdx << ")\n"); | 458 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 459 | 0 |
| 460 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 461 | 0 | if (!MO.isReg()) { | 462 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 463 | 0 | dbgs() << CurrentIdx << ": Not a register\n"); | 464 | 0 | if (handleReject() == RejectAndGiveUp) | 465 | 0 | return false; | 466 | 0 | break; | 467 | 0 | } | 468 | 0 | | 469 | 0 | if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { | 470 | 0 | if (handleReject() == RejectAndGiveUp) | 471 | 0 | return false; | 472 | 0 | break; | 473 | 0 | } | 474 | 0 | | 475 | 0 | MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); | 476 | 0 |
| 477 | 0 | unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); | 478 | 0 | if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && | 479 | 0 | MMO->getSizeInBits() != Size) { | 480 | 0 | if (handleReject() == RejectAndGiveUp) | 481 | 0 | return false; | 482 | 0 | } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && | 483 | 0 | MMO->getSizeInBits() >= Size) { | 484 | 0 | if (handleReject() == RejectAndGiveUp) | 485 | 0 | return false; | 486 | 0 | } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && | 487 | 0 | MMO->getSizeInBits() <= Size) | 488 | 0 | if (handleReject() == RejectAndGiveUp) | 489 | 0 | return false; | 490 | 0 | | 491 | 0 | break; | 492 | 0 | } | 493 | 1.62k | case GIM_CheckType: { | 494 | 1.62k | int64_t InsnID = MatchTable[CurrentIdx++]; | 495 | 1.62k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 496 | 1.62k | int64_t TypeID = MatchTable[CurrentIdx++]; | 497 | 1.62k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 498 | 1.62k | dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID | 499 | 1.62k | << "]->getOperand(" << OpIdx | 500 | 1.62k | << "), TypeID=" << TypeID << ")\n"); | 501 | 1.62k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 502 | 1.62k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 503 | 1.62k | if (!MO.isReg() || | 504 | 1.62k | MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) { | 505 | 227 | if (handleReject() == RejectAndGiveUp) | 506 | 0 | return false; | 507 | 1.62k | } | 508 | 1.62k | break; | 509 | 1.62k | } | 510 | 1.62k | case GIM_CheckPointerToAny: { | 511 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 512 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 513 | 0 | int64_t SizeInBits = MatchTable[CurrentIdx++]; | 514 | 0 |
| 515 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 516 | 0 | dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs[" | 517 | 0 | << InsnID << "]->getOperand(" << OpIdx | 518 | 0 | << "), SizeInBits=" << SizeInBits << ")\n"); | 519 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 520 | 0 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 521 | 0 | const LLT Ty = MRI.getType(MO.getReg()); | 522 | 0 |
| 523 | 0 | // iPTR must be looked up in the target. | 524 | 0 | if (SizeInBits == 0) { | 525 | 0 | MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); | 526 | 0 | const unsigned AddrSpace = Ty.getAddressSpace(); | 527 | 0 | SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); | 528 | 0 | } | 529 | 0 |
| 530 | 0 | assert(SizeInBits != 0 && "Pointer size must be known"); | 531 | 0 |
| 532 | 0 | if (MO.isReg()) { | 533 | 0 | if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) | 534 | 0 | if (handleReject() == RejectAndGiveUp) | 535 | 0 | return false; | 536 | 0 | } else if (handleReject() == RejectAndGiveUp) | 537 | 0 | return false; | 538 | 0 | | 539 | 0 | break; | 540 | 0 | } | 541 | 2.17k | case GIM_CheckRegBankForClass: { | 542 | 2.17k | int64_t InsnID = MatchTable[CurrentIdx++]; | 543 | 2.17k | int64_t OpIdx = MatchTable[CurrentIdx++]; | 544 | 2.17k | int64_t RCEnum = MatchTable[CurrentIdx++]; | 545 | 2.17k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 546 | 2.17k | dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs[" | 547 | 2.17k | << InsnID << "]->getOperand(" << OpIdx | 548 | 2.17k | << "), RCEnum=" << RCEnum << ")\n"); | 549 | 2.17k | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 550 | 2.17k | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 551 | 2.17k | if (!MO.isReg() || | 552 | 2.17k | &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) != | 553 | 2.17k | RBI.getRegBank(MO.getReg(), MRI, TRI)) { | 554 | 0 | if (handleReject() == RejectAndGiveUp) | 555 | 0 | return false; | 556 | 2.17k | } | 557 | 2.17k | break; | 558 | 2.17k | } | 559 | 2.17k | | 560 | 2.17k | case GIM_CheckComplexPattern: { | 561 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 562 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 563 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 564 | 0 | int64_t ComplexPredicateID = MatchTable[CurrentIdx++]; | 565 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 566 | 0 | dbgs() << CurrentIdx << ": State.Renderers[" << RendererID | 567 | 0 | << "] = GIM_CheckComplexPattern(MIs[" << InsnID | 568 | 0 | << "]->getOperand(" << OpIdx | 569 | 0 | << "), ComplexPredicateID=" << ComplexPredicateID | 570 | 0 | << ")\n"); | 571 | 0 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 572 | 0 | // FIXME: Use std::invoke() when it's available. | 573 | 0 | ComplexRendererFns Renderer = | 574 | 0 | (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])( | 575 | 0 | State.MIs[InsnID]->getOperand(OpIdx)); | 576 | 0 | if (Renderer.hasValue()) | 577 | 0 | State.Renderers[RendererID] = Renderer.getValue(); | 578 | 0 | else | 579 | 0 | if (handleReject() == RejectAndGiveUp) | 580 | 0 | return false; | 581 | 0 | break; | 582 | 0 | } | 583 | 0 |
| 584 | 321 | case GIM_CheckConstantInt: { | 585 | 321 | int64_t InsnID = MatchTable[CurrentIdx++]; | 586 | 321 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 587 | 321 | int64_t Value = MatchTable[CurrentIdx++]; | 588 | 321 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 589 | 321 | dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs[" | 590 | 321 | << InsnID << "]->getOperand(" << OpIdx | 591 | 321 | << "), Value=" << Value << ")\n"); | 592 | 321 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 593 | 321 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 594 | 321 | if (MO.isReg()) { | 595 | 321 | // isOperandImmEqual() will sign-extend to 64-bits, so should we. | 596 | 321 | LLT Ty = MRI.getType(MO.getReg()); | 597 | 321 | Value = SignExtend64(Value, Ty.getSizeInBits()); | 598 | 321 | | 599 | 321 | if (!isOperandImmEqual(MO, Value, MRI)) { | 600 | 285 | if (handleReject() == RejectAndGiveUp) | 601 | 0 | return false; | 602 | 0 | } | 603 | 0 | } else if (handleReject() == RejectAndGiveUp) | 604 | 0 | return false; | 605 | 321 | | 606 | 321 | break; | 607 | 321 | } | 608 | 321 | | 609 | 321 | case GIM_CheckLiteralInt: { | 610 | 45 | int64_t InsnID = MatchTable[CurrentIdx++]; | 611 | 45 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 612 | 45 | int64_t Value = MatchTable[CurrentIdx++]; | 613 | 45 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 614 | 45 | dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs[" | 615 | 45 | << InsnID << "]->getOperand(" << OpIdx | 616 | 45 | << "), Value=" << Value << ")\n"); | 617 | 45 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 618 | 45 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 619 | 45 | if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) { | 620 | 23 | if (handleReject() == RejectAndGiveUp) | 621 | 0 | return false; | 622 | 45 | } | 623 | 45 | break; | 624 | 45 | } | 625 | 45 | | 626 | 45 | case GIM_CheckIntrinsicID: { | 627 | 24 | int64_t InsnID = MatchTable[CurrentIdx++]; | 628 | 24 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 629 | 24 | int64_t Value = MatchTable[CurrentIdx++]; | 630 | 24 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 631 | 24 | dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs[" | 632 | 24 | << InsnID << "]->getOperand(" << OpIdx | 633 | 24 | << "), Value=" << Value << ")\n"); | 634 | 24 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 635 | 24 | MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); | 636 | 24 | if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value17 ) | 637 | 21 | if (handleReject() == RejectAndGiveUp) | 638 | 0 | return false; | 639 | 24 | break; | 640 | 24 | } | 641 | 24 | | 642 | 24 | case GIM_CheckIsMBB: { | 643 | 19 | int64_t InsnID = MatchTable[CurrentIdx++]; | 644 | 19 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 645 | 19 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 646 | 19 | dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID | 647 | 19 | << "]->getOperand(" << OpIdx << "))\n"); | 648 | 19 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 649 | 19 | if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) { | 650 | 0 | if (handleReject() == RejectAndGiveUp) | 651 | 0 | return false; | 652 | 19 | } | 653 | 19 | break; | 654 | 19 | } | 655 | 19 | | 656 | 69 | case GIM_CheckIsSafeToFold: { | 657 | 69 | int64_t InsnID = MatchTable[CurrentIdx++]; | 658 | 69 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 659 | 69 | dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs[" | 660 | 69 | << InsnID << "])\n"); | 661 | 69 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 662 | 69 | if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) { | 663 | 0 | if (handleReject() == RejectAndGiveUp) | 664 | 0 | return false; | 665 | 69 | } | 666 | 69 | break; | 667 | 69 | } | 668 | 69 | case GIM_CheckIsSameOperand: { | 669 | 4 | int64_t InsnID = MatchTable[CurrentIdx++]; | 670 | 4 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 671 | 4 | int64_t OtherInsnID = MatchTable[CurrentIdx++]; | 672 | 4 | int64_t OtherOpIdx = MatchTable[CurrentIdx++]; | 673 | 4 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 674 | 4 | dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs[" | 675 | 4 | << InsnID << "][" << OpIdx << "], MIs[" | 676 | 4 | << OtherInsnID << "][" << OtherOpIdx << "])\n"); | 677 | 4 | assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); | 678 | 4 | assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined"); | 679 | 4 | if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo( | 680 | 4 | State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) { | 681 | 2 | if (handleReject() == RejectAndGiveUp) | 682 | 0 | return false; | 683 | 4 | } | 684 | 4 | break; | 685 | 4 | } | 686 | 2.29k | case GIM_Reject: | 687 | 2.29k | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 688 | 2.29k | dbgs() << CurrentIdx << ": GIM_Reject\n"); | 689 | 2.29k | if (handleReject() == RejectAndGiveUp) | 690 | 1.46k | return false; | 691 | 824 | break; | 692 | 824 | | 693 | 824 | case GIR_MutateOpcode: { | 694 | 473 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 695 | 473 | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 696 | 473 | int64_t NewOpcode = MatchTable[CurrentIdx++]; | 697 | 473 | if (NewInsnID >= OutMIs.size()) | 698 | 473 | OutMIs.resize(NewInsnID + 1); | 699 | 473 | | 700 | 473 | OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(), | 701 | 473 | State.MIs[OldInsnID]); | 702 | 473 | OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode)); | 703 | 473 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 704 | 473 | dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs[" | 705 | 473 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 706 | 473 | << NewOpcode << ")\n"); | 707 | 473 | break; | 708 | 824 | } | 709 | 824 | | 710 | 824 | case GIR_BuildMI: { | 711 | 334 | uint64_t NewInsnID = MatchTable[CurrentIdx++]; | 712 | 334 | int64_t Opcode = MatchTable[CurrentIdx++]; | 713 | 334 | if (NewInsnID >= OutMIs.size()) | 714 | 330 | OutMIs.resize(NewInsnID + 1); | 715 | 334 | | 716 | 334 | OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0], | 717 | 334 | State.MIs[0]->getDebugLoc(), TII.get(Opcode)); | 718 | 334 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 719 | 334 | dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs[" | 720 | 334 | << NewInsnID << "], " << Opcode << ")\n"); | 721 | 334 | break; | 722 | 824 | } | 723 | 824 | | 724 | 824 | case GIR_Copy: { | 725 | 435 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 726 | 435 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 727 | 435 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 728 | 435 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 729 | 435 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx)); | 730 | 435 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 731 | 435 | dbgs() | 732 | 435 | << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID | 733 | 435 | << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n"); | 734 | 435 | break; | 735 | 824 | } | 736 | 824 | | 737 | 824 | case GIR_CopyOrAddZeroReg: { | 738 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 739 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 740 | 0 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 741 | 0 | int64_t ZeroReg = MatchTable[CurrentIdx++]; | 742 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 743 | 0 | MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx); | 744 | 0 | if (isOperandImmEqual(MO, 0, MRI)) | 745 | 0 | OutMIs[NewInsnID].addReg(ZeroReg); | 746 | 0 | else | 747 | 0 | OutMIs[NewInsnID].add(MO); | 748 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 749 | 0 | dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs[" | 750 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 751 | 0 | << OpIdx << ", " << ZeroReg << ")\n"); | 752 | 0 | break; | 753 | 824 | } | 754 | 824 | | 755 | 824 | case GIR_CopySubReg: { | 756 | 170 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 757 | 170 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 758 | 170 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 759 | 170 | int64_t SubRegIdx = MatchTable[CurrentIdx++]; | 760 | 170 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 761 | 170 | OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(), | 762 | 170 | 0, SubRegIdx); | 763 | 170 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 764 | 170 | dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs[" | 765 | 170 | << NewInsnID << "], MIs[" << OldInsnID << "], " | 766 | 170 | << OpIdx << ", " << SubRegIdx << ")\n"); | 767 | 170 | break; | 768 | 824 | } | 769 | 824 | | 770 | 824 | case GIR_AddImplicitDef: { | 771 | 116 | int64_t InsnID = MatchTable[CurrentIdx++]; | 772 | 116 | int64_t RegNum = MatchTable[CurrentIdx++]; | 773 | 116 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 774 | 116 | OutMIs[InsnID].addDef(RegNum, RegState::Implicit); | 775 | 116 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 776 | 116 | dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs[" | 777 | 116 | << InsnID << "], " << RegNum << ")\n"); | 778 | 116 | break; | 779 | 824 | } | 780 | 824 | | 781 | 824 | case GIR_AddImplicitUse: { | 782 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 783 | 0 | int64_t RegNum = MatchTable[CurrentIdx++]; | 784 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 785 | 0 | OutMIs[InsnID].addUse(RegNum, RegState::Implicit); | 786 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 787 | 0 | dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs[" | 788 | 0 | << InsnID << "], " << RegNum << ")\n"); | 789 | 0 | break; | 790 | 824 | } | 791 | 824 | | 792 | 824 | case GIR_AddRegister: { | 793 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 794 | 0 | int64_t RegNum = MatchTable[CurrentIdx++]; | 795 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 796 | 0 | OutMIs[InsnID].addReg(RegNum); | 797 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 798 | 0 | dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs[" | 799 | 0 | << InsnID << "], " << RegNum << ")\n"); | 800 | 0 | break; | 801 | 824 | } | 802 | 824 | | 803 | 824 | case GIR_AddTempRegister: { | 804 | 8 | int64_t InsnID = MatchTable[CurrentIdx++]; | 805 | 8 | int64_t TempRegID = MatchTable[CurrentIdx++]; | 806 | 8 | uint64_t TempRegFlags = MatchTable[CurrentIdx++]; | 807 | 8 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 808 | 8 | OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags); | 809 | 8 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 810 | 8 | dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" | 811 | 8 | << InsnID << "], TempRegisters[" << TempRegID | 812 | 8 | << "], " << TempRegFlags << ")\n"); | 813 | 8 | break; | 814 | 824 | } | 815 | 824 | | 816 | 824 | case GIR_AddImm: { | 817 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 818 | 0 | int64_t Imm = MatchTable[CurrentIdx++]; | 819 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 820 | 0 | OutMIs[InsnID].addImm(Imm); | 821 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 822 | 0 | dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID | 823 | 0 | << "], " << Imm << ")\n"); | 824 | 0 | break; | 825 | 824 | } | 826 | 824 | | 827 | 824 | case GIR_ComplexRenderer: { | 828 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 829 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 830 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 831 | 0 | for (const auto &RenderOpFn : State.Renderers[RendererID]) | 832 | 0 | RenderOpFn(OutMIs[InsnID]); | 833 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 834 | 0 | dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs[" | 835 | 0 | << InsnID << "], " << RendererID << ")\n"); | 836 | 0 | break; | 837 | 824 | } | 838 | 824 | case GIR_ComplexSubOperandRenderer: { | 839 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 840 | 0 | int64_t RendererID = MatchTable[CurrentIdx++]; | 841 | 0 | int64_t RenderOpID = MatchTable[CurrentIdx++]; | 842 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 843 | 0 | State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]); | 844 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 845 | 0 | dbgs() << CurrentIdx | 846 | 0 | << ": GIR_ComplexSubOperandRenderer(OutMIs[" | 847 | 0 | << InsnID << "], " << RendererID << ", " | 848 | 0 | << RenderOpID << ")\n"); | 849 | 0 | break; | 850 | 824 | } | 851 | 824 | | 852 | 824 | case GIR_CopyConstantAsSImm: { | 853 | 107 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 854 | 107 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 855 | 107 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 856 | 107 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); | 857 | 107 | if (State.MIs[OldInsnID]->getOperand(1).isCImm()) { | 858 | 107 | OutMIs[NewInsnID].addImm( | 859 | 107 | State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue()); | 860 | 107 | } else if (0 State.MIs[OldInsnID]->getOperand(1).isImm()0 ) | 861 | 0 | OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1)); | 862 | 0 | else | 863 | 0 | llvm_unreachable("Expected Imm or CImm operand"); | 864 | 107 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 865 | 107 | dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs[" | 866 | 107 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 867 | 107 | break; | 868 | 107 | } | 869 | 107 | | 870 | 107 | // TODO: Needs a test case once we have a pattern that uses this. | 871 | 107 | case GIR_CopyFConstantAsFPImm: { | 872 | 0 | int64_t NewInsnID = MatchTable[CurrentIdx++]; | 873 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 874 | 0 | assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction"); | 875 | 0 | assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT"); | 876 | 0 | if (State.MIs[OldInsnID]->getOperand(1).isFPImm()) | 877 | 0 | OutMIs[NewInsnID].addFPImm( | 878 | 0 | State.MIs[OldInsnID]->getOperand(1).getFPImm()); | 879 | 0 | else | 880 | 0 | llvm_unreachable("Expected FPImm operand"); | 881 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 882 | 0 | dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs[" | 883 | 0 | << NewInsnID << "], MIs[" << OldInsnID << "])\n"); | 884 | 0 | break; | 885 | 0 | } | 886 | 0 |
| 887 | 0 | case GIR_CustomRenderer: { | 888 | 0 | int64_t InsnID = MatchTable[CurrentIdx++]; | 889 | 0 | int64_t OldInsnID = MatchTable[CurrentIdx++]; | 890 | 0 | int64_t RendererFnID = MatchTable[CurrentIdx++]; | 891 | 0 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 892 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 893 | 0 | dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs[" | 894 | 0 | << InsnID << "], MIs[" << OldInsnID << "], " | 895 | 0 | << RendererFnID << ")\n"); | 896 | 0 | (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID], | 897 | 0 | *State.MIs[OldInsnID]); | 898 | 0 | break; | 899 | 0 | } | 900 | 332 | case GIR_ConstrainOperandRC: { | 901 | 332 | int64_t InsnID = MatchTable[CurrentIdx++]; | 902 | 332 | int64_t OpIdx = MatchTable[CurrentIdx++]; | 903 | 332 | int64_t RCEnum = MatchTable[CurrentIdx++]; | 904 | 332 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 905 | 332 | constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx, | 906 | 332 | *TRI.getRegClass(RCEnum), TII, TRI, RBI); | 907 | 332 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 908 | 332 | dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs[" | 909 | 332 | << InsnID << "], " << OpIdx << ", " << RCEnum | 910 | 332 | << ")\n"); | 911 | 332 | break; | 912 | 0 | } | 913 | 0 |
| 914 | 641 | case GIR_ConstrainSelectedInstOperands: { | 915 | 641 | int64_t InsnID = MatchTable[CurrentIdx++]; | 916 | 641 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 917 | 641 | constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI, | 918 | 641 | RBI); | 919 | 641 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 920 | 641 | dbgs() << CurrentIdx | 921 | 641 | << ": GIR_ConstrainSelectedInstOperands(OutMIs[" | 922 | 641 | << InsnID << "])\n"); | 923 | 641 | break; | 924 | 0 | } | 925 | 0 |
| 926 | 1 | case GIR_MergeMemOperands: { | 927 | 1 | int64_t InsnID = MatchTable[CurrentIdx++]; | 928 | 1 | assert(OutMIs[InsnID] && "Attempted to add to undefined instruction"); | 929 | 1 | | 930 | 1 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 931 | 1 | dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs[" | 932 | 1 | << InsnID << "]"); | 933 | 1 | int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList; | 934 | 2 | while ((MergeInsnID = MatchTable[CurrentIdx++]) != | 935 | 2 | GIU_MergeMemOperands_EndOfList) { | 936 | 1 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 937 | 1 | dbgs() << ", MIs[" << MergeInsnID << "]"); | 938 | 1 | for (const auto &MMO : State.MIs[MergeInsnID]->memoperands()) | 939 | 0 | OutMIs[InsnID].addMemOperand(MMO); | 940 | 1 | } | 941 | 1 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n"); | 942 | 1 | break; | 943 | 0 | } | 944 | 0 |
| 945 | 330 | case GIR_EraseFromParent: { | 946 | 330 | int64_t InsnID = MatchTable[CurrentIdx++]; | 947 | 330 | assert(State.MIs[InsnID] && | 948 | 330 | "Attempted to erase an undefined instruction"); | 949 | 330 | State.MIs[InsnID]->eraseFromParent(); | 950 | 330 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 951 | 330 | dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs[" | 952 | 330 | << InsnID << "])\n"); | 953 | 330 | break; | 954 | 0 | } | 955 | 0 |
| 956 | 4 | case GIR_MakeTempReg: { | 957 | 4 | int64_t TempRegID = MatchTable[CurrentIdx++]; | 958 | 4 | int64_t TypeID = MatchTable[CurrentIdx++]; | 959 | 4 | | 960 | 4 | State.TempRegisters[TempRegID] = | 961 | 4 | MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]); | 962 | 4 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 963 | 4 | dbgs() << CurrentIdx << ": TempRegs[" << TempRegID | 964 | 4 | << "] = GIR_MakeTempReg(" << TypeID << ")\n"); | 965 | 4 | break; | 966 | 0 | } | 967 | 0 |
| 968 | 0 | case GIR_Coverage: { | 969 | 0 | int64_t RuleID = MatchTable[CurrentIdx++]; | 970 | 0 | CoverageInfo.setCovered(RuleID); | 971 | 0 |
| 972 | 0 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 973 | 0 | dbgs() | 974 | 0 | << CurrentIdx << ": GIR_Coverage(" << RuleID << ")"); | 975 | 0 | break; | 976 | 0 | } | 977 | 0 |
| 978 | 803 | case GIR_Done: | 979 | 803 | DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), | 980 | 803 | dbgs() << CurrentIdx << ": GIR_Done\n"); | 981 | 803 | return true; | 982 | 0 |
| 983 | 0 | default: | 984 | 0 | llvm_unreachable("Unexpected command"); | 985 | 19.3k | } | 986 | 19.3k | } | 987 | 2.27k | } |
|
988 | | |
989 | | } // end namespace llvm |
990 | | |
991 | | #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H |