/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- FastISel.cpp - Implementation of the FastISel class ----------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This file contains the implementation of the FastISel class. |
10 | | // |
11 | | // "Fast" instruction selection is designed to emit very poor code quickly. |
12 | | // Also, it is not designed to be able to do much lowering, so most illegal |
13 | | // types (e.g. i64 on 32-bit targets) and operations are not supported. It is |
14 | | // also not intended to be able to do much optimization, except in a few cases |
15 | | // where doing optimizations reduces overall compile time. For example, folding |
16 | | // constants into immediate fields is often done, because it's cheap and it |
17 | | // reduces the number of instructions later phases have to examine. |
18 | | // |
19 | | // "Fast" instruction selection is able to fail gracefully and transfer |
20 | | // control to the SelectionDAG selector for operations that it doesn't |
21 | | // support. In many cases, this allows us to avoid duplicating a lot of |
22 | | // the complicated lowering logic that SelectionDAG currently has. |
23 | | // |
24 | | // The intended use for "fast" instruction selection is "-O0" mode |
25 | | // compilation, where the quality of the generated code is irrelevant when |
26 | | // weighed against the speed at which the code can be generated. Also, |
27 | | // at -O0, the LLVM optimizers are not running, and this makes the |
28 | | // compile time of codegen a much higher portion of the overall compile |
29 | | // time. Despite its limitations, "fast" instruction selection is able to |
30 | | // handle enough code on its own to provide noticeable overall speedups |
31 | | // in -O0 compiles. |
32 | | // |
33 | | // Basic operations are supported in a target-independent way, by reading |
34 | | // the same instruction descriptions that the SelectionDAG selector reads, |
35 | | // and identifying simple arithmetic operations that can be directly selected |
36 | | // from simple operators. More complicated operations currently require |
37 | | // target-specific code. |
38 | | // |
39 | | //===----------------------------------------------------------------------===// |
40 | | |
41 | | #include "llvm/CodeGen/FastISel.h" |
42 | | #include "llvm/ADT/APFloat.h" |
43 | | #include "llvm/ADT/APSInt.h" |
44 | | #include "llvm/ADT/DenseMap.h" |
45 | | #include "llvm/ADT/Optional.h" |
46 | | #include "llvm/ADT/SmallPtrSet.h" |
47 | | #include "llvm/ADT/SmallString.h" |
48 | | #include "llvm/ADT/SmallVector.h" |
49 | | #include "llvm/ADT/Statistic.h" |
50 | | #include "llvm/Analysis/BranchProbabilityInfo.h" |
51 | | #include "llvm/Analysis/TargetLibraryInfo.h" |
52 | | #include "llvm/CodeGen/Analysis.h" |
53 | | #include "llvm/CodeGen/FunctionLoweringInfo.h" |
54 | | #include "llvm/CodeGen/ISDOpcodes.h" |
55 | | #include "llvm/CodeGen/MachineBasicBlock.h" |
56 | | #include "llvm/CodeGen/MachineFrameInfo.h" |
57 | | #include "llvm/CodeGen/MachineInstr.h" |
58 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
59 | | #include "llvm/CodeGen/MachineMemOperand.h" |
60 | | #include "llvm/CodeGen/MachineModuleInfo.h" |
61 | | #include "llvm/CodeGen/MachineOperand.h" |
62 | | #include "llvm/CodeGen/MachineRegisterInfo.h" |
63 | | #include "llvm/CodeGen/StackMaps.h" |
64 | | #include "llvm/CodeGen/TargetInstrInfo.h" |
65 | | #include "llvm/CodeGen/TargetLowering.h" |
66 | | #include "llvm/CodeGen/TargetSubtargetInfo.h" |
67 | | #include "llvm/CodeGen/ValueTypes.h" |
68 | | #include "llvm/IR/Argument.h" |
69 | | #include "llvm/IR/Attributes.h" |
70 | | #include "llvm/IR/BasicBlock.h" |
71 | | #include "llvm/IR/CallSite.h" |
72 | | #include "llvm/IR/CallingConv.h" |
73 | | #include "llvm/IR/Constant.h" |
74 | | #include "llvm/IR/Constants.h" |
75 | | #include "llvm/IR/DataLayout.h" |
76 | | #include "llvm/IR/DebugInfo.h" |
77 | | #include "llvm/IR/DebugLoc.h" |
78 | | #include "llvm/IR/DerivedTypes.h" |
79 | | #include "llvm/IR/Function.h" |
80 | | #include "llvm/IR/GetElementPtrTypeIterator.h" |
81 | | #include "llvm/IR/GlobalValue.h" |
82 | | #include "llvm/IR/InlineAsm.h" |
83 | | #include "llvm/IR/InstrTypes.h" |
84 | | #include "llvm/IR/Instruction.h" |
85 | | #include "llvm/IR/Instructions.h" |
86 | | #include "llvm/IR/IntrinsicInst.h" |
87 | | #include "llvm/IR/LLVMContext.h" |
88 | | #include "llvm/IR/Mangler.h" |
89 | | #include "llvm/IR/Metadata.h" |
90 | | #include "llvm/IR/Operator.h" |
91 | | #include "llvm/IR/PatternMatch.h" |
92 | | #include "llvm/IR/Type.h" |
93 | | #include "llvm/IR/User.h" |
94 | | #include "llvm/IR/Value.h" |
95 | | #include "llvm/MC/MCContext.h" |
96 | | #include "llvm/MC/MCInstrDesc.h" |
97 | | #include "llvm/MC/MCRegisterInfo.h" |
98 | | #include "llvm/Support/Casting.h" |
99 | | #include "llvm/Support/Debug.h" |
100 | | #include "llvm/Support/ErrorHandling.h" |
101 | | #include "llvm/Support/MachineValueType.h" |
102 | | #include "llvm/Support/MathExtras.h" |
103 | | #include "llvm/Support/raw_ostream.h" |
104 | | #include "llvm/Target/TargetMachine.h" |
105 | | #include "llvm/Target/TargetOptions.h" |
106 | | #include <algorithm> |
107 | | #include <cassert> |
108 | | #include <cstdint> |
109 | | #include <iterator> |
110 | | #include <utility> |
111 | | |
112 | | using namespace llvm; |
113 | | using namespace PatternMatch; |
114 | | |
115 | | #define DEBUG_TYPE "isel" |
116 | | |
117 | | // FIXME: Remove this after the feature has proven reliable. |
118 | | static cl::opt<bool> SinkLocalValues("fast-isel-sink-local-values", |
119 | | cl::init(true), cl::Hidden, |
120 | | cl::desc("Sink local values in FastISel")); |
121 | | |
122 | | STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by " |
123 | | "target-independent selector"); |
124 | | STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by " |
125 | | "target-specific selector"); |
126 | | STATISTIC(NumFastIselDead, "Number of dead insts removed on failure"); |
127 | | |
128 | | /// Set the current block to which generated machine instructions will be |
129 | | /// appended. |
130 | 15.8k | void FastISel::startNewBlock() { |
131 | 15.8k | assert(LocalValueMap.empty() && |
132 | 15.8k | "local values should be cleared after finishing a BB"); |
133 | 15.8k | |
134 | 15.8k | // Instructions are appended to FuncInfo.MBB. If the basic block already |
135 | 15.8k | // contains labels or copies, use the last instruction as the last local |
136 | 15.8k | // value. |
137 | 15.8k | EmitStartPt = nullptr; |
138 | 15.8k | if (!FuncInfo.MBB->empty()) |
139 | 290 | EmitStartPt = &FuncInfo.MBB->back(); |
140 | 15.8k | LastLocalValue = EmitStartPt; |
141 | 15.8k | } |
142 | | |
143 | | /// Flush the local CSE map and sink anything we can. |
144 | 15.8k | void FastISel::finishBasicBlock() { flushLocalValueMap(); } |
145 | | |
146 | 13.4k | bool FastISel::lowerArguments() { |
147 | 13.4k | if (!FuncInfo.CanLowerReturn) |
148 | 152 | // Fallback to SDISel argument lowering code to deal with sret pointer |
149 | 152 | // parameter. |
150 | 152 | return false; |
151 | 13.2k | |
152 | 13.2k | if (!fastLowerArguments()) |
153 | 8.09k | return false; |
154 | 5.18k | |
155 | 5.18k | // Enter arguments into ValueMap for uses in non-entry BBs. |
156 | 5.18k | for (Function::const_arg_iterator I = FuncInfo.Fn->arg_begin(), |
157 | 5.18k | E = FuncInfo.Fn->arg_end(); |
158 | 11.4k | I != E; ++I6.31k ) { |
159 | 6.31k | DenseMap<const Value *, unsigned>::iterator VI = LocalValueMap.find(&*I); |
160 | 6.31k | assert(VI != LocalValueMap.end() && "Missed an argument?"); |
161 | 6.31k | FuncInfo.ValueMap[&*I] = VI->second; |
162 | 6.31k | } |
163 | 5.18k | return true; |
164 | 5.18k | } |
165 | | |
166 | | /// Return the defined register if this instruction defines exactly one |
167 | | /// virtual register and uses no other virtual registers. Otherwise return 0. |
168 | 3.73k | static unsigned findSinkableLocalRegDef(MachineInstr &MI) { |
169 | 3.73k | unsigned RegDef = 0; |
170 | 11.0k | for (const MachineOperand &MO : MI.operands()) { |
171 | 11.0k | if (!MO.isReg()) |
172 | 4.12k | continue; |
173 | 6.91k | if (MO.isDef()) { |
174 | 4.21k | if (RegDef) |
175 | 481 | return 0; |
176 | 3.73k | RegDef = MO.getReg(); |
177 | 3.73k | } else if (2.69k TargetRegisterInfo::isVirtualRegister(MO.getReg())2.69k ) { |
178 | 453 | // This is another use of a vreg. Don't try to sink it. |
179 | 453 | return 0; |
180 | 453 | } |
181 | 6.91k | } |
182 | 3.73k | return RegDef2.80k ; |
183 | 3.73k | } |
184 | | |
185 | 18.1k | void FastISel::flushLocalValueMap() { |
186 | 18.1k | // Try to sink local values down to their first use so that we can give them a |
187 | 18.1k | // better debug location. This has the side effect of shrinking local value |
188 | 18.1k | // live ranges, which helps out fast regalloc. |
189 | 18.1k | if (SinkLocalValues && LastLocalValue != EmitStartPt) { |
190 | 2.72k | // Sink local value materialization instructions between EmitStartPt and |
191 | 2.72k | // LastLocalValue. Visit them bottom-up, starting from LastLocalValue, to |
192 | 2.72k | // avoid inserting into the range that we're iterating over. |
193 | 2.72k | MachineBasicBlock::reverse_iterator RE = |
194 | 2.72k | EmitStartPt ? MachineBasicBlock::reverse_iterator(EmitStartPt)576 |
195 | 2.72k | : FuncInfo.MBB->rend()2.14k ; |
196 | 2.72k | MachineBasicBlock::reverse_iterator RI(LastLocalValue); |
197 | 2.72k | |
198 | 2.72k | InstOrderMap OrderMap; |
199 | 7.53k | for (; RI != RE;) { |
200 | 4.81k | MachineInstr &LocalMI = *RI; |
201 | 4.81k | ++RI; |
202 | 4.81k | bool Store = true; |
203 | 4.81k | if (!LocalMI.isSafeToMove(nullptr, Store)) |
204 | 1.07k | continue; |
205 | 3.73k | unsigned DefReg = findSinkableLocalRegDef(LocalMI); |
206 | 3.73k | if (DefReg == 0) |
207 | 934 | continue; |
208 | 2.80k | |
209 | 2.80k | sinkLocalValueMaterialization(LocalMI, DefReg, OrderMap); |
210 | 2.80k | } |
211 | 2.72k | } |
212 | 18.1k | |
213 | 18.1k | LocalValueMap.clear(); |
214 | 18.1k | LastLocalValue = EmitStartPt; |
215 | 18.1k | recomputeInsertPt(); |
216 | 18.1k | SavedInsertPt = FuncInfo.InsertPt; |
217 | 18.1k | LastFlushPoint = FuncInfo.InsertPt; |
218 | 18.1k | } |
219 | | |
220 | | static bool isRegUsedByPhiNodes(unsigned DefReg, |
221 | 2.76k | FunctionLoweringInfo &FuncInfo) { |
222 | 2.76k | for (auto &P : FuncInfo.PHINodesToUpdate) |
223 | 76 | if (P.second == DefReg) |
224 | 37 | return true; |
225 | 2.76k | return false2.72k ; |
226 | 2.76k | } |
227 | | |
228 | | /// Build a map of instruction orders. Return the first terminator and its |
229 | | /// order. Consider EH_LABEL instructions to be terminators as well, since local |
230 | | /// values for phis after invokes must be materialized before the call. |
231 | | void FastISel::InstOrderMap::initialize( |
232 | 1.92k | MachineBasicBlock *MBB, MachineBasicBlock::iterator LastFlushPoint) { |
233 | 1.92k | unsigned Order = 0; |
234 | 17.2k | for (MachineInstr &I : *MBB) { |
235 | 17.2k | if (!FirstTerminator && |
236 | 17.2k | (17.1k I.isTerminator()17.1k || (15.7k I.isEHLabel()15.7k && &I != &MBB->front()6 ))) { |
237 | 1.41k | FirstTerminator = &I; |
238 | 1.41k | FirstTerminatorOrder = Order; |
239 | 1.41k | } |
240 | 17.2k | Orders[&I] = Order++; |
241 | 17.2k | |
242 | 17.2k | // We don't need to order instructions past the last flush point. |
243 | 17.2k | if (I.getIterator() == LastFlushPoint) |
244 | 737 | break; |
245 | 17.2k | } |
246 | 1.92k | } |
247 | | |
248 | | void FastISel::sinkLocalValueMaterialization(MachineInstr &LocalMI, |
249 | | unsigned DefReg, |
250 | 2.80k | InstOrderMap &OrderMap) { |
251 | 2.80k | // If this register is used by a register fixup, MRI will not contain all |
252 | 2.80k | // the uses until after register fixups, so don't attempt to sink or DCE |
253 | 2.80k | // this instruction. Register fixups typically come from no-op cast |
254 | 2.80k | // instructions, which replace the cast instruction vreg with the local |
255 | 2.80k | // value vreg. |
256 | 2.80k | if (FuncInfo.RegsWithFixups.count(DefReg)) |
257 | 37 | return; |
258 | 2.76k | |
259 | 2.76k | // We can DCE this instruction if there are no uses and it wasn't a |
260 | 2.76k | // materialized for a successor PHI node. |
261 | 2.76k | bool UsedByPHI = isRegUsedByPhiNodes(DefReg, FuncInfo); |
262 | 2.76k | if (!UsedByPHI && MRI.use_nodbg_empty(DefReg)2.72k ) { |
263 | 129 | if (EmitStartPt == &LocalMI) |
264 | 0 | EmitStartPt = EmitStartPt->getPrevNode(); |
265 | 129 | LLVM_DEBUG(dbgs() << "removing dead local value materialization " |
266 | 129 | << LocalMI); |
267 | 129 | OrderMap.Orders.erase(&LocalMI); |
268 | 129 | LocalMI.eraseFromParent(); |
269 | 129 | return; |
270 | 129 | } |
271 | 2.63k | |
272 | 2.63k | // Number the instructions if we haven't yet so we can efficiently find the |
273 | 2.63k | // earliest use. |
274 | 2.63k | if (OrderMap.Orders.empty()) |
275 | 1.92k | OrderMap.initialize(FuncInfo.MBB, LastFlushPoint); |
276 | 2.63k | |
277 | 2.63k | // Find the first user in the BB. |
278 | 2.63k | MachineInstr *FirstUser = nullptr; |
279 | 2.63k | unsigned FirstOrder = std::numeric_limits<unsigned>::max(); |
280 | 2.84k | for (MachineInstr &UseInst : MRI.use_nodbg_instructions(DefReg)) { |
281 | 2.84k | auto I = OrderMap.Orders.find(&UseInst); |
282 | 2.84k | assert(I != OrderMap.Orders.end() && |
283 | 2.84k | "local value used by instruction outside local region"); |
284 | 2.84k | unsigned UseOrder = I->second; |
285 | 2.84k | if (UseOrder < FirstOrder) { |
286 | 2.63k | FirstOrder = UseOrder; |
287 | 2.63k | FirstUser = &UseInst; |
288 | 2.63k | } |
289 | 2.84k | } |
290 | 2.63k | |
291 | 2.63k | // The insertion point will be the first terminator or the first user, |
292 | 2.63k | // whichever came first. If there was no terminator, this must be a |
293 | 2.63k | // fallthrough block and the insertion point is the end of the block. |
294 | 2.63k | MachineBasicBlock::instr_iterator SinkPos; |
295 | 2.63k | if (UsedByPHI && OrderMap.FirstTerminatorOrder < FirstOrder37 ) { |
296 | 33 | FirstOrder = OrderMap.FirstTerminatorOrder; |
297 | 33 | SinkPos = OrderMap.FirstTerminator->getIterator(); |
298 | 2.60k | } else if (FirstUser) { |
299 | 2.59k | SinkPos = FirstUser->getIterator(); |
300 | 2.59k | } else { |
301 | 4 | assert(UsedByPHI && "must be users if not used by a phi"); |
302 | 4 | SinkPos = FuncInfo.MBB->instr_end(); |
303 | 4 | } |
304 | 2.63k | |
305 | 2.63k | // Collect all DBG_VALUEs before the new insertion position so that we can |
306 | 2.63k | // sink them. |
307 | 2.63k | SmallVector<MachineInstr *, 1> DbgValues; |
308 | 2.85k | for (MachineInstr &DbgVal : MRI.use_instructions(DefReg)) { |
309 | 2.85k | if (!DbgVal.isDebugValue()) |
310 | 2.84k | continue; |
311 | 14 | unsigned UseOrder = OrderMap.Orders[&DbgVal]; |
312 | 14 | if (UseOrder < FirstOrder) |
313 | 14 | DbgValues.push_back(&DbgVal); |
314 | 14 | } |
315 | 2.63k | |
316 | 2.63k | // Sink LocalMI before SinkPos and assign it the same DebugLoc. |
317 | 2.63k | LLVM_DEBUG(dbgs() << "sinking local value to first use " << LocalMI); |
318 | 2.63k | FuncInfo.MBB->remove(&LocalMI); |
319 | 2.63k | FuncInfo.MBB->insert(SinkPos, &LocalMI); |
320 | 2.63k | if (SinkPos != FuncInfo.MBB->end()) |
321 | 2.63k | LocalMI.setDebugLoc(SinkPos->getDebugLoc()); |
322 | 2.63k | |
323 | 2.63k | // Sink any debug values that we've collected. |
324 | 2.63k | for (MachineInstr *DI : DbgValues) { |
325 | 14 | FuncInfo.MBB->remove(DI); |
326 | 14 | FuncInfo.MBB->insert(SinkPos, DI); |
327 | 14 | } |
328 | 2.63k | } |
329 | | |
330 | 13.0k | bool FastISel::hasTrivialKill(const Value *V) { |
331 | 13.0k | // Don't consider constants or arguments to have trivial kills. |
332 | 13.0k | const Instruction *I = dyn_cast<Instruction>(V); |
333 | 13.0k | if (!I) |
334 | 6.79k | return false; |
335 | 6.26k | |
336 | 6.26k | // No-op casts are trivially coalesced by fast-isel. |
337 | 6.26k | if (const auto *Cast = dyn_cast<CastInst>(I)) |
338 | 1.06k | if (Cast->isNoopCast(DL) && !hasTrivialKill(Cast->getOperand(0))455 ) |
339 | 316 | return false; |
340 | 5.94k | |
341 | 5.94k | // Even the value might have only one use in the LLVM IR, it is possible that |
342 | 5.94k | // FastISel might fold the use into another instruction and now there is more |
343 | 5.94k | // than one use at the Machine Instruction level. |
344 | 5.94k | unsigned Reg = lookUpRegForValue(V); |
345 | 5.94k | if (Reg && !MRI.use_empty(Reg)5.78k ) |
346 | 210 | return false; |
347 | 5.73k | |
348 | 5.73k | // GEPs with all zero indices are trivially coalesced by fast-isel. |
349 | 5.73k | if (const auto *GEP = dyn_cast<GetElementPtrInst>(I)) |
350 | 56 | if (GEP->hasAllZeroIndices() && !hasTrivialKill(GEP->getOperand(0))24 ) |
351 | 9 | return false; |
352 | 5.73k | |
353 | 5.73k | // Only instructions with a single use in the same basic block are considered |
354 | 5.73k | // to have trivial kills. |
355 | 5.73k | return I->hasOneUse() && |
356 | 5.73k | !(5.22k I->getOpcode() == Instruction::BitCast5.22k || |
357 | 5.22k | I->getOpcode() == Instruction::PtrToInt5.19k || |
358 | 5.22k | I->getOpcode() == Instruction::IntToPtr5.18k ) && |
359 | 5.73k | cast<Instruction>(*I->user_begin())->getParent() == I->getParent()5.18k ; |
360 | 5.73k | } |
361 | | |
362 | 41.6k | unsigned FastISel::getRegForValue(const Value *V) { |
363 | 41.6k | EVT RealVT = TLI.getValueType(DL, V->getType(), /*AllowUnknown=*/true); |
364 | 41.6k | // Don't handle non-simple values in FastISel. |
365 | 41.6k | if (!RealVT.isSimple()) |
366 | 9 | return 0; |
367 | 41.6k | |
368 | 41.6k | // Ignore illegal types. We must do this before looking up the value |
369 | 41.6k | // in ValueMap because Arguments are given virtual registers regardless |
370 | 41.6k | // of whether FastISel can handle them. |
371 | 41.6k | MVT VT = RealVT.getSimpleVT(); |
372 | 41.6k | if (!TLI.isTypeLegal(VT)) { |
373 | 3.54k | // Handle integer promotions, though, because they're common and easy. |
374 | 3.54k | if (VT == MVT::i1 || VT == MVT::i82.25k || VT == MVT::i161.29k ) |
375 | 3.04k | VT = TLI.getTypeToTransformTo(V->getContext(), VT).getSimpleVT(); |
376 | 497 | else |
377 | 497 | return 0; |
378 | 41.1k | } |
379 | 41.1k | |
380 | 41.1k | // Look up the value to see if we already have a register for it. |
381 | 41.1k | unsigned Reg = lookUpRegForValue(V); |
382 | 41.1k | if (Reg) |
383 | 19.1k | return Reg; |
384 | 21.9k | |
385 | 21.9k | // In bottom-up mode, just create the virtual register which will be used |
386 | 21.9k | // to hold the value. It will be materialized later. |
387 | 21.9k | if (isa<Instruction>(V) && |
388 | 21.9k | (17.6k !isa<AllocaInst>(V)17.6k || |
389 | 17.6k | !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(V))478 )) |
390 | 17.1k | return FuncInfo.InitializeRegForValue(V); |
391 | 4.77k | |
392 | 4.77k | SavePoint SaveInsertPt = enterLocalValueArea(); |
393 | 4.77k | |
394 | 4.77k | // Materialize the value in a register. Emit any instructions in the |
395 | 4.77k | // local value area. |
396 | 4.77k | Reg = materializeRegForValue(V, VT); |
397 | 4.77k | |
398 | 4.77k | leaveLocalValueArea(SaveInsertPt); |
399 | 4.77k | |
400 | 4.77k | return Reg; |
401 | 4.77k | } |
402 | | |
403 | 1.40k | unsigned FastISel::materializeConstant(const Value *V, MVT VT) { |
404 | 1.40k | unsigned Reg = 0; |
405 | 1.40k | if (const auto *CI = dyn_cast<ConstantInt>(V)) { |
406 | 144 | if (CI->getValue().getActiveBits() <= 64) |
407 | 144 | Reg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue()); |
408 | 1.25k | } else if (isa<AllocaInst>(V)) |
409 | 464 | Reg = fastMaterializeAlloca(cast<AllocaInst>(V)); |
410 | 795 | else if (isa<ConstantPointerNull>(V)) |
411 | 39 | // Translate this as an integer zero so that it can be |
412 | 39 | // local-CSE'd with actual integer zeros. |
413 | 39 | Reg = getRegForValue( |
414 | 39 | Constant::getNullValue(DL.getIntPtrType(V->getContext()))); |
415 | 756 | else if (const auto *CF = dyn_cast<ConstantFP>(V)) { |
416 | 9 | if (CF->isNullValue()) |
417 | 1 | Reg = fastMaterializeFloatZero(CF); |
418 | 8 | else |
419 | 8 | // Try to emit the constant directly. |
420 | 8 | Reg = fastEmit_f(VT, VT, ISD::ConstantFP, CF); |
421 | 9 | |
422 | 9 | if (!Reg) { |
423 | 1 | // Try to emit the constant by using an integer constant with a cast. |
424 | 1 | const APFloat &Flt = CF->getValueAPF(); |
425 | 1 | EVT IntVT = TLI.getPointerTy(DL); |
426 | 1 | uint32_t IntBitWidth = IntVT.getSizeInBits(); |
427 | 1 | APSInt SIntVal(IntBitWidth, /*isUnsigned=*/false); |
428 | 1 | bool isExact; |
429 | 1 | (void)Flt.convertToInteger(SIntVal, APFloat::rmTowardZero, &isExact); |
430 | 1 | if (isExact) { |
431 | 1 | unsigned IntegerReg = |
432 | 1 | getRegForValue(ConstantInt::get(V->getContext(), SIntVal)); |
433 | 1 | if (IntegerReg != 0) |
434 | 1 | Reg = fastEmit_r(IntVT.getSimpleVT(), VT, ISD::SINT_TO_FP, IntegerReg, |
435 | 1 | /*Kill=*/false); |
436 | 1 | } |
437 | 1 | } |
438 | 747 | } else if (const auto *Op = dyn_cast<Operator>(V)) { |
439 | 280 | if (!selectOperator(Op, Op->getOpcode())) |
440 | 67 | if (!isa<Instruction>(Op) || |
441 | 67 | !fastSelectInstruction(cast<Instruction>(Op))0 ) |
442 | 67 | return 0; |
443 | 213 | Reg = lookUpRegForValue(Op); |
444 | 467 | } else if (isa<UndefValue>(V)) { |
445 | 192 | Reg = createResultReg(TLI.getRegClassFor(VT)); |
446 | 192 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
447 | 192 | TII.get(TargetOpcode::IMPLICIT_DEF), Reg); |
448 | 192 | } |
449 | 1.40k | return Reg1.33k ; |
450 | 1.40k | } |
451 | | |
452 | | /// Helper for getRegForValue. This function is called when the value isn't |
453 | | /// already available in a register and must be materialized with new |
454 | | /// instructions. |
455 | 4.77k | unsigned FastISel::materializeRegForValue(const Value *V, MVT VT) { |
456 | 4.77k | unsigned Reg = 0; |
457 | 4.77k | // Give the target-specific code a try first. |
458 | 4.77k | if (isa<Constant>(V)) |
459 | 4.30k | Reg = fastMaterializeConstant(cast<Constant>(V)); |
460 | 4.77k | |
461 | 4.77k | // If target-specific code couldn't or didn't want to handle the value, then |
462 | 4.77k | // give target-independent code a try. |
463 | 4.77k | if (!Reg) |
464 | 1.40k | Reg = materializeConstant(V, VT); |
465 | 4.77k | |
466 | 4.77k | // Don't cache constant materializations in the general ValueMap. |
467 | 4.77k | // To do so would require tracking what uses they dominate. |
468 | 4.77k | if (Reg) { |
469 | 4.42k | LocalValueMap[V] = Reg; |
470 | 4.42k | LastLocalValue = MRI.getVRegDef(Reg); |
471 | 4.42k | } |
472 | 4.77k | return Reg; |
473 | 4.77k | } |
474 | | |
475 | 48.0k | unsigned FastISel::lookUpRegForValue(const Value *V) { |
476 | 48.0k | // Look up the value to see if we already have a register for it. We |
477 | 48.0k | // cache values defined by Instructions across blocks, and other values |
478 | 48.0k | // only locally. This is because Instructions already have the SSA |
479 | 48.0k | // def-dominates-use requirement enforced. |
480 | 48.0k | DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(V); |
481 | 48.0k | if (I != FuncInfo.ValueMap.end()) |
482 | 23.3k | return I->second; |
483 | 24.7k | return LocalValueMap[V]; |
484 | 24.7k | } |
485 | | |
486 | 18.7k | void FastISel::updateValueMap(const Value *I, unsigned Reg, unsigned NumRegs) { |
487 | 18.7k | if (!isa<Instruction>(I)) { |
488 | 6.52k | LocalValueMap[I] = Reg; |
489 | 6.52k | return; |
490 | 6.52k | } |
491 | 12.2k | |
492 | 12.2k | unsigned &AssignedReg = FuncInfo.ValueMap[I]; |
493 | 12.2k | if (AssignedReg == 0) |
494 | 320 | // Use the new register. |
495 | 320 | AssignedReg = Reg; |
496 | 11.9k | else if (Reg != AssignedReg) { |
497 | 11.9k | // Arrange for uses of AssignedReg to be replaced by uses of Reg. |
498 | 23.9k | for (unsigned i = 0; i < NumRegs; i++12.0k ) { |
499 | 12.0k | FuncInfo.RegFixups[AssignedReg + i] = Reg + i; |
500 | 12.0k | FuncInfo.RegsWithFixups.insert(Reg + i); |
501 | 12.0k | } |
502 | 11.9k | |
503 | 11.9k | AssignedReg = Reg; |
504 | 11.9k | } |
505 | 12.2k | } |
506 | | |
507 | 141 | std::pair<unsigned, bool> FastISel::getRegForGEPIndex(const Value *Idx) { |
508 | 141 | unsigned IdxN = getRegForValue(Idx); |
509 | 141 | if (IdxN == 0) |
510 | 9 | // Unhandled operand. Halt "fast" selection and bail. |
511 | 9 | return std::pair<unsigned, bool>(0, false); |
512 | 132 | |
513 | 132 | bool IdxNIsKill = hasTrivialKill(Idx); |
514 | 132 | |
515 | 132 | // If the index is smaller or larger than intptr_t, truncate or extend it. |
516 | 132 | MVT PtrVT = TLI.getPointerTy(DL); |
517 | 132 | EVT IdxVT = EVT::getEVT(Idx->getType(), /*HandleUnknown=*/false); |
518 | 132 | if (IdxVT.bitsLT(PtrVT)) { |
519 | 12 | IdxN = fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::SIGN_EXTEND, IdxN, |
520 | 12 | IdxNIsKill); |
521 | 12 | IdxNIsKill = true; |
522 | 120 | } else if (IdxVT.bitsGT(PtrVT)) { |
523 | 9 | IdxN = |
524 | 9 | fastEmit_r(IdxVT.getSimpleVT(), PtrVT, ISD::TRUNCATE, IdxN, IdxNIsKill); |
525 | 9 | IdxNIsKill = true; |
526 | 9 | } |
527 | 132 | return std::pair<unsigned, bool>(IdxN, IdxNIsKill); |
528 | 132 | } |
529 | | |
530 | 120k | void FastISel::recomputeInsertPt() { |
531 | 120k | if (getLastLocalValue()) { |
532 | 59.0k | FuncInfo.InsertPt = getLastLocalValue(); |
533 | 59.0k | FuncInfo.MBB = FuncInfo.InsertPt->getParent(); |
534 | 59.0k | ++FuncInfo.InsertPt; |
535 | 59.0k | } else |
536 | 61.2k | FuncInfo.InsertPt = FuncInfo.MBB->getFirstNonPHI(); |
537 | 120k | |
538 | 120k | // Now skip past any EH_LABELs, which must remain at the beginning. |
539 | 120k | while (FuncInfo.InsertPt != FuncInfo.MBB->end() && |
540 | 120k | FuncInfo.InsertPt->getOpcode() == TargetOpcode::EH_LABEL86.9k ) |
541 | 43 | ++FuncInfo.InsertPt; |
542 | 120k | } |
543 | | |
544 | | void FastISel::removeDeadCode(MachineBasicBlock::iterator I, |
545 | 721 | MachineBasicBlock::iterator E) { |
546 | 721 | assert(I.isValid() && E.isValid() && std::distance(I, E) > 0 && |
547 | 721 | "Invalid iterator!"); |
548 | 1.57k | while (I != E) { |
549 | 851 | if (LastFlushPoint == I) |
550 | 98 | LastFlushPoint = E; |
551 | 851 | if (SavedInsertPt == I) |
552 | 175 | SavedInsertPt = E; |
553 | 851 | if (EmitStartPt == I) |
554 | 0 | EmitStartPt = E.isValid() ? &*E : nullptr; |
555 | 851 | if (LastLocalValue == I) |
556 | 0 | LastLocalValue = E.isValid() ? &*E : nullptr; |
557 | 851 | |
558 | 851 | MachineInstr *Dead = &*I; |
559 | 851 | ++I; |
560 | 851 | Dead->eraseFromParent(); |
561 | 851 | ++NumFastIselDead; |
562 | 851 | } |
563 | 721 | recomputeInsertPt(); |
564 | 721 | } |
565 | | |
566 | 4.84k | FastISel::SavePoint FastISel::enterLocalValueArea() { |
567 | 4.84k | MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt; |
568 | 4.84k | DebugLoc OldDL = DbgLoc; |
569 | 4.84k | recomputeInsertPt(); |
570 | 4.84k | DbgLoc = DebugLoc(); |
571 | 4.84k | SavePoint SP = {OldInsertPt, OldDL}; |
572 | 4.84k | return SP; |
573 | 4.84k | } |
574 | | |
575 | 4.84k | void FastISel::leaveLocalValueArea(SavePoint OldInsertPt) { |
576 | 4.84k | if (FuncInfo.InsertPt != FuncInfo.MBB->begin()) |
577 | 4.62k | LastLocalValue = &*std::prev(FuncInfo.InsertPt); |
578 | 4.84k | |
579 | 4.84k | // Restore the previous insert position. |
580 | 4.84k | FuncInfo.InsertPt = OldInsertPt.InsertPt; |
581 | 4.84k | DbgLoc = OldInsertPt.DL; |
582 | 4.84k | } |
583 | | |
584 | 2.06k | bool FastISel::selectBinaryOp(const User *I, unsigned ISDOpcode) { |
585 | 2.06k | EVT VT = EVT::getEVT(I->getType(), /*HandleUnknown=*/true); |
586 | 2.06k | if (VT == MVT::Other || !VT.isSimple()) |
587 | 0 | // Unhandled type. Halt "fast" selection and bail. |
588 | 0 | return false; |
589 | 2.06k | |
590 | 2.06k | // We only handle legal types. For example, on x86-32 the instruction |
591 | 2.06k | // selector contains all of the 64-bit instructions from x86-64, |
592 | 2.06k | // under the assumption that i64 won't be used if the target doesn't |
593 | 2.06k | // support it. |
594 | 2.06k | if (!TLI.isTypeLegal(VT)) { |
595 | 148 | // MVT::i1 is special. Allow AND, OR, or XOR because they |
596 | 148 | // don't require additional zeroing, which makes them easy. |
597 | 148 | if (VT == MVT::i1 && (41 ISDOpcode == ISD::AND41 || ISDOpcode == ISD::OR36 || |
598 | 41 | ISDOpcode == ISD::XOR28 )) |
599 | 34 | VT = TLI.getTypeToTransformTo(I->getContext(), VT); |
600 | 114 | else |
601 | 114 | return false; |
602 | 1.95k | } |
603 | 1.95k | |
604 | 1.95k | // Check if the first operand is a constant, and handle it as "ri". At -O0, |
605 | 1.95k | // we don't have anything that canonicalizes operand order. |
606 | 1.95k | if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(0))) |
607 | 66 | if (isa<Instruction>(I) && cast<Instruction>(I)->isCommutative()) { |
608 | 35 | unsigned Op1 = getRegForValue(I->getOperand(1)); |
609 | 35 | if (!Op1) |
610 | 0 | return false; |
611 | 35 | bool Op1IsKill = hasTrivialKill(I->getOperand(1)); |
612 | 35 | |
613 | 35 | unsigned ResultReg = |
614 | 35 | fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op1, Op1IsKill, |
615 | 35 | CI->getZExtValue(), VT.getSimpleVT()); |
616 | 35 | if (!ResultReg) |
617 | 0 | return false; |
618 | 35 | |
619 | 35 | // We successfully emitted code for the given LLVM Instruction. |
620 | 35 | updateValueMap(I, ResultReg); |
621 | 35 | return true; |
622 | 35 | } |
623 | 1.91k | |
624 | 1.91k | unsigned Op0 = getRegForValue(I->getOperand(0)); |
625 | 1.91k | if (!Op0) // Unhandled operand. Halt "fast" selection and bail. |
626 | 29 | return false; |
627 | 1.88k | bool Op0IsKill = hasTrivialKill(I->getOperand(0)); |
628 | 1.88k | |
629 | 1.88k | // Check if the second operand is a constant and handle it appropriately. |
630 | 1.88k | if (const auto *CI = dyn_cast<ConstantInt>(I->getOperand(1))) { |
631 | 470 | uint64_t Imm = CI->getSExtValue(); |
632 | 470 | |
633 | 470 | // Transform "sdiv exact X, 8" -> "sra X, 3". |
634 | 470 | if (ISDOpcode == ISD::SDIV && isa<BinaryOperator>(I)5 && |
635 | 470 | cast<BinaryOperator>(I)->isExact()5 && isPowerOf2_64(Imm)4 ) { |
636 | 4 | Imm = Log2_64(Imm); |
637 | 4 | ISDOpcode = ISD::SRA; |
638 | 4 | } |
639 | 470 | |
640 | 470 | // Transform "urem x, pow2" -> "and x, pow2-1". |
641 | 470 | if (ISDOpcode == ISD::UREM && isa<BinaryOperator>(I)5 && |
642 | 470 | isPowerOf2_64(Imm)5 ) { |
643 | 3 | --Imm; |
644 | 3 | ISDOpcode = ISD::AND; |
645 | 3 | } |
646 | 470 | |
647 | 470 | unsigned ResultReg = fastEmit_ri_(VT.getSimpleVT(), ISDOpcode, Op0, |
648 | 470 | Op0IsKill, Imm, VT.getSimpleVT()); |
649 | 470 | if (!ResultReg) |
650 | 16 | return false; |
651 | 454 | |
652 | 454 | // We successfully emitted code for the given LLVM Instruction. |
653 | 454 | updateValueMap(I, ResultReg); |
654 | 454 | return true; |
655 | 454 | } |
656 | 1.41k | |
657 | 1.41k | unsigned Op1 = getRegForValue(I->getOperand(1)); |
658 | 1.41k | if (!Op1) // Unhandled operand. Halt "fast" selection and bail. |
659 | 71 | return false; |
660 | 1.34k | bool Op1IsKill = hasTrivialKill(I->getOperand(1)); |
661 | 1.34k | |
662 | 1.34k | // Now we have both operands in registers. Emit the instruction. |
663 | 1.34k | unsigned ResultReg = fastEmit_rr(VT.getSimpleVT(), VT.getSimpleVT(), |
664 | 1.34k | ISDOpcode, Op0, Op0IsKill, Op1, Op1IsKill); |
665 | 1.34k | if (!ResultReg) |
666 | 252 | // Target-specific code wasn't able to find a machine opcode for |
667 | 252 | // the given ISD opcode and type. Halt "fast" selection and bail. |
668 | 252 | return false; |
669 | 1.09k | |
670 | 1.09k | // We successfully emitted code for the given LLVM Instruction. |
671 | 1.09k | updateValueMap(I, ResultReg); |
672 | 1.09k | return true; |
673 | 1.09k | } |
674 | | |
675 | 325 | bool FastISel::selectGetElementPtr(const User *I) { |
676 | 325 | unsigned N = getRegForValue(I->getOperand(0)); |
677 | 325 | if (!N) // Unhandled operand. Halt "fast" selection and bail. |
678 | 48 | return false; |
679 | 277 | bool NIsKill = hasTrivialKill(I->getOperand(0)); |
680 | 277 | |
681 | 277 | // Keep a running tab of the total offset to coalesce multiple N = N + Offset |
682 | 277 | // into a single N = N + TotalOffset. |
683 | 277 | uint64_t TotalOffs = 0; |
684 | 277 | // FIXME: What's a good SWAG number for MaxOffs? |
685 | 277 | uint64_t MaxOffs = 2048; |
686 | 277 | MVT VT = TLI.getPointerTy(DL); |
687 | 277 | for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I); |
688 | 836 | GTI != E; ++GTI559 ) { |
689 | 577 | const Value *Idx = GTI.getOperand(); |
690 | 577 | if (StructType *StTy = GTI.getStructTypeOrNull()) { |
691 | 53 | uint64_t Field = cast<ConstantInt>(Idx)->getZExtValue(); |
692 | 53 | if (Field) { |
693 | 27 | // N = N + Offset |
694 | 27 | TotalOffs += DL.getStructLayout(StTy)->getElementOffset(Field); |
695 | 27 | if (TotalOffs >= MaxOffs) { |
696 | 0 | N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT); |
697 | 0 | if (!N) // Unhandled operand. Halt "fast" selection and bail. |
698 | 0 | return false; |
699 | 0 | NIsKill = true; |
700 | 0 | TotalOffs = 0; |
701 | 0 | } |
702 | 27 | } |
703 | 524 | } else { |
704 | 524 | Type *Ty = GTI.getIndexedType(); |
705 | 524 | |
706 | 524 | // If this is a constant subscript, handle it quickly. |
707 | 524 | if (const auto *CI = dyn_cast<ConstantInt>(Idx)) { |
708 | 467 | if (CI->isZero()) |
709 | 318 | continue; |
710 | 149 | // N = N + Offset |
711 | 149 | uint64_t IdxN = CI->getValue().sextOrTrunc(64).getSExtValue(); |
712 | 149 | TotalOffs += DL.getTypeAllocSize(Ty) * IdxN; |
713 | 149 | if (TotalOffs >= MaxOffs) { |
714 | 12 | N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT); |
715 | 12 | if (!N) // Unhandled operand. Halt "fast" selection and bail. |
716 | 0 | return false; |
717 | 12 | NIsKill = true; |
718 | 12 | TotalOffs = 0; |
719 | 12 | } |
720 | 149 | continue; |
721 | 57 | } |
722 | 57 | if (TotalOffs) { |
723 | 0 | N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT); |
724 | 0 | if (!N) // Unhandled operand. Halt "fast" selection and bail. |
725 | 0 | return false; |
726 | 0 | NIsKill = true; |
727 | 0 | TotalOffs = 0; |
728 | 0 | } |
729 | 57 | |
730 | 57 | // N = N + Idx * ElementSize; |
731 | 57 | uint64_t ElementSize = DL.getTypeAllocSize(Ty); |
732 | 57 | std::pair<unsigned, bool> Pair = getRegForGEPIndex(Idx); |
733 | 57 | unsigned IdxN = Pair.first; |
734 | 57 | bool IdxNIsKill = Pair.second; |
735 | 57 | if (!IdxN) // Unhandled operand. Halt "fast" selection and bail. |
736 | 15 | return false; |
737 | 42 | |
738 | 42 | if (ElementSize != 1) { |
739 | 18 | IdxN = fastEmit_ri_(VT, ISD::MUL, IdxN, IdxNIsKill, ElementSize, VT); |
740 | 18 | if (!IdxN) // Unhandled operand. Halt "fast" selection and bail. |
741 | 3 | return false; |
742 | 15 | IdxNIsKill = true; |
743 | 15 | } |
744 | 42 | N = fastEmit_rr(VT, VT, ISD::ADD, N, NIsKill, IdxN, IdxNIsKill); |
745 | 39 | if (!N) // Unhandled operand. Halt "fast" selection and bail. |
746 | 0 | return false; |
747 | 39 | } |
748 | 577 | } |
749 | 277 | if (259 TotalOffs259 ) { |
750 | 96 | N = fastEmit_ri_(VT, ISD::ADD, N, NIsKill, TotalOffs, VT); |
751 | 96 | if (!N) // Unhandled operand. Halt "fast" selection and bail. |
752 | 0 | return false; |
753 | 259 | } |
754 | 259 | |
755 | 259 | // We successfully emitted code for the given LLVM Instruction. |
756 | 259 | updateValueMap(I, N); |
757 | 259 | return true; |
758 | 259 | } |
759 | | |
760 | | bool FastISel::addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops, |
761 | 65 | const CallInst *CI, unsigned StartIdx) { |
762 | 184 | for (unsigned i = StartIdx, e = CI->getNumArgOperands(); i != e; ++i119 ) { |
763 | 119 | Value *Val = CI->getArgOperand(i); |
764 | 119 | // Check for constants and encode them with a StackMaps::ConstantOp prefix. |
765 | 119 | if (const auto *C = dyn_cast<ConstantInt>(Val)) { |
766 | 18 | Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp)); |
767 | 18 | Ops.push_back(MachineOperand::CreateImm(C->getSExtValue())); |
768 | 101 | } else if (isa<ConstantPointerNull>(Val)) { |
769 | 0 | Ops.push_back(MachineOperand::CreateImm(StackMaps::ConstantOp)); |
770 | 0 | Ops.push_back(MachineOperand::CreateImm(0)); |
771 | 101 | } else if (auto *AI = dyn_cast<AllocaInst>(Val)) { |
772 | 8 | // Values coming from a stack location also require a special encoding, |
773 | 8 | // but that is added later on by the target specific frame index |
774 | 8 | // elimination implementation. |
775 | 8 | auto SI = FuncInfo.StaticAllocaMap.find(AI); |
776 | 8 | if (SI != FuncInfo.StaticAllocaMap.end()) |
777 | 8 | Ops.push_back(MachineOperand::CreateFI(SI->second)); |
778 | 0 | else |
779 | 0 | return false; |
780 | 93 | } else { |
781 | 93 | unsigned Reg = getRegForValue(Val); |
782 | 93 | if (!Reg) |
783 | 0 | return false; |
784 | 93 | Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false)); |
785 | 93 | } |
786 | 119 | } |
787 | 65 | return true; |
788 | 65 | } |
789 | | |
790 | 26 | bool FastISel::selectStackmap(const CallInst *I) { |
791 | 26 | // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>, |
792 | 26 | // [live variables...]) |
793 | 26 | assert(I->getCalledFunction()->getReturnType()->isVoidTy() && |
794 | 26 | "Stackmap cannot return a value."); |
795 | 26 | |
796 | 26 | // The stackmap intrinsic only records the live variables (the arguments |
797 | 26 | // passed to it) and emits NOPS (if requested). Unlike the patchpoint |
798 | 26 | // intrinsic, this won't be lowered to a function call. This means we don't |
799 | 26 | // have to worry about calling conventions and target-specific lowering code. |
800 | 26 | // Instead we perform the call lowering right here. |
801 | 26 | // |
802 | 26 | // CALLSEQ_START(0, 0...) |
803 | 26 | // STACKMAP(id, nbytes, ...) |
804 | 26 | // CALLSEQ_END(0, 0) |
805 | 26 | // |
806 | 26 | SmallVector<MachineOperand, 32> Ops; |
807 | 26 | |
808 | 26 | // Add the <id> and <numBytes> constants. |
809 | 26 | assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) && |
810 | 26 | "Expected a constant integer."); |
811 | 26 | const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)); |
812 | 26 | Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue())); |
813 | 26 | |
814 | 26 | assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) && |
815 | 26 | "Expected a constant integer."); |
816 | 26 | const auto *NumBytes = |
817 | 26 | cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)); |
818 | 26 | Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue())); |
819 | 26 | |
820 | 26 | // Push live variables for the stack map (skipping the first two arguments |
821 | 26 | // <id> and <numBytes>). |
822 | 26 | if (!addStackMapLiveVars(Ops, I, 2)) |
823 | 0 | return false; |
824 | 26 | |
825 | 26 | // We are not adding any register mask info here, because the stackmap doesn't |
826 | 26 | // clobber anything. |
827 | 26 | |
828 | 26 | // Add scratch registers as implicit def and early clobber. |
829 | 26 | CallingConv::ID CC = I->getCallingConv(); |
830 | 26 | const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC); |
831 | 82 | for (unsigned i = 0; ScratchRegs[i]; ++i56 ) |
832 | 56 | Ops.push_back(MachineOperand::CreateReg( |
833 | 56 | ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false, |
834 | 56 | /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true)); |
835 | 26 | |
836 | 26 | // Issue CALLSEQ_START |
837 | 26 | unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); |
838 | 26 | auto Builder = |
839 | 26 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackDown)); |
840 | 26 | const MCInstrDesc &MCID = Builder.getInstr()->getDesc(); |
841 | 89 | for (unsigned I = 0, E = MCID.getNumOperands(); I < E; ++I63 ) |
842 | 63 | Builder.addImm(0); |
843 | 26 | |
844 | 26 | // Issue STACKMAP. |
845 | 26 | MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
846 | 26 | TII.get(TargetOpcode::STACKMAP)); |
847 | 26 | for (auto const &MO : Ops) |
848 | 205 | MIB.add(MO); |
849 | 26 | |
850 | 26 | // Issue CALLSEQ_END |
851 | 26 | unsigned AdjStackUp = TII.getCallFrameDestroyOpcode(); |
852 | 26 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AdjStackUp)) |
853 | 26 | .addImm(0) |
854 | 26 | .addImm(0); |
855 | 26 | |
856 | 26 | // Inform the Frame Information that we have a stackmap in this function. |
857 | 26 | FuncInfo.MF->getFrameInfo().setHasStackMap(); |
858 | 26 | |
859 | 26 | return true; |
860 | 26 | } |
861 | | |
862 | | /// Lower an argument list according to the target calling convention. |
863 | | /// |
864 | | /// This is a helper for lowering intrinsics that follow a target calling |
865 | | /// convention or require stack pointer adjustment. Only a subset of the |
866 | | /// intrinsic's operands need to participate in the calling convention. |
867 | | bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx, |
868 | | unsigned NumArgs, const Value *Callee, |
869 | 39 | bool ForceRetVoidTy, CallLoweringInfo &CLI) { |
870 | 39 | ArgListTy Args; |
871 | 39 | Args.reserve(NumArgs); |
872 | 39 | |
873 | 39 | // Populate the argument list. |
874 | 39 | ImmutableCallSite CS(CI); |
875 | 154 | for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI115 ) { |
876 | 115 | Value *V = CI->getOperand(ArgI); |
877 | 115 | |
878 | 115 | assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); |
879 | 115 | |
880 | 115 | ArgListEntry Entry; |
881 | 115 | Entry.Val = V; |
882 | 115 | Entry.Ty = V->getType(); |
883 | 115 | Entry.setAttributes(&CS, ArgI); |
884 | 115 | Args.push_back(Entry); |
885 | 115 | } |
886 | 39 | |
887 | 39 | Type *RetTy = ForceRetVoidTy ? Type::getVoidTy(CI->getType()->getContext())1 |
888 | 39 | : CI->getType()38 ; |
889 | 39 | CLI.setCallee(CI->getCallingConv(), RetTy, Callee, std::move(Args), NumArgs); |
890 | 39 | |
891 | 39 | return lowerCallTo(CLI); |
892 | 39 | } |
893 | | |
894 | | FastISel::CallLoweringInfo &FastISel::CallLoweringInfo::setCallee( |
895 | | const DataLayout &DL, MCContext &Ctx, CallingConv::ID CC, Type *ResultTy, |
896 | 16 | StringRef Target, ArgListTy &&ArgsList, unsigned FixedArgs) { |
897 | 16 | SmallString<32> MangledName; |
898 | 16 | Mangler::getNameWithPrefix(MangledName, Target, DL); |
899 | 16 | MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName); |
900 | 16 | return setCallee(CC, ResultTy, Sym, std::move(ArgsList), FixedArgs); |
901 | 16 | } |
902 | | |
903 | 39 | bool FastISel::selectPatchpoint(const CallInst *I) { |
904 | 39 | // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>, |
905 | 39 | // i32 <numBytes>, |
906 | 39 | // i8* <target>, |
907 | 39 | // i32 <numArgs>, |
908 | 39 | // [Args...], |
909 | 39 | // [live variables...]) |
910 | 39 | CallingConv::ID CC = I->getCallingConv(); |
911 | 39 | bool IsAnyRegCC = CC == CallingConv::AnyReg; |
912 | 39 | bool HasDef = !I->getType()->isVoidTy(); |
913 | 39 | Value *Callee = I->getOperand(PatchPointOpers::TargetPos)->stripPointerCasts(); |
914 | 39 | |
915 | 39 | // Get the real number of arguments participating in the call <numArgs> |
916 | 39 | assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)) && |
917 | 39 | "Expected a constant integer."); |
918 | 39 | const auto *NumArgsVal = |
919 | 39 | cast<ConstantInt>(I->getOperand(PatchPointOpers::NArgPos)); |
920 | 39 | unsigned NumArgs = NumArgsVal->getZExtValue(); |
921 | 39 | |
922 | 39 | // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> |
923 | 39 | // This includes all meta-operands up to but not including CC. |
924 | 39 | unsigned NumMetaOpers = PatchPointOpers::CCPos; |
925 | 39 | assert(I->getNumArgOperands() >= NumMetaOpers + NumArgs && |
926 | 39 | "Not enough arguments provided to the patchpoint intrinsic"); |
927 | 39 | |
928 | 39 | // For AnyRegCC the arguments are lowered later on manually. |
929 | 39 | unsigned NumCallArgs = IsAnyRegCC ? 01 : NumArgs38 ; |
930 | 39 | CallLoweringInfo CLI; |
931 | 39 | CLI.setIsPatchPoint(); |
932 | 39 | if (!lowerCallOperands(I, NumMetaOpers, NumCallArgs, Callee, IsAnyRegCC, CLI)) |
933 | 0 | return false; |
934 | 39 | |
935 | 39 | assert(CLI.Call && "No call instruction specified."); |
936 | 39 | |
937 | 39 | SmallVector<MachineOperand, 32> Ops; |
938 | 39 | |
939 | 39 | // Add an explicit result reg if we use the anyreg calling convention. |
940 | 39 | if (IsAnyRegCC && HasDef1 ) { |
941 | 0 | assert(CLI.NumResultRegs == 0 && "Unexpected result register."); |
942 | 0 | CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64)); |
943 | 0 | CLI.NumResultRegs = 1; |
944 | 0 | Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*isDef=*/true)); |
945 | 0 | } |
946 | 39 | |
947 | 39 | // Add the <id> and <numBytes> constants. |
948 | 39 | assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)) && |
949 | 39 | "Expected a constant integer."); |
950 | 39 | const auto *ID = cast<ConstantInt>(I->getOperand(PatchPointOpers::IDPos)); |
951 | 39 | Ops.push_back(MachineOperand::CreateImm(ID->getZExtValue())); |
952 | 39 | |
953 | 39 | assert(isa<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)) && |
954 | 39 | "Expected a constant integer."); |
955 | 39 | const auto *NumBytes = |
956 | 39 | cast<ConstantInt>(I->getOperand(PatchPointOpers::NBytesPos)); |
957 | 39 | Ops.push_back(MachineOperand::CreateImm(NumBytes->getZExtValue())); |
958 | 39 | |
959 | 39 | // Add the call target. |
960 | 39 | if (const auto *C = dyn_cast<IntToPtrInst>(Callee)) { |
961 | 18 | uint64_t CalleeConstAddr = |
962 | 18 | cast<ConstantInt>(C->getOperand(0))->getZExtValue(); |
963 | 18 | Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr)); |
964 | 21 | } else if (const auto *C = dyn_cast<ConstantExpr>(Callee)) { |
965 | 4 | if (C->getOpcode() == Instruction::IntToPtr) { |
966 | 4 | uint64_t CalleeConstAddr = |
967 | 4 | cast<ConstantInt>(C->getOperand(0))->getZExtValue(); |
968 | 4 | Ops.push_back(MachineOperand::CreateImm(CalleeConstAddr)); |
969 | 4 | } else |
970 | 4 | llvm_unreachable0 ("Unsupported ConstantExpr."); |
971 | 17 | } else if (const auto *GV = dyn_cast<GlobalValue>(Callee)) { |
972 | 4 | Ops.push_back(MachineOperand::CreateGA(GV, 0)); |
973 | 13 | } else if (isa<ConstantPointerNull>(Callee)) |
974 | 13 | Ops.push_back(MachineOperand::CreateImm(0)); |
975 | 13 | else |
976 | 13 | llvm_unreachable0 ("Unsupported callee address."); |
977 | 39 | |
978 | 39 | // Adjust <numArgs> to account for any arguments that have been passed on |
979 | 39 | // the stack instead. |
980 | 39 | unsigned NumCallRegArgs = IsAnyRegCC ? NumArgs1 : CLI.OutRegs.size()38 ; |
981 | 39 | Ops.push_back(MachineOperand::CreateImm(NumCallRegArgs)); |
982 | 39 | |
983 | 39 | // Add the calling convention |
984 | 39 | Ops.push_back(MachineOperand::CreateImm((unsigned)CC)); |
985 | 39 | |
986 | 39 | // Add the arguments we omitted previously. The register allocator should |
987 | 39 | // place these in any free register. |
988 | 39 | if (IsAnyRegCC) { |
989 | 3 | for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i2 ) { |
990 | 2 | unsigned Reg = getRegForValue(I->getArgOperand(i)); |
991 | 2 | if (!Reg) |
992 | 0 | return false; |
993 | 2 | Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false)); |
994 | 2 | } |
995 | 1 | } |
996 | 39 | |
997 | 39 | // Push the arguments from the call instruction. |
998 | 39 | for (auto Reg : CLI.OutRegs) |
999 | 51 | Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false)); |
1000 | 39 | |
1001 | 39 | // Push live variables for the stack map. |
1002 | 39 | if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs)) |
1003 | 0 | return false; |
1004 | 39 | |
1005 | 39 | // Push the register mask info. |
1006 | 39 | Ops.push_back(MachineOperand::CreateRegMask( |
1007 | 39 | TRI.getCallPreservedMask(*FuncInfo.MF, CC))); |
1008 | 39 | |
1009 | 39 | // Add scratch registers as implicit def and early clobber. |
1010 | 39 | const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC); |
1011 | 132 | for (unsigned i = 0; ScratchRegs[i]; ++i93 ) |
1012 | 93 | Ops.push_back(MachineOperand::CreateReg( |
1013 | 93 | ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false, |
1014 | 93 | /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true)); |
1015 | 39 | |
1016 | 39 | // Add implicit defs (return values). |
1017 | 39 | for (auto Reg : CLI.InRegs) |
1018 | 21 | Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/true, |
1019 | 21 | /*isImp=*/true)); |
1020 | 39 | |
1021 | 39 | // Insert the patchpoint instruction before the call generated by the target. |
1022 | 39 | MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc, |
1023 | 39 | TII.get(TargetOpcode::PATCHPOINT)); |
1024 | 39 | |
1025 | 39 | for (auto &MO : Ops) |
1026 | 441 | MIB.add(MO); |
1027 | 39 | |
1028 | 39 | MIB->setPhysRegsDeadExcept(CLI.InRegs, TRI); |
1029 | 39 | |
1030 | 39 | // Delete the original call instruction. |
1031 | 39 | CLI.Call->eraseFromParent(); |
1032 | 39 | |
1033 | 39 | // Inform the Frame Information that we have a patchpoint in this function. |
1034 | 39 | FuncInfo.MF->getFrameInfo().setHasPatchPoint(); |
1035 | 39 | |
1036 | 39 | if (CLI.NumResultRegs) |
1037 | 21 | updateValueMap(I, CLI.ResultReg, CLI.NumResultRegs); |
1038 | 39 | return true; |
1039 | 39 | } |
1040 | | |
1041 | 0 | bool FastISel::selectXRayCustomEvent(const CallInst *I) { |
1042 | 0 | const auto &Triple = TM.getTargetTriple(); |
1043 | 0 | if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux()) |
1044 | 0 | return true; // don't do anything to this instruction. |
1045 | 0 | SmallVector<MachineOperand, 8> Ops; |
1046 | 0 | Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)), |
1047 | 0 | /*isDef=*/false)); |
1048 | 0 | Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)), |
1049 | 0 | /*isDef=*/false)); |
1050 | 0 | MachineInstrBuilder MIB = |
1051 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
1052 | 0 | TII.get(TargetOpcode::PATCHABLE_EVENT_CALL)); |
1053 | 0 | for (auto &MO : Ops) |
1054 | 0 | MIB.add(MO); |
1055 | 0 |
|
1056 | 0 | // Insert the Patchable Event Call instruction, that gets lowered properly. |
1057 | 0 | return true; |
1058 | 0 | } |
1059 | | |
1060 | 0 | bool FastISel::selectXRayTypedEvent(const CallInst *I) { |
1061 | 0 | const auto &Triple = TM.getTargetTriple(); |
1062 | 0 | if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux()) |
1063 | 0 | return true; // don't do anything to this instruction. |
1064 | 0 | SmallVector<MachineOperand, 8> Ops; |
1065 | 0 | Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)), |
1066 | 0 | /*isDef=*/false)); |
1067 | 0 | Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)), |
1068 | 0 | /*isDef=*/false)); |
1069 | 0 | Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)), |
1070 | 0 | /*isDef=*/false)); |
1071 | 0 | MachineInstrBuilder MIB = |
1072 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
1073 | 0 | TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL)); |
1074 | 0 | for (auto &MO : Ops) |
1075 | 0 | MIB.add(MO); |
1076 | 0 |
|
1077 | 0 | // Insert the Patchable Typed Event Call instruction, that gets lowered properly. |
1078 | 0 | return true; |
1079 | 0 | } |
1080 | | |
1081 | | /// Returns an AttributeList representing the attributes applied to the return |
1082 | | /// value of the given call. |
1083 | 2.19k | static AttributeList getReturnAttrs(FastISel::CallLoweringInfo &CLI) { |
1084 | 2.19k | SmallVector<Attribute::AttrKind, 2> Attrs; |
1085 | 2.19k | if (CLI.RetSExt) |
1086 | 54 | Attrs.push_back(Attribute::SExt); |
1087 | 2.19k | if (CLI.RetZExt) |
1088 | 45 | Attrs.push_back(Attribute::ZExt); |
1089 | 2.19k | if (CLI.IsInReg) |
1090 | 0 | Attrs.push_back(Attribute::InReg); |
1091 | 2.19k | |
1092 | 2.19k | return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex, |
1093 | 2.19k | Attrs); |
1094 | 2.19k | } |
1095 | | |
1096 | | bool FastISel::lowerCallTo(const CallInst *CI, const char *SymName, |
1097 | 34 | unsigned NumArgs) { |
1098 | 34 | MCContext &Ctx = MF->getContext(); |
1099 | 34 | SmallString<32> MangledName; |
1100 | 34 | Mangler::getNameWithPrefix(MangledName, SymName, DL); |
1101 | 34 | MCSymbol *Sym = Ctx.getOrCreateSymbol(MangledName); |
1102 | 34 | return lowerCallTo(CI, Sym, NumArgs); |
1103 | 34 | } |
1104 | | |
1105 | | bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol, |
1106 | 34 | unsigned NumArgs) { |
1107 | 34 | ImmutableCallSite CS(CI); |
1108 | 34 | |
1109 | 34 | FunctionType *FTy = CS.getFunctionType(); |
1110 | 34 | Type *RetTy = CS.getType(); |
1111 | 34 | |
1112 | 34 | ArgListTy Args; |
1113 | 34 | Args.reserve(NumArgs); |
1114 | 34 | |
1115 | 34 | // Populate the argument list. |
1116 | 34 | // Attributes for args start at offset 1, after the return attribute. |
1117 | 136 | for (unsigned ArgI = 0; ArgI != NumArgs; ++ArgI102 ) { |
1118 | 102 | Value *V = CI->getOperand(ArgI); |
1119 | 102 | |
1120 | 102 | assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); |
1121 | 102 | |
1122 | 102 | ArgListEntry Entry; |
1123 | 102 | Entry.Val = V; |
1124 | 102 | Entry.Ty = V->getType(); |
1125 | 102 | Entry.setAttributes(&CS, ArgI); |
1126 | 102 | Args.push_back(Entry); |
1127 | 102 | } |
1128 | 34 | TLI.markLibCallAttributes(MF, CS.getCallingConv(), Args); |
1129 | 34 | |
1130 | 34 | CallLoweringInfo CLI; |
1131 | 34 | CLI.setCallee(RetTy, FTy, Symbol, std::move(Args), CS, NumArgs); |
1132 | 34 | |
1133 | 34 | return lowerCallTo(CLI); |
1134 | 34 | } |
1135 | | |
1136 | 2.19k | bool FastISel::lowerCallTo(CallLoweringInfo &CLI) { |
1137 | 2.19k | // Handle the incoming return values from the call. |
1138 | 2.19k | CLI.clearIns(); |
1139 | 2.19k | SmallVector<EVT, 4> RetTys; |
1140 | 2.19k | ComputeValueVTs(TLI, DL, CLI.RetTy, RetTys); |
1141 | 2.19k | |
1142 | 2.19k | SmallVector<ISD::OutputArg, 4> Outs; |
1143 | 2.19k | GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, TLI, DL); |
1144 | 2.19k | |
1145 | 2.19k | bool CanLowerReturn = TLI.CanLowerReturn( |
1146 | 2.19k | CLI.CallConv, *FuncInfo.MF, CLI.IsVarArg, Outs, CLI.RetTy->getContext()); |
1147 | 2.19k | |
1148 | 2.19k | // FIXME: sret demotion isn't supported yet - bail out. |
1149 | 2.19k | if (!CanLowerReturn) |
1150 | 5 | return false; |
1151 | 2.19k | |
1152 | 3.14k | for (unsigned I = 0, E = RetTys.size(); 2.19k I != E; ++I951 ) { |
1153 | 951 | EVT VT = RetTys[I]; |
1154 | 951 | MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT); |
1155 | 951 | unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT); |
1156 | 1.92k | for (unsigned i = 0; i != NumRegs; ++i971 ) { |
1157 | 971 | ISD::InputArg MyFlags; |
1158 | 971 | MyFlags.VT = RegisterVT; |
1159 | 971 | MyFlags.ArgVT = VT; |
1160 | 971 | MyFlags.Used = CLI.IsReturnValueUsed; |
1161 | 971 | if (CLI.RetSExt) |
1162 | 54 | MyFlags.Flags.setSExt(); |
1163 | 971 | if (CLI.RetZExt) |
1164 | 45 | MyFlags.Flags.setZExt(); |
1165 | 971 | if (CLI.IsInReg) |
1166 | 0 | MyFlags.Flags.setInReg(); |
1167 | 971 | CLI.Ins.push_back(MyFlags); |
1168 | 971 | } |
1169 | 951 | } |
1170 | 2.19k | |
1171 | 2.19k | // Handle all of the outgoing arguments. |
1172 | 2.19k | CLI.clearOuts(); |
1173 | 4.46k | for (auto &Arg : CLI.getArgs()) { |
1174 | 4.46k | Type *FinalType = Arg.Ty; |
1175 | 4.46k | if (Arg.IsByVal) |
1176 | 35 | FinalType = cast<PointerType>(Arg.Ty)->getElementType(); |
1177 | 4.46k | bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( |
1178 | 4.46k | FinalType, CLI.CallConv, CLI.IsVarArg); |
1179 | 4.46k | |
1180 | 4.46k | ISD::ArgFlagsTy Flags; |
1181 | 4.46k | if (Arg.IsZExt) |
1182 | 186 | Flags.setZExt(); |
1183 | 4.46k | if (Arg.IsSExt) |
1184 | 56 | Flags.setSExt(); |
1185 | 4.46k | if (Arg.IsInReg) |
1186 | 16 | Flags.setInReg(); |
1187 | 4.46k | if (Arg.IsSRet) |
1188 | 72 | Flags.setSRet(); |
1189 | 4.46k | if (Arg.IsSwiftSelf) |
1190 | 24 | Flags.setSwiftSelf(); |
1191 | 4.46k | if (Arg.IsSwiftError) |
1192 | 5 | Flags.setSwiftError(); |
1193 | 4.46k | if (Arg.IsByVal) |
1194 | 35 | Flags.setByVal(); |
1195 | 4.46k | if (Arg.IsInAlloca) { |
1196 | 6 | Flags.setInAlloca(); |
1197 | 6 | // Set the byval flag for CCAssignFn callbacks that don't know about |
1198 | 6 | // inalloca. This way we can know how many bytes we should've allocated |
1199 | 6 | // and how many bytes a callee cleanup function will pop. If we port |
1200 | 6 | // inalloca to more targets, we'll have to add custom inalloca handling in |
1201 | 6 | // the various CC lowering callbacks. |
1202 | 6 | Flags.setByVal(); |
1203 | 6 | } |
1204 | 4.46k | if (Arg.IsByVal || Arg.IsInAlloca4.43k ) { |
1205 | 41 | PointerType *Ty = cast<PointerType>(Arg.Ty); |
1206 | 41 | Type *ElementTy = Ty->getElementType(); |
1207 | 41 | unsigned FrameSize = |
1208 | 41 | DL.getTypeAllocSize(Arg.ByValType ? Arg.ByValType35 : ElementTy6 ); |
1209 | 41 | |
1210 | 41 | // For ByVal, alignment should come from FE. BE will guess if this info |
1211 | 41 | // is not there, but there are cases it cannot get right. |
1212 | 41 | unsigned FrameAlign = Arg.Alignment; |
1213 | 41 | if (!FrameAlign) |
1214 | 12 | FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL); |
1215 | 41 | Flags.setByValSize(FrameSize); |
1216 | 41 | Flags.setByValAlign(FrameAlign); |
1217 | 41 | } |
1218 | 4.46k | if (Arg.IsNest) |
1219 | 1 | Flags.setNest(); |
1220 | 4.46k | if (NeedsRegBlock) |
1221 | 10 | Flags.setInConsecutiveRegs(); |
1222 | 4.46k | unsigned OriginalAlignment = DL.getABITypeAlignment(Arg.Ty); |
1223 | 4.46k | Flags.setOrigAlign(OriginalAlignment); |
1224 | 4.46k | |
1225 | 4.46k | CLI.OutVals.push_back(Arg.Val); |
1226 | 4.46k | CLI.OutFlags.push_back(Flags); |
1227 | 4.46k | } |
1228 | 2.19k | |
1229 | 2.19k | if (!fastLowerCall(CLI)) |
1230 | 969 | return false; |
1231 | 1.22k | |
1232 | 1.22k | // Set all unused physreg defs as dead. |
1233 | 1.22k | assert(CLI.Call && "No call instruction specified."); |
1234 | 1.22k | CLI.Call->setPhysRegsDeadExcept(CLI.InRegs, TRI); |
1235 | 1.22k | |
1236 | 1.22k | if (CLI.NumResultRegs && CLI.CS379 ) |
1237 | 342 | updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs); |
1238 | 1.22k | |
1239 | 1.22k | // Set labels for heapallocsite call. |
1240 | 1.22k | if (CLI.CS && CLI.CS->getInstruction()->getMetadata("heapallocsite")1.16k ) { |
1241 | 3 | const MDNode *MD = CLI.CS->getInstruction()->getMetadata("heapallocsite"); |
1242 | 3 | MF->addCodeViewHeapAllocSite(CLI.Call, MD); |
1243 | 3 | } |
1244 | 1.22k | |
1245 | 1.22k | return true; |
1246 | 1.22k | } |
1247 | | |
1248 | 2.10k | bool FastISel::lowerCall(const CallInst *CI) { |
1249 | 2.10k | ImmutableCallSite CS(CI); |
1250 | 2.10k | |
1251 | 2.10k | FunctionType *FuncTy = CS.getFunctionType(); |
1252 | 2.10k | Type *RetTy = CS.getType(); |
1253 | 2.10k | |
1254 | 2.10k | ArgListTy Args; |
1255 | 2.10k | ArgListEntry Entry; |
1256 | 2.10k | Args.reserve(CS.arg_size()); |
1257 | 2.10k | |
1258 | 2.10k | for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); |
1259 | 6.33k | i != e; ++i4.23k ) { |
1260 | 4.23k | Value *V = *i; |
1261 | 4.23k | |
1262 | 4.23k | // Skip empty types |
1263 | 4.23k | if (V->getType()->isEmptyTy()) |
1264 | 0 | continue; |
1265 | 4.23k | |
1266 | 4.23k | Entry.Val = V; |
1267 | 4.23k | Entry.Ty = V->getType(); |
1268 | 4.23k | |
1269 | 4.23k | // Skip the first return-type Attribute to get to params. |
1270 | 4.23k | Entry.setAttributes(&CS, i - CS.arg_begin()); |
1271 | 4.23k | Args.push_back(Entry); |
1272 | 4.23k | } |
1273 | 2.10k | |
1274 | 2.10k | // Check if target-independent constraints permit a tail call here. |
1275 | 2.10k | // Target-dependent constraints are checked within fastLowerCall. |
1276 | 2.10k | bool IsTailCall = CI->isTailCall(); |
1277 | 2.10k | if (IsTailCall && !isInTailCallPosition(CS, TM)383 ) |
1278 | 246 | IsTailCall = false; |
1279 | 2.10k | |
1280 | 2.10k | CallLoweringInfo CLI; |
1281 | 2.10k | CLI.setCallee(RetTy, FuncTy, CI->getCalledValue(), std::move(Args), CS) |
1282 | 2.10k | .setTailCall(IsTailCall); |
1283 | 2.10k | |
1284 | 2.10k | return lowerCallTo(CLI); |
1285 | 2.10k | } |
1286 | | |
1287 | 5.67k | bool FastISel::selectCall(const User *I) { |
1288 | 5.67k | const CallInst *Call = cast<CallInst>(I); |
1289 | 5.67k | |
1290 | 5.67k | // Handle simple inline asms. |
1291 | 5.67k | if (const InlineAsm *IA = dyn_cast<InlineAsm>(Call->getCalledValue())) { |
1292 | 140 | // If the inline asm has side effects, then make sure that no local value |
1293 | 140 | // lives across by flushing the local value map. |
1294 | 140 | if (IA->hasSideEffects()) |
1295 | 126 | flushLocalValueMap(); |
1296 | 140 | |
1297 | 140 | // Don't attempt to handle constraints. |
1298 | 140 | if (!IA->getConstraintString().empty()) |
1299 | 135 | return false; |
1300 | 5 | |
1301 | 5 | unsigned ExtraInfo = 0; |
1302 | 5 | if (IA->hasSideEffects()) |
1303 | 5 | ExtraInfo |= InlineAsm::Extra_HasSideEffects; |
1304 | 5 | if (IA->isAlignStack()) |
1305 | 0 | ExtraInfo |= InlineAsm::Extra_IsAlignStack; |
1306 | 5 | |
1307 | 5 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
1308 | 5 | TII.get(TargetOpcode::INLINEASM)) |
1309 | 5 | .addExternalSymbol(IA->getAsmString().c_str()) |
1310 | 5 | .addImm(ExtraInfo); |
1311 | 5 | return true; |
1312 | 5 | } |
1313 | 5.53k | |
1314 | 5.53k | // Handle intrinsic function calls. |
1315 | 5.53k | if (const auto *II = dyn_cast<IntrinsicInst>(Call)) |
1316 | 3.42k | return selectIntrinsicCall(II); |
1317 | 2.10k | |
1318 | 2.10k | // Usually, it does not make sense to initialize a value, |
1319 | 2.10k | // make an unrelated function call and use the value, because |
1320 | 2.10k | // it tends to be spilled on the stack. So, we move the pointer |
1321 | 2.10k | // to the last local value to the beginning of the block, so that |
1322 | 2.10k | // all the values which have already been materialized, |
1323 | 2.10k | // appear after the call. It also makes sense to skip intrinsics |
1324 | 2.10k | // since they tend to be inlined. |
1325 | 2.10k | flushLocalValueMap(); |
1326 | 2.10k | |
1327 | 2.10k | return lowerCall(Call); |
1328 | 2.10k | } |
1329 | | |
1330 | 3.42k | bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) { |
1331 | 3.42k | switch (II->getIntrinsicID()) { |
1332 | 3.42k | default: |
1333 | 2.75k | break; |
1334 | 3.42k | // At -O0 we don't care about the lifetime intrinsics. |
1335 | 3.42k | case Intrinsic::lifetime_start: |
1336 | 17 | case Intrinsic::lifetime_end: |
1337 | 17 | // The donothing intrinsic does, well, nothing. |
1338 | 17 | case Intrinsic::donothing: |
1339 | 17 | // Neither does the sideeffect intrinsic. |
1340 | 17 | case Intrinsic::sideeffect: |
1341 | 17 | // Neither does the assume intrinsic; it's also OK not to codegen its operand. |
1342 | 17 | case Intrinsic::assume: |
1343 | 17 | return true; |
1344 | 487 | case Intrinsic::dbg_declare: { |
1345 | 487 | const DbgDeclareInst *DI = cast<DbgDeclareInst>(II); |
1346 | 487 | assert(DI->getVariable() && "Missing variable"); |
1347 | 487 | if (!FuncInfo.MF->getMMI().hasDebugInfo()) { |
1348 | 0 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); |
1349 | 0 | return true; |
1350 | 0 | } |
1351 | 487 | |
1352 | 487 | const Value *Address = DI->getAddress(); |
1353 | 487 | if (!Address || isa<UndefValue>(Address)) { |
1354 | 4 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); |
1355 | 4 | return true; |
1356 | 4 | } |
1357 | 483 | |
1358 | 483 | // Byval arguments with frame indices were already handled after argument |
1359 | 483 | // lowering and before isel. |
1360 | 483 | const auto *Arg = |
1361 | 483 | dyn_cast<Argument>(Address->stripInBoundsConstantOffsets()); |
1362 | 483 | if (Arg && FuncInfo.getArgumentFrameIndex(Arg) != INT_MAX25 ) |
1363 | 483 | return true10 ; |
1364 | 473 | |
1365 | 473 | Optional<MachineOperand> Op; |
1366 | 473 | if (unsigned Reg = lookUpRegForValue(Address)) |
1367 | 35 | Op = MachineOperand::CreateReg(Reg, false); |
1368 | 473 | |
1369 | 473 | // If we have a VLA that has a "use" in a metadata node that's then used |
1370 | 473 | // here but it has no other uses, then we have a problem. E.g., |
1371 | 473 | // |
1372 | 473 | // int foo (const int *x) { |
1373 | 473 | // char a[*x]; |
1374 | 473 | // return 0; |
1375 | 473 | // } |
1376 | 473 | // |
1377 | 473 | // If we assign 'a' a vreg and fast isel later on has to use the selection |
1378 | 473 | // DAG isel, it will want to copy the value to the vreg. However, there are |
1379 | 473 | // no uses, which goes counter to what selection DAG isel expects. |
1380 | 473 | if (!Op && !Address->use_empty()438 && isa<Instruction>(Address)399 && |
1381 | 473 | (399 !isa<AllocaInst>(Address)399 || |
1382 | 399 | !FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(Address))398 )) |
1383 | 1 | Op = MachineOperand::CreateReg(FuncInfo.InitializeRegForValue(Address), |
1384 | 1 | false); |
1385 | 473 | |
1386 | 473 | if (Op) { |
1387 | 36 | assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) && |
1388 | 36 | "Expected inlined-at fields to agree"); |
1389 | 36 | // A dbg.declare describes the address of a source variable, so lower it |
1390 | 36 | // into an indirect DBG_VALUE. |
1391 | 36 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
1392 | 36 | TII.get(TargetOpcode::DBG_VALUE), /*IsIndirect*/ true, |
1393 | 36 | *Op, DI->getVariable(), DI->getExpression()); |
1394 | 437 | } else { |
1395 | 437 | // We can't yet handle anything else here because it would require |
1396 | 437 | // generating code, thus altering codegen because of debug info. |
1397 | 437 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); |
1398 | 437 | } |
1399 | 473 | return true; |
1400 | 473 | } |
1401 | 473 | case Intrinsic::dbg_value: { |
1402 | 85 | // This form of DBG_VALUE is target-independent. |
1403 | 85 | const DbgValueInst *DI = cast<DbgValueInst>(II); |
1404 | 85 | const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); |
1405 | 85 | const Value *V = DI->getValue(); |
1406 | 85 | assert(DI->getVariable()->isValidLocationForIntrinsic(DbgLoc) && |
1407 | 85 | "Expected inlined-at fields to agree"); |
1408 | 85 | if (!V) { |
1409 | 0 | // Currently the optimizer can produce this; insert an undef to |
1410 | 0 | // help debugging. Probably the optimizer should not do this. |
1411 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, false, 0U, |
1412 | 0 | DI->getVariable(), DI->getExpression()); |
1413 | 85 | } else if (const auto *CI = dyn_cast<ConstantInt>(V)) { |
1414 | 16 | if (CI->getBitWidth() > 64) |
1415 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
1416 | 0 | .addCImm(CI) |
1417 | 0 | .addImm(0U) |
1418 | 0 | .addMetadata(DI->getVariable()) |
1419 | 0 | .addMetadata(DI->getExpression()); |
1420 | 16 | else |
1421 | 16 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
1422 | 16 | .addImm(CI->getZExtValue()) |
1423 | 16 | .addImm(0U) |
1424 | 16 | .addMetadata(DI->getVariable()) |
1425 | 16 | .addMetadata(DI->getExpression()); |
1426 | 69 | } else if (const auto *CF = dyn_cast<ConstantFP>(V)) { |
1427 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
1428 | 0 | .addFPImm(CF) |
1429 | 0 | .addImm(0U) |
1430 | 0 | .addMetadata(DI->getVariable()) |
1431 | 0 | .addMetadata(DI->getExpression()); |
1432 | 69 | } else if (unsigned Reg = lookUpRegForValue(V)) { |
1433 | 53 | // FIXME: This does not handle register-indirect values at offset 0. |
1434 | 53 | bool IsIndirect = false; |
1435 | 53 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, IsIndirect, Reg, |
1436 | 53 | DI->getVariable(), DI->getExpression()); |
1437 | 53 | } else { |
1438 | 16 | // We can't yet handle anything else here because it would require |
1439 | 16 | // generating code, thus altering codegen because of debug info. |
1440 | 16 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); |
1441 | 16 | } |
1442 | 85 | return true; |
1443 | 473 | } |
1444 | 473 | case Intrinsic::dbg_label: { |
1445 | 5 | const DbgLabelInst *DI = cast<DbgLabelInst>(II); |
1446 | 5 | assert(DI->getLabel() && "Missing label"); |
1447 | 5 | if (!FuncInfo.MF->getMMI().hasDebugInfo()) { |
1448 | 0 | LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); |
1449 | 0 | return true; |
1450 | 0 | } |
1451 | 5 | |
1452 | 5 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
1453 | 5 | TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel()); |
1454 | 5 | return true; |
1455 | 5 | } |
1456 | 5 | case Intrinsic::objectsize: { |
1457 | 3 | ConstantInt *CI = cast<ConstantInt>(II->getArgOperand(1)); |
1458 | 3 | unsigned long long Res = CI->isZero() ? -1ULL1 : 02 ; |
1459 | 3 | Constant *ResCI = ConstantInt::get(II->getType(), Res); |
1460 | 3 | unsigned ResultReg = getRegForValue(ResCI); |
1461 | 3 | if (!ResultReg) |
1462 | 0 | return false; |
1463 | 3 | updateValueMap(II, ResultReg); |
1464 | 3 | return true; |
1465 | 3 | } |
1466 | 3 | case Intrinsic::is_constant: { |
1467 | 2 | Constant *ResCI = ConstantInt::get(II->getType(), 0); |
1468 | 2 | unsigned ResultReg = getRegForValue(ResCI); |
1469 | 2 | if (!ResultReg) |
1470 | 0 | return false; |
1471 | 2 | updateValueMap(II, ResultReg); |
1472 | 2 | return true; |
1473 | 2 | } |
1474 | 8 | case Intrinsic::launder_invariant_group: |
1475 | 8 | case Intrinsic::strip_invariant_group: |
1476 | 8 | case Intrinsic::expect: { |
1477 | 8 | unsigned ResultReg = getRegForValue(II->getArgOperand(0)); |
1478 | 8 | if (!ResultReg) |
1479 | 0 | return false; |
1480 | 8 | updateValueMap(II, ResultReg); |
1481 | 8 | return true; |
1482 | 8 | } |
1483 | 26 | case Intrinsic::experimental_stackmap: |
1484 | 26 | return selectStackmap(II); |
1485 | 39 | case Intrinsic::experimental_patchpoint_void: |
1486 | 39 | case Intrinsic::experimental_patchpoint_i64: |
1487 | 39 | return selectPatchpoint(II); |
1488 | 39 | |
1489 | 39 | case Intrinsic::xray_customevent: |
1490 | 0 | return selectXRayCustomEvent(II); |
1491 | 39 | case Intrinsic::xray_typedevent: |
1492 | 0 | return selectXRayTypedEvent(II); |
1493 | 2.75k | } |
1494 | 2.75k | |
1495 | 2.75k | return fastLowerIntrinsicCall(II); |
1496 | 2.75k | } |
1497 | | |
1498 | 1.22k | bool FastISel::selectCast(const User *I, unsigned Opcode) { |
1499 | 1.22k | EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType()); |
1500 | 1.22k | EVT DstVT = TLI.getValueType(DL, I->getType()); |
1501 | 1.22k | |
1502 | 1.22k | if (SrcVT == MVT::Other || !SrcVT.isSimple() || DstVT == MVT::Other1.22k || |
1503 | 1.22k | !DstVT.isSimple()1.22k ) |
1504 | 2 | // Unhandled type. Halt "fast" selection and bail. |
1505 | 2 | return false; |
1506 | 1.22k | |
1507 | 1.22k | // Check if the destination type is legal. |
1508 | 1.22k | if (!TLI.isTypeLegal(DstVT)) |
1509 | 172 | return false; |
1510 | 1.05k | |
1511 | 1.05k | // Check if the source operand is legal. |
1512 | 1.05k | if (!TLI.isTypeLegal(SrcVT)) |
1513 | 596 | return false; |
1514 | 455 | |
1515 | 455 | unsigned InputReg = getRegForValue(I->getOperand(0)); |
1516 | 455 | if (!InputReg) |
1517 | 5 | // Unhandled operand. Halt "fast" selection and bail. |
1518 | 5 | return false; |
1519 | 450 | |
1520 | 450 | bool InputRegIsKill = hasTrivialKill(I->getOperand(0)); |
1521 | 450 | |
1522 | 450 | unsigned ResultReg = fastEmit_r(SrcVT.getSimpleVT(), DstVT.getSimpleVT(), |
1523 | 450 | Opcode, InputReg, InputRegIsKill); |
1524 | 450 | if (!ResultReg) |
1525 | 111 | return false; |
1526 | 339 | |
1527 | 339 | updateValueMap(I, ResultReg); |
1528 | 339 | return true; |
1529 | 339 | } |
1530 | | |
1531 | 4.49k | bool FastISel::selectBitCast(const User *I) { |
1532 | 4.49k | // If the bitcast doesn't change the type, just use the operand value. |
1533 | 4.49k | if (I->getType() == I->getOperand(0)->getType()) { |
1534 | 3 | unsigned Reg = getRegForValue(I->getOperand(0)); |
1535 | 3 | if (!Reg) |
1536 | 0 | return false; |
1537 | 3 | updateValueMap(I, Reg); |
1538 | 3 | return true; |
1539 | 3 | } |
1540 | 4.49k | |
1541 | 4.49k | // Bitcasts of other values become reg-reg copies or BITCAST operators. |
1542 | 4.49k | EVT SrcEVT = TLI.getValueType(DL, I->getOperand(0)->getType()); |
1543 | 4.49k | EVT DstEVT = TLI.getValueType(DL, I->getType()); |
1544 | 4.49k | if (SrcEVT == MVT::Other || DstEVT == MVT::Other || |
1545 | 4.49k | !TLI.isTypeLegal(SrcEVT) || !TLI.isTypeLegal(DstEVT)4.45k ) |
1546 | 45 | // Unhandled type. Halt "fast" selection and bail. |
1547 | 45 | return false; |
1548 | 4.44k | |
1549 | 4.44k | MVT SrcVT = SrcEVT.getSimpleVT(); |
1550 | 4.44k | MVT DstVT = DstEVT.getSimpleVT(); |
1551 | 4.44k | unsigned Op0 = getRegForValue(I->getOperand(0)); |
1552 | 4.44k | if (!Op0) // Unhandled operand. Halt "fast" selection and bail. |
1553 | 16 | return false; |
1554 | 4.43k | bool Op0IsKill = hasTrivialKill(I->getOperand(0)); |
1555 | 4.43k | |
1556 | 4.43k | // First, try to perform the bitcast by inserting a reg-reg copy. |
1557 | 4.43k | unsigned ResultReg = 0; |
1558 | 4.43k | if (SrcVT == DstVT) { |
1559 | 414 | const TargetRegisterClass *SrcClass = TLI.getRegClassFor(SrcVT); |
1560 | 414 | const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT); |
1561 | 414 | // Don't attempt a cross-class copy. It will likely fail. |
1562 | 414 | if (SrcClass == DstClass) { |
1563 | 414 | ResultReg = createResultReg(DstClass); |
1564 | 414 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
1565 | 414 | TII.get(TargetOpcode::COPY), ResultReg).addReg(Op0); |
1566 | 414 | } |
1567 | 414 | } |
1568 | 4.43k | |
1569 | 4.43k | // If the reg-reg copy failed, select a BITCAST opcode. |
1570 | 4.43k | if (!ResultReg) |
1571 | 4.01k | ResultReg = fastEmit_r(SrcVT, DstVT, ISD::BITCAST, Op0, Op0IsKill); |
1572 | 4.43k | |
1573 | 4.43k | if (!ResultReg) |
1574 | 3.99k | return false; |
1575 | 434 | |
1576 | 434 | updateValueMap(I, ResultReg); |
1577 | 434 | return true; |
1578 | 434 | } |
1579 | | |
1580 | | // Remove local value instructions starting from the instruction after |
1581 | | // SavedLastLocalValue to the current function insert point. |
1582 | | void FastISel::removeDeadLocalValueCode(MachineInstr *SavedLastLocalValue) |
1583 | 1.14k | { |
1584 | 1.14k | MachineInstr *CurLastLocalValue = getLastLocalValue(); |
1585 | 1.14k | if (CurLastLocalValue != SavedLastLocalValue) { |
1586 | 25 | // Find the first local value instruction to be deleted. |
1587 | 25 | // This is the instruction after SavedLastLocalValue if it is non-NULL. |
1588 | 25 | // Otherwise it's the first instruction in the block. |
1589 | 25 | MachineBasicBlock::iterator FirstDeadInst(SavedLastLocalValue); |
1590 | 25 | if (SavedLastLocalValue) |
1591 | 4 | ++FirstDeadInst; |
1592 | 21 | else |
1593 | 21 | FirstDeadInst = FuncInfo.MBB->getFirstNonPHI(); |
1594 | 25 | setLastLocalValue(SavedLastLocalValue); |
1595 | 25 | removeDeadCode(FirstDeadInst, FuncInfo.InsertPt); |
1596 | 25 | } |
1597 | 1.14k | } |
1598 | | |
1599 | 41.2k | bool FastISel::selectInstruction(const Instruction *I) { |
1600 | 41.2k | MachineInstr *SavedLastLocalValue = getLastLocalValue(); |
1601 | 41.2k | // Just before the terminator instruction, insert instructions to |
1602 | 41.2k | // feed PHI nodes in successor blocks. |
1603 | 41.2k | if (I->isTerminator()) { |
1604 | 15.8k | if (!handlePHINodesInSuccessorBlocks(I->getParent())) { |
1605 | 68 | // PHI node handling may have generated local value instructions, |
1606 | 68 | // even though it failed to handle all PHI nodes. |
1607 | 68 | // We remove these instructions because SelectionDAGISel will generate |
1608 | 68 | // them again. |
1609 | 68 | removeDeadLocalValueCode(SavedLastLocalValue); |
1610 | 68 | return false; |
1611 | 68 | } |
1612 | 41.1k | } |
1613 | 41.1k | |
1614 | 41.1k | // FastISel does not handle any operand bundles except OB_funclet. |
1615 | 41.1k | if (ImmutableCallSite CS = ImmutableCallSite(I)) |
1616 | 5.87k | for (unsigned i = 0, e = CS.getNumOperandBundles(); 5.86k i != e; ++i13 ) |
1617 | 13 | if (CS.getOperandBundleAt(i).getTagID() != LLVMContext::OB_funclet) |
1618 | 0 | return false; |
1619 | 41.1k | |
1620 | 41.1k | DbgLoc = I->getDebugLoc(); |
1621 | 41.1k | |
1622 | 41.1k | SavedInsertPt = FuncInfo.InsertPt; |
1623 | 41.1k | |
1624 | 41.1k | if (const auto *Call = dyn_cast<CallInst>(I)) { |
1625 | 5.71k | const Function *F = Call->getCalledFunction(); |
1626 | 5.71k | LibFunc Func; |
1627 | 5.71k | |
1628 | 5.71k | // As a special case, don't handle calls to builtin library functions that |
1629 | 5.71k | // may be translated directly to target instructions. |
1630 | 5.71k | if (F && !F->hasLocalLinkage()5.43k && F->hasName()5.36k && |
1631 | 5.71k | LibInfo->getLibFunc(F->getName(), Func)5.36k && |
1632 | 5.71k | LibInfo->hasOptimizedCodeGen(Func)125 ) |
1633 | 10 | return false; |
1634 | 5.70k | |
1635 | 5.70k | // Don't handle Intrinsic::trap if a trap function is specified. |
1636 | 5.70k | if (F && F->getIntrinsicID() == Intrinsic::trap5.42k && |
1637 | 5.70k | Call->hasFnAttr("trap-func-name")16 ) |
1638 | 0 | return false; |
1639 | 41.1k | } |
1640 | 41.1k | |
1641 | 41.1k | // First, try doing target-independent selection. |
1642 | 41.1k | if (!SkipTargetIndependentISel) { |
1643 | 35.6k | if (selectOperator(I, I->getOpcode())) { |
1644 | 5.28k | ++NumFastIselSuccessIndependent; |
1645 | 5.28k | DbgLoc = DebugLoc(); |
1646 | 5.28k | return true; |
1647 | 5.28k | } |
1648 | 30.3k | // Remove dead code. |
1649 | 30.3k | recomputeInsertPt(); |
1650 | 30.3k | if (SavedInsertPt != FuncInfo.InsertPt) |
1651 | 61 | removeDeadCode(FuncInfo.InsertPt, SavedInsertPt); |
1652 | 30.3k | SavedInsertPt = FuncInfo.InsertPt; |
1653 | 30.3k | } |
1654 | 41.1k | // Next, try calling the target to attempt to handle the instruction. |
1655 | 41.1k | if (35.8k fastSelectInstruction(I)35.8k ) { |
1656 | 26.7k | ++NumFastIselSuccessTarget; |
1657 | 26.7k | DbgLoc = DebugLoc(); |
1658 | 26.7k | return true; |
1659 | 26.7k | } |
1660 | 9.15k | // Remove dead code. |
1661 | 9.15k | recomputeInsertPt(); |
1662 | 9.15k | if (SavedInsertPt != FuncInfo.InsertPt) |
1663 | 17 | removeDeadCode(FuncInfo.InsertPt, SavedInsertPt); |
1664 | 9.15k | |
1665 | 9.15k | DbgLoc = DebugLoc(); |
1666 | 9.15k | // Undo phi node updates, because they will be added again by SelectionDAG. |
1667 | 9.15k | if (I->isTerminator()) { |
1668 | 1.07k | // PHI node handling may have generated local value instructions. |
1669 | 1.07k | // We remove them because SelectionDAGISel will generate them again. |
1670 | 1.07k | removeDeadLocalValueCode(SavedLastLocalValue); |
1671 | 1.07k | FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate); |
1672 | 1.07k | } |
1673 | 9.15k | return false; |
1674 | 9.15k | } |
1675 | | |
1676 | | /// Emit an unconditional branch to the given block, unless it is the immediate |
1677 | | /// (fall-through) successor, and update the CFG. |
1678 | | void FastISel::fastEmitBranch(MachineBasicBlock *MSucc, |
1679 | 1.81k | const DebugLoc &DbgLoc) { |
1680 | 1.81k | if (FuncInfo.MBB->getBasicBlock()->size() > 1 && |
1681 | 1.81k | FuncInfo.MBB->isLayoutSuccessor(MSucc)1.52k ) { |
1682 | 1.10k | // For more accurate line information if this is the only instruction |
1683 | 1.10k | // in the block then emit it, otherwise we have the unconditional |
1684 | 1.10k | // fall-through case, which needs no instructions. |
1685 | 1.10k | } else { |
1686 | 714 | // The unconditional branch case. |
1687 | 714 | TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr, |
1688 | 714 | SmallVector<MachineOperand, 0>(), DbgLoc); |
1689 | 714 | } |
1690 | 1.81k | if (FuncInfo.BPI) { |
1691 | 271 | auto BranchProbability = FuncInfo.BPI->getEdgeProbability( |
1692 | 271 | FuncInfo.MBB->getBasicBlock(), MSucc->getBasicBlock()); |
1693 | 271 | FuncInfo.MBB->addSuccessor(MSucc, BranchProbability); |
1694 | 271 | } else |
1695 | 1.54k | FuncInfo.MBB->addSuccessorWithoutProb(MSucc); |
1696 | 1.81k | } |
1697 | | |
1698 | | void FastISel::finishCondBranch(const BasicBlock *BranchBB, |
1699 | | MachineBasicBlock *TrueMBB, |
1700 | 710 | MachineBasicBlock *FalseMBB) { |
1701 | 710 | // Add TrueMBB as successor unless it is equal to the FalseMBB: This can |
1702 | 710 | // happen in degenerate IR and MachineIR forbids to have a block twice in the |
1703 | 710 | // successor/predecessor lists. |
1704 | 710 | if (TrueMBB != FalseMBB) { |
1705 | 706 | if (FuncInfo.BPI) { |
1706 | 176 | auto BranchProbability = |
1707 | 176 | FuncInfo.BPI->getEdgeProbability(BranchBB, TrueMBB->getBasicBlock()); |
1708 | 176 | FuncInfo.MBB->addSuccessor(TrueMBB, BranchProbability); |
1709 | 176 | } else |
1710 | 530 | FuncInfo.MBB->addSuccessorWithoutProb(TrueMBB); |
1711 | 706 | } |
1712 | 710 | |
1713 | 710 | fastEmitBranch(FalseMBB, DbgLoc); |
1714 | 710 | } |
1715 | | |
1716 | | /// Emit an FNeg operation. |
1717 | 54 | bool FastISel::selectFNeg(const User *I, const Value *In) { |
1718 | 54 | unsigned OpReg = getRegForValue(In); |
1719 | 54 | if (!OpReg) |
1720 | 0 | return false; |
1721 | 54 | bool OpRegIsKill = hasTrivialKill(In); |
1722 | 54 | |
1723 | 54 | // If the target has ISD::FNEG, use it. |
1724 | 54 | EVT VT = TLI.getValueType(DL, I->getType()); |
1725 | 54 | unsigned ResultReg = fastEmit_r(VT.getSimpleVT(), VT.getSimpleVT(), ISD::FNEG, |
1726 | 54 | OpReg, OpRegIsKill); |
1727 | 54 | if (ResultReg) { |
1728 | 5 | updateValueMap(I, ResultReg); |
1729 | 5 | return true; |
1730 | 5 | } |
1731 | 49 | |
1732 | 49 | // Bitcast the value to integer, twiddle the sign bit with xor, |
1733 | 49 | // and then bitcast it back to floating-point. |
1734 | 49 | if (VT.getSizeInBits() > 64) |
1735 | 40 | return false; |
1736 | 9 | EVT IntVT = EVT::getIntegerVT(I->getContext(), VT.getSizeInBits()); |
1737 | 9 | if (!TLI.isTypeLegal(IntVT)) |
1738 | 1 | return false; |
1739 | 8 | |
1740 | 8 | unsigned IntReg = fastEmit_r(VT.getSimpleVT(), IntVT.getSimpleVT(), |
1741 | 8 | ISD::BITCAST, OpReg, OpRegIsKill); |
1742 | 8 | if (!IntReg) |
1743 | 0 | return false; |
1744 | 8 | |
1745 | 8 | unsigned IntResultReg = fastEmit_ri_( |
1746 | 8 | IntVT.getSimpleVT(), ISD::XOR, IntReg, /*IsKill=*/true, |
1747 | 8 | UINT64_C(1) << (VT.getSizeInBits() - 1), IntVT.getSimpleVT()); |
1748 | 8 | if (!IntResultReg) |
1749 | 0 | return false; |
1750 | 8 | |
1751 | 8 | ResultReg = fastEmit_r(IntVT.getSimpleVT(), VT.getSimpleVT(), ISD::BITCAST, |
1752 | 8 | IntResultReg, /*IsKill=*/true); |
1753 | 8 | if (!ResultReg) |
1754 | 0 | return false; |
1755 | 8 | |
1756 | 8 | updateValueMap(I, ResultReg); |
1757 | 8 | return true; |
1758 | 8 | } |
1759 | | |
1760 | 513 | bool FastISel::selectExtractValue(const User *U) { |
1761 | 513 | const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(U); |
1762 | 513 | if (!EVI) |
1763 | 0 | return false; |
1764 | 513 | |
1765 | 513 | // Make sure we only try to handle extracts with a legal result. But also |
1766 | 513 | // allow i1 because it's easy. |
1767 | 513 | EVT RealVT = TLI.getValueType(DL, EVI->getType(), /*AllowUnknown=*/true); |
1768 | 513 | if (!RealVT.isSimple()) |
1769 | 0 | return false; |
1770 | 513 | MVT VT = RealVT.getSimpleVT(); |
1771 | 513 | if (!TLI.isTypeLegal(VT) && VT != MVT::i1210 ) |
1772 | 5 | return false; |
1773 | 508 | |
1774 | 508 | const Value *Op0 = EVI->getOperand(0); |
1775 | 508 | Type *AggTy = Op0->getType(); |
1776 | 508 | |
1777 | 508 | // Get the base result register. |
1778 | 508 | unsigned ResultReg; |
1779 | 508 | DenseMap<const Value *, unsigned>::iterator I = FuncInfo.ValueMap.find(Op0); |
1780 | 508 | if (I != FuncInfo.ValueMap.end()) |
1781 | 247 | ResultReg = I->second; |
1782 | 261 | else if (isa<Instruction>(Op0)) |
1783 | 258 | ResultReg = FuncInfo.InitializeRegForValue(Op0); |
1784 | 3 | else |
1785 | 3 | return false; // fast-isel can't handle aggregate constants at the moment |
1786 | 505 | |
1787 | 505 | // Get the actual result register, which is an offset from the base register. |
1788 | 505 | unsigned VTIndex = ComputeLinearIndex(AggTy, EVI->getIndices()); |
1789 | 505 | |
1790 | 505 | SmallVector<EVT, 4> AggValueVTs; |
1791 | 505 | ComputeValueVTs(TLI, DL, AggTy, AggValueVTs); |
1792 | 505 | |
1793 | 884 | for (unsigned i = 0; i < VTIndex; i++379 ) |
1794 | 379 | ResultReg += TLI.getNumRegisters(FuncInfo.Fn->getContext(), AggValueVTs[i]); |
1795 | 505 | |
1796 | 505 | updateValueMap(EVI, ResultReg); |
1797 | 505 | return true; |
1798 | 505 | } |
1799 | | |
1800 | 37.0k | bool FastISel::selectOperator(const User *I, unsigned Opcode) { |
1801 | 37.0k | switch (Opcode) { |
1802 | 37.0k | case Instruction::Add: |
1803 | 571 | return selectBinaryOp(I, ISD::ADD); |
1804 | 37.0k | case Instruction::FAdd: |
1805 | 261 | return selectBinaryOp(I, ISD::FADD); |
1806 | 37.0k | case Instruction::Sub: |
1807 | 133 | return selectBinaryOp(I, ISD::SUB); |
1808 | 37.0k | case Instruction::FSub: { |
1809 | 75 | // FNeg is currently represented in LLVM IR as a special case of FSub. |
1810 | 75 | Value *X; |
1811 | 75 | if (match(I, m_FNeg(m_Value(X)))) |
1812 | 52 | return selectFNeg(I, X); |
1813 | 23 | return selectBinaryOp(I, ISD::FSUB); |
1814 | 23 | } |
1815 | 109 | case Instruction::Mul: |
1816 | 109 | return selectBinaryOp(I, ISD::MUL); |
1817 | 29 | case Instruction::FMul: |
1818 | 29 | return selectBinaryOp(I, ISD::FMUL); |
1819 | 23 | case Instruction::SDiv: |
1820 | 21 | return selectBinaryOp(I, ISD::SDIV); |
1821 | 26 | case Instruction::UDiv: |
1822 | 26 | return selectBinaryOp(I, ISD::UDIV); |
1823 | 23 | case Instruction::FDiv: |
1824 | 23 | return selectBinaryOp(I, ISD::FDIV); |
1825 | 44 | case Instruction::SRem: |
1826 | 44 | return selectBinaryOp(I, ISD::SREM); |
1827 | 23 | case Instruction::URem: |
1828 | 22 | return selectBinaryOp(I, ISD::UREM); |
1829 | 23 | case Instruction::FRem: |
1830 | 1 | return selectBinaryOp(I, ISD::FREM); |
1831 | 75 | case Instruction::Shl: |
1832 | 75 | return selectBinaryOp(I, ISD::SHL); |
1833 | 63 | case Instruction::LShr: |
1834 | 63 | return selectBinaryOp(I, ISD::SRL); |
1835 | 60 | case Instruction::AShr: |
1836 | 60 | return selectBinaryOp(I, ISD::SRA); |
1837 | 232 | case Instruction::And: |
1838 | 232 | return selectBinaryOp(I, ISD::AND); |
1839 | 150 | case Instruction::Or: |
1840 | 150 | return selectBinaryOp(I, ISD::OR); |
1841 | 196 | case Instruction::Xor: |
1842 | 196 | return selectBinaryOp(I, ISD::XOR); |
1843 | 23 | |
1844 | 23 | case Instruction::FNeg: |
1845 | 2 | return selectFNeg(I, I->getOperand(0)); |
1846 | 23 | |
1847 | 325 | case Instruction::GetElementPtr: |
1848 | 325 | return selectGetElementPtr(I); |
1849 | 23 | |
1850 | 1.46k | case Instruction::Br: { |
1851 | 1.46k | const BranchInst *BI = cast<BranchInst>(I); |
1852 | 1.46k | |
1853 | 1.46k | if (BI->isUnconditional()) { |
1854 | 847 | const BasicBlock *LLVMSucc = BI->getSuccessor(0); |
1855 | 847 | MachineBasicBlock *MSucc = FuncInfo.MBBMap[LLVMSucc]; |
1856 | 847 | fastEmitBranch(MSucc, BI->getDebugLoc()); |
1857 | 847 | return true; |
1858 | 847 | } |
1859 | 617 | |
1860 | 617 | // Conditional branches are not handed yet. |
1861 | 617 | // Halt "fast" selection and bail. |
1862 | 617 | return false; |
1863 | 617 | } |
1864 | 617 | |
1865 | 617 | case Instruction::Unreachable: |
1866 | 109 | if (TM.Options.TrapUnreachable) |
1867 | 62 | return fastEmit_(MVT::Other, MVT::Other, ISD::TRAP) != 0; |
1868 | 47 | else |
1869 | 47 | return true; |
1870 | 0 | |
1871 | 16 | case Instruction::Alloca: |
1872 | 16 | // FunctionLowering has the static-sized case covered. |
1873 | 16 | if (FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(I))) |
1874 | 0 | return true; |
1875 | 16 | |
1876 | 16 | // Dynamic-sized alloca is not handled yet. |
1877 | 16 | return false; |
1878 | 16 | |
1879 | 5.62k | case Instruction::Call: |
1880 | 5.62k | // On AIX, call lowering uses the DAG-ISEL path currently so that the |
1881 | 5.62k | // callee of the direct function call instruction will be mapped to the |
1882 | 5.62k | // symbol for the function's entry point, which is distinct from the |
1883 | 5.62k | // function descriptor symbol. The latter is the symbol whose XCOFF symbol |
1884 | 5.62k | // name is the C-linkage name of the source level function. |
1885 | 5.62k | if (TM.getTargetTriple().isOSAIX()) |
1886 | 0 | return false; |
1887 | 5.62k | return selectCall(I); |
1888 | 5.62k | |
1889 | 5.62k | case Instruction::BitCast: |
1890 | 4.44k | return selectBitCast(I); |
1891 | 5.62k | |
1892 | 5.62k | case Instruction::FPToSI: |
1893 | 34 | return selectCast(I, ISD::FP_TO_SINT); |
1894 | 5.62k | case Instruction::ZExt: |
1895 | 386 | return selectCast(I, ISD::ZERO_EXTEND); |
1896 | 5.62k | case Instruction::SExt: |
1897 | 485 | return selectCast(I, ISD::SIGN_EXTEND); |
1898 | 5.62k | case Instruction::Trunc: |
1899 | 143 | return selectCast(I, ISD::TRUNCATE); |
1900 | 5.62k | case Instruction::SIToFP: |
1901 | 109 | return selectCast(I, ISD::SINT_TO_FP); |
1902 | 5.62k | |
1903 | 5.62k | case Instruction::IntToPtr: // Deliberate fall-through. |
1904 | 88 | case Instruction::PtrToInt: { |
1905 | 88 | EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType()); |
1906 | 88 | EVT DstVT = TLI.getValueType(DL, I->getType()); |
1907 | 88 | if (DstVT.bitsGT(SrcVT)) |
1908 | 3 | return selectCast(I, ISD::ZERO_EXTEND); |
1909 | 85 | if (DstVT.bitsLT(SrcVT)) |
1910 | 18 | return selectCast(I, ISD::TRUNCATE); |
1911 | 67 | unsigned Reg = getRegForValue(I->getOperand(0)); |
1912 | 67 | if (!Reg) |
1913 | 8 | return false; |
1914 | 59 | updateValueMap(I, Reg); |
1915 | 59 | return true; |
1916 | 59 | } |
1917 | 59 | |
1918 | 513 | case Instruction::ExtractValue: |
1919 | 513 | return selectExtractValue(I); |
1920 | 59 | |
1921 | 59 | case Instruction::PHI: |
1922 | 0 | llvm_unreachable("FastISel shouldn't visit PHI nodes!"); |
1923 | 59 | |
1924 | 21.1k | default: |
1925 | 21.1k | // Unhandled instruction. Halt "fast" selection and bail. |
1926 | 21.1k | return false; |
1927 | 37.0k | } |
1928 | 37.0k | } |
1929 | | |
1930 | | FastISel::FastISel(FunctionLoweringInfo &FuncInfo, |
1931 | | const TargetLibraryInfo *LibInfo, |
1932 | | bool SkipTargetIndependentISel) |
1933 | | : FuncInfo(FuncInfo), MF(FuncInfo.MF), MRI(FuncInfo.MF->getRegInfo()), |
1934 | | MFI(FuncInfo.MF->getFrameInfo()), MCP(*FuncInfo.MF->getConstantPool()), |
1935 | | TM(FuncInfo.MF->getTarget()), DL(MF->getDataLayout()), |
1936 | | TII(*MF->getSubtarget().getInstrInfo()), |
1937 | | TLI(*MF->getSubtarget().getTargetLowering()), |
1938 | | TRI(*MF->getSubtarget().getRegisterInfo()), LibInfo(LibInfo), |
1939 | 13.4k | SkipTargetIndependentISel(SkipTargetIndependentISel) {} |
1940 | | |
1941 | 13.4k | FastISel::~FastISel() = default; |
1942 | | |
1943 | 0 | bool FastISel::fastLowerArguments() { return false; } |
1944 | | |
1945 | 617 | bool FastISel::fastLowerCall(CallLoweringInfo & /*CLI*/) { return false; } |
1946 | | |
1947 | 191 | bool FastISel::fastLowerIntrinsicCall(const IntrinsicInst * /*II*/) { |
1948 | 191 | return false; |
1949 | 191 | } |
1950 | | |
1951 | 58 | unsigned FastISel::fastEmit_(MVT, MVT, unsigned) { return 0; } |
1952 | | |
1953 | | unsigned FastISel::fastEmit_r(MVT, MVT, unsigned, unsigned /*Op0*/, |
1954 | 0 | bool /*Op0IsKill*/) { |
1955 | 0 | return 0; |
1956 | 0 | } |
1957 | | |
1958 | | unsigned FastISel::fastEmit_rr(MVT, MVT, unsigned, unsigned /*Op0*/, |
1959 | | bool /*Op0IsKill*/, unsigned /*Op1*/, |
1960 | 0 | bool /*Op1IsKill*/) { |
1961 | 0 | return 0; |
1962 | 0 | } |
1963 | | |
1964 | 0 | unsigned FastISel::fastEmit_i(MVT, MVT, unsigned, uint64_t /*Imm*/) { |
1965 | 0 | return 0; |
1966 | 0 | } |
1967 | | |
1968 | | unsigned FastISel::fastEmit_f(MVT, MVT, unsigned, |
1969 | 0 | const ConstantFP * /*FPImm*/) { |
1970 | 0 | return 0; |
1971 | 0 | } |
1972 | | |
1973 | | unsigned FastISel::fastEmit_ri(MVT, MVT, unsigned, unsigned /*Op0*/, |
1974 | 22 | bool /*Op0IsKill*/, uint64_t /*Imm*/) { |
1975 | 22 | return 0; |
1976 | 22 | } |
1977 | | |
1978 | | /// This method is a wrapper of fastEmit_ri. It first tries to emit an |
1979 | | /// instruction with an immediate operand using fastEmit_ri. |
1980 | | /// If that fails, it materializes the immediate into a register and try |
1981 | | /// fastEmit_rr instead. |
1982 | | unsigned FastISel::fastEmit_ri_(MVT VT, unsigned Opcode, unsigned Op0, |
1983 | 1.04k | bool Op0IsKill, uint64_t Imm, MVT ImmType) { |
1984 | 1.04k | // If this is a multiply by a power of two, emit this as a shift left. |
1985 | 1.04k | if (Opcode == ISD::MUL && isPowerOf2_64(Imm)61 ) { |
1986 | 37 | Opcode = ISD::SHL; |
1987 | 37 | Imm = Log2_64(Imm); |
1988 | 1.00k | } else if (Opcode == ISD::UDIV && isPowerOf2_64(Imm)4 ) { |
1989 | 3 | // div x, 8 -> srl x, 3 |
1990 | 3 | Opcode = ISD::SRL; |
1991 | 3 | Imm = Log2_64(Imm); |
1992 | 3 | } |
1993 | 1.04k | |
1994 | 1.04k | // Horrible hack (to be removed), check to make sure shift amounts are |
1995 | 1.04k | // in-range. |
1996 | 1.04k | if ((Opcode == ISD::SHL || Opcode == ISD::SRA986 || Opcode == ISD::SRL967 ) && |
1997 | 1.04k | Imm >= VT.getSizeInBits()100 ) |
1998 | 4 | return 0; |
1999 | 1.03k | |
2000 | 1.03k | // First check if immediate type is legal. If not, we can't use the ri form. |
2001 | 1.03k | unsigned ResultReg = fastEmit_ri(VT, VT, Opcode, Op0, Op0IsKill, Imm); |
2002 | 1.03k | if (ResultReg) |
2003 | 952 | return ResultReg; |
2004 | 87 | unsigned MaterialReg = fastEmit_i(ImmType, ImmType, ISD::Constant, Imm); |
2005 | 87 | bool IsImmKill = true; |
2006 | 87 | if (!MaterialReg) { |
2007 | 48 | // This is a bit ugly/slow, but failing here means falling out of |
2008 | 48 | // fast-isel, which would be very slow. |
2009 | 48 | IntegerType *ITy = |
2010 | 48 | IntegerType::get(FuncInfo.Fn->getContext(), VT.getSizeInBits()); |
2011 | 48 | MaterialReg = getRegForValue(ConstantInt::get(ITy, Imm)); |
2012 | 48 | if (!MaterialReg) |
2013 | 0 | return 0; |
2014 | 48 | // FIXME: If the materialized register here has no uses yet then this |
2015 | 48 | // will be the first use and we should be able to mark it as killed. |
2016 | 48 | // However, the local value area for materialising constant expressions |
2017 | 48 | // grows down, not up, which means that any constant expressions we generate |
2018 | 48 | // later which also use 'Imm' could be after this instruction and therefore |
2019 | 48 | // after this kill. |
2020 | 48 | IsImmKill = false; |
2021 | 48 | } |
2022 | 87 | return fastEmit_rr(VT, VT, Opcode, Op0, Op0IsKill, MaterialReg, IsImmKill); |
2023 | 87 | } |
2024 | | |
2025 | 22.5k | unsigned FastISel::createResultReg(const TargetRegisterClass *RC) { |
2026 | 22.5k | return MRI.createVirtualRegister(RC); |
2027 | 22.5k | } |
2028 | | |
2029 | | unsigned FastISel::constrainOperandRegClass(const MCInstrDesc &II, unsigned Op, |
2030 | 17.2k | unsigned OpNum) { |
2031 | 17.2k | if (TargetRegisterInfo::isVirtualRegister(Op)) { |
2032 | 11.6k | const TargetRegisterClass *RegClass = |
2033 | 11.6k | TII.getRegClass(II, OpNum, &TRI, *FuncInfo.MF); |
2034 | 11.6k | if (!MRI.constrainRegClass(Op, RegClass)) { |
2035 | 0 | // If it's not legal to COPY between the register classes, something |
2036 | 0 | // has gone very wrong before we got here. |
2037 | 0 | unsigned NewOp = createResultReg(RegClass); |
2038 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2039 | 0 | TII.get(TargetOpcode::COPY), NewOp).addReg(Op); |
2040 | 0 | return NewOp; |
2041 | 0 | } |
2042 | 17.2k | } |
2043 | 17.2k | return Op; |
2044 | 17.2k | } |
2045 | | |
2046 | | unsigned FastISel::fastEmitInst_(unsigned MachineInstOpcode, |
2047 | 406 | const TargetRegisterClass *RC) { |
2048 | 406 | unsigned ResultReg = createResultReg(RC); |
2049 | 406 | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2050 | 406 | |
2051 | 406 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg); |
2052 | 406 | return ResultReg; |
2053 | 406 | } |
2054 | | |
2055 | | unsigned FastISel::fastEmitInst_r(unsigned MachineInstOpcode, |
2056 | | const TargetRegisterClass *RC, unsigned Op0, |
2057 | 823 | bool Op0IsKill) { |
2058 | 823 | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2059 | 823 | |
2060 | 823 | unsigned ResultReg = createResultReg(RC); |
2061 | 823 | Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs()); |
2062 | 823 | |
2063 | 823 | if (II.getNumDefs() >= 1) |
2064 | 799 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) |
2065 | 799 | .addReg(Op0, getKillRegState(Op0IsKill)); |
2066 | 24 | else { |
2067 | 24 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
2068 | 24 | .addReg(Op0, getKillRegState(Op0IsKill)); |
2069 | 24 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2070 | 24 | TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]); |
2071 | 24 | } |
2072 | 823 | |
2073 | 823 | return ResultReg; |
2074 | 823 | } |
2075 | | |
2076 | | unsigned FastISel::fastEmitInst_rr(unsigned MachineInstOpcode, |
2077 | | const TargetRegisterClass *RC, unsigned Op0, |
2078 | | bool Op0IsKill, unsigned Op1, |
2079 | 1.29k | bool Op1IsKill) { |
2080 | 1.29k | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2081 | 1.29k | |
2082 | 1.29k | unsigned ResultReg = createResultReg(RC); |
2083 | 1.29k | Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs()); |
2084 | 1.29k | Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1); |
2085 | 1.29k | |
2086 | 1.29k | if (II.getNumDefs() >= 1) |
2087 | 1.29k | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) |
2088 | 1.29k | .addReg(Op0, getKillRegState(Op0IsKill)) |
2089 | 1.29k | .addReg(Op1, getKillRegState(Op1IsKill)); |
2090 | 0 | else { |
2091 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
2092 | 0 | .addReg(Op0, getKillRegState(Op0IsKill)) |
2093 | 0 | .addReg(Op1, getKillRegState(Op1IsKill)); |
2094 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2095 | 0 | TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]); |
2096 | 0 | } |
2097 | 1.29k | return ResultReg; |
2098 | 1.29k | } |
2099 | | |
2100 | | unsigned FastISel::fastEmitInst_rrr(unsigned MachineInstOpcode, |
2101 | | const TargetRegisterClass *RC, unsigned Op0, |
2102 | | bool Op0IsKill, unsigned Op1, |
2103 | | bool Op1IsKill, unsigned Op2, |
2104 | 59 | bool Op2IsKill) { |
2105 | 59 | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2106 | 59 | |
2107 | 59 | unsigned ResultReg = createResultReg(RC); |
2108 | 59 | Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs()); |
2109 | 59 | Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1); |
2110 | 59 | Op2 = constrainOperandRegClass(II, Op2, II.getNumDefs() + 2); |
2111 | 59 | |
2112 | 59 | if (II.getNumDefs() >= 1) |
2113 | 59 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) |
2114 | 59 | .addReg(Op0, getKillRegState(Op0IsKill)) |
2115 | 59 | .addReg(Op1, getKillRegState(Op1IsKill)) |
2116 | 59 | .addReg(Op2, getKillRegState(Op2IsKill)); |
2117 | 0 | else { |
2118 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
2119 | 0 | .addReg(Op0, getKillRegState(Op0IsKill)) |
2120 | 0 | .addReg(Op1, getKillRegState(Op1IsKill)) |
2121 | 0 | .addReg(Op2, getKillRegState(Op2IsKill)); |
2122 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2123 | 0 | TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]); |
2124 | 0 | } |
2125 | 59 | return ResultReg; |
2126 | 59 | } |
2127 | | |
2128 | | unsigned FastISel::fastEmitInst_ri(unsigned MachineInstOpcode, |
2129 | | const TargetRegisterClass *RC, unsigned Op0, |
2130 | 1.06k | bool Op0IsKill, uint64_t Imm) { |
2131 | 1.06k | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2132 | 1.06k | |
2133 | 1.06k | unsigned ResultReg = createResultReg(RC); |
2134 | 1.06k | Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs()); |
2135 | 1.06k | |
2136 | 1.06k | if (II.getNumDefs() >= 1) |
2137 | 1.06k | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) |
2138 | 1.06k | .addReg(Op0, getKillRegState(Op0IsKill)) |
2139 | 1.06k | .addImm(Imm); |
2140 | 0 | else { |
2141 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
2142 | 0 | .addReg(Op0, getKillRegState(Op0IsKill)) |
2143 | 0 | .addImm(Imm); |
2144 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2145 | 0 | TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]); |
2146 | 0 | } |
2147 | 1.06k | return ResultReg; |
2148 | 1.06k | } |
2149 | | |
2150 | | unsigned FastISel::fastEmitInst_rii(unsigned MachineInstOpcode, |
2151 | | const TargetRegisterClass *RC, unsigned Op0, |
2152 | | bool Op0IsKill, uint64_t Imm1, |
2153 | 361 | uint64_t Imm2) { |
2154 | 361 | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2155 | 361 | |
2156 | 361 | unsigned ResultReg = createResultReg(RC); |
2157 | 361 | Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs()); |
2158 | 361 | |
2159 | 361 | if (II.getNumDefs() >= 1) |
2160 | 361 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) |
2161 | 361 | .addReg(Op0, getKillRegState(Op0IsKill)) |
2162 | 361 | .addImm(Imm1) |
2163 | 361 | .addImm(Imm2); |
2164 | 0 | else { |
2165 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
2166 | 0 | .addReg(Op0, getKillRegState(Op0IsKill)) |
2167 | 0 | .addImm(Imm1) |
2168 | 0 | .addImm(Imm2); |
2169 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2170 | 0 | TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]); |
2171 | 0 | } |
2172 | 361 | return ResultReg; |
2173 | 361 | } |
2174 | | |
2175 | | unsigned FastISel::fastEmitInst_f(unsigned MachineInstOpcode, |
2176 | | const TargetRegisterClass *RC, |
2177 | 8 | const ConstantFP *FPImm) { |
2178 | 8 | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2179 | 8 | |
2180 | 8 | unsigned ResultReg = createResultReg(RC); |
2181 | 8 | |
2182 | 8 | if (II.getNumDefs() >= 1) |
2183 | 8 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) |
2184 | 8 | .addFPImm(FPImm); |
2185 | 0 | else { |
2186 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
2187 | 0 | .addFPImm(FPImm); |
2188 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2189 | 0 | TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]); |
2190 | 0 | } |
2191 | 8 | return ResultReg; |
2192 | 8 | } |
2193 | | |
2194 | | unsigned FastISel::fastEmitInst_rri(unsigned MachineInstOpcode, |
2195 | | const TargetRegisterClass *RC, unsigned Op0, |
2196 | | bool Op0IsKill, unsigned Op1, |
2197 | 355 | bool Op1IsKill, uint64_t Imm) { |
2198 | 355 | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2199 | 355 | |
2200 | 355 | unsigned ResultReg = createResultReg(RC); |
2201 | 355 | Op0 = constrainOperandRegClass(II, Op0, II.getNumDefs()); |
2202 | 355 | Op1 = constrainOperandRegClass(II, Op1, II.getNumDefs() + 1); |
2203 | 355 | |
2204 | 355 | if (II.getNumDefs() >= 1) |
2205 | 355 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) |
2206 | 355 | .addReg(Op0, getKillRegState(Op0IsKill)) |
2207 | 355 | .addReg(Op1, getKillRegState(Op1IsKill)) |
2208 | 355 | .addImm(Imm); |
2209 | 0 | else { |
2210 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) |
2211 | 0 | .addReg(Op0, getKillRegState(Op0IsKill)) |
2212 | 0 | .addReg(Op1, getKillRegState(Op1IsKill)) |
2213 | 0 | .addImm(Imm); |
2214 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2215 | 0 | TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]); |
2216 | 0 | } |
2217 | 355 | return ResultReg; |
2218 | 355 | } |
2219 | | |
2220 | | unsigned FastISel::fastEmitInst_i(unsigned MachineInstOpcode, |
2221 | 739 | const TargetRegisterClass *RC, uint64_t Imm) { |
2222 | 739 | unsigned ResultReg = createResultReg(RC); |
2223 | 739 | const MCInstrDesc &II = TII.get(MachineInstOpcode); |
2224 | 739 | |
2225 | 739 | if (II.getNumDefs() >= 1) |
2226 | 739 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II, ResultReg) |
2227 | 739 | .addImm(Imm); |
2228 | 0 | else { |
2229 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II).addImm(Imm); |
2230 | 0 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, |
2231 | 0 | TII.get(TargetOpcode::COPY), ResultReg).addReg(II.ImplicitDefs[0]); |
2232 | 0 | } |
2233 | 739 | return ResultReg; |
2234 | 739 | } |
2235 | | |
2236 | | unsigned FastISel::fastEmitInst_extractsubreg(MVT RetVT, unsigned Op0, |
2237 | 178 | bool Op0IsKill, uint32_t Idx) { |
2238 | 178 | unsigned ResultReg = createResultReg(TLI.getRegClassFor(RetVT)); |
2239 | 178 | assert(TargetRegisterInfo::isVirtualRegister(Op0) && |
2240 | 178 | "Cannot yet extract from physregs"); |
2241 | 178 | const TargetRegisterClass *RC = MRI.getRegClass(Op0); |
2242 | 178 | MRI.constrainRegClass(Op0, TRI.getSubClassWithSubReg(RC, Idx)); |
2243 | 178 | BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), |
2244 | 178 | ResultReg).addReg(Op0, getKillRegState(Op0IsKill), Idx); |
2245 | 178 | return ResultReg; |
2246 | 178 | } |
2247 | | |
2248 | | /// Emit MachineInstrs to compute the value of Op with all but the least |
2249 | | /// significant bit set to zero. |
2250 | 355 | unsigned FastISel::fastEmitZExtFromI1(MVT VT, unsigned Op0, bool Op0IsKill) { |
2251 | 355 | return fastEmit_ri(VT, VT, ISD::AND, Op0, Op0IsKill, 1); |
2252 | 355 | } |
2253 | | |
2254 | | /// HandlePHINodesInSuccessorBlocks - Handle PHI nodes in successor blocks. |
2255 | | /// Emit code to ensure constants are copied into registers when needed. |
2256 | | /// Remember the virtual registers that need to be added to the Machine PHI |
2257 | | /// nodes as input. We cannot just directly add them, because expansion |
2258 | | /// might result in multiple MBB's for one BB. As such, the start of the |
2259 | | /// BB might correspond to a different MBB than the end. |
2260 | 15.8k | bool FastISel::handlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { |
2261 | 15.8k | const Instruction *TI = LLVMBB->getTerminator(); |
2262 | 15.8k | |
2263 | 15.8k | SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; |
2264 | 15.8k | FuncInfo.OrigNumPHINodesToUpdate = FuncInfo.PHINodesToUpdate.size(); |
2265 | 15.8k | |
2266 | 15.8k | // Check successor nodes' PHI nodes that expect a constant to be available |
2267 | 15.8k | // from this block. |
2268 | 19.3k | for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ3.48k ) { |
2269 | 3.54k | const BasicBlock *SuccBB = TI->getSuccessor(succ); |
2270 | 3.54k | if (!isa<PHINode>(SuccBB->begin())) |
2271 | 3.15k | continue; |
2272 | 396 | MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB]; |
2273 | 396 | |
2274 | 396 | // If this terminator has multiple identical successors (common for |
2275 | 396 | // switches), only handle each succ once. |
2276 | 396 | if (!SuccsHandled.insert(SuccMBB).second) |
2277 | 1 | continue; |
2278 | 395 | |
2279 | 395 | MachineBasicBlock::iterator MBBI = SuccMBB->begin(); |
2280 | 395 | |
2281 | 395 | // At this point we know that there is a 1-1 correspondence between LLVM PHI |
2282 | 395 | // nodes and Machine PHI nodes, but the incoming operands have not been |
2283 | 395 | // emitted yet. |
2284 | 430 | for (const PHINode &PN : SuccBB->phis()) { |
2285 | 430 | // Ignore dead phi's. |
2286 | 430 | if (PN.use_empty()) |
2287 | 8 | continue; |
2288 | 422 | |
2289 | 422 | // Only handle legal types. Two interesting things to note here. First, |
2290 | 422 | // by bailing out early, we may leave behind some dead instructions, |
2291 | 422 | // since SelectionDAG's HandlePHINodesInSuccessorBlocks will insert its |
2292 | 422 | // own moves. Second, this check is necessary because FastISel doesn't |
2293 | 422 | // use CreateRegs to create registers, so it always creates |
2294 | 422 | // exactly one register for each non-void instruction. |
2295 | 422 | EVT VT = TLI.getValueType(DL, PN.getType(), /*AllowUnknown=*/true); |
2296 | 422 | if (VT == MVT::Other || !TLI.isTypeLegal(VT)) { |
2297 | 89 | // Handle integer promotions, though, because they're common and easy. |
2298 | 89 | if (!(VT == MVT::i1 || VT == MVT::i869 || VT == MVT::i1667 )) { |
2299 | 65 | FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate); |
2300 | 65 | return false; |
2301 | 65 | } |
2302 | 357 | } |
2303 | 357 | |
2304 | 357 | const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB); |
2305 | 357 | |
2306 | 357 | // Set the DebugLoc for the copy. Prefer the location of the operand |
2307 | 357 | // if there is one; use the location of the PHI otherwise. |
2308 | 357 | DbgLoc = PN.getDebugLoc(); |
2309 | 357 | if (const auto *Inst = dyn_cast<Instruction>(PHIOp)) |
2310 | 267 | DbgLoc = Inst->getDebugLoc(); |
2311 | 357 | |
2312 | 357 | unsigned Reg = getRegForValue(PHIOp); |
2313 | 357 | if (!Reg) { |
2314 | 3 | FuncInfo.PHINodesToUpdate.resize(FuncInfo.OrigNumPHINodesToUpdate); |
2315 | 3 | return false; |
2316 | 3 | } |
2317 | 354 | FuncInfo.PHINodesToUpdate.push_back(std::make_pair(&*MBBI++, Reg)); |
2318 | 354 | DbgLoc = DebugLoc(); |
2319 | 354 | } |
2320 | 395 | } |
2321 | 15.8k | |
2322 | 15.8k | return true15.8k ; |
2323 | 15.8k | } |
2324 | | |
2325 | 2.61k | bool FastISel::tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst) { |
2326 | 2.61k | assert(LI->hasOneUse() && |
2327 | 2.61k | "tryToFoldLoad expected a LoadInst with a single use"); |
2328 | 2.61k | // We know that the load has a single use, but don't know what it is. If it |
2329 | 2.61k | // isn't one of the folded instructions, then we can't succeed here. Handle |
2330 | 2.61k | // this by scanning the single-use users of the load until we get to FoldInst. |
2331 | 2.61k | unsigned MaxUsers = 6; // Don't scan down huge single-use chains of instrs. |
2332 | 2.61k | |
2333 | 2.61k | const Instruction *TheUser = LI->user_back(); |
2334 | 3.25k | while (TheUser != FoldInst && // Scan up until we find FoldInst. |
2335 | 3.25k | // Stay in the right block. |
2336 | 3.25k | TheUser->getParent() == FoldInst->getParent()1.16k && |
2337 | 3.25k | --MaxUsers1.07k ) { // Don't scan too far. |
2338 | 1.06k | // If there are multiple or no uses of this instruction, then bail out. |
2339 | 1.06k | if (!TheUser->hasOneUse()) |
2340 | 428 | return false; |
2341 | 641 | |
2342 | 641 | TheUser = TheUser->user_back(); |
2343 | 641 | } |
2344 | 2.61k | |
2345 | 2.61k | // If we didn't find the fold instruction, then we failed to collapse the |
2346 | 2.61k | // sequence. |
2347 | 2.61k | if (2.18k TheUser != FoldInst2.18k ) |
2348 | 96 | return false; |
2349 | 2.09k | |
2350 | 2.09k | // Don't try to fold volatile loads. Target has to deal with alignment |
2351 | 2.09k | // constraints. |
2352 | 2.09k | if (LI->isVolatile()) |
2353 | 70 | return false; |
2354 | 2.02k | |
2355 | 2.02k | // Figure out which vreg this is going into. If there is no assigned vreg yet |
2356 | 2.02k | // then there actually was no reference to it. Perhaps the load is referenced |
2357 | 2.02k | // by a dead instruction. |
2358 | 2.02k | unsigned LoadReg = getRegForValue(LI); |
2359 | 2.02k | if (!LoadReg) |
2360 | 0 | return false; |
2361 | 2.02k | |
2362 | 2.02k | // We can't fold if this vreg has no uses or more than one use. Multiple uses |
2363 | 2.02k | // may mean that the instruction got lowered to multiple MIs, or the use of |
2364 | 2.02k | // the loaded value ended up being multiple operands of the result. |
2365 | 2.02k | if (!MRI.hasOneUse(LoadReg)) |
2366 | 49 | return false; |
2367 | 1.97k | |
2368 | 1.97k | MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(LoadReg); |
2369 | 1.97k | MachineInstr *User = RI->getParent(); |
2370 | 1.97k | |
2371 | 1.97k | // Set the insertion point properly. Folding the load can cause generation of |
2372 | 1.97k | // other random instructions (like sign extends) for addressing modes; make |
2373 | 1.97k | // sure they get inserted in a logical place before the new instruction. |
2374 | 1.97k | FuncInfo.InsertPt = User; |
2375 | 1.97k | FuncInfo.MBB = User->getParent(); |
2376 | 1.97k | |
2377 | 1.97k | // Ask the target to try folding the load. |
2378 | 1.97k | return tryToFoldLoadIntoMI(User, RI.getOperandNo(), LI); |
2379 | 1.97k | } |
2380 | | |
2381 | 105 | bool FastISel::canFoldAddIntoGEP(const User *GEP, const Value *Add) { |
2382 | 105 | // Must be an add. |
2383 | 105 | if (!isa<AddOperator>(Add)) |
2384 | 89 | return false; |
2385 | 16 | // Type size needs to match. |
2386 | 16 | if (DL.getTypeSizeInBits(GEP->getType()) != |
2387 | 16 | DL.getTypeSizeInBits(Add->getType())) |
2388 | 5 | return false; |
2389 | 11 | // Must be in the same basic block. |
2390 | 11 | if (isa<Instruction>(Add) && |
2391 | 11 | FuncInfo.MBBMap[cast<Instruction>(Add)->getParent()] != FuncInfo.MBB) |
2392 | 1 | return false; |
2393 | 10 | // Must have a constant operand. |
2394 | 10 | return isa<ConstantInt>(cast<AddOperator>(Add)->getOperand(1)); |
2395 | 10 | } |
2396 | | |
2397 | | MachineMemOperand * |
2398 | 4.47k | FastISel::createMachineMemOperandFor(const Instruction *I) const { |
2399 | 4.47k | const Value *Ptr; |
2400 | 4.47k | Type *ValTy; |
2401 | 4.47k | unsigned Alignment; |
2402 | 4.47k | MachineMemOperand::Flags Flags; |
2403 | 4.47k | bool IsVolatile; |
2404 | 4.47k | |
2405 | 4.47k | if (const auto *LI = dyn_cast<LoadInst>(I)) { |
2406 | 2.06k | Alignment = LI->getAlignment(); |
2407 | 2.06k | IsVolatile = LI->isVolatile(); |
2408 | 2.06k | Flags = MachineMemOperand::MOLoad; |
2409 | 2.06k | Ptr = LI->getPointerOperand(); |
2410 | 2.06k | ValTy = LI->getType(); |
2411 | 2.41k | } else if (const auto *SI = dyn_cast<StoreInst>(I)) { |
2412 | 2.41k | Alignment = SI->getAlignment(); |
2413 | 2.41k | IsVolatile = SI->isVolatile(); |
2414 | 2.41k | Flags = MachineMemOperand::MOStore; |
2415 | 2.41k | Ptr = SI->getPointerOperand(); |
2416 | 2.41k | ValTy = SI->getValueOperand()->getType(); |
2417 | 2.41k | } else |
2418 | 0 | return nullptr; |
2419 | 4.47k | |
2420 | 4.47k | bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr; |
2421 | 4.47k | bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr; |
2422 | 4.47k | bool IsDereferenceable = |
2423 | 4.47k | I->getMetadata(LLVMContext::MD_dereferenceable) != nullptr; |
2424 | 4.47k | const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range); |
2425 | 4.47k | |
2426 | 4.47k | AAMDNodes AAInfo; |
2427 | 4.47k | I->getAAMetadata(AAInfo); |
2428 | 4.47k | |
2429 | 4.47k | if (Alignment == 0) // Ensure that codegen never sees alignment 0. |
2430 | 1.14k | Alignment = DL.getABITypeAlignment(ValTy); |
2431 | 4.47k | |
2432 | 4.47k | unsigned Size = DL.getTypeStoreSize(ValTy); |
2433 | 4.47k | |
2434 | 4.47k | if (IsVolatile) |
2435 | 82 | Flags |= MachineMemOperand::MOVolatile; |
2436 | 4.47k | if (IsNonTemporal) |
2437 | 273 | Flags |= MachineMemOperand::MONonTemporal; |
2438 | 4.47k | if (IsDereferenceable) |
2439 | 0 | Flags |= MachineMemOperand::MODereferenceable; |
2440 | 4.47k | if (IsInvariant) |
2441 | 13 | Flags |= MachineMemOperand::MOInvariant; |
2442 | 4.47k | |
2443 | 4.47k | return FuncInfo.MF->getMachineMemOperand(MachinePointerInfo(Ptr), Flags, Size, |
2444 | 4.47k | Alignment, AAInfo, Ranges); |
2445 | 4.47k | } |
2446 | | |
2447 | 1.28k | CmpInst::Predicate FastISel::optimizeCmpPredicate(const CmpInst *CI) const { |
2448 | 1.28k | // If both operands are the same, then try to optimize or fold the cmp. |
2449 | 1.28k | CmpInst::Predicate Predicate = CI->getPredicate(); |
2450 | 1.28k | if (CI->getOperand(0) != CI->getOperand(1)) |
2451 | 1.18k | return Predicate; |
2452 | 101 | |
2453 | 101 | switch (Predicate) { |
2454 | 101 | default: 0 llvm_unreachable0 ("Invalid predicate!"); |
2455 | 101 | case CmpInst::FCMP_FALSE: Predicate = CmpInst::FCMP_FALSE; break0 ; |
2456 | 101 | case CmpInst::FCMP_OEQ: Predicate = CmpInst::FCMP_ORD; break4 ; |
2457 | 101 | case CmpInst::FCMP_OGT: Predicate = CmpInst::FCMP_FALSE; break6 ; |
2458 | 101 | case CmpInst::FCMP_OGE: Predicate = CmpInst::FCMP_ORD; break4 ; |
2459 | 101 | case CmpInst::FCMP_OLT: Predicate = CmpInst::FCMP_FALSE; break4 ; |
2460 | 101 | case CmpInst::FCMP_OLE: Predicate = CmpInst::FCMP_ORD; break4 ; |
2461 | 101 | case CmpInst::FCMP_ONE: Predicate = CmpInst::FCMP_FALSE; break4 ; |
2462 | 101 | case CmpInst::FCMP_ORD: Predicate = CmpInst::FCMP_ORD; break4 ; |
2463 | 101 | case CmpInst::FCMP_UNO: Predicate = CmpInst::FCMP_UNO; break4 ; |
2464 | 101 | case CmpInst::FCMP_UEQ: Predicate = CmpInst::FCMP_TRUE; break6 ; |
2465 | 101 | case CmpInst::FCMP_UGT: Predicate = CmpInst::FCMP_UNO; break4 ; |
2466 | 101 | case CmpInst::FCMP_UGE: Predicate = CmpInst::FCMP_TRUE; break4 ; |
2467 | 101 | case CmpInst::FCMP_ULT: Predicate = CmpInst::FCMP_UNO; break4 ; |
2468 | 101 | case CmpInst::FCMP_ULE: Predicate = CmpInst::FCMP_TRUE; break4 ; |
2469 | 101 | case CmpInst::FCMP_UNE: Predicate = CmpInst::FCMP_UNO; break4 ; |
2470 | 101 | case CmpInst::FCMP_TRUE: Predicate = CmpInst::FCMP_TRUE; break0 ; |
2471 | 101 | |
2472 | 101 | case CmpInst::ICMP_EQ: Predicate = CmpInst::FCMP_TRUE; break4 ; |
2473 | 101 | case CmpInst::ICMP_NE: Predicate = CmpInst::FCMP_FALSE; break4 ; |
2474 | 101 | case CmpInst::ICMP_UGT: Predicate = CmpInst::FCMP_FALSE; break4 ; |
2475 | 101 | case CmpInst::ICMP_UGE: Predicate = CmpInst::FCMP_TRUE; break4 ; |
2476 | 101 | case CmpInst::ICMP_ULT: Predicate = CmpInst::FCMP_FALSE; break4 ; |
2477 | 101 | case CmpInst::ICMP_ULE: Predicate = CmpInst::FCMP_TRUE; break4 ; |
2478 | 101 | case CmpInst::ICMP_SGT: Predicate = CmpInst::FCMP_FALSE; break5 ; |
2479 | 101 | case CmpInst::ICMP_SGE: Predicate = CmpInst::FCMP_TRUE; break4 ; |
2480 | 101 | case CmpInst::ICMP_SLT: Predicate = CmpInst::FCMP_FALSE; break4 ; |
2481 | 101 | case CmpInst::ICMP_SLE: Predicate = CmpInst::FCMP_TRUE; break4 ; |
2482 | 101 | } |
2483 | 101 | |
2484 | 101 | return Predicate; |
2485 | 101 | } |