/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/TargetLoweringBase.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // This implements the TargetLoweringBase class. |
10 | | // |
11 | | //===----------------------------------------------------------------------===// |
12 | | |
13 | | #include "llvm/ADT/BitVector.h" |
14 | | #include "llvm/ADT/STLExtras.h" |
15 | | #include "llvm/ADT/SmallVector.h" |
16 | | #include "llvm/ADT/StringExtras.h" |
17 | | #include "llvm/ADT/StringRef.h" |
18 | | #include "llvm/ADT/Triple.h" |
19 | | #include "llvm/ADT/Twine.h" |
20 | | #include "llvm/CodeGen/Analysis.h" |
21 | | #include "llvm/CodeGen/ISDOpcodes.h" |
22 | | #include "llvm/CodeGen/MachineBasicBlock.h" |
23 | | #include "llvm/CodeGen/MachineFrameInfo.h" |
24 | | #include "llvm/CodeGen/MachineFunction.h" |
25 | | #include "llvm/CodeGen/MachineInstr.h" |
26 | | #include "llvm/CodeGen/MachineInstrBuilder.h" |
27 | | #include "llvm/CodeGen/MachineMemOperand.h" |
28 | | #include "llvm/CodeGen/MachineOperand.h" |
29 | | #include "llvm/CodeGen/MachineRegisterInfo.h" |
30 | | #include "llvm/CodeGen/RuntimeLibcalls.h" |
31 | | #include "llvm/CodeGen/StackMaps.h" |
32 | | #include "llvm/CodeGen/TargetLowering.h" |
33 | | #include "llvm/CodeGen/TargetOpcodes.h" |
34 | | #include "llvm/CodeGen/TargetRegisterInfo.h" |
35 | | #include "llvm/CodeGen/ValueTypes.h" |
36 | | #include "llvm/IR/Attributes.h" |
37 | | #include "llvm/IR/CallingConv.h" |
38 | | #include "llvm/IR/DataLayout.h" |
39 | | #include "llvm/IR/DerivedTypes.h" |
40 | | #include "llvm/IR/Function.h" |
41 | | #include "llvm/IR/GlobalValue.h" |
42 | | #include "llvm/IR/GlobalVariable.h" |
43 | | #include "llvm/IR/IRBuilder.h" |
44 | | #include "llvm/IR/Module.h" |
45 | | #include "llvm/IR/Type.h" |
46 | | #include "llvm/Support/BranchProbability.h" |
47 | | #include "llvm/Support/Casting.h" |
48 | | #include "llvm/Support/CommandLine.h" |
49 | | #include "llvm/Support/Compiler.h" |
50 | | #include "llvm/Support/ErrorHandling.h" |
51 | | #include "llvm/Support/MachineValueType.h" |
52 | | #include "llvm/Support/MathExtras.h" |
53 | | #include "llvm/Target/TargetMachine.h" |
54 | | #include <algorithm> |
55 | | #include <cassert> |
56 | | #include <cstddef> |
57 | | #include <cstdint> |
58 | | #include <cstring> |
59 | | #include <iterator> |
60 | | #include <string> |
61 | | #include <tuple> |
62 | | #include <utility> |
63 | | |
64 | | using namespace llvm; |
65 | | |
66 | | static cl::opt<bool> JumpIsExpensiveOverride( |
67 | | "jump-is-expensive", cl::init(false), |
68 | | cl::desc("Do not create extra branches to split comparison logic."), |
69 | | cl::Hidden); |
70 | | |
71 | | static cl::opt<unsigned> MinimumJumpTableEntries |
72 | | ("min-jump-table-entries", cl::init(4), cl::Hidden, |
73 | | cl::desc("Set minimum number of entries to use a jump table.")); |
74 | | |
75 | | static cl::opt<unsigned> MaximumJumpTableSize |
76 | | ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden, |
77 | | cl::desc("Set maximum size of jump tables.")); |
78 | | |
79 | | /// Minimum jump table density for normal functions. |
80 | | static cl::opt<unsigned> |
81 | | JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden, |
82 | | cl::desc("Minimum density for building a jump table in " |
83 | | "a normal function")); |
84 | | |
85 | | /// Minimum jump table density for -Os or -Oz functions. |
86 | | static cl::opt<unsigned> OptsizeJumpTableDensity( |
87 | | "optsize-jump-table-density", cl::init(40), cl::Hidden, |
88 | | cl::desc("Minimum density for building a jump table in " |
89 | | "an optsize function")); |
90 | | |
91 | 26.3k | static bool darwinHasSinCos(const Triple &TT) { |
92 | 26.3k | assert(TT.isOSDarwin() && "should be called with darwin triple"); |
93 | 26.3k | // Don't bother with 32 bit x86. |
94 | 26.3k | if (TT.getArch() == Triple::x86) |
95 | 1.05k | return false; |
96 | 25.2k | // Macos < 10.9 has no sincos_stret. |
97 | 25.2k | if (TT.isMacOSX()) |
98 | 16.0k | return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit()13.1k ; |
99 | 9.21k | // iOS < 7.0 has no sincos_stret. |
100 | 9.21k | if (TT.isiOS()) |
101 | 9.19k | return !TT.isOSVersionLT(7, 0); |
102 | 21 | // Any other darwin such as WatchOS/TvOS is new enough. |
103 | 21 | return true; |
104 | 21 | } |
105 | | |
106 | | // Although this default value is arbitrary, it is not random. It is assumed |
107 | | // that a condition that evaluates the same way by a higher percentage than this |
108 | | // is best represented as control flow. Therefore, the default value N should be |
109 | | // set such that the win from N% correct executions is greater than the loss |
110 | | // from (100 - N)% mispredicted executions for the majority of intended targets. |
111 | | static cl::opt<int> MinPercentageForPredictableBranch( |
112 | | "min-predictable-branch", cl::init(99), |
113 | | cl::desc("Minimum percentage (0-100) that a condition must be either true " |
114 | | "or false to assume that the condition is predictable"), |
115 | | cl::Hidden); |
116 | | |
117 | 53.2k | void TargetLoweringBase::InitLibcalls(const Triple &TT) { |
118 | 53.2k | #define HANDLE_LIBCALL(code, name) \ |
119 | 26.2M | setLibcallName(RTLIB::code, name); |
120 | 53.2k | #include "llvm/IR/RuntimeLibcalls.def" |
121 | 53.2k | #undef HANDLE_LIBCALL |
122 | 53.2k | // Initialize calling conventions to their default. |
123 | 26.2M | for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC26.1M ) |
124 | 26.1M | setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C); |
125 | 53.2k | |
126 | 53.2k | // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf". |
127 | 53.2k | if (TT.getArch() == Triple::ppc || TT.isPPC64()52.8k ) { |
128 | 1.85k | setLibcallName(RTLIB::ADD_F128, "__addkf3"); |
129 | 1.85k | setLibcallName(RTLIB::SUB_F128, "__subkf3"); |
130 | 1.85k | setLibcallName(RTLIB::MUL_F128, "__mulkf3"); |
131 | 1.85k | setLibcallName(RTLIB::DIV_F128, "__divkf3"); |
132 | 1.85k | setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2"); |
133 | 1.85k | setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2"); |
134 | 1.85k | setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2"); |
135 | 1.85k | setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2"); |
136 | 1.85k | setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi"); |
137 | 1.85k | setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi"); |
138 | 1.85k | setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi"); |
139 | 1.85k | setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi"); |
140 | 1.85k | setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf"); |
141 | 1.85k | setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf"); |
142 | 1.85k | setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf"); |
143 | 1.85k | setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf"); |
144 | 1.85k | setLibcallName(RTLIB::OEQ_F128, "__eqkf2"); |
145 | 1.85k | setLibcallName(RTLIB::UNE_F128, "__nekf2"); |
146 | 1.85k | setLibcallName(RTLIB::OGE_F128, "__gekf2"); |
147 | 1.85k | setLibcallName(RTLIB::OLT_F128, "__ltkf2"); |
148 | 1.85k | setLibcallName(RTLIB::OLE_F128, "__lekf2"); |
149 | 1.85k | setLibcallName(RTLIB::OGT_F128, "__gtkf2"); |
150 | 1.85k | setLibcallName(RTLIB::UO_F128, "__unordkf2"); |
151 | 1.85k | setLibcallName(RTLIB::O_F128, "__unordkf2"); |
152 | 1.85k | } |
153 | 53.2k | |
154 | 53.2k | // A few names are different on particular architectures or environments. |
155 | 53.2k | if (TT.isOSDarwin()) { |
156 | 26.3k | // For f16/f32 conversions, Darwin uses the standard naming scheme, instead |
157 | 26.3k | // of the gnueabi-style __gnu_*_ieee. |
158 | 26.3k | // FIXME: What about other targets? |
159 | 26.3k | setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2"); |
160 | 26.3k | setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2"); |
161 | 26.3k | |
162 | 26.3k | // Some darwins have an optimized __bzero/bzero function. |
163 | 26.3k | switch (TT.getArch()) { |
164 | 26.3k | case Triple::x86: |
165 | 6.50k | case Triple::x86_64: |
166 | 6.50k | if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6)5.39k ) |
167 | 3.16k | setLibcallName(RTLIB::BZERO, "__bzero"); |
168 | 6.50k | break; |
169 | 7.51k | case Triple::aarch64: |
170 | 7.51k | setLibcallName(RTLIB::BZERO, "bzero"); |
171 | 7.51k | break; |
172 | 12.3k | default: |
173 | 12.3k | break; |
174 | 26.3k | } |
175 | 26.3k | |
176 | 26.3k | if (darwinHasSinCos(TT)) { |
177 | 14.7k | setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret"); |
178 | 14.7k | setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret"); |
179 | 14.7k | if (TT.isWatchABI()) { |
180 | 156 | setLibcallCallingConv(RTLIB::SINCOS_STRET_F32, |
181 | 156 | CallingConv::ARM_AAPCS_VFP); |
182 | 156 | setLibcallCallingConv(RTLIB::SINCOS_STRET_F64, |
183 | 156 | CallingConv::ARM_AAPCS_VFP); |
184 | 156 | } |
185 | 14.7k | } |
186 | 26.9k | } else { |
187 | 26.9k | setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee"); |
188 | 26.9k | setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee"); |
189 | 26.9k | } |
190 | 53.2k | |
191 | 53.2k | if (53.2k TT.isGNUEnvironment()53.2k || TT.isOSFuchsia()42.0k || |
192 | 53.2k | (42.0k TT.isAndroid()42.0k && !TT.isAndroidVersionLT(9)218 )) { |
193 | 11.3k | setLibcallName(RTLIB::SINCOS_F32, "sincosf"); |
194 | 11.3k | setLibcallName(RTLIB::SINCOS_F64, "sincos"); |
195 | 11.3k | setLibcallName(RTLIB::SINCOS_F80, "sincosl"); |
196 | 11.3k | setLibcallName(RTLIB::SINCOS_F128, "sincosl"); |
197 | 11.3k | setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl"); |
198 | 11.3k | } |
199 | 53.2k | |
200 | 53.2k | if (TT.isOSOpenBSD()) { |
201 | 7 | setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr); |
202 | 7 | } |
203 | 53.2k | } |
204 | | |
205 | | /// getFPEXT - Return the FPEXT_*_* value for the given types, or |
206 | | /// UNKNOWN_LIBCALL if there is none. |
207 | 164 | RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) { |
208 | 164 | if (OpVT == MVT::f16) { |
209 | 0 | if (RetVT == MVT::f32) |
210 | 0 | return FPEXT_F16_F32; |
211 | 164 | } else if (OpVT == MVT::f32) { |
212 | 135 | if (RetVT == MVT::f64) |
213 | 115 | return FPEXT_F32_F64; |
214 | 20 | if (RetVT == MVT::f128) |
215 | 20 | return FPEXT_F32_F128; |
216 | 0 | if (RetVT == MVT::ppcf128) |
217 | 0 | return FPEXT_F32_PPCF128; |
218 | 29 | } else if (OpVT == MVT::f64) { |
219 | 25 | if (RetVT == MVT::f128) |
220 | 24 | return FPEXT_F64_F128; |
221 | 1 | else if (RetVT == MVT::ppcf128) |
222 | 1 | return FPEXT_F64_PPCF128; |
223 | 4 | } else if (OpVT == MVT::f80) { |
224 | 4 | if (RetVT == MVT::f128) |
225 | 4 | return FPEXT_F80_F128; |
226 | 0 | } |
227 | 0 | |
228 | 0 | return UNKNOWN_LIBCALL; |
229 | 0 | } |
230 | | |
231 | | /// getFPROUND - Return the FPROUND_*_* value for the given types, or |
232 | | /// UNKNOWN_LIBCALL if there is none. |
233 | 786 | RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) { |
234 | 786 | if (RetVT == MVT::f16) { |
235 | 715 | if (OpVT == MVT::f32) |
236 | 414 | return FPROUND_F32_F16; |
237 | 301 | if (OpVT == MVT::f64) |
238 | 296 | return FPROUND_F64_F16; |
239 | 5 | if (OpVT == MVT::f80) |
240 | 3 | return FPROUND_F80_F16; |
241 | 2 | if (OpVT == MVT::f128) |
242 | 2 | return FPROUND_F128_F16; |
243 | 0 | if (OpVT == MVT::ppcf128) |
244 | 0 | return FPROUND_PPCF128_F16; |
245 | 71 | } else if (RetVT == MVT::f32) { |
246 | 42 | if (OpVT == MVT::f64) |
247 | 24 | return FPROUND_F64_F32; |
248 | 18 | if (OpVT == MVT::f80) |
249 | 0 | return FPROUND_F80_F32; |
250 | 18 | if (OpVT == MVT::f128) |
251 | 17 | return FPROUND_F128_F32; |
252 | 1 | if (OpVT == MVT::ppcf128) |
253 | 1 | return FPROUND_PPCF128_F32; |
254 | 29 | } else if (RetVT == MVT::f64) { |
255 | 25 | if (OpVT == MVT::f80) |
256 | 0 | return FPROUND_F80_F64; |
257 | 25 | if (OpVT == MVT::f128) |
258 | 24 | return FPROUND_F128_F64; |
259 | 1 | if (OpVT == MVT::ppcf128) |
260 | 1 | return FPROUND_PPCF128_F64; |
261 | 4 | } else if (RetVT == MVT::f80) { |
262 | 4 | if (OpVT == MVT::f128) |
263 | 4 | return FPROUND_F128_F80; |
264 | 0 | } |
265 | 0 | |
266 | 0 | return UNKNOWN_LIBCALL; |
267 | 0 | } |
268 | | |
269 | | /// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or |
270 | | /// UNKNOWN_LIBCALL if there is none. |
271 | 310 | RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) { |
272 | 310 | if (OpVT == MVT::f32) { |
273 | 100 | if (RetVT == MVT::i32) |
274 | 49 | return FPTOSINT_F32_I32; |
275 | 51 | if (RetVT == MVT::i64) |
276 | 42 | return FPTOSINT_F32_I64; |
277 | 9 | if (RetVT == MVT::i128) |
278 | 3 | return FPTOSINT_F32_I128; |
279 | 210 | } else if (OpVT == MVT::f64) { |
280 | 98 | if (RetVT == MVT::i32) |
281 | 47 | return FPTOSINT_F64_I32; |
282 | 51 | if (RetVT == MVT::i64) |
283 | 51 | return FPTOSINT_F64_I64; |
284 | 0 | if (RetVT == MVT::i128) |
285 | 0 | return FPTOSINT_F64_I128; |
286 | 112 | } else if (OpVT == MVT::f80) { |
287 | 0 | if (RetVT == MVT::i32) |
288 | 0 | return FPTOSINT_F80_I32; |
289 | 0 | if (RetVT == MVT::i64) |
290 | 0 | return FPTOSINT_F80_I64; |
291 | 0 | if (RetVT == MVT::i128) |
292 | 0 | return FPTOSINT_F80_I128; |
293 | 112 | } else if (OpVT == MVT::f128) { |
294 | 110 | if (RetVT == MVT::i32) |
295 | 59 | return FPTOSINT_F128_I32; |
296 | 51 | if (RetVT == MVT::i64) |
297 | 48 | return FPTOSINT_F128_I64; |
298 | 3 | if (RetVT == MVT::i128) |
299 | 3 | return FPTOSINT_F128_I128; |
300 | 2 | } else if (OpVT == MVT::ppcf128) { |
301 | 2 | if (RetVT == MVT::i32) |
302 | 0 | return FPTOSINT_PPCF128_I32; |
303 | 2 | if (RetVT == MVT::i64) |
304 | 2 | return FPTOSINT_PPCF128_I64; |
305 | 0 | if (RetVT == MVT::i128) |
306 | 0 | return FPTOSINT_PPCF128_I128; |
307 | 6 | } |
308 | 6 | return UNKNOWN_LIBCALL; |
309 | 6 | } |
310 | | |
311 | | /// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or |
312 | | /// UNKNOWN_LIBCALL if there is none. |
313 | 252 | RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) { |
314 | 252 | if (OpVT == MVT::f32) { |
315 | 77 | if (RetVT == MVT::i32) |
316 | 32 | return FPTOUINT_F32_I32; |
317 | 45 | if (RetVT == MVT::i64) |
318 | 39 | return FPTOUINT_F32_I64; |
319 | 6 | if (RetVT == MVT::i128) |
320 | 0 | return FPTOUINT_F32_I128; |
321 | 175 | } else if (OpVT == MVT::f64) { |
322 | 118 | if (RetVT == MVT::i32) |
323 | 72 | return FPTOUINT_F64_I32; |
324 | 46 | if (RetVT == MVT::i64) |
325 | 44 | return FPTOUINT_F64_I64; |
326 | 2 | if (RetVT == MVT::i128) |
327 | 2 | return FPTOUINT_F64_I128; |
328 | 57 | } else if (OpVT == MVT::f80) { |
329 | 1 | if (RetVT == MVT::i32) |
330 | 0 | return FPTOUINT_F80_I32; |
331 | 1 | if (RetVT == MVT::i64) |
332 | 0 | return FPTOUINT_F80_I64; |
333 | 1 | if (RetVT == MVT::i128) |
334 | 1 | return FPTOUINT_F80_I128; |
335 | 56 | } else if (OpVT == MVT::f128) { |
336 | 54 | if (RetVT == MVT::i32) |
337 | 18 | return FPTOUINT_F128_I32; |
338 | 36 | if (RetVT == MVT::i64) |
339 | 33 | return FPTOUINT_F128_I64; |
340 | 3 | if (RetVT == MVT::i128) |
341 | 3 | return FPTOUINT_F128_I128; |
342 | 2 | } else if (OpVT == MVT::ppcf128) { |
343 | 2 | if (RetVT == MVT::i32) |
344 | 0 | return FPTOUINT_PPCF128_I32; |
345 | 2 | if (RetVT == MVT::i64) |
346 | 1 | return FPTOUINT_PPCF128_I64; |
347 | 1 | if (RetVT == MVT::i128) |
348 | 1 | return FPTOUINT_PPCF128_I128; |
349 | 6 | } |
350 | 6 | return UNKNOWN_LIBCALL; |
351 | 6 | } |
352 | | |
353 | | /// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or |
354 | | /// UNKNOWN_LIBCALL if there is none. |
355 | 268 | RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) { |
356 | 268 | if (OpVT == MVT::i32) { |
357 | 137 | if (RetVT == MVT::f32) |
358 | 47 | return SINTTOFP_I32_F32; |
359 | 90 | if (RetVT == MVT::f64) |
360 | 63 | return SINTTOFP_I32_F64; |
361 | 27 | if (RetVT == MVT::f80) |
362 | 0 | return SINTTOFP_I32_F80; |
363 | 27 | if (RetVT == MVT::f128) |
364 | 26 | return SINTTOFP_I32_F128; |
365 | 1 | if (RetVT == MVT::ppcf128) |
366 | 1 | return SINTTOFP_I32_PPCF128; |
367 | 131 | } else if (OpVT == MVT::i64) { |
368 | 116 | if (RetVT == MVT::f32) |
369 | 43 | return SINTTOFP_I64_F32; |
370 | 73 | if (RetVT == MVT::f64) |
371 | 52 | return SINTTOFP_I64_F64; |
372 | 21 | if (RetVT == MVT::f80) |
373 | 0 | return SINTTOFP_I64_F80; |
374 | 21 | if (RetVT == MVT::f128) |
375 | 21 | return SINTTOFP_I64_F128; |
376 | 0 | if (RetVT == MVT::ppcf128) |
377 | 0 | return SINTTOFP_I64_PPCF128; |
378 | 15 | } else if (OpVT == MVT::i128) { |
379 | 3 | if (RetVT == MVT::f32) |
380 | 1 | return SINTTOFP_I128_F32; |
381 | 2 | if (RetVT == MVT::f64) |
382 | 2 | return SINTTOFP_I128_F64; |
383 | 0 | if (RetVT == MVT::f80) |
384 | 0 | return SINTTOFP_I128_F80; |
385 | 0 | if (RetVT == MVT::f128) |
386 | 0 | return SINTTOFP_I128_F128; |
387 | 0 | if (RetVT == MVT::ppcf128) |
388 | 0 | return SINTTOFP_I128_PPCF128; |
389 | 12 | } |
390 | 12 | return UNKNOWN_LIBCALL; |
391 | 12 | } |
392 | | |
393 | | /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or |
394 | | /// UNKNOWN_LIBCALL if there is none. |
395 | 558 | RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) { |
396 | 558 | if (OpVT == MVT::i32) { |
397 | 194 | if (RetVT == MVT::f32) |
398 | 81 | return UINTTOFP_I32_F32; |
399 | 113 | if (RetVT == MVT::f64) |
400 | 91 | return UINTTOFP_I32_F64; |
401 | 22 | if (RetVT == MVT::f80) |
402 | 0 | return UINTTOFP_I32_F80; |
403 | 22 | if (RetVT == MVT::f128) |
404 | 21 | return UINTTOFP_I32_F128; |
405 | 1 | if (RetVT == MVT::ppcf128) |
406 | 1 | return UINTTOFP_I32_PPCF128; |
407 | 364 | } else if (OpVT == MVT::i64) { |
408 | 102 | if (RetVT == MVT::f32) |
409 | 40 | return UINTTOFP_I64_F32; |
410 | 62 | if (RetVT == MVT::f64) |
411 | 43 | return UINTTOFP_I64_F64; |
412 | 19 | if (RetVT == MVT::f80) |
413 | 0 | return UINTTOFP_I64_F80; |
414 | 19 | if (RetVT == MVT::f128) |
415 | 19 | return UINTTOFP_I64_F128; |
416 | 0 | if (RetVT == MVT::ppcf128) |
417 | 0 | return UINTTOFP_I64_PPCF128; |
418 | 262 | } else if (OpVT == MVT::i128) { |
419 | 4 | if (RetVT == MVT::f32) |
420 | 2 | return UINTTOFP_I128_F32; |
421 | 2 | if (RetVT == MVT::f64) |
422 | 0 | return UINTTOFP_I128_F64; |
423 | 2 | if (RetVT == MVT::f80) |
424 | 0 | return UINTTOFP_I128_F80; |
425 | 2 | if (RetVT == MVT::f128) |
426 | 2 | return UINTTOFP_I128_F128; |
427 | 0 | if (RetVT == MVT::ppcf128) |
428 | 0 | return UINTTOFP_I128_PPCF128; |
429 | 258 | } |
430 | 258 | return UNKNOWN_LIBCALL; |
431 | 258 | } |
432 | | |
433 | 172 | RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) { |
434 | 172 | #define OP_TO_LIBCALL(Name, Enum) \ |
435 | 172 | case Name: \ |
436 | 172 | switch (VT.SimpleTy) { \ |
437 | 172 | default: \ |
438 | 0 | return UNKNOWN_LIBCALL; \ |
439 | 172 | case MVT::i8: \ |
440 | 28 | return Enum##_1; \ |
441 | 172 | case MVT::i16: \ |
442 | 9 | return Enum##_2; \ |
443 | 172 | case MVT::i32: \ |
444 | 49 | return Enum##_4; \ |
445 | 172 | case MVT::i64: \ |
446 | 30 | return Enum##_8; \ |
447 | 172 | case MVT::i128: \ |
448 | 56 | return Enum##_16; \ |
449 | 172 | } |
450 | 172 | |
451 | 172 | switch (Opc) { |
452 | 172 | OP_TO_LIBCALL44 (ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET) |
453 | 46 | OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP) |
454 | 46 | OP_TO_LIBCALL16 (ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD) |
455 | 16 | OP_TO_LIBCALL7 (ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB) |
456 | 7 | OP_TO_LIBCALL4 (ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND) |
457 | 9 | OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR) |
458 | 9 | OP_TO_LIBCALL3 (ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR) |
459 | 3 | OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND) |
460 | 6 | OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX) |
461 | 14 | OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX) |
462 | 14 | OP_TO_LIBCALL6 (ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN) |
463 | 14 | OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN) |
464 | 172 | } |
465 | 172 | |
466 | 172 | #undef OP_TO_LIBCALL |
467 | 172 | |
468 | 172 | return UNKNOWN_LIBCALL0 ; |
469 | 172 | } |
470 | | |
471 | 15 | RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { |
472 | 15 | switch (ElementSize) { |
473 | 15 | case 1: |
474 | 6 | return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1; |
475 | 15 | case 2: |
476 | 2 | return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2; |
477 | 15 | case 4: |
478 | 3 | return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4; |
479 | 15 | case 8: |
480 | 2 | return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8; |
481 | 15 | case 16: |
482 | 2 | return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16; |
483 | 15 | default: |
484 | 0 | return UNKNOWN_LIBCALL; |
485 | 15 | } |
486 | 15 | } |
487 | | |
488 | 13 | RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { |
489 | 13 | switch (ElementSize) { |
490 | 13 | case 1: |
491 | 4 | return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1; |
492 | 13 | case 2: |
493 | 2 | return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2; |
494 | 13 | case 4: |
495 | 3 | return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4; |
496 | 13 | case 8: |
497 | 2 | return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8; |
498 | 13 | case 16: |
499 | 2 | return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16; |
500 | 13 | default: |
501 | 0 | return UNKNOWN_LIBCALL; |
502 | 13 | } |
503 | 13 | } |
504 | | |
505 | 23 | RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) { |
506 | 23 | switch (ElementSize) { |
507 | 23 | case 1: |
508 | 3 | return MEMSET_ELEMENT_UNORDERED_ATOMIC_1; |
509 | 23 | case 2: |
510 | 2 | return MEMSET_ELEMENT_UNORDERED_ATOMIC_2; |
511 | 23 | case 4: |
512 | 13 | return MEMSET_ELEMENT_UNORDERED_ATOMIC_4; |
513 | 23 | case 8: |
514 | 2 | return MEMSET_ELEMENT_UNORDERED_ATOMIC_8; |
515 | 23 | case 16: |
516 | 3 | return MEMSET_ELEMENT_UNORDERED_ATOMIC_16; |
517 | 23 | default: |
518 | 0 | return UNKNOWN_LIBCALL; |
519 | 23 | } |
520 | 23 | } |
521 | | |
522 | | /// InitCmpLibcallCCs - Set default comparison libcall CC. |
523 | 53.2k | static void InitCmpLibcallCCs(ISD::CondCode *CCs) { |
524 | 53.2k | memset(CCs, ISD::SETCC_INVALID, sizeof(ISD::CondCode)*RTLIB::UNKNOWN_LIBCALL); |
525 | 53.2k | CCs[RTLIB::OEQ_F32] = ISD::SETEQ; |
526 | 53.2k | CCs[RTLIB::OEQ_F64] = ISD::SETEQ; |
527 | 53.2k | CCs[RTLIB::OEQ_F128] = ISD::SETEQ; |
528 | 53.2k | CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ; |
529 | 53.2k | CCs[RTLIB::UNE_F32] = ISD::SETNE; |
530 | 53.2k | CCs[RTLIB::UNE_F64] = ISD::SETNE; |
531 | 53.2k | CCs[RTLIB::UNE_F128] = ISD::SETNE; |
532 | 53.2k | CCs[RTLIB::UNE_PPCF128] = ISD::SETNE; |
533 | 53.2k | CCs[RTLIB::OGE_F32] = ISD::SETGE; |
534 | 53.2k | CCs[RTLIB::OGE_F64] = ISD::SETGE; |
535 | 53.2k | CCs[RTLIB::OGE_F128] = ISD::SETGE; |
536 | 53.2k | CCs[RTLIB::OGE_PPCF128] = ISD::SETGE; |
537 | 53.2k | CCs[RTLIB::OLT_F32] = ISD::SETLT; |
538 | 53.2k | CCs[RTLIB::OLT_F64] = ISD::SETLT; |
539 | 53.2k | CCs[RTLIB::OLT_F128] = ISD::SETLT; |
540 | 53.2k | CCs[RTLIB::OLT_PPCF128] = ISD::SETLT; |
541 | 53.2k | CCs[RTLIB::OLE_F32] = ISD::SETLE; |
542 | 53.2k | CCs[RTLIB::OLE_F64] = ISD::SETLE; |
543 | 53.2k | CCs[RTLIB::OLE_F128] = ISD::SETLE; |
544 | 53.2k | CCs[RTLIB::OLE_PPCF128] = ISD::SETLE; |
545 | 53.2k | CCs[RTLIB::OGT_F32] = ISD::SETGT; |
546 | 53.2k | CCs[RTLIB::OGT_F64] = ISD::SETGT; |
547 | 53.2k | CCs[RTLIB::OGT_F128] = ISD::SETGT; |
548 | 53.2k | CCs[RTLIB::OGT_PPCF128] = ISD::SETGT; |
549 | 53.2k | CCs[RTLIB::UO_F32] = ISD::SETNE; |
550 | 53.2k | CCs[RTLIB::UO_F64] = ISD::SETNE; |
551 | 53.2k | CCs[RTLIB::UO_F128] = ISD::SETNE; |
552 | 53.2k | CCs[RTLIB::UO_PPCF128] = ISD::SETNE; |
553 | 53.2k | CCs[RTLIB::O_F32] = ISD::SETEQ; |
554 | 53.2k | CCs[RTLIB::O_F64] = ISD::SETEQ; |
555 | 53.2k | CCs[RTLIB::O_F128] = ISD::SETEQ; |
556 | 53.2k | CCs[RTLIB::O_PPCF128] = ISD::SETEQ; |
557 | 53.2k | } |
558 | | |
559 | | /// NOTE: The TargetMachine owns TLOF. |
560 | 53.2k | TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) { |
561 | 53.2k | initActions(); |
562 | 53.2k | |
563 | 53.2k | // Perform these initializations only once. |
564 | 53.2k | MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove = |
565 | 53.2k | MaxLoadsPerMemcmp = 8; |
566 | 53.2k | MaxGluedStoresPerMemcpy = 0; |
567 | 53.2k | MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize = |
568 | 53.2k | MaxStoresPerMemmoveOptSize = MaxLoadsPerMemcmpOptSize = 4; |
569 | 53.2k | UseUnderscoreSetJmp = false; |
570 | 53.2k | UseUnderscoreLongJmp = false; |
571 | 53.2k | HasMultipleConditionRegisters = false; |
572 | 53.2k | HasExtractBitsInsn = false; |
573 | 53.2k | JumpIsExpensive = JumpIsExpensiveOverride; |
574 | 53.2k | PredictableSelectIsExpensive = false; |
575 | 53.2k | EnableExtLdPromotion = false; |
576 | 53.2k | StackPointerRegisterToSaveRestore = 0; |
577 | 53.2k | BooleanContents = UndefinedBooleanContent; |
578 | 53.2k | BooleanFloatContents = UndefinedBooleanContent; |
579 | 53.2k | BooleanVectorContents = UndefinedBooleanContent; |
580 | 53.2k | SchedPreferenceInfo = Sched::ILP; |
581 | 53.2k | JumpBufSize = 0; |
582 | 53.2k | JumpBufAlignment = 0; |
583 | 53.2k | MinFunctionAlignment = 0; |
584 | 53.2k | PrefFunctionAlignment = 0; |
585 | 53.2k | PrefLoopAlignment = 0; |
586 | 53.2k | GatherAllAliasesMaxDepth = 18; |
587 | 53.2k | MinStackArgumentAlignment = 1; |
588 | 53.2k | // TODO: the default will be switched to 0 in the next commit, along |
589 | 53.2k | // with the Target-specific changes necessary. |
590 | 53.2k | MaxAtomicSizeInBitsSupported = 1024; |
591 | 53.2k | |
592 | 53.2k | MinCmpXchgSizeInBits = 0; |
593 | 53.2k | SupportsUnalignedAtomics = false; |
594 | 53.2k | |
595 | 53.2k | std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr); |
596 | 53.2k | |
597 | 53.2k | InitLibcalls(TM.getTargetTriple()); |
598 | 53.2k | InitCmpLibcallCCs(CmpLibcallCCs); |
599 | 53.2k | } |
600 | | |
601 | 53.2k | void TargetLoweringBase::initActions() { |
602 | 53.2k | // All operations default to being supported. |
603 | 53.2k | memset(OpActions, 0, sizeof(OpActions)); |
604 | 53.2k | memset(LoadExtActions, 0, sizeof(LoadExtActions)); |
605 | 53.2k | memset(TruncStoreActions, 0, sizeof(TruncStoreActions)); |
606 | 53.2k | memset(IndexedModeActions, 0, sizeof(IndexedModeActions)); |
607 | 53.2k | memset(CondCodeActions, 0, sizeof(CondCodeActions)); |
608 | 53.2k | std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr); |
609 | 53.2k | std::fill(std::begin(TargetDAGCombineArray), |
610 | 53.2k | std::end(TargetDAGCombineArray), 0); |
611 | 53.2k | |
612 | 319k | for (MVT VT : MVT::fp_valuetypes()) { |
613 | 319k | MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits()); |
614 | 319k | if (IntVT.isValid()) { |
615 | 266k | setOperationAction(ISD::ATOMIC_SWAP, VT, Promote); |
616 | 266k | AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT); |
617 | 266k | } |
618 | 319k | } |
619 | 53.2k | |
620 | 53.2k | // Set default actions for various operations. |
621 | 6.87M | for (MVT VT : MVT::all_valuetypes()) { |
622 | 6.87M | // Default all indexed load / store to expand. |
623 | 6.87M | for (unsigned IM = (unsigned)ISD::PRE_INC; |
624 | 34.3M | IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM27.4M ) { |
625 | 27.4M | setIndexedLoadAction(IM, VT, Expand); |
626 | 27.4M | setIndexedStoreAction(IM, VT, Expand); |
627 | 27.4M | } |
628 | 6.87M | |
629 | 6.87M | // Most backends expect to see the node which just returns the value loaded. |
630 | 6.87M | setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Expand); |
631 | 6.87M | |
632 | 6.87M | // These operations default to expand. |
633 | 6.87M | setOperationAction(ISD::FGETSIGN, VT, Expand); |
634 | 6.87M | setOperationAction(ISD::CONCAT_VECTORS, VT, Expand); |
635 | 6.87M | setOperationAction(ISD::FMINNUM, VT, Expand); |
636 | 6.87M | setOperationAction(ISD::FMAXNUM, VT, Expand); |
637 | 6.87M | setOperationAction(ISD::FMINNUM_IEEE, VT, Expand); |
638 | 6.87M | setOperationAction(ISD::FMAXNUM_IEEE, VT, Expand); |
639 | 6.87M | setOperationAction(ISD::FMINIMUM, VT, Expand); |
640 | 6.87M | setOperationAction(ISD::FMAXIMUM, VT, Expand); |
641 | 6.87M | setOperationAction(ISD::FMAD, VT, Expand); |
642 | 6.87M | setOperationAction(ISD::SMIN, VT, Expand); |
643 | 6.87M | setOperationAction(ISD::SMAX, VT, Expand); |
644 | 6.87M | setOperationAction(ISD::UMIN, VT, Expand); |
645 | 6.87M | setOperationAction(ISD::UMAX, VT, Expand); |
646 | 6.87M | setOperationAction(ISD::ABS, VT, Expand); |
647 | 6.87M | setOperationAction(ISD::FSHL, VT, Expand); |
648 | 6.87M | setOperationAction(ISD::FSHR, VT, Expand); |
649 | 6.87M | setOperationAction(ISD::SADDSAT, VT, Expand); |
650 | 6.87M | setOperationAction(ISD::UADDSAT, VT, Expand); |
651 | 6.87M | setOperationAction(ISD::SSUBSAT, VT, Expand); |
652 | 6.87M | setOperationAction(ISD::USUBSAT, VT, Expand); |
653 | 6.87M | setOperationAction(ISD::SMULFIX, VT, Expand); |
654 | 6.87M | setOperationAction(ISD::SMULFIXSAT, VT, Expand); |
655 | 6.87M | setOperationAction(ISD::UMULFIX, VT, Expand); |
656 | 6.87M | |
657 | 6.87M | // Overflow operations default to expand |
658 | 6.87M | setOperationAction(ISD::SADDO, VT, Expand); |
659 | 6.87M | setOperationAction(ISD::SSUBO, VT, Expand); |
660 | 6.87M | setOperationAction(ISD::UADDO, VT, Expand); |
661 | 6.87M | setOperationAction(ISD::USUBO, VT, Expand); |
662 | 6.87M | setOperationAction(ISD::SMULO, VT, Expand); |
663 | 6.87M | setOperationAction(ISD::UMULO, VT, Expand); |
664 | 6.87M | |
665 | 6.87M | // ADDCARRY operations default to expand |
666 | 6.87M | setOperationAction(ISD::ADDCARRY, VT, Expand); |
667 | 6.87M | setOperationAction(ISD::SUBCARRY, VT, Expand); |
668 | 6.87M | setOperationAction(ISD::SETCCCARRY, VT, Expand); |
669 | 6.87M | |
670 | 6.87M | // ADDC/ADDE/SUBC/SUBE default to expand. |
671 | 6.87M | setOperationAction(ISD::ADDC, VT, Expand); |
672 | 6.87M | setOperationAction(ISD::ADDE, VT, Expand); |
673 | 6.87M | setOperationAction(ISD::SUBC, VT, Expand); |
674 | 6.87M | setOperationAction(ISD::SUBE, VT, Expand); |
675 | 6.87M | |
676 | 6.87M | // These default to Expand so they will be expanded to CTLZ/CTTZ by default. |
677 | 6.87M | setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); |
678 | 6.87M | setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); |
679 | 6.87M | |
680 | 6.87M | setOperationAction(ISD::BITREVERSE, VT, Expand); |
681 | 6.87M | |
682 | 6.87M | // These library functions default to expand. |
683 | 6.87M | setOperationAction(ISD::FROUND, VT, Expand); |
684 | 6.87M | setOperationAction(ISD::FPOWI, VT, Expand); |
685 | 6.87M | |
686 | 6.87M | // These operations default to expand for vector types. |
687 | 6.87M | if (VT.isVector()) { |
688 | 5.91M | setOperationAction(ISD::FCOPYSIGN, VT, Expand); |
689 | 5.91M | setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, VT, Expand); |
690 | 5.91M | setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Expand); |
691 | 5.91M | setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Expand); |
692 | 5.91M | } |
693 | 6.87M | |
694 | 6.87M | // Constrained floating-point operations default to expand. |
695 | 6.87M | setOperationAction(ISD::STRICT_FADD, VT, Expand); |
696 | 6.87M | setOperationAction(ISD::STRICT_FSUB, VT, Expand); |
697 | 6.87M | setOperationAction(ISD::STRICT_FMUL, VT, Expand); |
698 | 6.87M | setOperationAction(ISD::STRICT_FDIV, VT, Expand); |
699 | 6.87M | setOperationAction(ISD::STRICT_FREM, VT, Expand); |
700 | 6.87M | setOperationAction(ISD::STRICT_FMA, VT, Expand); |
701 | 6.87M | setOperationAction(ISD::STRICT_FSQRT, VT, Expand); |
702 | 6.87M | setOperationAction(ISD::STRICT_FPOW, VT, Expand); |
703 | 6.87M | setOperationAction(ISD::STRICT_FPOWI, VT, Expand); |
704 | 6.87M | setOperationAction(ISD::STRICT_FSIN, VT, Expand); |
705 | 6.87M | setOperationAction(ISD::STRICT_FCOS, VT, Expand); |
706 | 6.87M | setOperationAction(ISD::STRICT_FEXP, VT, Expand); |
707 | 6.87M | setOperationAction(ISD::STRICT_FEXP2, VT, Expand); |
708 | 6.87M | setOperationAction(ISD::STRICT_FLOG, VT, Expand); |
709 | 6.87M | setOperationAction(ISD::STRICT_FLOG10, VT, Expand); |
710 | 6.87M | setOperationAction(ISD::STRICT_FLOG2, VT, Expand); |
711 | 6.87M | setOperationAction(ISD::STRICT_FRINT, VT, Expand); |
712 | 6.87M | setOperationAction(ISD::STRICT_FNEARBYINT, VT, Expand); |
713 | 6.87M | setOperationAction(ISD::STRICT_FCEIL, VT, Expand); |
714 | 6.87M | setOperationAction(ISD::STRICT_FFLOOR, VT, Expand); |
715 | 6.87M | setOperationAction(ISD::STRICT_FROUND, VT, Expand); |
716 | 6.87M | setOperationAction(ISD::STRICT_FTRUNC, VT, Expand); |
717 | 6.87M | setOperationAction(ISD::STRICT_FMAXNUM, VT, Expand); |
718 | 6.87M | setOperationAction(ISD::STRICT_FMINNUM, VT, Expand); |
719 | 6.87M | setOperationAction(ISD::STRICT_FP_ROUND, VT, Expand); |
720 | 6.87M | setOperationAction(ISD::STRICT_FP_EXTEND, VT, Expand); |
721 | 6.87M | |
722 | 6.87M | // For most targets @llvm.get.dynamic.area.offset just returns 0. |
723 | 6.87M | setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, VT, Expand); |
724 | 6.87M | |
725 | 6.87M | // Vector reduction default to expand. |
726 | 6.87M | setOperationAction(ISD::VECREDUCE_FADD, VT, Expand); |
727 | 6.87M | setOperationAction(ISD::VECREDUCE_FMUL, VT, Expand); |
728 | 6.87M | setOperationAction(ISD::VECREDUCE_ADD, VT, Expand); |
729 | 6.87M | setOperationAction(ISD::VECREDUCE_MUL, VT, Expand); |
730 | 6.87M | setOperationAction(ISD::VECREDUCE_AND, VT, Expand); |
731 | 6.87M | setOperationAction(ISD::VECREDUCE_OR, VT, Expand); |
732 | 6.87M | setOperationAction(ISD::VECREDUCE_XOR, VT, Expand); |
733 | 6.87M | setOperationAction(ISD::VECREDUCE_SMAX, VT, Expand); |
734 | 6.87M | setOperationAction(ISD::VECREDUCE_SMIN, VT, Expand); |
735 | 6.87M | setOperationAction(ISD::VECREDUCE_UMAX, VT, Expand); |
736 | 6.87M | setOperationAction(ISD::VECREDUCE_UMIN, VT, Expand); |
737 | 6.87M | setOperationAction(ISD::VECREDUCE_FMAX, VT, Expand); |
738 | 6.87M | setOperationAction(ISD::VECREDUCE_FMIN, VT, Expand); |
739 | 6.87M | } |
740 | 53.2k | |
741 | 53.2k | // Most targets ignore the @llvm.prefetch intrinsic. |
742 | 53.2k | setOperationAction(ISD::PREFETCH, MVT::Other, Expand); |
743 | 53.2k | |
744 | 53.2k | // Most targets also ignore the @llvm.readcyclecounter intrinsic. |
745 | 53.2k | setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand); |
746 | 53.2k | |
747 | 53.2k | // ConstantFP nodes default to expand. Targets can either change this to |
748 | 53.2k | // Legal, in which case all fp constants are legal, or use isFPImmLegal() |
749 | 53.2k | // to optimize expansions for certain constants. |
750 | 53.2k | setOperationAction(ISD::ConstantFP, MVT::f16, Expand); |
751 | 53.2k | setOperationAction(ISD::ConstantFP, MVT::f32, Expand); |
752 | 53.2k | setOperationAction(ISD::ConstantFP, MVT::f64, Expand); |
753 | 53.2k | setOperationAction(ISD::ConstantFP, MVT::f80, Expand); |
754 | 53.2k | setOperationAction(ISD::ConstantFP, MVT::f128, Expand); |
755 | 53.2k | |
756 | 53.2k | // These library functions default to expand. |
757 | 159k | for (MVT VT : {MVT::f32, MVT::f64, MVT::f128}) { |
758 | 159k | setOperationAction(ISD::FCBRT, VT, Expand); |
759 | 159k | setOperationAction(ISD::FLOG , VT, Expand); |
760 | 159k | setOperationAction(ISD::FLOG2, VT, Expand); |
761 | 159k | setOperationAction(ISD::FLOG10, VT, Expand); |
762 | 159k | setOperationAction(ISD::FEXP , VT, Expand); |
763 | 159k | setOperationAction(ISD::FEXP2, VT, Expand); |
764 | 159k | setOperationAction(ISD::FFLOOR, VT, Expand); |
765 | 159k | setOperationAction(ISD::FNEARBYINT, VT, Expand); |
766 | 159k | setOperationAction(ISD::FCEIL, VT, Expand); |
767 | 159k | setOperationAction(ISD::FRINT, VT, Expand); |
768 | 159k | setOperationAction(ISD::FTRUNC, VT, Expand); |
769 | 159k | setOperationAction(ISD::FROUND, VT, Expand); |
770 | 159k | setOperationAction(ISD::LROUND, VT, Expand); |
771 | 159k | setOperationAction(ISD::LLROUND, VT, Expand); |
772 | 159k | setOperationAction(ISD::LRINT, VT, Expand); |
773 | 159k | setOperationAction(ISD::LLRINT, VT, Expand); |
774 | 159k | } |
775 | 53.2k | |
776 | 53.2k | // Default ISD::TRAP to expand (which turns it into abort). |
777 | 53.2k | setOperationAction(ISD::TRAP, MVT::Other, Expand); |
778 | 53.2k | |
779 | 53.2k | // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand" |
780 | 53.2k | // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP. |
781 | 53.2k | setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand); |
782 | 53.2k | } |
783 | | |
784 | | MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL, |
785 | 159k | EVT) const { |
786 | 159k | return MVT::getIntegerVT(DL.getPointerSizeInBits(0)); |
787 | 159k | } |
788 | | |
789 | | EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy, const DataLayout &DL, |
790 | 742k | bool LegalTypes) const { |
791 | 742k | assert(LHSTy.isInteger() && "Shift amount is not an integer type!"); |
792 | 742k | if (LHSTy.isVector()) |
793 | 14.6k | return LHSTy; |
794 | 728k | return LegalTypes ? getScalarShiftAmountTy(DL, LHSTy)699k |
795 | 728k | : getPointerTy(DL)28.9k ; |
796 | 728k | } |
797 | | |
798 | 191 | bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const { |
799 | 191 | assert(isTypeLegal(VT)); |
800 | 191 | switch (Op) { |
801 | 191 | default: |
802 | 140 | return false; |
803 | 191 | case ISD::SDIV: |
804 | 51 | case ISD::UDIV: |
805 | 51 | case ISD::SREM: |
806 | 51 | case ISD::UREM: |
807 | 51 | return true; |
808 | 191 | } |
809 | 191 | } |
810 | | |
811 | 6.03k | void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) { |
812 | 6.03k | // If the command-line option was specified, ignore this request. |
813 | 6.03k | if (!JumpIsExpensiveOverride.getNumOccurrences()) |
814 | 6.03k | JumpIsExpensive = isExpensive; |
815 | 6.03k | } |
816 | | |
817 | | TargetLoweringBase::LegalizeKind |
818 | 86.9M | TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const { |
819 | 86.9M | // If this is a simple type, use the ComputeRegisterProp mechanism. |
820 | 86.9M | if (VT.isSimple()) { |
821 | 86.2M | MVT SVT = VT.getSimpleVT(); |
822 | 86.2M | assert((unsigned)SVT.SimpleTy < array_lengthof(TransformToType)); |
823 | 86.2M | MVT NVT = TransformToType[SVT.SimpleTy]; |
824 | 86.2M | LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT); |
825 | 86.2M | |
826 | 86.2M | assert((LA == TypeLegal || LA == TypeSoftenFloat || |
827 | 86.2M | ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger) && |
828 | 86.2M | "Promote may not follow Expand or Promote"); |
829 | 86.2M | |
830 | 86.2M | if (LA == TypeSplitVector) |
831 | 1.37M | return LegalizeKind(LA, |
832 | 1.37M | EVT::getVectorVT(Context, SVT.getVectorElementType(), |
833 | 1.37M | SVT.getVectorNumElements() / 2)); |
834 | 84.8M | if (LA == TypeScalarizeVector) |
835 | 205k | return LegalizeKind(LA, SVT.getVectorElementType()); |
836 | 84.6M | return LegalizeKind(LA, NVT); |
837 | 84.6M | } |
838 | 663k | |
839 | 663k | // Handle Extended Scalar Types. |
840 | 663k | if (!VT.isVector()) { |
841 | 531k | assert(VT.isInteger() && "Float types must be simple"); |
842 | 531k | unsigned BitSize = VT.getSizeInBits(); |
843 | 531k | // First promote to a power-of-two size, then expand if necessary. |
844 | 531k | if (BitSize < 8 || !isPowerOf2_32(BitSize)491k ) { |
845 | 311k | EVT NVT = VT.getRoundIntegerType(Context); |
846 | 311k | assert(NVT != VT && "Unable to round integer VT"); |
847 | 311k | LegalizeKind NextStep = getTypeConversion(Context, NVT); |
848 | 311k | // Avoid multi-step promotion. |
849 | 311k | if (NextStep.first == TypePromoteInteger) |
850 | 28.4k | return NextStep; |
851 | 283k | // Return rounded integer type. |
852 | 283k | return LegalizeKind(TypePromoteInteger, NVT); |
853 | 283k | } |
854 | 219k | |
855 | 219k | return LegalizeKind(TypeExpandInteger, |
856 | 219k | EVT::getIntegerVT(Context, VT.getSizeInBits() / 2)); |
857 | 219k | } |
858 | 131k | |
859 | 131k | // Handle vector types. |
860 | 131k | unsigned NumElts = VT.getVectorNumElements(); |
861 | 131k | EVT EltVT = VT.getVectorElementType(); |
862 | 131k | |
863 | 131k | // Vectors with only one element are always scalarized. |
864 | 131k | if (NumElts == 1) |
865 | 13.0k | return LegalizeKind(TypeScalarizeVector, EltVT); |
866 | 118k | |
867 | 118k | // Try to widen vector elements until the element type is a power of two and |
868 | 118k | // promote it to a legal type later on, for example: |
869 | 118k | // <3 x i8> -> <4 x i8> -> <4 x i32> |
870 | 118k | if (EltVT.isInteger()) { |
871 | 59.3k | // Vectors with a number of elements that is not a power of two are always |
872 | 59.3k | // widened, for example <3 x i8> -> <4 x i8>. |
873 | 59.3k | if (!VT.isPow2VectorType()) { |
874 | 34.0k | NumElts = (unsigned)NextPowerOf2(NumElts); |
875 | 34.0k | EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts); |
876 | 34.0k | return LegalizeKind(TypeWidenVector, NVT); |
877 | 34.0k | } |
878 | 25.3k | |
879 | 25.3k | // Examine the element type. |
880 | 25.3k | LegalizeKind LK = getTypeConversion(Context, EltVT); |
881 | 25.3k | |
882 | 25.3k | // If type is to be expanded, split the vector. |
883 | 25.3k | // <4 x i140> -> <2 x i140> |
884 | 25.3k | if (LK.first == TypeExpandInteger) |
885 | 14.6k | return LegalizeKind(TypeSplitVector, |
886 | 14.6k | EVT::getVectorVT(Context, EltVT, NumElts / 2)); |
887 | 10.6k | |
888 | 10.6k | // Promote the integer element types until a legal vector type is found |
889 | 10.6k | // or until the element integer type is too big. If a legal type was not |
890 | 10.6k | // found, fallback to the usual mechanism of widening/splitting the |
891 | 10.6k | // vector. |
892 | 10.6k | EVT OldEltVT = EltVT; |
893 | 24.2k | while (true) { |
894 | 24.2k | // Increase the bitwidth of the element to the next pow-of-two |
895 | 24.2k | // (which is greater than 8 bits). |
896 | 24.2k | EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits()) |
897 | 24.2k | .getRoundIntegerType(Context); |
898 | 24.2k | |
899 | 24.2k | // Stop trying when getting a non-simple element type. |
900 | 24.2k | // Note that vector elements may be greater than legal vector element |
901 | 24.2k | // types. Example: X86 XMM registers hold 64bit element on 32bit |
902 | 24.2k | // systems. |
903 | 24.2k | if (!EltVT.isSimple()) |
904 | 2.99k | break; |
905 | 21.2k | |
906 | 21.2k | // Build a new vector type and check if it is legal. |
907 | 21.2k | MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts); |
908 | 21.2k | // Found a legal promoted vector type. |
909 | 21.2k | if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal17.8k ) |
910 | 7.70k | return LegalizeKind(TypePromoteInteger, |
911 | 7.70k | EVT::getVectorVT(Context, EltVT, NumElts)); |
912 | 21.2k | } |
913 | 10.6k | |
914 | 10.6k | // Reset the type to the unexpanded type if we did not find a legal vector |
915 | 10.6k | // type with a promoted vector element type. |
916 | 10.6k | EltVT = OldEltVT; |
917 | 2.99k | } |
918 | 118k | |
919 | 118k | // Try to widen the vector until a legal type is found. |
920 | 118k | // If there is no wider legal type, split the vector. |
921 | 118k | while (62.3k true73.2k ) { |
922 | 73.2k | // Round up to the next power of 2. |
923 | 73.2k | NumElts = (unsigned)NextPowerOf2(NumElts); |
924 | 73.2k | |
925 | 73.2k | // If there is no simple vector type with this many elements then there |
926 | 73.2k | // cannot be a larger legal vector type. Note that this assumes that |
927 | 73.2k | // there are no skipped intermediate vector types in the simple types. |
928 | 73.2k | if (!EltVT.isSimple()) |
929 | 363 | break; |
930 | 72.8k | MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts); |
931 | 72.8k | if (LargerVector == MVT()) |
932 | 61.0k | break; |
933 | 11.8k | |
934 | 11.8k | // If this type is legal then widen the vector. |
935 | 11.8k | if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal) |
936 | 1.01k | return LegalizeKind(TypeWidenVector, LargerVector); |
937 | 11.8k | } |
938 | 62.3k | |
939 | 62.3k | // Widen odd vectors to next power of two. |
940 | 62.3k | if (61.3k !VT.isPow2VectorType()61.3k ) { |
941 | 6.71k | EVT NVT = VT.getPow2VectorType(Context); |
942 | 6.71k | return LegalizeKind(TypeWidenVector, NVT); |
943 | 6.71k | } |
944 | 54.6k | |
945 | 54.6k | // Vectors with illegal element types are expanded. |
946 | 54.6k | EVT NVT = EVT::getVectorVT(Context, EltVT, VT.getVectorNumElements() / 2); |
947 | 54.6k | return LegalizeKind(TypeSplitVector, NVT); |
948 | 54.6k | } |
949 | | |
950 | | static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT, |
951 | | unsigned &NumIntermediates, |
952 | | MVT &RegisterVT, |
953 | 5.13M | TargetLoweringBase *TLI) { |
954 | 5.13M | // Figure out the right, legal destination reg to copy into. |
955 | 5.13M | unsigned NumElts = VT.getVectorNumElements(); |
956 | 5.13M | MVT EltTy = VT.getVectorElementType(); |
957 | 5.13M | |
958 | 5.13M | unsigned NumVectorRegs = 1; |
959 | 5.13M | |
960 | 5.13M | // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we |
961 | 5.13M | // could break down into LHS/RHS like LegalizeDAG does. |
962 | 5.13M | if (!isPowerOf2_32(NumElts)) { |
963 | 130k | NumVectorRegs = NumElts; |
964 | 130k | NumElts = 1; |
965 | 130k | } |
966 | 5.13M | |
967 | 5.13M | // Divide the input until we get to a supported size. This will always |
968 | 5.13M | // end with a scalar if the target doesn't support vectors. |
969 | 19.3M | while (NumElts > 1 && !TLI->isTypeLegal(MVT::getVectorVT(EltTy, NumElts))15.9M ) { |
970 | 14.2M | NumElts >>= 1; |
971 | 14.2M | NumVectorRegs <<= 1; |
972 | 14.2M | } |
973 | 5.13M | |
974 | 5.13M | NumIntermediates = NumVectorRegs; |
975 | 5.13M | |
976 | 5.13M | MVT NewVT = MVT::getVectorVT(EltTy, NumElts); |
977 | 5.13M | if (!TLI->isTypeLegal(NewVT)) |
978 | 3.33M | NewVT = EltTy; |
979 | 5.13M | IntermediateVT = NewVT; |
980 | 5.13M | |
981 | 5.13M | unsigned NewVTSize = NewVT.getSizeInBits(); |
982 | 5.13M | |
983 | 5.13M | // Convert sizes such as i33 to i64. |
984 | 5.13M | if (!isPowerOf2_32(NewVTSize)) |
985 | 0 | NewVTSize = NextPowerOf2(NewVTSize); |
986 | 5.13M | |
987 | 5.13M | MVT DestVT = TLI->getRegisterType(NewVT); |
988 | 5.13M | RegisterVT = DestVT; |
989 | 5.13M | if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16. |
990 | 285k | return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); |
991 | 4.85M | |
992 | 4.85M | // Otherwise, promotion or legal types use the same number of registers as |
993 | 4.85M | // the vector decimated to the appropriate level. |
994 | 4.85M | return NumVectorRegs; |
995 | 4.85M | } |
996 | | |
997 | | /// isLegalRC - Return true if the value types that can be represented by the |
998 | | /// specified register class are all legal. |
999 | | bool TargetLoweringBase::isLegalRC(const TargetRegisterInfo &TRI, |
1000 | 32.5M | const TargetRegisterClass &RC) const { |
1001 | 52.0M | for (auto I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I19.5M ) |
1002 | 35.5M | if (isTypeLegal(*I)) |
1003 | 16.0M | return true; |
1004 | 32.5M | return false16.5M ; |
1005 | 32.5M | } |
1006 | | |
1007 | | /// Replace/modify any TargetFrameIndex operands with a targte-dependent |
1008 | | /// sequence of memory operands that is recognized by PrologEpilogInserter. |
1009 | | MachineBasicBlock * |
1010 | | TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI, |
1011 | 439 | MachineBasicBlock *MBB) const { |
1012 | 439 | MachineInstr *MI = &InitialMI; |
1013 | 439 | MachineFunction &MF = *MI->getMF(); |
1014 | 439 | MachineFrameInfo &MFI = MF.getFrameInfo(); |
1015 | 439 | |
1016 | 439 | // We're handling multiple types of operands here: |
1017 | 439 | // PATCHPOINT MetaArgs - live-in, read only, direct |
1018 | 439 | // STATEPOINT Deopt Spill - live-through, read only, indirect |
1019 | 439 | // STATEPOINT Deopt Alloca - live-through, read only, direct |
1020 | 439 | // (We're currently conservative and mark the deopt slots read/write in |
1021 | 439 | // practice.) |
1022 | 439 | // STATEPOINT GC Spill - live-through, read/write, indirect |
1023 | 439 | // STATEPOINT GC Alloca - live-through, read/write, direct |
1024 | 439 | // The live-in vs live-through is handled already (the live through ones are |
1025 | 439 | // all stack slots), but we need to handle the different type of stackmap |
1026 | 439 | // operands and memory effects here. |
1027 | 439 | |
1028 | 439 | // MI changes inside this loop as we grow operands. |
1029 | 5.82k | for(unsigned OperIdx = 0; OperIdx != MI->getNumOperands(); ++OperIdx5.38k ) { |
1030 | 5.38k | MachineOperand &MO = MI->getOperand(OperIdx); |
1031 | 5.38k | if (!MO.isFI()) |
1032 | 5.20k | continue; |
1033 | 181 | |
1034 | 181 | // foldMemoryOperand builds a new MI after replacing a single FI operand |
1035 | 181 | // with the canonical set of five x86 addressing-mode operands. |
1036 | 181 | int FI = MO.getIndex(); |
1037 | 181 | MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc()); |
1038 | 181 | |
1039 | 181 | // Copy operands before the frame-index. |
1040 | 3.34k | for (unsigned i = 0; i < OperIdx; ++i3.16k ) |
1041 | 3.16k | MIB.add(MI->getOperand(i)); |
1042 | 181 | // Add frame index operands recognized by stackmaps.cpp |
1043 | 181 | if (MFI.isStatepointSpillSlotObjectIndex(FI)) { |
1044 | 142 | // indirect-mem-ref tag, size, #FI, offset. |
1045 | 142 | // Used for spills inserted by StatepointLowering. This codepath is not |
1046 | 142 | // used for patchpoints/stackmaps at all, for these spilling is done via |
1047 | 142 | // foldMemoryOperand callback only. |
1048 | 142 | assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity"); |
1049 | 142 | MIB.addImm(StackMaps::IndirectMemRefOp); |
1050 | 142 | MIB.addImm(MFI.getObjectSize(FI)); |
1051 | 142 | MIB.add(MI->getOperand(OperIdx)); |
1052 | 142 | MIB.addImm(0); |
1053 | 142 | } else { |
1054 | 39 | // direct-mem-ref tag, #FI, offset. |
1055 | 39 | // Used by patchpoint, and direct alloca arguments to statepoints |
1056 | 39 | MIB.addImm(StackMaps::DirectMemRefOp); |
1057 | 39 | MIB.add(MI->getOperand(OperIdx)); |
1058 | 39 | MIB.addImm(0); |
1059 | 39 | } |
1060 | 181 | // Copy the operands after the frame index. |
1061 | 1.02k | for (unsigned i = OperIdx + 1; i != MI->getNumOperands(); ++i844 ) |
1062 | 844 | MIB.add(MI->getOperand(i)); |
1063 | 181 | |
1064 | 181 | // Inherit previous memory operands. |
1065 | 181 | MIB.cloneMemRefs(*MI); |
1066 | 181 | assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!"); |
1067 | 181 | |
1068 | 181 | // Add a new memory operand for this FI. |
1069 | 181 | assert(MFI.getObjectOffset(FI) != -1); |
1070 | 181 | |
1071 | 181 | // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and |
1072 | 181 | // PATCHPOINT should be updated to do the same. (TODO) |
1073 | 181 | if (MI->getOpcode() != TargetOpcode::STATEPOINT) { |
1074 | 29 | auto Flags = MachineMemOperand::MOLoad; |
1075 | 29 | MachineMemOperand *MMO = MF.getMachineMemOperand( |
1076 | 29 | MachinePointerInfo::getFixedStack(MF, FI), Flags, |
1077 | 29 | MF.getDataLayout().getPointerSize(), MFI.getObjectAlignment(FI)); |
1078 | 29 | MIB->addMemOperand(MF, MMO); |
1079 | 29 | } |
1080 | 181 | |
1081 | 181 | // Replace the instruction and update the operand index. |
1082 | 181 | MBB->insert(MachineBasicBlock::iterator(MI), MIB); |
1083 | 181 | OperIdx += (MIB->getNumOperands() - MI->getNumOperands()) - 1; |
1084 | 181 | MI->eraseFromParent(); |
1085 | 181 | MI = MIB; |
1086 | 181 | } |
1087 | 439 | return MBB; |
1088 | 439 | } |
1089 | | |
1090 | | MachineBasicBlock * |
1091 | | TargetLoweringBase::emitXRayCustomEvent(MachineInstr &MI, |
1092 | 2 | MachineBasicBlock *MBB) const { |
1093 | 2 | assert(MI.getOpcode() == TargetOpcode::PATCHABLE_EVENT_CALL && |
1094 | 2 | "Called emitXRayCustomEvent on the wrong MI!"); |
1095 | 2 | auto &MF = *MI.getMF(); |
1096 | 2 | auto MIB = BuildMI(MF, MI.getDebugLoc(), MI.getDesc()); |
1097 | 6 | for (unsigned OpIdx = 0; OpIdx != MI.getNumOperands(); ++OpIdx4 ) |
1098 | 4 | MIB.add(MI.getOperand(OpIdx)); |
1099 | 2 | |
1100 | 2 | MBB->insert(MachineBasicBlock::iterator(MI), MIB); |
1101 | 2 | MI.eraseFromParent(); |
1102 | 2 | return MBB; |
1103 | 2 | } |
1104 | | |
1105 | | MachineBasicBlock * |
1106 | | TargetLoweringBase::emitXRayTypedEvent(MachineInstr &MI, |
1107 | 2 | MachineBasicBlock *MBB) const { |
1108 | 2 | assert(MI.getOpcode() == TargetOpcode::PATCHABLE_TYPED_EVENT_CALL && |
1109 | 2 | "Called emitXRayTypedEvent on the wrong MI!"); |
1110 | 2 | auto &MF = *MI.getMF(); |
1111 | 2 | auto MIB = BuildMI(MF, MI.getDebugLoc(), MI.getDesc()); |
1112 | 8 | for (unsigned OpIdx = 0; OpIdx != MI.getNumOperands(); ++OpIdx6 ) |
1113 | 6 | MIB.add(MI.getOperand(OpIdx)); |
1114 | 2 | |
1115 | 2 | MBB->insert(MachineBasicBlock::iterator(MI), MIB); |
1116 | 2 | MI.eraseFromParent(); |
1117 | 2 | return MBB; |
1118 | 2 | } |
1119 | | |
1120 | | /// findRepresentativeClass - Return the largest legal super-reg register class |
1121 | | /// of the register class for the specified type and its associated "cost". |
1122 | | // This function is in TargetLowering because it uses RegClassForVT which would |
1123 | | // need to be moved to TargetRegisterInfo and would necessitate moving |
1124 | | // isTypeLegal over as well - a massive change that would just require |
1125 | | // TargetLowering having a TargetRegisterInfo class member that it would use. |
1126 | | std::pair<const TargetRegisterClass *, uint8_t> |
1127 | | TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo *TRI, |
1128 | 6.42M | MVT VT) const { |
1129 | 6.42M | const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy]; |
1130 | 6.42M | if (!RC) |
1131 | 6.04M | return std::make_pair(RC, 0); |
1132 | 386k | |
1133 | 386k | // Compute the set of all super-register classes. |
1134 | 386k | BitVector SuperRegRC(TRI->getNumRegClasses()); |
1135 | 3.87M | for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI3.49M ) |
1136 | 3.49M | SuperRegRC.setBitsInMask(RCI.getMask()); |
1137 | 386k | |
1138 | 386k | // Find the first legal register class with the largest spill size. |
1139 | 386k | const TargetRegisterClass *BestRC = RC; |
1140 | 15.0M | for (unsigned i : SuperRegRC.set_bits()) { |
1141 | 15.0M | const TargetRegisterClass *SuperRC = TRI->getRegClass(i); |
1142 | 15.0M | // We want the largest possible spill size. |
1143 | 15.0M | if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC)) |
1144 | 2.53M | continue; |
1145 | 12.5M | if (!isLegalRC(*TRI, *SuperRC)) |
1146 | 12.0M | continue; |
1147 | 460k | BestRC = SuperRC; |
1148 | 460k | } |
1149 | 386k | return std::make_pair(BestRC, 1); |
1150 | 386k | } |
1151 | | |
1152 | | /// computeRegisterProperties - Once all of the register classes are added, |
1153 | | /// this allows us to compute derived properties we expose. |
1154 | | void TargetLoweringBase::computeRegisterProperties( |
1155 | 53.2k | const TargetRegisterInfo *TRI) { |
1156 | 53.2k | static_assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE, |
1157 | 53.2k | "Too many value types for ValueTypeActions to hold!"); |
1158 | 53.2k | |
1159 | 53.2k | // Everything defaults to needing one register. |
1160 | 6.97M | for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i6.92M ) { |
1161 | 6.92M | NumRegistersForVT[i] = 1; |
1162 | 6.92M | RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i; |
1163 | 6.92M | } |
1164 | 53.2k | // ...except isVoid, which doesn't need any registers. |
1165 | 53.2k | NumRegistersForVT[MVT::isVoid] = 0; |
1166 | 53.2k | |
1167 | 53.2k | // Find the largest integer register class. |
1168 | 53.2k | unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE; |
1169 | 126k | for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg73.7k ) |
1170 | 53.2k | assert(LargestIntReg != MVT::i1 && "No integer registers defined!"); |
1171 | 53.2k | |
1172 | 53.2k | // Every integer value type larger than this largest register takes twice as |
1173 | 53.2k | // many registers to represent as the previous ValueType. |
1174 | 53.2k | for (unsigned ExpandedReg = LargestIntReg + 1; |
1175 | 126k | ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg73.7k ) { |
1176 | 73.7k | NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1]; |
1177 | 73.7k | RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg; |
1178 | 73.7k | TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1); |
1179 | 73.7k | ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg, |
1180 | 73.7k | TypeExpandInteger); |
1181 | 73.7k | } |
1182 | 53.2k | |
1183 | 53.2k | // Inspect all of the ValueType's smaller than the largest integer |
1184 | 53.2k | // register to see which ones need promotion. |
1185 | 53.2k | unsigned LegalIntReg = LargestIntReg; |
1186 | 53.2k | for (unsigned IntReg = LargestIntReg - 1; |
1187 | 245k | IntReg >= (unsigned)MVT::i1; --IntReg192k ) { |
1188 | 192k | MVT IVT = (MVT::SimpleValueType)IntReg; |
1189 | 192k | if (isTypeLegal(IVT)) { |
1190 | 72.3k | LegalIntReg = IntReg; |
1191 | 120k | } else { |
1192 | 120k | RegisterTypeForVT[IntReg] = TransformToType[IntReg] = |
1193 | 120k | (MVT::SimpleValueType)LegalIntReg; |
1194 | 120k | ValueTypeActions.setTypeAction(IVT, TypePromoteInteger); |
1195 | 120k | } |
1196 | 192k | } |
1197 | 53.2k | |
1198 | 53.2k | // ppcf128 type is really two f64's. |
1199 | 53.2k | if (!isTypeLegal(MVT::ppcf128)53.2k ) { |
1200 | 53.2k | if (isTypeLegal(MVT::f64)) { |
1201 | 46.1k | NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64]; |
1202 | 46.1k | RegisterTypeForVT[MVT::ppcf128] = MVT::f64; |
1203 | 46.1k | TransformToType[MVT::ppcf128] = MVT::f64; |
1204 | 46.1k | ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat); |
1205 | 46.1k | } else { |
1206 | 7.06k | NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128]; |
1207 | 7.06k | RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128]; |
1208 | 7.06k | TransformToType[MVT::ppcf128] = MVT::i128; |
1209 | 7.06k | ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat); |
1210 | 7.06k | } |
1211 | 53.2k | } |
1212 | 53.2k | |
1213 | 53.2k | // Decide how to handle f128. If the target does not have native f128 support, |
1214 | 53.2k | // expand it to i128 and we will be generating soft float library calls. |
1215 | 53.2k | if (!isTypeLegal(MVT::f128)) { |
1216 | 37.8k | NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128]; |
1217 | 37.8k | RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128]; |
1218 | 37.8k | TransformToType[MVT::f128] = MVT::i128; |
1219 | 37.8k | ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat); |
1220 | 37.8k | } |
1221 | 53.2k | |
1222 | 53.2k | // Decide how to handle f64. If the target does not have native f64 support, |
1223 | 53.2k | // expand it to i64 and we will be generating soft float library calls. |
1224 | 53.2k | if (!isTypeLegal(MVT::f64)) { |
1225 | 7.06k | NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64]; |
1226 | 7.06k | RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64]; |
1227 | 7.06k | TransformToType[MVT::f64] = MVT::i64; |
1228 | 7.06k | ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat); |
1229 | 7.06k | } |
1230 | 53.2k | |
1231 | 53.2k | // Decide how to handle f32. If the target does not have native f32 support, |
1232 | 53.2k | // expand it to i32 and we will be generating soft float library calls. |
1233 | 53.2k | if (!isTypeLegal(MVT::f32)) { |
1234 | 6.69k | NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32]; |
1235 | 6.69k | RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32]; |
1236 | 6.69k | TransformToType[MVT::f32] = MVT::i32; |
1237 | 6.69k | ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat); |
1238 | 6.69k | } |
1239 | 53.2k | |
1240 | 53.2k | // Decide how to handle f16. If the target does not have native f16 support, |
1241 | 53.2k | // promote it to f32, because there are no f16 library calls (except for |
1242 | 53.2k | // conversions). |
1243 | 53.2k | if (!isTypeLegal(MVT::f16)) { |
1244 | 40.7k | NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32]; |
1245 | 40.7k | RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32]; |
1246 | 40.7k | TransformToType[MVT::f16] = MVT::f32; |
1247 | 40.7k | ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat); |
1248 | 40.7k | } |
1249 | 53.2k | |
1250 | 53.2k | // Loop over all of the vector value types to see which need transformations. |
1251 | 53.2k | for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE; |
1252 | 5.96M | i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i5.91M ) { |
1253 | 5.91M | MVT VT = (MVT::SimpleValueType) i; |
1254 | 5.91M | if (isTypeLegal(VT)) |
1255 | 383k | continue; |
1256 | 5.52M | |
1257 | 5.52M | MVT EltVT = VT.getVectorElementType(); |
1258 | 5.52M | unsigned NElts = VT.getVectorNumElements(); |
1259 | 5.52M | bool IsLegalWiderType = false; |
1260 | 5.52M | LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT); |
1261 | 5.52M | switch (PreferredAction) { |
1262 | 5.52M | case TypePromoteInteger: |
1263 | 4.06M | // Try to promote the elements of integer vectors. If no legal |
1264 | 4.06M | // promotion was found, fall through to the widen-vector method. |
1265 | 100M | for (unsigned nVT = i + 1; nVT <= MVT::LAST_INTEGER_VECTOR_VALUETYPE; ++nVT96.0M ) { |
1266 | 96.3M | MVT SVT = (MVT::SimpleValueType) nVT; |
1267 | 96.3M | // Promote vectors of integers to vectors with the same number |
1268 | 96.3M | // of elements, with a wider element type. |
1269 | 96.3M | if (SVT.getScalarSizeInBits() > EltVT.getSizeInBits() && |
1270 | 96.3M | SVT.getVectorNumElements() == NElts61.4M && isTypeLegal(SVT)6.00M ) { |
1271 | 238k | TransformToType[i] = SVT; |
1272 | 238k | RegisterTypeForVT[i] = SVT; |
1273 | 238k | NumRegistersForVT[i] = 1; |
1274 | 238k | ValueTypeActions.setTypeAction(VT, TypePromoteInteger); |
1275 | 238k | IsLegalWiderType = true; |
1276 | 238k | break; |
1277 | 238k | } |
1278 | 96.3M | } |
1279 | 4.06M | if (IsLegalWiderType) |
1280 | 238k | break; |
1281 | 3.82M | LLVM_FALLTHROUGH; |
1282 | 3.82M | |
1283 | 4.35M | case TypeWidenVector: |
1284 | 4.35M | // Try to widen the vector. |
1285 | 219M | for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT215M ) { |
1286 | 215M | MVT SVT = (MVT::SimpleValueType) nVT; |
1287 | 215M | if (SVT.getVectorElementType() == EltVT |
1288 | 215M | && SVT.getVectorNumElements() > NElts25.2M && isTypeLegal(SVT)13.9M ) { |
1289 | 153k | TransformToType[i] = SVT; |
1290 | 153k | RegisterTypeForVT[i] = SVT; |
1291 | 153k | NumRegistersForVT[i] = 1; |
1292 | 153k | ValueTypeActions.setTypeAction(VT, TypeWidenVector); |
1293 | 153k | IsLegalWiderType = true; |
1294 | 153k | break; |
1295 | 153k | } |
1296 | 215M | } |
1297 | 4.35M | if (IsLegalWiderType) |
1298 | 153k | break; |
1299 | 4.19M | LLVM_FALLTHROUGH; |
1300 | 4.19M | |
1301 | 5.13M | case TypeSplitVector: |
1302 | 5.13M | case TypeScalarizeVector: { |
1303 | 5.13M | MVT IntermediateVT; |
1304 | 5.13M | MVT RegisterVT; |
1305 | 5.13M | unsigned NumIntermediates; |
1306 | 5.13M | NumRegistersForVT[i] = getVectorTypeBreakdownMVT(VT, IntermediateVT, |
1307 | 5.13M | NumIntermediates, RegisterVT, this); |
1308 | 5.13M | RegisterTypeForVT[i] = RegisterVT; |
1309 | 5.13M | |
1310 | 5.13M | MVT NVT = VT.getPow2VectorType(); |
1311 | 5.13M | if (NVT == VT) { |
1312 | 5.00M | // Type is already a power of 2. The default action is to split. |
1313 | 5.00M | TransformToType[i] = MVT::Other; |
1314 | 5.00M | if (PreferredAction == TypeScalarizeVector) |
1315 | 700k | ValueTypeActions.setTypeAction(VT, TypeScalarizeVector); |
1316 | 4.30M | else if (PreferredAction == TypeSplitVector) |
1317 | 234k | ValueTypeActions.setTypeAction(VT, TypeSplitVector); |
1318 | 4.07M | else |
1319 | 4.07M | // Set type action according to the number of elements. |
1320 | 4.07M | ValueTypeActions.setTypeAction(VT, NElts == 1 ? TypeScalarizeVector28.0k |
1321 | 4.07M | : TypeSplitVector4.04M ); |
1322 | 5.00M | } else { |
1323 | 130k | TransformToType[i] = NVT; |
1324 | 130k | ValueTypeActions.setTypeAction(VT, TypeWidenVector); |
1325 | 130k | } |
1326 | 5.13M | break; |
1327 | 5.13M | } |
1328 | 5.13M | default: |
1329 | 0 | llvm_unreachable("Unknown vector legalization action!"); |
1330 | 5.52M | } |
1331 | 5.52M | } |
1332 | 53.2k | |
1333 | 53.2k | // Determine the 'representative' register class for each value type. |
1334 | 53.2k | // An representative register class is the largest (meaning one which is |
1335 | 53.2k | // not a sub-register class / subreg register class) legal register class for |
1336 | 53.2k | // a group of value types. For example, on i386, i8, i16, and i32 |
1337 | 53.2k | // representative would be GR32; while on x86_64 it's GR64. |
1338 | 6.97M | for (unsigned i = 0; 53.2k i != MVT::LAST_VALUETYPE; ++i6.92M ) { |
1339 | 6.92M | const TargetRegisterClass* RRC; |
1340 | 6.92M | uint8_t Cost; |
1341 | 6.92M | std::tie(RRC, Cost) = findRepresentativeClass(TRI, (MVT::SimpleValueType)i); |
1342 | 6.92M | RepRegClassForVT[i] = RRC; |
1343 | 6.92M | RepRegClassCostForVT[i] = Cost; |
1344 | 6.92M | } |
1345 | 53.2k | } |
1346 | | |
1347 | | EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &, |
1348 | 3.85k | EVT VT) const { |
1349 | 3.85k | assert(!VT.isVector() && "No default SetCC type for vectors!"); |
1350 | 3.85k | return getPointerTy(DL).SimpleTy; |
1351 | 3.85k | } |
1352 | | |
1353 | 1.10k | MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const { |
1354 | 1.10k | return MVT::i32; // return the default value |
1355 | 1.10k | } |
1356 | | |
1357 | | /// getVectorTypeBreakdown - Vector types are broken down into some number of |
1358 | | /// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32 |
1359 | | /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack. |
1360 | | /// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86. |
1361 | | /// |
1362 | | /// This method returns the number of registers needed, and the VT for each |
1363 | | /// register. It also returns the VT and quantity of the intermediate values |
1364 | | /// before they are promoted/expanded. |
1365 | | unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context, EVT VT, |
1366 | | EVT &IntermediateVT, |
1367 | | unsigned &NumIntermediates, |
1368 | 50.5k | MVT &RegisterVT) const { |
1369 | 50.5k | unsigned NumElts = VT.getVectorNumElements(); |
1370 | 50.5k | |
1371 | 50.5k | // If there is a wider vector type with the same element type as this one, |
1372 | 50.5k | // or a promoted vector type that has the same number of elements which |
1373 | 50.5k | // are wider, then we should convert to that legal vector type. |
1374 | 50.5k | // This handles things like <2 x float> -> <4 x float> and |
1375 | 50.5k | // <4 x i1> -> <4 x i32>. |
1376 | 50.5k | LegalizeTypeAction TA = getTypeAction(Context, VT); |
1377 | 50.5k | if (NumElts != 1 && (49.9k TA == TypeWidenVector49.9k || TA == TypePromoteInteger45.7k )) { |
1378 | 5.05k | EVT RegisterEVT = getTypeToTransformTo(Context, VT); |
1379 | 5.05k | if (isTypeLegal(RegisterEVT)) { |
1380 | 1.96k | IntermediateVT = RegisterEVT; |
1381 | 1.96k | RegisterVT = RegisterEVT.getSimpleVT(); |
1382 | 1.96k | NumIntermediates = 1; |
1383 | 1.96k | return 1; |
1384 | 1.96k | } |
1385 | 48.6k | } |
1386 | 48.6k | |
1387 | 48.6k | // Figure out the right, legal destination reg to copy into. |
1388 | 48.6k | EVT EltTy = VT.getVectorElementType(); |
1389 | 48.6k | |
1390 | 48.6k | unsigned NumVectorRegs = 1; |
1391 | 48.6k | |
1392 | 48.6k | // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we |
1393 | 48.6k | // could break down into LHS/RHS like LegalizeDAG does. |
1394 | 48.6k | if (!isPowerOf2_32(NumElts)) { |
1395 | 3.09k | NumVectorRegs = NumElts; |
1396 | 3.09k | NumElts = 1; |
1397 | 3.09k | } |
1398 | 48.6k | |
1399 | 48.6k | // Divide the input until we get to a supported size. This will always |
1400 | 48.6k | // end with a scalar if the target doesn't support vectors. |
1401 | 107k | while (NumElts > 1 && !isTypeLegal( |
1402 | 97.7k | EVT::getVectorVT(Context, EltTy, NumElts))) { |
1403 | 58.5k | NumElts >>= 1; |
1404 | 58.5k | NumVectorRegs <<= 1; |
1405 | 58.5k | } |
1406 | 48.6k | |
1407 | 48.6k | NumIntermediates = NumVectorRegs; |
1408 | 48.6k | |
1409 | 48.6k | EVT NewVT = EVT::getVectorVT(Context, EltTy, NumElts); |
1410 | 48.6k | if (!isTypeLegal(NewVT)) |
1411 | 9.39k | NewVT = EltTy; |
1412 | 48.6k | IntermediateVT = NewVT; |
1413 | 48.6k | |
1414 | 48.6k | MVT DestVT = getRegisterType(Context, NewVT); |
1415 | 48.6k | RegisterVT = DestVT; |
1416 | 48.6k | unsigned NewVTSize = NewVT.getSizeInBits(); |
1417 | 48.6k | |
1418 | 48.6k | // Convert sizes such as i33 to i64. |
1419 | 48.6k | if (!isPowerOf2_32(NewVTSize)) |
1420 | 165 | NewVTSize = NextPowerOf2(NewVTSize); |
1421 | 48.6k | |
1422 | 48.6k | if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16. |
1423 | 2.46k | return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits()); |
1424 | 46.1k | |
1425 | 46.1k | // Otherwise, promotion or legal types use the same number of registers as |
1426 | 46.1k | // the vector decimated to the appropriate level. |
1427 | 46.1k | return NumVectorRegs; |
1428 | 46.1k | } |
1429 | | |
1430 | | /// Get the EVTs and ArgFlags collections that represent the legalized return |
1431 | | /// type of the given function. This does not require a DAG or a return value, |
1432 | | /// and is suitable for use before any DAGs for the function are constructed. |
1433 | | /// TODO: Move this out of TargetLowering.cpp. |
1434 | | void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType, |
1435 | | AttributeList attr, |
1436 | | SmallVectorImpl<ISD::OutputArg> &Outs, |
1437 | 750k | const TargetLowering &TLI, const DataLayout &DL) { |
1438 | 750k | SmallVector<EVT, 4> ValueVTs; |
1439 | 750k | ComputeValueVTs(TLI, DL, ReturnType, ValueVTs); |
1440 | 750k | unsigned NumValues = ValueVTs.size(); |
1441 | 750k | if (NumValues == 0) return282k ; |
1442 | 467k | |
1443 | 939k | for (unsigned j = 0, f = NumValues; 467k j != f; ++j471k ) { |
1444 | 471k | EVT VT = ValueVTs[j]; |
1445 | 471k | ISD::NodeType ExtendKind = ISD::ANY_EXTEND; |
1446 | 471k | |
1447 | 471k | if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) |
1448 | 5.90k | ExtendKind = ISD::SIGN_EXTEND; |
1449 | 465k | else if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt)) |
1450 | 34.5k | ExtendKind = ISD::ZERO_EXTEND; |
1451 | 471k | |
1452 | 471k | // FIXME: C calling convention requires the return type to be promoted to |
1453 | 471k | // at least 32-bit. But this is not necessary for non-C calling |
1454 | 471k | // conventions. The frontend should mark functions whose return values |
1455 | 471k | // require promoting with signext or zeroext attributes. |
1456 | 471k | if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()40.4k ) { |
1457 | 35.3k | MVT MinVT = TLI.getRegisterType(ReturnType->getContext(), MVT::i32); |
1458 | 35.3k | if (VT.bitsLT(MinVT)) |
1459 | 25.2k | VT = MinVT; |
1460 | 35.3k | } |
1461 | 471k | |
1462 | 471k | unsigned NumParts = |
1463 | 471k | TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT); |
1464 | 471k | MVT PartVT = |
1465 | 471k | TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT); |
1466 | 471k | |
1467 | 471k | // 'inreg' on function refers to return value |
1468 | 471k | ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); |
1469 | 471k | if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::InReg)) |
1470 | 474 | Flags.setInReg(); |
1471 | 471k | |
1472 | 471k | // Propagate extension type if any |
1473 | 471k | if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) |
1474 | 5.90k | Flags.setSExt(); |
1475 | 465k | else if (attr.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt)) |
1476 | 34.5k | Flags.setZExt(); |
1477 | 471k | |
1478 | 976k | for (unsigned i = 0; i < NumParts; ++i504k ) |
1479 | 504k | Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0)); |
1480 | 471k | } |
1481 | 467k | } |
1482 | | |
1483 | | /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate |
1484 | | /// function arguments in the caller parameter area. This is the actual |
1485 | | /// alignment, not its logarithm. |
1486 | | unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty, |
1487 | 407 | const DataLayout &DL) const { |
1488 | 407 | return DL.getABITypeAlignment(Ty); |
1489 | 407 | } |
1490 | | |
1491 | | bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, |
1492 | | const DataLayout &DL, EVT VT, |
1493 | | unsigned AddrSpace, |
1494 | | unsigned Alignment, |
1495 | | MachineMemOperand::Flags Flags, |
1496 | 3.87M | bool *Fast) const { |
1497 | 3.87M | // Check if the specified alignment is sufficient based on the data layout. |
1498 | 3.87M | // TODO: While using the data layout works in practice, a better solution |
1499 | 3.87M | // would be to implement this check directly (make this a virtual function). |
1500 | 3.87M | // For example, the ABI alignment may change based on software platform while |
1501 | 3.87M | // this function should only be affected by hardware implementation. |
1502 | 3.87M | Type *Ty = VT.getTypeForEVT(Context); |
1503 | 3.87M | if (Alignment >= DL.getABITypeAlignment(Ty)) { |
1504 | 3.38M | // Assume that an access that meets the ABI-specified alignment is fast. |
1505 | 3.38M | if (Fast != nullptr) |
1506 | 230k | *Fast = true; |
1507 | 3.38M | return true; |
1508 | 3.38M | } |
1509 | 487k | |
1510 | 487k | // This is a misaligned access. |
1511 | 487k | return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast); |
1512 | 487k | } |
1513 | | |
1514 | | bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, |
1515 | | const DataLayout &DL, EVT VT, |
1516 | | const MachineMemOperand &MMO, |
1517 | 3.85M | bool *Fast) const { |
1518 | 3.85M | return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), |
1519 | 3.85M | MMO.getAlignment(), MMO.getFlags(), Fast); |
1520 | 3.85M | } |
1521 | | |
1522 | 14 | BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const { |
1523 | 14 | return BranchProbability(MinPercentageForPredictableBranch, 100); |
1524 | 14 | } |
1525 | | |
1526 | | //===----------------------------------------------------------------------===// |
1527 | | // TargetTransformInfo Helpers |
1528 | | //===----------------------------------------------------------------------===// |
1529 | | |
1530 | 2.31M | int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const { |
1531 | 2.31M | enum InstructionOpcodes { |
1532 | 2.31M | #define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM, |
1533 | 2.31M | #define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM |
1534 | 2.31M | #include "llvm/IR/Instruction.def" |
1535 | 2.31M | }; |
1536 | 2.31M | switch (static_cast<InstructionOpcodes>(Opcode)) { |
1537 | 2.31M | case Ret: return 016 ; |
1538 | 2.31M | case Br: return 0127 ; |
1539 | 2.31M | case Switch: return 05 ; |
1540 | 2.31M | case IndirectBr: return 00 ; |
1541 | 2.31M | case Invoke: return 015 ; |
1542 | 2.31M | case CallBr: return 00 ; |
1543 | 2.31M | case Resume: return 00 ; |
1544 | 2.31M | case Unreachable: return 00 ; |
1545 | 2.31M | case CleanupRet: return 00 ; |
1546 | 2.31M | case CatchRet: return 00 ; |
1547 | 2.31M | case CatchPad: return 00 ; |
1548 | 2.31M | case CatchSwitch: return 00 ; |
1549 | 2.31M | case CleanupPad: return 00 ; |
1550 | 2.31M | case FNeg: return ISD::FNEG172 ; |
1551 | 2.31M | case Add: return ISD::ADD356k ; |
1552 | 2.31M | case FAdd: return ISD::FADD148k ; |
1553 | 2.31M | case Sub: return ISD::SUB161k ; |
1554 | 2.31M | case FSub: return ISD::FSUB107k ; |
1555 | 2.31M | case Mul: return ISD::MUL99.2k ; |
1556 | 2.31M | case FMul: return ISD::FMUL224k ; |
1557 | 2.31M | case UDiv: return ISD::UDIV11.6k ; |
1558 | 2.31M | case SDiv: return ISD::SDIV18.2k ; |
1559 | 2.31M | case FDiv: return ISD::FDIV50.5k ; |
1560 | 2.31M | case URem: return ISD::UREM11.7k ; |
1561 | 2.31M | case SRem: return ISD::SREM2.42k ; |
1562 | 2.31M | case FRem: return ISD::FREM283 ; |
1563 | 2.31M | case Shl: return ISD::SHL99.6k ; |
1564 | 2.31M | case LShr: return ISD::SRL89.0k ; |
1565 | 2.31M | case AShr: return ISD::SRA35.2k ; |
1566 | 2.31M | case And: return ISD::AND88.3k ; |
1567 | 2.31M | case Or: return ISD::OR45.9k ; |
1568 | 2.31M | case Xor: return ISD::XOR32.5k ; |
1569 | 2.31M | case Alloca: return 00 ; |
1570 | 2.31M | case Load: return ISD::LOAD151 ; |
1571 | 2.31M | case Store: return ISD::STORE5.77k ; |
1572 | 2.31M | case GetElementPtr: return 0273 ; |
1573 | 2.31M | case Fence: return 00 ; |
1574 | 2.31M | case AtomicCmpXchg: return 00 ; |
1575 | 2.31M | case AtomicRMW: return 00 ; |
1576 | 2.31M | case Trunc: return ISD::TRUNCATE100k ; |
1577 | 2.31M | case ZExt: return ISD::ZERO_EXTEND49.3k ; |
1578 | 2.31M | case SExt: return ISD::SIGN_EXTEND33.1k ; |
1579 | 2.31M | case FPToUI: return ISD::FP_TO_UINT3.61k ; |
1580 | 2.31M | case FPToSI: return ISD::FP_TO_SINT4.86k ; |
1581 | 2.31M | case UIToFP: return ISD::UINT_TO_FP5.47k ; |
1582 | 2.31M | case SIToFP: return ISD::SINT_TO_FP37.7k ; |
1583 | 2.31M | case FPTrunc: return ISD::FP_ROUND2.78k ; |
1584 | 2.31M | case FPExt: return ISD::FP_EXTEND2.91k ; |
1585 | 2.31M | case PtrToInt: return ISD::BITCAST9.90k ; |
1586 | 2.31M | case IntToPtr: return ISD::BITCAST3.04k ; |
1587 | 2.31M | case BitCast: return ISD::BITCAST23.9k ; |
1588 | 2.31M | case AddrSpaceCast: return ISD::ADDRSPACECAST15 ; |
1589 | 2.31M | case ICmp: return ISD::SETCC267k ; |
1590 | 2.31M | case FCmp: return ISD::SETCC12.2k ; |
1591 | 2.31M | case PHI: return 0403 ; |
1592 | 2.31M | case Call: return 0214 ; |
1593 | 2.31M | case Select: return ISD::SELECT152k ; |
1594 | 2.31M | case UserOp1: return 00 ; |
1595 | 2.31M | case UserOp2: return 00 ; |
1596 | 2.31M | case VAArg: return 00 ; |
1597 | 2.31M | case ExtractElement: return ISD::EXTRACT_VECTOR_ELT7.28k ; |
1598 | 2.31M | case InsertElement: return ISD::INSERT_VECTOR_ELT7.13k ; |
1599 | 2.31M | case ShuffleVector: return ISD::VECTOR_SHUFFLE3 ; |
1600 | 2.31M | case ExtractValue: return ISD::MERGE_VALUES5 ; |
1601 | 2.31M | case InsertValue: return ISD::MERGE_VALUES1 ; |
1602 | 2.31M | case LandingPad: return 00 ; |
1603 | 0 | } |
1604 | 0 | |
1605 | 0 | llvm_unreachable("Unknown instruction type encountered!"); |
1606 | 0 | } |
1607 | | |
1608 | | std::pair<int, MVT> |
1609 | | TargetLoweringBase::getTypeLegalizationCost(const DataLayout &DL, |
1610 | 5.74M | Type *Ty) const { |
1611 | 5.74M | LLVMContext &C = Ty->getContext(); |
1612 | 5.74M | EVT MTy = getValueType(DL, Ty); |
1613 | 5.74M | |
1614 | 5.74M | int Cost = 1; |
1615 | 5.74M | // We keep legalizing the type until we find a legal kind. We assume that |
1616 | 5.74M | // the only operation that costs anything is the split. After splitting |
1617 | 5.74M | // we need to handle two types. |
1618 | 6.64M | while (true) { |
1619 | 6.64M | LegalizeKind LK = getTypeConversion(C, MTy); |
1620 | 6.64M | |
1621 | 6.64M | if (LK.first == TypeLegal) |
1622 | 5.74M | return std::make_pair(Cost, MTy.getSimpleVT()); |
1623 | 894k | |
1624 | 894k | if (LK.first == TypeSplitVector || LK.first == TypeExpandInteger369k ) |
1625 | 536k | Cost *= 2; |
1626 | 894k | |
1627 | 894k | // Do not loop with f128 type. |
1628 | 894k | if (MTy == LK.second) |
1629 | 0 | return std::make_pair(Cost, MTy.getSimpleVT()); |
1630 | 894k | |
1631 | 894k | // Keep legalizing the type. |
1632 | 894k | MTy = LK.second; |
1633 | 894k | } |
1634 | 5.74M | } |
1635 | | |
1636 | | Value *TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilder<> &IRB, |
1637 | 141 | bool UseTLS) const { |
1638 | 141 | // compiler-rt provides a variable with a magic name. Targets that do not |
1639 | 141 | // link with compiler-rt may also provide such a variable. |
1640 | 141 | Module *M = IRB.GetInsertBlock()->getParent()->getParent(); |
1641 | 141 | const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr"; |
1642 | 141 | auto UnsafeStackPtr = |
1643 | 141 | dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar)); |
1644 | 141 | |
1645 | 141 | Type *StackPtrTy = Type::getInt8PtrTy(M->getContext()); |
1646 | 141 | |
1647 | 141 | if (!UnsafeStackPtr) { |
1648 | 80 | auto TLSModel = UseTLS ? |
1649 | 79 | GlobalValue::InitialExecTLSModel : |
1650 | 80 | GlobalValue::NotThreadLocal1 ; |
1651 | 80 | // The global variable is not defined yet, define it ourselves. |
1652 | 80 | // We use the initial-exec TLS model because we do not support the |
1653 | 80 | // variable living anywhere other than in the main executable. |
1654 | 80 | UnsafeStackPtr = new GlobalVariable( |
1655 | 80 | *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr, |
1656 | 80 | UnsafeStackPtrVar, nullptr, TLSModel); |
1657 | 80 | } else { |
1658 | 61 | // The variable exists, check its type and attributes. |
1659 | 61 | if (UnsafeStackPtr->getValueType() != StackPtrTy) |
1660 | 0 | report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type"); |
1661 | 61 | if (UseTLS != UnsafeStackPtr->isThreadLocal()) |
1662 | 0 | report_fatal_error(Twine(UnsafeStackPtrVar) + " must " + |
1663 | 0 | (UseTLS ? "" : "not ") + "be thread-local"); |
1664 | 141 | } |
1665 | 141 | return UnsafeStackPtr; |
1666 | 141 | } |
1667 | | |
1668 | 139 | Value *TargetLoweringBase::getSafeStackPointerLocation(IRBuilder<> &IRB) const { |
1669 | 139 | if (!TM.getTargetTriple().isAndroid()) |
1670 | 137 | return getDefaultSafeStackPointerLocation(IRB, true); |
1671 | 2 | |
1672 | 2 | // Android provides a libc function to retrieve the address of the current |
1673 | 2 | // thread's unsafe stack pointer. |
1674 | 2 | Module *M = IRB.GetInsertBlock()->getParent()->getParent(); |
1675 | 2 | Type *StackPtrTy = Type::getInt8PtrTy(M->getContext()); |
1676 | 2 | FunctionCallee Fn = M->getOrInsertFunction("__safestack_pointer_address", |
1677 | 2 | StackPtrTy->getPointerTo(0)); |
1678 | 2 | return IRB.CreateCall(Fn); |
1679 | 2 | } |
1680 | | |
1681 | | //===----------------------------------------------------------------------===// |
1682 | | // Loop Strength Reduction hooks |
1683 | | //===----------------------------------------------------------------------===// |
1684 | | |
1685 | | /// isLegalAddressingMode - Return true if the addressing mode represented |
1686 | | /// by AM is legal for this target, for a load/store of the specified type. |
1687 | | bool TargetLoweringBase::isLegalAddressingMode(const DataLayout &DL, |
1688 | | const AddrMode &AM, Type *Ty, |
1689 | 529k | unsigned AS, Instruction *I) const { |
1690 | 529k | // The default implementation of this implements a conservative RISCy, r+r and |
1691 | 529k | // r+i addr mode. |
1692 | 529k | |
1693 | 529k | // Allows a sign-extended 16-bit immediate field. |
1694 | 529k | if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1529k ) |
1695 | 6 | return false; |
1696 | 529k | |
1697 | 529k | // No global is ever allowed as a base. |
1698 | 529k | if (AM.BaseGV) |
1699 | 1.39k | return false; |
1700 | 527k | |
1701 | 527k | // Only support r+r, |
1702 | 527k | switch (AM.Scale) { |
1703 | 527k | case 0: // "r+i" or just "i", depending on HasBaseReg. |
1704 | 525k | break; |
1705 | 527k | case 1: |
1706 | 784 | if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. |
1707 | 14 | return false; |
1708 | 770 | // Otherwise we have r+r or r+i. |
1709 | 770 | break; |
1710 | 770 | case 2: |
1711 | 170 | if (AM.HasBaseReg || AM.BaseOffs10 ) // 2*r+r or 2*r+i is not allowed. |
1712 | 160 | return false; |
1713 | 10 | // Allow 2*r as r+r. |
1714 | 10 | break; |
1715 | 1.88k | default: // Don't allow n * r |
1716 | 1.88k | return false; |
1717 | 525k | } |
1718 | 525k | |
1719 | 525k | return true; |
1720 | 525k | } |
1721 | | |
1722 | | //===----------------------------------------------------------------------===// |
1723 | | // Stack Protector |
1724 | | //===----------------------------------------------------------------------===// |
1725 | | |
1726 | | // For OpenBSD return its special guard variable. Otherwise return nullptr, |
1727 | | // so that SelectionDAG handle SSP. |
1728 | 4.54k | Value *TargetLoweringBase::getIRStackGuard(IRBuilder<> &IRB) const { |
1729 | 4.54k | if (getTargetMachine().getTargetTriple().isOSOpenBSD()) { |
1730 | 120 | Module &M = *IRB.GetInsertBlock()->getParent()->getParent(); |
1731 | 120 | PointerType *PtrTy = Type::getInt8PtrTy(M.getContext()); |
1732 | 120 | return M.getOrInsertGlobal("__guard_local", PtrTy); |
1733 | 120 | } |
1734 | 4.42k | return nullptr; |
1735 | 4.42k | } |
1736 | | |
1737 | | // Currently only support "standard" __stack_chk_guard. |
1738 | | // TODO: add LOAD_STACK_GUARD support. |
1739 | 4.33k | void TargetLoweringBase::insertSSPDeclarations(Module &M) const { |
1740 | 4.33k | if (!M.getNamedValue("__stack_chk_guard")) |
1741 | 872 | new GlobalVariable(M, Type::getInt8PtrTy(M.getContext()), false, |
1742 | 872 | GlobalVariable::ExternalLinkage, |
1743 | 872 | nullptr, "__stack_chk_guard"); |
1744 | 4.33k | } |
1745 | | |
1746 | | // Currently only support "standard" __stack_chk_guard. |
1747 | | // TODO: add LOAD_STACK_GUARD support. |
1748 | 8.11k | Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const { |
1749 | 8.11k | return M.getNamedValue("__stack_chk_guard"); |
1750 | 8.11k | } |
1751 | | |
1752 | 3.24k | Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const { |
1753 | 3.24k | return nullptr; |
1754 | 3.24k | } |
1755 | | |
1756 | 50.7k | unsigned TargetLoweringBase::getMinimumJumpTableEntries() const { |
1757 | 50.7k | return MinimumJumpTableEntries; |
1758 | 50.7k | } |
1759 | | |
1760 | 1.95k | void TargetLoweringBase::setMinimumJumpTableEntries(unsigned Val) { |
1761 | 1.95k | MinimumJumpTableEntries = Val; |
1762 | 1.95k | } |
1763 | | |
1764 | 39.7k | unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const { |
1765 | 39.7k | return OptForSize ? OptsizeJumpTableDensity118 : JumpTableDensity39.6k ; |
1766 | 39.7k | } |
1767 | | |
1768 | 39.8k | unsigned TargetLoweringBase::getMaximumJumpTableSize() const { |
1769 | 39.8k | return MaximumJumpTableSize; |
1770 | 39.8k | } |
1771 | | |
1772 | 46 | void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) { |
1773 | 46 | MaximumJumpTableSize = Val; |
1774 | 46 | } |
1775 | | |
1776 | | //===----------------------------------------------------------------------===// |
1777 | | // Reciprocal Estimates |
1778 | | //===----------------------------------------------------------------------===// |
1779 | | |
1780 | | /// Get the reciprocal estimate attribute string for a function that will |
1781 | | /// override the target defaults. |
1782 | 2.46k | static StringRef getRecipEstimateForFunc(MachineFunction &MF) { |
1783 | 2.46k | const Function &F = MF.getFunction(); |
1784 | 2.46k | return F.getFnAttribute("reciprocal-estimates").getValueAsString(); |
1785 | 2.46k | } |
1786 | | |
1787 | | /// Construct a string for the given reciprocal operation of the given type. |
1788 | | /// This string should match the corresponding option to the front-end's |
1789 | | /// "-mrecip" flag assuming those strings have been passed through in an |
1790 | | /// attribute string. For example, "vec-divf" for a division of a vXf32. |
1791 | 1.03k | static std::string getReciprocalOpName(bool IsSqrt, EVT VT) { |
1792 | 1.03k | std::string Name = VT.isVector() ? "vec-"831 : ""208 ; |
1793 | 1.03k | |
1794 | 1.03k | Name += IsSqrt ? "sqrt"143 : "div"896 ; |
1795 | 1.03k | |
1796 | 1.03k | // TODO: Handle "half" or other float types? |
1797 | 1.03k | if (VT.getScalarType() == MVT::f64) { |
1798 | 32 | Name += "d"; |
1799 | 1.00k | } else { |
1800 | 1.00k | assert(VT.getScalarType() == MVT::f32 && |
1801 | 1.00k | "Unexpected FP type for reciprocal estimate"); |
1802 | 1.00k | Name += "f"; |
1803 | 1.00k | } |
1804 | 1.03k | |
1805 | 1.03k | return Name; |
1806 | 1.03k | } |
1807 | | |
1808 | | /// Return the character position and value (a single numeric character) of a |
1809 | | /// customized refinement operation in the input string if it exists. Return |
1810 | | /// false if there is no customized refinement step count. |
1811 | | static bool parseRefinementStep(StringRef In, size_t &Position, |
1812 | 2.02k | uint8_t &Value) { |
1813 | 2.02k | const char RefStepToken = ':'; |
1814 | 2.02k | Position = In.find(RefStepToken); |
1815 | 2.02k | if (Position == StringRef::npos) |
1816 | 1.17k | return false; |
1817 | 850 | |
1818 | 850 | StringRef RefStepString = In.substr(Position + 1); |
1819 | 850 | // Allow exactly one numeric character for the additional refinement |
1820 | 850 | // step parameter. |
1821 | 850 | if (RefStepString.size() == 1) { |
1822 | 850 | char RefStepChar = RefStepString[0]; |
1823 | 850 | if (RefStepChar >= '0' && RefStepChar <= '9') { |
1824 | 850 | Value = RefStepChar - '0'; |
1825 | 850 | return true; |
1826 | 850 | } |
1827 | 0 | } |
1828 | 0 | report_fatal_error("Invalid refinement step for -recip."); |
1829 | 0 | } |
1830 | | |
1831 | | /// For the input attribute string, return one of the ReciprocalEstimate enum |
1832 | | /// status values (enabled, disabled, or not specified) for this operation on |
1833 | | /// the specified data type. |
1834 | 1.29k | static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) { |
1835 | 1.29k | if (Override.empty()) |
1836 | 713 | return TargetLoweringBase::ReciprocalEstimate::Unspecified; |
1837 | 581 | |
1838 | 581 | SmallVector<StringRef, 4> OverrideVector; |
1839 | 581 | Override.split(OverrideVector, ','); |
1840 | 581 | unsigned NumArgs = OverrideVector.size(); |
1841 | 581 | |
1842 | 581 | // Check if "all", "none", or "default" was specified. |
1843 | 581 | if (NumArgs == 1) { |
1844 | 2 | // Look for an optional setting of the number of refinement steps needed |
1845 | 2 | // for this type of reciprocal operation. |
1846 | 2 | size_t RefPos; |
1847 | 2 | uint8_t RefSteps; |
1848 | 2 | if (parseRefinementStep(Override, RefPos, RefSteps)) { |
1849 | 2 | // Split the string for further processing. |
1850 | 2 | Override = Override.substr(0, RefPos); |
1851 | 2 | } |
1852 | 2 | |
1853 | 2 | // All reciprocal types are enabled. |
1854 | 2 | if (Override == "all") |
1855 | 0 | return TargetLoweringBase::ReciprocalEstimate::Enabled; |
1856 | 2 | |
1857 | 2 | // All reciprocal types are disabled. |
1858 | 2 | if (Override == "none") |
1859 | 0 | return TargetLoweringBase::ReciprocalEstimate::Disabled; |
1860 | 2 | |
1861 | 2 | // Target defaults for enablement are used. |
1862 | 2 | if (Override == "default") |
1863 | 0 | return TargetLoweringBase::ReciprocalEstimate::Unspecified; |
1864 | 581 | } |
1865 | 581 | |
1866 | 581 | // The attribute string may omit the size suffix ('f'/'d'). |
1867 | 581 | std::string VTName = getReciprocalOpName(IsSqrt, VT); |
1868 | 581 | std::string VTNameNoSize = VTName; |
1869 | 581 | VTNameNoSize.pop_back(); |
1870 | 581 | static const char DisabledPrefix = '!'; |
1871 | 581 | |
1872 | 1.11k | for (StringRef RecipType : OverrideVector) { |
1873 | 1.11k | size_t RefPos; |
1874 | 1.11k | uint8_t RefSteps; |
1875 | 1.11k | if (parseRefinementStep(RecipType, RefPos, RefSteps)) |
1876 | 423 | RecipType = RecipType.substr(0, RefPos); |
1877 | 1.11k | |
1878 | 1.11k | // Ignore the disablement token for string matching. |
1879 | 1.11k | bool IsDisabled = RecipType[0] == DisabledPrefix; |
1880 | 1.11k | if (IsDisabled) |
1881 | 276 | RecipType = RecipType.substr(1); |
1882 | 1.11k | |
1883 | 1.11k | if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize)590 ) |
1884 | 575 | return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled123 |
1885 | 575 | : TargetLoweringBase::ReciprocalEstimate::Enabled452 ; |
1886 | 1.11k | } |
1887 | 581 | |
1888 | 581 | return TargetLoweringBase::ReciprocalEstimate::Unspecified6 ; |
1889 | 581 | } |
1890 | | |
1891 | | /// For the input attribute string, return the customized refinement step count |
1892 | | /// for this operation on the specified data type. If the step count does not |
1893 | | /// exist, return the ReciprocalEstimate enum value for unspecified. |
1894 | 1.17k | static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) { |
1895 | 1.17k | if (Override.empty()) |
1896 | 713 | return TargetLoweringBase::ReciprocalEstimate::Unspecified; |
1897 | 458 | |
1898 | 458 | SmallVector<StringRef, 4> OverrideVector; |
1899 | 458 | Override.split(OverrideVector, ','); |
1900 | 458 | unsigned NumArgs = OverrideVector.size(); |
1901 | 458 | |
1902 | 458 | // Check if "all", "default", or "none" was specified. |
1903 | 458 | if (NumArgs == 1) { |
1904 | 2 | // Look for an optional setting of the number of refinement steps needed |
1905 | 2 | // for this type of reciprocal operation. |
1906 | 2 | size_t RefPos; |
1907 | 2 | uint8_t RefSteps; |
1908 | 2 | if (!parseRefinementStep(Override, RefPos, RefSteps)) |
1909 | 0 | return TargetLoweringBase::ReciprocalEstimate::Unspecified; |
1910 | 2 | |
1911 | 2 | // Split the string for further processing. |
1912 | 2 | Override = Override.substr(0, RefPos); |
1913 | 2 | assert(Override != "none" && |
1914 | 2 | "Disabled reciprocals, but specifed refinement steps?"); |
1915 | 2 | |
1916 | 2 | // If this is a general override, return the specified number of steps. |
1917 | 2 | if (Override == "all" || Override == "default") |
1918 | 0 | return RefSteps; |
1919 | 458 | } |
1920 | 458 | |
1921 | 458 | // The attribute string may omit the size suffix ('f'/'d'). |
1922 | 458 | std::string VTName = getReciprocalOpName(IsSqrt, VT); |
1923 | 458 | std::string VTNameNoSize = VTName; |
1924 | 458 | VTNameNoSize.pop_back(); |
1925 | 458 | |
1926 | 901 | for (StringRef RecipType : OverrideVector) { |
1927 | 901 | size_t RefPos; |
1928 | 901 | uint8_t RefSteps; |
1929 | 901 | if (!parseRefinementStep(RecipType, RefPos, RefSteps)) |
1930 | 478 | continue; |
1931 | 423 | |
1932 | 423 | RecipType = RecipType.substr(0, RefPos); |
1933 | 423 | if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize)203 ) |
1934 | 222 | return RefSteps; |
1935 | 423 | } |
1936 | 458 | |
1937 | 458 | return TargetLoweringBase::ReciprocalEstimate::Unspecified236 ; |
1938 | 458 | } |
1939 | | |
1940 | | int TargetLoweringBase::getRecipEstimateSqrtEnabled(EVT VT, |
1941 | 267 | MachineFunction &MF) const { |
1942 | 267 | return getOpEnabled(true, VT, getRecipEstimateForFunc(MF)); |
1943 | 267 | } |
1944 | | |
1945 | | int TargetLoweringBase::getRecipEstimateDivEnabled(EVT VT, |
1946 | 1.02k | MachineFunction &MF) const { |
1947 | 1.02k | return getOpEnabled(false, VT, getRecipEstimateForFunc(MF)); |
1948 | 1.02k | } |
1949 | | |
1950 | | int TargetLoweringBase::getSqrtRefinementSteps(EVT VT, |
1951 | 224 | MachineFunction &MF) const { |
1952 | 224 | return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF)); |
1953 | 224 | } |
1954 | | |
1955 | | int TargetLoweringBase::getDivRefinementSteps(EVT VT, |
1956 | 947 | MachineFunction &MF) const { |
1957 | 947 | return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF)); |
1958 | 947 | } |
1959 | | |
1960 | 1.00M | void TargetLoweringBase::finalizeLowering(MachineFunction &MF) const { |
1961 | 1.00M | MF.getRegInfo().freezeReservedRegs(MF); |
1962 | 1.00M | } |