/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/Support/RISCVVIntrinsicUtils.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===- RISCVVIntrinsicUtils.cpp - RISC-V Vector Intrinsic Utils -*- C++ -*-===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | |
9 | | #include "clang/Support/RISCVVIntrinsicUtils.h" |
10 | | #include "llvm/ADT/ArrayRef.h" |
11 | | #include "llvm/ADT/SmallSet.h" |
12 | | #include "llvm/ADT/StringExtras.h" |
13 | | #include "llvm/ADT/StringMap.h" |
14 | | #include "llvm/ADT/StringSet.h" |
15 | | #include "llvm/ADT/Twine.h" |
16 | | #include "llvm/Support/ErrorHandling.h" |
17 | | #include "llvm/Support/raw_ostream.h" |
18 | | #include <numeric> |
19 | | #include <optional> |
20 | | |
21 | | using namespace llvm; |
22 | | |
23 | | namespace clang { |
24 | | namespace RISCV { |
25 | | |
26 | | const PrototypeDescriptor PrototypeDescriptor::Mask = PrototypeDescriptor( |
27 | | BaseTypeModifier::Vector, VectorTypeModifier::MaskVector); |
28 | | const PrototypeDescriptor PrototypeDescriptor::VL = |
29 | | PrototypeDescriptor(BaseTypeModifier::SizeT); |
30 | | const PrototypeDescriptor PrototypeDescriptor::Vector = |
31 | | PrototypeDescriptor(BaseTypeModifier::Vector); |
32 | | |
33 | | //===----------------------------------------------------------------------===// |
34 | | // Type implementation |
35 | | //===----------------------------------------------------------------------===// |
36 | | |
37 | 3.38k | LMULType::LMULType(int NewLog2LMUL) { |
38 | | // Check Log2LMUL is -3, -2, -1, 0, 1, 2, 3 |
39 | 3.38k | assert(NewLog2LMUL <= 3 && NewLog2LMUL >= -3 && "Bad LMUL number!"); |
40 | 3.38k | Log2LMUL = NewLog2LMUL; |
41 | 3.38k | } |
42 | | |
43 | 3.49k | std::string LMULType::str() const { |
44 | 3.49k | if (Log2LMUL < 0) |
45 | 944 | return "mf" + utostr(1ULL << (-Log2LMUL)); |
46 | 2.54k | return "m" + utostr(1ULL << Log2LMUL); |
47 | 3.49k | } |
48 | | |
49 | 3.54k | VScaleVal LMULType::getScale(unsigned ElementBitwidth) const { |
50 | 3.54k | int Log2ScaleResult = 0; |
51 | 3.54k | switch (ElementBitwidth) { |
52 | 8 | default: |
53 | 8 | break; |
54 | 744 | case 8: |
55 | 744 | Log2ScaleResult = Log2LMUL + 3; |
56 | 744 | break; |
57 | 757 | case 16: |
58 | 757 | Log2ScaleResult = Log2LMUL + 2; |
59 | 757 | break; |
60 | 1.03k | case 32: |
61 | 1.03k | Log2ScaleResult = Log2LMUL + 1; |
62 | 1.03k | break; |
63 | 1.00k | case 64: |
64 | 1.00k | Log2ScaleResult = Log2LMUL; |
65 | 1.00k | break; |
66 | 3.54k | } |
67 | | // Illegal vscale result would be less than 1 |
68 | 3.54k | if (Log2ScaleResult < 0) |
69 | 734 | return std::nullopt; |
70 | 2.81k | return 1 << Log2ScaleResult; |
71 | 3.54k | } |
72 | | |
73 | 331 | void LMULType::MulLog2LMUL(int log2LMUL) { Log2LMUL += log2LMUL; } |
74 | | |
75 | | RVVType::RVVType(BasicType BT, int Log2LMUL, |
76 | | const PrototypeDescriptor &prototype) |
77 | 2.80k | : BT(BT), LMUL(LMULType(Log2LMUL)) { |
78 | 2.80k | applyBasicType(); |
79 | 2.80k | applyModifier(prototype); |
80 | 2.80k | Valid = verifyType(); |
81 | 2.80k | if (Valid) { |
82 | 1.75k | initBuiltinStr(); |
83 | 1.75k | initTypeStr(); |
84 | 1.75k | if (isVector()) { |
85 | 1.34k | initClangBuiltinStr(); |
86 | 1.34k | } |
87 | 1.75k | } |
88 | 2.80k | } |
89 | | |
90 | | // clang-format off |
91 | | // boolean type are encoded the ratio of n (SEW/LMUL) |
92 | | // SEW/LMUL | 1 | 2 | 4 | 8 | 16 | 32 | 64 |
93 | | // c type | vbool64_t | vbool32_t | vbool16_t | vbool8_t | vbool4_t | vbool2_t | vbool1_t |
94 | | // IR type | nxv1i1 | nxv2i1 | nxv4i1 | nxv8i1 | nxv16i1 | nxv32i1 | nxv64i1 |
95 | | |
96 | | // type\lmul | 1/8 | 1/4 | 1/2 | 1 | 2 | 4 | 8 |
97 | | // -------- |------ | -------- | ------- | ------- | -------- | -------- | -------- |
98 | | // i64 | N/A | N/A | N/A | nxv1i64 | nxv2i64 | nxv4i64 | nxv8i64 |
99 | | // i32 | N/A | N/A | nxv1i32 | nxv2i32 | nxv4i32 | nxv8i32 | nxv16i32 |
100 | | // i16 | N/A | nxv1i16 | nxv2i16 | nxv4i16 | nxv8i16 | nxv16i16 | nxv32i16 |
101 | | // i8 | nxv1i8 | nxv2i8 | nxv4i8 | nxv8i8 | nxv16i8 | nxv32i8 | nxv64i8 |
102 | | // double | N/A | N/A | N/A | nxv1f64 | nxv2f64 | nxv4f64 | nxv8f64 |
103 | | // float | N/A | N/A | nxv1f32 | nxv2f32 | nxv4f32 | nxv8f32 | nxv16f32 |
104 | | // half | N/A | nxv1f16 | nxv2f16 | nxv4f16 | nxv8f16 | nxv16f16 | nxv32f16 |
105 | | // clang-format on |
106 | | |
107 | 2.80k | bool RVVType::verifyType() const { |
108 | 2.80k | if (ScalarType == Invalid) |
109 | 476 | return false; |
110 | 2.32k | if (isScalar()) |
111 | 417 | return true; |
112 | 1.90k | if (!Scale) |
113 | 365 | return false; |
114 | 1.54k | if (isFloat() && ElementBitwidth == 8216 ) |
115 | 6 | return false; |
116 | 1.53k | if (IsTuple && (572 NF == 1572 || NF > 8572 )) |
117 | 0 | return false; |
118 | 1.53k | if (IsTuple && (1 << std::max(0, LMUL.Log2LMUL)) * NF > 8572 ) |
119 | 170 | return false; |
120 | 1.36k | unsigned V = *Scale; |
121 | 1.36k | switch (ElementBitwidth) { |
122 | 31 | case 1: |
123 | 350 | case 8: |
124 | | // Check Scale is 1,2,4,8,16,32,64 |
125 | 350 | return (V <= 64 && isPowerOf2_32(V)); |
126 | 324 | case 16: |
127 | | // Check Scale is 1,2,4,8,16,32 |
128 | 324 | return (V <= 32 && isPowerOf2_32(V)321 ); |
129 | 369 | case 32: |
130 | | // Check Scale is 1,2,4,8,16 |
131 | 369 | return (V <= 16 && isPowerOf2_32(V)364 ); |
132 | 315 | case 64: |
133 | | // Check Scale is 1,2,4,8 |
134 | 315 | return (V <= 8 && isPowerOf2_32(V)306 ); |
135 | 1.36k | } |
136 | 8 | return false; |
137 | 1.36k | } |
138 | | |
139 | 1.75k | void RVVType::initBuiltinStr() { |
140 | 1.75k | assert(isValid() && "RVVType is invalid"); |
141 | 1.75k | switch (ScalarType) { |
142 | 42 | case ScalarTypeKind::Void: |
143 | 42 | BuiltinStr = "v"; |
144 | 42 | return; |
145 | 93 | case ScalarTypeKind::Size_t: |
146 | 93 | BuiltinStr = "z"; |
147 | 93 | if (IsImmediate) |
148 | 31 | BuiltinStr = "I" + BuiltinStr; |
149 | 93 | if (IsPointer) |
150 | 31 | BuiltinStr += "*"; |
151 | 93 | return; |
152 | 42 | case ScalarTypeKind::Ptrdiff_t: |
153 | 42 | BuiltinStr = "Y"; |
154 | 42 | return; |
155 | 31 | case ScalarTypeKind::UnsignedLong: |
156 | 31 | BuiltinStr = "ULi"; |
157 | 31 | return; |
158 | 7 | case ScalarTypeKind::SignedLong: |
159 | 7 | BuiltinStr = "Li"; |
160 | 7 | return; |
161 | 31 | case ScalarTypeKind::Boolean: |
162 | 31 | assert(ElementBitwidth == 1); |
163 | 31 | BuiltinStr += "b"; |
164 | 31 | break; |
165 | 596 | case ScalarTypeKind::SignedInteger: |
166 | 1.29k | case ScalarTypeKind::UnsignedInteger: |
167 | 1.29k | switch (ElementBitwidth) { |
168 | 361 | case 8: |
169 | 361 | BuiltinStr += "c"; |
170 | 361 | break; |
171 | 349 | case 16: |
172 | 349 | BuiltinStr += "s"; |
173 | 349 | break; |
174 | 319 | case 32: |
175 | 319 | BuiltinStr += "i"; |
176 | 319 | break; |
177 | 270 | case 64: |
178 | 270 | BuiltinStr += "Wi"; |
179 | 270 | break; |
180 | 0 | default: |
181 | 0 | llvm_unreachable("Unhandled ElementBitwidth!"); |
182 | 1.29k | } |
183 | 1.29k | if (isSignedInteger()) |
184 | 596 | BuiltinStr = "S" + BuiltinStr; |
185 | 703 | else |
186 | 703 | BuiltinStr = "U" + BuiltinStr; |
187 | 1.29k | break; |
188 | 213 | case ScalarTypeKind::Float: |
189 | 213 | switch (ElementBitwidth) { |
190 | 12 | case 16: |
191 | 12 | BuiltinStr += "x"; |
192 | 12 | break; |
193 | 107 | case 32: |
194 | 107 | BuiltinStr += "f"; |
195 | 107 | break; |
196 | 94 | case 64: |
197 | 94 | BuiltinStr += "d"; |
198 | 94 | break; |
199 | 0 | default: |
200 | 0 | llvm_unreachable("Unhandled ElementBitwidth!"); |
201 | 213 | } |
202 | 213 | break; |
203 | 213 | default: |
204 | 0 | llvm_unreachable("ScalarType is invalid!"); |
205 | 1.75k | } |
206 | 1.54k | if (IsImmediate) |
207 | 0 | BuiltinStr = "I" + BuiltinStr; |
208 | 1.54k | if (isScalar()) { |
209 | 202 | if (IsConstant) |
210 | 53 | BuiltinStr += "C"; |
211 | 202 | if (IsPointer) |
212 | 123 | BuiltinStr += "*"; |
213 | 202 | return; |
214 | 202 | } |
215 | 1.34k | BuiltinStr = "q" + utostr(*Scale) + BuiltinStr; |
216 | | // Pointer to vector types. Defined for segment load intrinsics. |
217 | | // segment load intrinsics have pointer type arguments to store the loaded |
218 | | // vector values. |
219 | 1.34k | if (IsPointer) |
220 | 0 | BuiltinStr += "*"; |
221 | | |
222 | 1.34k | if (IsTuple) |
223 | 402 | BuiltinStr = "T" + utostr(NF) + BuiltinStr; |
224 | 1.34k | } |
225 | | |
226 | 1.34k | void RVVType::initClangBuiltinStr() { |
227 | 1.34k | assert(isValid() && "RVVType is invalid"); |
228 | 1.34k | assert(isVector() && "Handle Vector type only"); |
229 | | |
230 | 1.34k | ClangBuiltinStr = "__rvv_"; |
231 | 1.34k | switch (ScalarType) { |
232 | 31 | case ScalarTypeKind::Boolean: |
233 | 31 | ClangBuiltinStr += "bool" + utostr(64 / *Scale) + "_t"; |
234 | 31 | return; |
235 | 176 | case ScalarTypeKind::Float: |
236 | 176 | ClangBuiltinStr += "float"; |
237 | 176 | break; |
238 | 518 | case ScalarTypeKind::SignedInteger: |
239 | 518 | ClangBuiltinStr += "int"; |
240 | 518 | break; |
241 | 616 | case ScalarTypeKind::UnsignedInteger: |
242 | 616 | ClangBuiltinStr += "uint"; |
243 | 616 | break; |
244 | 0 | default: |
245 | 0 | llvm_unreachable("ScalarTypeKind is invalid"); |
246 | 1.34k | } |
247 | 1.31k | ClangBuiltinStr += utostr(ElementBitwidth) + LMUL.str() + |
248 | 1.31k | (IsTuple ? "x" + utostr(NF)402 : ""908 ) + "_t"; |
249 | 1.31k | } |
250 | | |
251 | 1.75k | void RVVType::initTypeStr() { |
252 | 1.75k | assert(isValid() && "RVVType is invalid"); |
253 | | |
254 | 1.75k | if (IsConstant) |
255 | 285 | Str += "const "; |
256 | | |
257 | 1.75k | auto getTypeString = [&](StringRef TypeStr) { |
258 | 1.47k | if (isScalar()) |
259 | 165 | return Twine(TypeStr + Twine(ElementBitwidth) + "_t").str(); |
260 | 1.31k | return Twine("v" + TypeStr + Twine(ElementBitwidth) + LMUL.str() + |
261 | 1.31k | (IsTuple ? "x" + utostr(NF)402 : ""908 ) + "_t") |
262 | 1.31k | .str(); |
263 | 1.47k | }; |
264 | | |
265 | 1.75k | switch (ScalarType) { |
266 | 42 | case ScalarTypeKind::Void: |
267 | 42 | Str = "void"; |
268 | 42 | return; |
269 | 93 | case ScalarTypeKind::Size_t: |
270 | 93 | Str = "size_t"; |
271 | 93 | if (IsPointer) |
272 | 31 | Str += " *"; |
273 | 93 | return; |
274 | 42 | case ScalarTypeKind::Ptrdiff_t: |
275 | 42 | Str = "ptrdiff_t"; |
276 | 42 | return; |
277 | 31 | case ScalarTypeKind::UnsignedLong: |
278 | 31 | Str = "unsigned long"; |
279 | 31 | return; |
280 | 7 | case ScalarTypeKind::SignedLong: |
281 | 7 | Str = "long"; |
282 | 7 | return; |
283 | 31 | case ScalarTypeKind::Boolean: |
284 | 31 | if (isScalar()) |
285 | 0 | Str += "bool"; |
286 | 31 | else |
287 | | // Vector bool is special case, the formulate is |
288 | | // `vbool<N>_t = MVT::nxv<64/N>i1` ex. vbool16_t = MVT::4i1 |
289 | 31 | Str += "vbool" + utostr(64 / *Scale) + "_t"; |
290 | 31 | break; |
291 | 213 | case ScalarTypeKind::Float: |
292 | 213 | if (isScalar()) { |
293 | 37 | if (ElementBitwidth == 64) |
294 | 18 | Str += "double"; |
295 | 19 | else if (ElementBitwidth == 32) |
296 | 19 | Str += "float"; |
297 | 0 | else if (ElementBitwidth == 16) |
298 | 0 | Str += "_Float16"; |
299 | 0 | else |
300 | 0 | llvm_unreachable("Unhandled floating type."); |
301 | 37 | } else |
302 | 176 | Str += getTypeString("float"); |
303 | 213 | break; |
304 | 596 | case ScalarTypeKind::SignedInteger: |
305 | 596 | Str += getTypeString("int"); |
306 | 596 | break; |
307 | 703 | case ScalarTypeKind::UnsignedInteger: |
308 | 703 | Str += getTypeString("uint"); |
309 | 703 | break; |
310 | 0 | default: |
311 | 0 | llvm_unreachable("ScalarType is invalid!"); |
312 | 1.75k | } |
313 | 1.54k | if (IsPointer) |
314 | 123 | Str += " *"; |
315 | 1.54k | } |
316 | | |
317 | 957 | void RVVType::initShortStr() { |
318 | 957 | switch (ScalarType) { |
319 | 31 | case ScalarTypeKind::Boolean: |
320 | 31 | assert(isVector()); |
321 | 31 | ShortStr = "b" + utostr(64 / *Scale); |
322 | 31 | return; |
323 | 127 | case ScalarTypeKind::Float: |
324 | 127 | ShortStr = "f" + utostr(ElementBitwidth); |
325 | 127 | break; |
326 | 407 | case ScalarTypeKind::SignedInteger: |
327 | 407 | ShortStr = "i" + utostr(ElementBitwidth); |
328 | 407 | break; |
329 | 392 | case ScalarTypeKind::UnsignedInteger: |
330 | 392 | ShortStr = "u" + utostr(ElementBitwidth); |
331 | 392 | break; |
332 | 0 | default: |
333 | 0 | llvm_unreachable("Unhandled case!"); |
334 | 957 | } |
335 | 926 | if (isVector()) |
336 | 873 | ShortStr += LMUL.str(); |
337 | 926 | if (isTuple()) |
338 | 201 | ShortStr += "x" + utostr(NF); |
339 | 926 | } |
340 | | |
341 | 8.74k | static VectorTypeModifier getTupleVTM(unsigned NF) { |
342 | 8.74k | assert(2 <= NF && NF <= 8 && "2 <= NF <= 8"); |
343 | 8.74k | return static_cast<VectorTypeModifier>( |
344 | 8.74k | static_cast<uint8_t>(VectorTypeModifier::Tuple2) + (NF - 2)); |
345 | 8.74k | } |
346 | | |
347 | 2.80k | void RVVType::applyBasicType() { |
348 | 2.80k | switch (BT) { |
349 | 646 | case BasicType::Int8: |
350 | 646 | ElementBitwidth = 8; |
351 | 646 | ScalarType = ScalarTypeKind::SignedInteger; |
352 | 646 | break; |
353 | 570 | case BasicType::Int16: |
354 | 570 | ElementBitwidth = 16; |
355 | 570 | ScalarType = ScalarTypeKind::SignedInteger; |
356 | 570 | break; |
357 | 540 | case BasicType::Int32: |
358 | 540 | ElementBitwidth = 32; |
359 | 540 | ScalarType = ScalarTypeKind::SignedInteger; |
360 | 540 | break; |
361 | 488 | case BasicType::Int64: |
362 | 488 | ElementBitwidth = 64; |
363 | 488 | ScalarType = ScalarTypeKind::SignedInteger; |
364 | 488 | break; |
365 | 0 | case BasicType::Float16: |
366 | 0 | ElementBitwidth = 16; |
367 | 0 | ScalarType = ScalarTypeKind::Float; |
368 | 0 | break; |
369 | 297 | case BasicType::Float32: |
370 | 297 | ElementBitwidth = 32; |
371 | 297 | ScalarType = ScalarTypeKind::Float; |
372 | 297 | break; |
373 | 259 | case BasicType::Float64: |
374 | 259 | ElementBitwidth = 64; |
375 | 259 | ScalarType = ScalarTypeKind::Float; |
376 | 259 | break; |
377 | 0 | default: |
378 | 0 | llvm_unreachable("Unhandled type code!"); |
379 | 2.80k | } |
380 | 2.80k | assert(ElementBitwidth != 0 && "Bad element bitwidth!"); |
381 | 2.80k | } |
382 | | |
383 | | std::optional<PrototypeDescriptor> |
384 | | PrototypeDescriptor::parsePrototypeDescriptor( |
385 | 0 | llvm::StringRef PrototypeDescriptorStr) { |
386 | 0 | PrototypeDescriptor PD; |
387 | 0 | BaseTypeModifier PT = BaseTypeModifier::Invalid; |
388 | 0 | VectorTypeModifier VTM = VectorTypeModifier::NoModifier; |
389 | |
|
390 | 0 | if (PrototypeDescriptorStr.empty()) |
391 | 0 | return PD; |
392 | | |
393 | | // Handle base type modifier |
394 | 0 | auto PType = PrototypeDescriptorStr.back(); |
395 | 0 | switch (PType) { |
396 | 0 | case 'e': |
397 | 0 | PT = BaseTypeModifier::Scalar; |
398 | 0 | break; |
399 | 0 | case 'v': |
400 | 0 | PT = BaseTypeModifier::Vector; |
401 | 0 | break; |
402 | 0 | case 'w': |
403 | 0 | PT = BaseTypeModifier::Vector; |
404 | 0 | VTM = VectorTypeModifier::Widening2XVector; |
405 | 0 | break; |
406 | 0 | case 'q': |
407 | 0 | PT = BaseTypeModifier::Vector; |
408 | 0 | VTM = VectorTypeModifier::Widening4XVector; |
409 | 0 | break; |
410 | 0 | case 'o': |
411 | 0 | PT = BaseTypeModifier::Vector; |
412 | 0 | VTM = VectorTypeModifier::Widening8XVector; |
413 | 0 | break; |
414 | 0 | case 'm': |
415 | 0 | PT = BaseTypeModifier::Vector; |
416 | 0 | VTM = VectorTypeModifier::MaskVector; |
417 | 0 | break; |
418 | 0 | case '0': |
419 | 0 | PT = BaseTypeModifier::Void; |
420 | 0 | break; |
421 | 0 | case 'z': |
422 | 0 | PT = BaseTypeModifier::SizeT; |
423 | 0 | break; |
424 | 0 | case 't': |
425 | 0 | PT = BaseTypeModifier::Ptrdiff; |
426 | 0 | break; |
427 | 0 | case 'u': |
428 | 0 | PT = BaseTypeModifier::UnsignedLong; |
429 | 0 | break; |
430 | 0 | case 'l': |
431 | 0 | PT = BaseTypeModifier::SignedLong; |
432 | 0 | break; |
433 | 0 | default: |
434 | 0 | llvm_unreachable("Illegal primitive type transformers!"); |
435 | 0 | } |
436 | 0 | PD.PT = static_cast<uint8_t>(PT); |
437 | 0 | PrototypeDescriptorStr = PrototypeDescriptorStr.drop_back(); |
438 | | |
439 | | // Compute the vector type transformers, it can only appear one time. |
440 | 0 | if (PrototypeDescriptorStr.startswith("(")) { |
441 | 0 | assert(VTM == VectorTypeModifier::NoModifier && |
442 | 0 | "VectorTypeModifier should only have one modifier"); |
443 | 0 | size_t Idx = PrototypeDescriptorStr.find(')'); |
444 | 0 | assert(Idx != StringRef::npos); |
445 | 0 | StringRef ComplexType = PrototypeDescriptorStr.slice(1, Idx); |
446 | 0 | PrototypeDescriptorStr = PrototypeDescriptorStr.drop_front(Idx + 1); |
447 | 0 | assert(!PrototypeDescriptorStr.contains('(') && |
448 | 0 | "Only allow one vector type modifier"); |
449 | | |
450 | 0 | auto ComplexTT = ComplexType.split(":"); |
451 | 0 | if (ComplexTT.first == "Log2EEW") { |
452 | 0 | uint32_t Log2EEW; |
453 | 0 | if (ComplexTT.second.getAsInteger(10, Log2EEW)) { |
454 | 0 | llvm_unreachable("Invalid Log2EEW value!"); |
455 | 0 | return std::nullopt; |
456 | 0 | } |
457 | 0 | switch (Log2EEW) { |
458 | 0 | case 3: |
459 | 0 | VTM = VectorTypeModifier::Log2EEW3; |
460 | 0 | break; |
461 | 0 | case 4: |
462 | 0 | VTM = VectorTypeModifier::Log2EEW4; |
463 | 0 | break; |
464 | 0 | case 5: |
465 | 0 | VTM = VectorTypeModifier::Log2EEW5; |
466 | 0 | break; |
467 | 0 | case 6: |
468 | 0 | VTM = VectorTypeModifier::Log2EEW6; |
469 | 0 | break; |
470 | 0 | default: |
471 | 0 | llvm_unreachable("Invalid Log2EEW value, should be [3-6]"); |
472 | 0 | return std::nullopt; |
473 | 0 | } |
474 | 0 | } else if (ComplexTT.first == "FixedSEW") { |
475 | 0 | uint32_t NewSEW; |
476 | 0 | if (ComplexTT.second.getAsInteger(10, NewSEW)) { |
477 | 0 | llvm_unreachable("Invalid FixedSEW value!"); |
478 | 0 | return std::nullopt; |
479 | 0 | } |
480 | 0 | switch (NewSEW) { |
481 | 0 | case 8: |
482 | 0 | VTM = VectorTypeModifier::FixedSEW8; |
483 | 0 | break; |
484 | 0 | case 16: |
485 | 0 | VTM = VectorTypeModifier::FixedSEW16; |
486 | 0 | break; |
487 | 0 | case 32: |
488 | 0 | VTM = VectorTypeModifier::FixedSEW32; |
489 | 0 | break; |
490 | 0 | case 64: |
491 | 0 | VTM = VectorTypeModifier::FixedSEW64; |
492 | 0 | break; |
493 | 0 | default: |
494 | 0 | llvm_unreachable("Invalid FixedSEW value, should be 8, 16, 32 or 64"); |
495 | 0 | return std::nullopt; |
496 | 0 | } |
497 | 0 | } else if (ComplexTT.first == "LFixedLog2LMUL") { |
498 | 0 | int32_t Log2LMUL; |
499 | 0 | if (ComplexTT.second.getAsInteger(10, Log2LMUL)) { |
500 | 0 | llvm_unreachable("Invalid LFixedLog2LMUL value!"); |
501 | 0 | return std::nullopt; |
502 | 0 | } |
503 | 0 | switch (Log2LMUL) { |
504 | 0 | case -3: |
505 | 0 | VTM = VectorTypeModifier::LFixedLog2LMULN3; |
506 | 0 | break; |
507 | 0 | case -2: |
508 | 0 | VTM = VectorTypeModifier::LFixedLog2LMULN2; |
509 | 0 | break; |
510 | 0 | case -1: |
511 | 0 | VTM = VectorTypeModifier::LFixedLog2LMULN1; |
512 | 0 | break; |
513 | 0 | case 0: |
514 | 0 | VTM = VectorTypeModifier::LFixedLog2LMUL0; |
515 | 0 | break; |
516 | 0 | case 1: |
517 | 0 | VTM = VectorTypeModifier::LFixedLog2LMUL1; |
518 | 0 | break; |
519 | 0 | case 2: |
520 | 0 | VTM = VectorTypeModifier::LFixedLog2LMUL2; |
521 | 0 | break; |
522 | 0 | case 3: |
523 | 0 | VTM = VectorTypeModifier::LFixedLog2LMUL3; |
524 | 0 | break; |
525 | 0 | default: |
526 | 0 | llvm_unreachable("Invalid LFixedLog2LMUL value, should be [-3, 3]"); |
527 | 0 | return std::nullopt; |
528 | 0 | } |
529 | 0 | } else if (ComplexTT.first == "SFixedLog2LMUL") { |
530 | 0 | int32_t Log2LMUL; |
531 | 0 | if (ComplexTT.second.getAsInteger(10, Log2LMUL)) { |
532 | 0 | llvm_unreachable("Invalid SFixedLog2LMUL value!"); |
533 | 0 | return std::nullopt; |
534 | 0 | } |
535 | 0 | switch (Log2LMUL) { |
536 | 0 | case -3: |
537 | 0 | VTM = VectorTypeModifier::SFixedLog2LMULN3; |
538 | 0 | break; |
539 | 0 | case -2: |
540 | 0 | VTM = VectorTypeModifier::SFixedLog2LMULN2; |
541 | 0 | break; |
542 | 0 | case -1: |
543 | 0 | VTM = VectorTypeModifier::SFixedLog2LMULN1; |
544 | 0 | break; |
545 | 0 | case 0: |
546 | 0 | VTM = VectorTypeModifier::SFixedLog2LMUL0; |
547 | 0 | break; |
548 | 0 | case 1: |
549 | 0 | VTM = VectorTypeModifier::SFixedLog2LMUL1; |
550 | 0 | break; |
551 | 0 | case 2: |
552 | 0 | VTM = VectorTypeModifier::SFixedLog2LMUL2; |
553 | 0 | break; |
554 | 0 | case 3: |
555 | 0 | VTM = VectorTypeModifier::SFixedLog2LMUL3; |
556 | 0 | break; |
557 | 0 | default: |
558 | 0 | llvm_unreachable("Invalid LFixedLog2LMUL value, should be [-3, 3]"); |
559 | 0 | return std::nullopt; |
560 | 0 | } |
561 | |
|
562 | 0 | } else if (ComplexTT.first == "SEFixedLog2LMUL") { |
563 | 0 | int32_t Log2LMUL; |
564 | 0 | if (ComplexTT.second.getAsInteger(10, Log2LMUL)) { |
565 | 0 | llvm_unreachable("Invalid SEFixedLog2LMUL value!"); |
566 | 0 | return std::nullopt; |
567 | 0 | } |
568 | 0 | switch (Log2LMUL) { |
569 | 0 | case -3: |
570 | 0 | VTM = VectorTypeModifier::SEFixedLog2LMULN3; |
571 | 0 | break; |
572 | 0 | case -2: |
573 | 0 | VTM = VectorTypeModifier::SEFixedLog2LMULN2; |
574 | 0 | break; |
575 | 0 | case -1: |
576 | 0 | VTM = VectorTypeModifier::SEFixedLog2LMULN1; |
577 | 0 | break; |
578 | 0 | case 0: |
579 | 0 | VTM = VectorTypeModifier::SEFixedLog2LMUL0; |
580 | 0 | break; |
581 | 0 | case 1: |
582 | 0 | VTM = VectorTypeModifier::SEFixedLog2LMUL1; |
583 | 0 | break; |
584 | 0 | case 2: |
585 | 0 | VTM = VectorTypeModifier::SEFixedLog2LMUL2; |
586 | 0 | break; |
587 | 0 | case 3: |
588 | 0 | VTM = VectorTypeModifier::SEFixedLog2LMUL3; |
589 | 0 | break; |
590 | 0 | default: |
591 | 0 | llvm_unreachable("Invalid LFixedLog2LMUL value, should be [-3, 3]"); |
592 | 0 | return std::nullopt; |
593 | 0 | } |
594 | 0 | } else if (ComplexTT.first == "Tuple") { |
595 | 0 | unsigned NF = 0; |
596 | 0 | if (ComplexTT.second.getAsInteger(10, NF)) { |
597 | 0 | llvm_unreachable("Invalid NF value!"); |
598 | 0 | return std::nullopt; |
599 | 0 | } |
600 | 0 | VTM = getTupleVTM(NF); |
601 | 0 | } else { |
602 | 0 | llvm_unreachable("Illegal complex type transformers!"); |
603 | 0 | } |
604 | 0 | } |
605 | 0 | PD.VTM = static_cast<uint8_t>(VTM); |
606 | | |
607 | | // Compute the remain type transformers |
608 | 0 | TypeModifier TM = TypeModifier::NoModifier; |
609 | 0 | for (char I : PrototypeDescriptorStr) { |
610 | 0 | switch (I) { |
611 | 0 | case 'P': |
612 | 0 | if ((TM & TypeModifier::Const) == TypeModifier::Const) |
613 | 0 | llvm_unreachable("'P' transformer cannot be used after 'C'"); |
614 | 0 | if ((TM & TypeModifier::Pointer) == TypeModifier::Pointer) |
615 | 0 | llvm_unreachable("'P' transformer cannot be used twice"); |
616 | 0 | TM |= TypeModifier::Pointer; |
617 | 0 | break; |
618 | 0 | case 'C': |
619 | 0 | TM |= TypeModifier::Const; |
620 | 0 | break; |
621 | 0 | case 'K': |
622 | 0 | TM |= TypeModifier::Immediate; |
623 | 0 | break; |
624 | 0 | case 'U': |
625 | 0 | TM |= TypeModifier::UnsignedInteger; |
626 | 0 | break; |
627 | 0 | case 'I': |
628 | 0 | TM |= TypeModifier::SignedInteger; |
629 | 0 | break; |
630 | 0 | case 'F': |
631 | 0 | TM |= TypeModifier::Float; |
632 | 0 | break; |
633 | 0 | case 'S': |
634 | 0 | TM |= TypeModifier::LMUL1; |
635 | 0 | break; |
636 | 0 | default: |
637 | 0 | llvm_unreachable("Illegal non-primitive type transformer!"); |
638 | 0 | } |
639 | 0 | } |
640 | 0 | PD.TM = static_cast<uint8_t>(TM); |
641 | |
|
642 | 0 | return PD; |
643 | 0 | } |
644 | | |
645 | 2.80k | void RVVType::applyModifier(const PrototypeDescriptor &Transformer) { |
646 | | // Handle primitive type transformer |
647 | 2.80k | switch (static_cast<BaseTypeModifier>(Transformer.PT)) { |
648 | 202 | case BaseTypeModifier::Scalar: |
649 | 202 | Scale = 0; |
650 | 202 | break; |
651 | 2.38k | case BaseTypeModifier::Vector: |
652 | 2.38k | Scale = LMUL.getScale(ElementBitwidth); |
653 | 2.38k | break; |
654 | 42 | case BaseTypeModifier::Void: |
655 | 42 | ScalarType = ScalarTypeKind::Void; |
656 | 42 | break; |
657 | 93 | case BaseTypeModifier::SizeT: |
658 | 93 | ScalarType = ScalarTypeKind::Size_t; |
659 | 93 | break; |
660 | 42 | case BaseTypeModifier::Ptrdiff: |
661 | 42 | ScalarType = ScalarTypeKind::Ptrdiff_t; |
662 | 42 | break; |
663 | 31 | case BaseTypeModifier::UnsignedLong: |
664 | 31 | ScalarType = ScalarTypeKind::UnsignedLong; |
665 | 31 | break; |
666 | 7 | case BaseTypeModifier::SignedLong: |
667 | 7 | ScalarType = ScalarTypeKind::SignedLong; |
668 | 7 | break; |
669 | 0 | case BaseTypeModifier::Invalid: |
670 | 0 | ScalarType = ScalarTypeKind::Invalid; |
671 | 0 | return; |
672 | 2.80k | } |
673 | | |
674 | 2.80k | switch (static_cast<VectorTypeModifier>(Transformer.VTM)) { |
675 | 135 | case VectorTypeModifier::Widening2XVector: |
676 | 135 | ElementBitwidth *= 2; |
677 | 135 | LMUL.MulLog2LMUL(1); |
678 | 135 | Scale = LMUL.getScale(ElementBitwidth); |
679 | 135 | break; |
680 | 20 | case VectorTypeModifier::Widening4XVector: |
681 | 20 | ElementBitwidth *= 4; |
682 | 20 | LMUL.MulLog2LMUL(2); |
683 | 20 | Scale = LMUL.getScale(ElementBitwidth); |
684 | 20 | break; |
685 | 8 | case VectorTypeModifier::Widening8XVector: |
686 | 8 | ElementBitwidth *= 8; |
687 | 8 | LMUL.MulLog2LMUL(3); |
688 | 8 | Scale = LMUL.getScale(ElementBitwidth); |
689 | 8 | break; |
690 | 42 | case VectorTypeModifier::MaskVector: |
691 | 42 | ScalarType = ScalarTypeKind::Boolean; |
692 | 42 | Scale = LMUL.getScale(ElementBitwidth); |
693 | 42 | ElementBitwidth = 1; |
694 | 42 | break; |
695 | 42 | case VectorTypeModifier::Log2EEW3: |
696 | 42 | applyLog2EEW(3); |
697 | 42 | break; |
698 | 42 | case VectorTypeModifier::Log2EEW4: |
699 | 42 | applyLog2EEW(4); |
700 | 42 | break; |
701 | 42 | case VectorTypeModifier::Log2EEW5: |
702 | 42 | applyLog2EEW(5); |
703 | 42 | break; |
704 | 42 | case VectorTypeModifier::Log2EEW6: |
705 | 42 | applyLog2EEW(6); |
706 | 42 | break; |
707 | 56 | case VectorTypeModifier::FixedSEW8: |
708 | 56 | applyFixedSEW(8); |
709 | 56 | break; |
710 | 70 | case VectorTypeModifier::FixedSEW16: |
711 | 70 | applyFixedSEW(16); |
712 | 70 | break; |
713 | 70 | case VectorTypeModifier::FixedSEW32: |
714 | 70 | applyFixedSEW(32); |
715 | 70 | break; |
716 | 70 | case VectorTypeModifier::FixedSEW64: |
717 | 70 | applyFixedSEW(64); |
718 | 70 | break; |
719 | 0 | case VectorTypeModifier::LFixedLog2LMULN3: |
720 | 0 | applyFixedLog2LMUL(-3, FixedLMULType::LargerThan); |
721 | 0 | break; |
722 | 70 | case VectorTypeModifier::LFixedLog2LMULN2: |
723 | 70 | applyFixedLog2LMUL(-2, FixedLMULType::LargerThan); |
724 | 70 | break; |
725 | 70 | case VectorTypeModifier::LFixedLog2LMULN1: |
726 | 70 | applyFixedLog2LMUL(-1, FixedLMULType::LargerThan); |
727 | 70 | break; |
728 | 70 | case VectorTypeModifier::LFixedLog2LMUL0: |
729 | 70 | applyFixedLog2LMUL(0, FixedLMULType::LargerThan); |
730 | 70 | break; |
731 | 70 | case VectorTypeModifier::LFixedLog2LMUL1: |
732 | 70 | applyFixedLog2LMUL(1, FixedLMULType::LargerThan); |
733 | 70 | break; |
734 | 70 | case VectorTypeModifier::LFixedLog2LMUL2: |
735 | 70 | applyFixedLog2LMUL(2, FixedLMULType::LargerThan); |
736 | 70 | break; |
737 | 70 | case VectorTypeModifier::LFixedLog2LMUL3: |
738 | 70 | applyFixedLog2LMUL(3, FixedLMULType::LargerThan); |
739 | 70 | break; |
740 | 70 | case VectorTypeModifier::SFixedLog2LMULN3: |
741 | 70 | applyFixedLog2LMUL(-3, FixedLMULType::SmallerThan); |
742 | 70 | break; |
743 | 70 | case VectorTypeModifier::SFixedLog2LMULN2: |
744 | 70 | applyFixedLog2LMUL(-2, FixedLMULType::SmallerThan); |
745 | 70 | break; |
746 | 70 | case VectorTypeModifier::SFixedLog2LMULN1: |
747 | 70 | applyFixedLog2LMUL(-1, FixedLMULType::SmallerThan); |
748 | 70 | break; |
749 | 70 | case VectorTypeModifier::SFixedLog2LMUL0: |
750 | 70 | applyFixedLog2LMUL(0, FixedLMULType::SmallerThan); |
751 | 70 | break; |
752 | 70 | case VectorTypeModifier::SFixedLog2LMUL1: |
753 | 70 | applyFixedLog2LMUL(1, FixedLMULType::SmallerThan); |
754 | 70 | break; |
755 | 70 | case VectorTypeModifier::SFixedLog2LMUL2: |
756 | 70 | applyFixedLog2LMUL(2, FixedLMULType::SmallerThan); |
757 | 70 | break; |
758 | 0 | case VectorTypeModifier::SFixedLog2LMUL3: |
759 | 0 | applyFixedLog2LMUL(3, FixedLMULType::SmallerThan); |
760 | 0 | break; |
761 | 0 | case VectorTypeModifier::SEFixedLog2LMULN3: |
762 | 0 | applyFixedLog2LMUL(-3, FixedLMULType::SmallerOrEqual); |
763 | 0 | break; |
764 | 0 | case VectorTypeModifier::SEFixedLog2LMULN2: |
765 | 0 | applyFixedLog2LMUL(-2, FixedLMULType::SmallerOrEqual); |
766 | 0 | break; |
767 | 0 | case VectorTypeModifier::SEFixedLog2LMULN1: |
768 | 0 | applyFixedLog2LMUL(-1, FixedLMULType::SmallerOrEqual); |
769 | 0 | break; |
770 | 0 | case VectorTypeModifier::SEFixedLog2LMUL0: |
771 | 0 | applyFixedLog2LMUL(0, FixedLMULType::SmallerOrEqual); |
772 | 0 | break; |
773 | 0 | case VectorTypeModifier::SEFixedLog2LMUL1: |
774 | 0 | applyFixedLog2LMUL(1, FixedLMULType::SmallerOrEqual); |
775 | 0 | break; |
776 | 0 | case VectorTypeModifier::SEFixedLog2LMUL2: |
777 | 0 | applyFixedLog2LMUL(2, FixedLMULType::SmallerOrEqual); |
778 | 0 | break; |
779 | 0 | case VectorTypeModifier::SEFixedLog2LMUL3: |
780 | 0 | applyFixedLog2LMUL(3, FixedLMULType::SmallerOrEqual); |
781 | 0 | break; |
782 | 113 | case VectorTypeModifier::Tuple2: |
783 | 216 | case VectorTypeModifier::Tuple3: |
784 | 319 | case VectorTypeModifier::Tuple4: |
785 | 412 | case VectorTypeModifier::Tuple5: |
786 | 505 | case VectorTypeModifier::Tuple6: |
787 | 598 | case VectorTypeModifier::Tuple7: |
788 | 691 | case VectorTypeModifier::Tuple8: { |
789 | 691 | IsTuple = true; |
790 | 691 | NF = 2 + static_cast<uint8_t>(Transformer.VTM) - |
791 | 691 | static_cast<uint8_t>(VectorTypeModifier::Tuple2); |
792 | 691 | break; |
793 | 598 | } |
794 | 630 | case VectorTypeModifier::NoModifier: |
795 | 630 | break; |
796 | 2.80k | } |
797 | | |
798 | | // Early return if the current type modifier is already invalid. |
799 | 2.80k | if (ScalarType == Invalid) |
800 | 476 | return; |
801 | | |
802 | 2.32k | for (unsigned TypeModifierMaskShift = 0; |
803 | 18.5k | TypeModifierMaskShift <= static_cast<unsigned>(TypeModifier::MaxOffset); |
804 | 16.2k | ++TypeModifierMaskShift) { |
805 | 16.2k | unsigned TypeModifierMask = 1 << TypeModifierMaskShift; |
806 | 16.2k | if ((static_cast<unsigned>(Transformer.TM) & TypeModifierMask) != |
807 | 16.2k | TypeModifierMask) |
808 | 14.6k | continue; |
809 | 1.62k | switch (static_cast<TypeModifier>(TypeModifierMask)) { |
810 | 154 | case TypeModifier::Pointer: |
811 | 154 | IsPointer = true; |
812 | 154 | break; |
813 | 254 | case TypeModifier::Const: |
814 | 254 | IsConstant = true; |
815 | 254 | break; |
816 | 31 | case TypeModifier::Immediate: |
817 | 31 | IsImmediate = true; |
818 | 31 | IsConstant = true; |
819 | 31 | break; |
820 | 946 | case TypeModifier::UnsignedInteger: |
821 | 946 | ScalarType = ScalarTypeKind::UnsignedInteger; |
822 | 946 | break; |
823 | 38 | case TypeModifier::SignedInteger: |
824 | 38 | ScalarType = ScalarTypeKind::SignedInteger; |
825 | 38 | break; |
826 | 45 | case TypeModifier::Float: |
827 | 45 | ScalarType = ScalarTypeKind::Float; |
828 | 45 | break; |
829 | 161 | case TypeModifier::LMUL1: |
830 | 161 | LMUL = LMULType(0); |
831 | | // Update ElementBitwidth need to update Scale too. |
832 | 161 | Scale = LMUL.getScale(ElementBitwidth); |
833 | 161 | break; |
834 | 0 | default: |
835 | 0 | llvm_unreachable("Unknown type modifier mask!"); |
836 | 1.62k | } |
837 | 1.62k | } |
838 | 2.32k | } |
839 | | |
840 | 168 | void RVVType::applyLog2EEW(unsigned Log2EEW) { |
841 | | // update new elmul = (eew/sew) * lmul |
842 | 168 | LMUL.MulLog2LMUL(Log2EEW - Log2_32(ElementBitwidth)); |
843 | | // update new eew |
844 | 168 | ElementBitwidth = 1 << Log2EEW; |
845 | 168 | ScalarType = ScalarTypeKind::SignedInteger; |
846 | 168 | Scale = LMUL.getScale(ElementBitwidth); |
847 | 168 | } |
848 | | |
849 | 266 | void RVVType::applyFixedSEW(unsigned NewSEW) { |
850 | | // Set invalid type if src and dst SEW are same. |
851 | 266 | if (ElementBitwidth == NewSEW) { |
852 | 56 | ScalarType = ScalarTypeKind::Invalid; |
853 | 56 | return; |
854 | 56 | } |
855 | | // Update new SEW |
856 | 210 | ElementBitwidth = NewSEW; |
857 | 210 | Scale = LMUL.getScale(ElementBitwidth); |
858 | 210 | } |
859 | | |
860 | 840 | void RVVType::applyFixedLog2LMUL(int Log2LMUL, enum FixedLMULType Type) { |
861 | 840 | switch (Type) { |
862 | 420 | case FixedLMULType::LargerThan: |
863 | 420 | if (Log2LMUL <= LMUL.Log2LMUL) { |
864 | 210 | ScalarType = ScalarTypeKind::Invalid; |
865 | 210 | return; |
866 | 210 | } |
867 | 210 | break; |
868 | 420 | case FixedLMULType::SmallerThan: |
869 | 420 | if (Log2LMUL >= LMUL.Log2LMUL) { |
870 | 210 | ScalarType = ScalarTypeKind::Invalid; |
871 | 210 | return; |
872 | 210 | } |
873 | 210 | break; |
874 | 210 | case FixedLMULType::SmallerOrEqual: |
875 | 0 | if (Log2LMUL > LMUL.Log2LMUL) { |
876 | 0 | ScalarType = ScalarTypeKind::Invalid; |
877 | 0 | return; |
878 | 0 | } |
879 | 0 | break; |
880 | 840 | } |
881 | | |
882 | | // Update new LMUL |
883 | 420 | LMUL = LMULType(Log2LMUL); |
884 | 420 | Scale = LMUL.getScale(ElementBitwidth); |
885 | 420 | } |
886 | | |
887 | | std::optional<RVVTypes> |
888 | | RVVTypeCache::computeTypes(BasicType BT, int Log2LMUL, unsigned NF, |
889 | 64.0k | ArrayRef<PrototypeDescriptor> Prototype) { |
890 | 64.0k | RVVTypes Types; |
891 | 300k | for (const PrototypeDescriptor &Proto : Prototype) { |
892 | 300k | auto T = computeType(BT, Log2LMUL, Proto); |
893 | 300k | if (!T) |
894 | 10.4k | return std::nullopt; |
895 | | // Record legal type index |
896 | 289k | Types.push_back(*T); |
897 | 289k | } |
898 | 53.6k | return Types; |
899 | 64.0k | } |
900 | | |
901 | | // Compute the hash value of RVVType, used for cache the result of computeType. |
902 | | static uint64_t computeRVVTypeHashValue(BasicType BT, int Log2LMUL, |
903 | 316k | PrototypeDescriptor Proto) { |
904 | | // Layout of hash value: |
905 | | // 0 8 16 24 32 40 |
906 | | // | Log2LMUL + 3 | BT | Proto.PT | Proto.TM | Proto.VTM | |
907 | 316k | assert(Log2LMUL >= -3 && Log2LMUL <= 3); |
908 | 316k | return (Log2LMUL + 3) | (static_cast<uint64_t>(BT) & 0xff) << 8 | |
909 | 316k | ((uint64_t)(Proto.PT & 0xff) << 16) | |
910 | 316k | ((uint64_t)(Proto.TM & 0xff) << 24) | |
911 | 316k | ((uint64_t)(Proto.VTM & 0xff) << 32); |
912 | 316k | } |
913 | | |
914 | | std::optional<RVVTypePtr> RVVTypeCache::computeType(BasicType BT, int Log2LMUL, |
915 | 316k | PrototypeDescriptor Proto) { |
916 | 316k | uint64_t Idx = computeRVVTypeHashValue(BT, Log2LMUL, Proto); |
917 | | // Search first |
918 | 316k | auto It = LegalTypes.find(Idx); |
919 | 316k | if (It != LegalTypes.end()) |
920 | 304k | return &(It->second); |
921 | | |
922 | 12.1k | if (IllegalTypes.count(Idx)) |
923 | 9.39k | return std::nullopt; |
924 | | |
925 | | // Compute type and record the result. |
926 | 2.80k | RVVType T(BT, Log2LMUL, Proto); |
927 | 2.80k | if (T.isValid()) { |
928 | | // Record legal type index and value. |
929 | 1.75k | std::pair<std::unordered_map<uint64_t, RVVType>::iterator, bool> |
930 | 1.75k | InsertResult = LegalTypes.insert({Idx, T}); |
931 | 1.75k | return &(InsertResult.first->second); |
932 | 1.75k | } |
933 | | // Record illegal type index. |
934 | 1.04k | IllegalTypes.insert(Idx); |
935 | 1.04k | return std::nullopt; |
936 | 2.80k | } |
937 | | |
938 | | //===----------------------------------------------------------------------===// |
939 | | // RVVIntrinsic implementation |
940 | | //===----------------------------------------------------------------------===// |
941 | | RVVIntrinsic::RVVIntrinsic( |
942 | | StringRef NewName, StringRef Suffix, StringRef NewOverloadedName, |
943 | | StringRef OverloadedSuffix, StringRef IRName, bool IsMasked, |
944 | | bool HasMaskedOffOperand, bool HasVL, PolicyScheme Scheme, |
945 | | bool SupportOverloading, bool HasBuiltinAlias, StringRef ManualCodegen, |
946 | | const RVVTypes &OutInTypes, const std::vector<int64_t> &NewIntrinsicTypes, |
947 | | const std::vector<StringRef> &RequiredFeatures, unsigned NF, |
948 | | Policy NewPolicyAttrs, bool HasFRMRoundModeOp) |
949 | 0 | : IRName(IRName), IsMasked(IsMasked), |
950 | 0 | HasMaskedOffOperand(HasMaskedOffOperand), HasVL(HasVL), Scheme(Scheme), |
951 | 0 | SupportOverloading(SupportOverloading), HasBuiltinAlias(HasBuiltinAlias), |
952 | 0 | ManualCodegen(ManualCodegen.str()), NF(NF), PolicyAttrs(NewPolicyAttrs) { |
953 | | |
954 | | // Init BuiltinName, Name and OverloadedName |
955 | 0 | BuiltinName = NewName.str(); |
956 | 0 | Name = BuiltinName; |
957 | 0 | if (NewOverloadedName.empty()) |
958 | 0 | OverloadedName = NewName.split("_").first.str(); |
959 | 0 | else |
960 | 0 | OverloadedName = NewOverloadedName.str(); |
961 | 0 | if (!Suffix.empty()) |
962 | 0 | Name += "_" + Suffix.str(); |
963 | 0 | if (!OverloadedSuffix.empty()) |
964 | 0 | OverloadedName += "_" + OverloadedSuffix.str(); |
965 | |
|
966 | 0 | updateNamesAndPolicy(IsMasked, hasPolicy(), Name, BuiltinName, OverloadedName, |
967 | 0 | PolicyAttrs, HasFRMRoundModeOp); |
968 | | |
969 | | // Init OutputType and InputTypes |
970 | 0 | OutputType = OutInTypes[0]; |
971 | 0 | InputTypes.assign(OutInTypes.begin() + 1, OutInTypes.end()); |
972 | | |
973 | | // IntrinsicTypes is unmasked TA version index. Need to update it |
974 | | // if there is merge operand (It is always in first operand). |
975 | 0 | IntrinsicTypes = NewIntrinsicTypes; |
976 | 0 | if ((IsMasked && hasMaskedOffOperand()) || |
977 | 0 | (!IsMasked && hasPassthruOperand())) { |
978 | 0 | for (auto &I : IntrinsicTypes) { |
979 | 0 | if (I >= 0) |
980 | 0 | I += NF; |
981 | 0 | } |
982 | 0 | } |
983 | 0 | } |
984 | | |
985 | 0 | std::string RVVIntrinsic::getBuiltinTypeStr() const { |
986 | 0 | std::string S; |
987 | 0 | S += OutputType->getBuiltinStr(); |
988 | 0 | for (const auto &T : InputTypes) { |
989 | 0 | S += T->getBuiltinStr(); |
990 | 0 | } |
991 | 0 | return S; |
992 | 0 | } |
993 | | |
994 | | std::string RVVIntrinsic::getSuffixStr( |
995 | | RVVTypeCache &TypeCache, BasicType Type, int Log2LMUL, |
996 | 25.8k | llvm::ArrayRef<PrototypeDescriptor> PrototypeDescriptors) { |
997 | 25.8k | SmallVector<std::string> SuffixStrs; |
998 | 25.8k | for (auto PD : PrototypeDescriptors) { |
999 | 16.2k | auto T = TypeCache.computeType(Type, Log2LMUL, PD); |
1000 | 16.2k | SuffixStrs.push_back((*T)->getShortStr()); |
1001 | 16.2k | } |
1002 | 25.8k | return join(SuffixStrs, "_"); |
1003 | 25.8k | } |
1004 | | |
1005 | | llvm::SmallVector<PrototypeDescriptor> RVVIntrinsic::computeBuiltinTypes( |
1006 | | llvm::ArrayRef<PrototypeDescriptor> Prototype, bool IsMasked, |
1007 | | bool HasMaskedOffOperand, bool HasVL, unsigned NF, |
1008 | 34.7k | PolicyScheme DefaultScheme, Policy PolicyAttrs, bool IsTuple) { |
1009 | 34.7k | SmallVector<PrototypeDescriptor> NewPrototype(Prototype.begin(), |
1010 | 34.7k | Prototype.end()); |
1011 | 34.7k | bool HasPassthruOp = DefaultScheme == PolicyScheme::HasPassthruOperand; |
1012 | 34.7k | if (IsMasked) { |
1013 | | // If HasMaskedOffOperand, insert result type as first input operand if |
1014 | | // need. |
1015 | 24.4k | if (HasMaskedOffOperand && !PolicyAttrs.isTAMAPolicy()20.8k ) { |
1016 | 19.6k | if (NF == 1) { |
1017 | 13.0k | NewPrototype.insert(NewPrototype.begin() + 1, NewPrototype[0]); |
1018 | 13.0k | } else if (6.56k NF > 16.56k ) { |
1019 | 6.56k | if (IsTuple) { |
1020 | 6.56k | PrototypeDescriptor BasePtrOperand = Prototype[1]; |
1021 | 6.56k | PrototypeDescriptor MaskoffType = PrototypeDescriptor( |
1022 | 6.56k | static_cast<uint8_t>(BaseTypeModifier::Vector), |
1023 | 6.56k | static_cast<uint8_t>(getTupleVTM(NF)), |
1024 | 6.56k | BasePtrOperand.TM & ~static_cast<uint8_t>(TypeModifier::Pointer)); |
1025 | 6.56k | NewPrototype.insert(NewPrototype.begin() + 1, MaskoffType); |
1026 | 6.56k | } else { |
1027 | | // Convert |
1028 | | // (void, op0 address, op1 address, ...) |
1029 | | // to |
1030 | | // (void, op0 address, op1 address, ..., maskedoff0, maskedoff1, ...) |
1031 | 0 | PrototypeDescriptor MaskoffType = NewPrototype[1]; |
1032 | 0 | MaskoffType.TM &= ~static_cast<uint8_t>(TypeModifier::Pointer); |
1033 | 0 | NewPrototype.insert(NewPrototype.begin() + NF + 1, NF, MaskoffType); |
1034 | 0 | } |
1035 | 6.56k | } |
1036 | 19.6k | } |
1037 | 24.4k | if (HasMaskedOffOperand && NF > 120.8k ) { |
1038 | | // Convert |
1039 | | // (void, op0 address, op1 address, ..., maskedoff0, maskedoff1, ...) |
1040 | | // to |
1041 | | // (void, op0 address, op1 address, ..., mask, maskedoff0, maskedoff1, |
1042 | | // ...) |
1043 | 7.40k | if (IsTuple) |
1044 | 7.40k | NewPrototype.insert(NewPrototype.begin() + 1, |
1045 | 7.40k | PrototypeDescriptor::Mask); |
1046 | 0 | else |
1047 | 0 | NewPrototype.insert(NewPrototype.begin() + NF + 1, |
1048 | 0 | PrototypeDescriptor::Mask); |
1049 | 16.9k | } else { |
1050 | | // If IsMasked, insert PrototypeDescriptor:Mask as first input operand. |
1051 | 16.9k | NewPrototype.insert(NewPrototype.begin() + 1, PrototypeDescriptor::Mask); |
1052 | 16.9k | } |
1053 | 24.4k | } else { |
1054 | 10.3k | if (NF == 1) { |
1055 | 6.50k | if (PolicyAttrs.isTUPolicy() && HasPassthruOp5.74k ) |
1056 | 4.88k | NewPrototype.insert(NewPrototype.begin(), NewPrototype[0]); |
1057 | 6.50k | } else if (3.81k PolicyAttrs.isTUPolicy()3.81k && HasPassthruOp2.18k ) { |
1058 | 2.18k | if (IsTuple) { |
1059 | 2.18k | PrototypeDescriptor BasePtrOperand = Prototype[0]; |
1060 | 2.18k | PrototypeDescriptor MaskoffType = PrototypeDescriptor( |
1061 | 2.18k | static_cast<uint8_t>(BaseTypeModifier::Vector), |
1062 | 2.18k | static_cast<uint8_t>(getTupleVTM(NF)), |
1063 | 2.18k | BasePtrOperand.TM & ~static_cast<uint8_t>(TypeModifier::Pointer)); |
1064 | 2.18k | NewPrototype.insert(NewPrototype.begin(), MaskoffType); |
1065 | 2.18k | } else { |
1066 | | // NF > 1 cases for segment load operations. |
1067 | | // Convert |
1068 | | // (void, op0 address, op1 address, ...) |
1069 | | // to |
1070 | | // (void, op0 address, op1 address, maskedoff0, maskedoff1, ...) |
1071 | 0 | PrototypeDescriptor MaskoffType = Prototype[1]; |
1072 | 0 | MaskoffType.TM &= ~static_cast<uint8_t>(TypeModifier::Pointer); |
1073 | 0 | NewPrototype.insert(NewPrototype.begin() + NF + 1, NF, MaskoffType); |
1074 | 0 | } |
1075 | 2.18k | } |
1076 | 10.3k | } |
1077 | | |
1078 | | // If HasVL, append PrototypeDescriptor:VL to last operand |
1079 | 34.7k | if (HasVL) |
1080 | 34.6k | NewPrototype.push_back(PrototypeDescriptor::VL); |
1081 | | |
1082 | 34.7k | return NewPrototype; |
1083 | 34.7k | } |
1084 | | |
1085 | 2.39k | llvm::SmallVector<Policy> RVVIntrinsic::getSupportedUnMaskedPolicies() { |
1086 | 2.39k | return {Policy(Policy::PolicyType::Undisturbed)}; // TU |
1087 | 2.39k | } |
1088 | | |
1089 | | llvm::SmallVector<Policy> |
1090 | | RVVIntrinsic::getSupportedMaskedPolicies(bool HasTailPolicy, |
1091 | 2.39k | bool HasMaskPolicy) { |
1092 | 2.39k | if (HasTailPolicy && HasMaskPolicy2.35k ) |
1093 | 2.32k | return {Policy(Policy::PolicyType::Undisturbed, |
1094 | 2.32k | Policy::PolicyType::Agnostic), // TUM |
1095 | 2.32k | Policy(Policy::PolicyType::Undisturbed, |
1096 | 2.32k | Policy::PolicyType::Undisturbed), // TUMU |
1097 | 2.32k | Policy(Policy::PolicyType::Agnostic, |
1098 | 2.32k | Policy::PolicyType::Undisturbed)}; // MU |
1099 | 63 | if (HasTailPolicy && !HasMaskPolicy24 ) |
1100 | 24 | return {Policy(Policy::PolicyType::Undisturbed, |
1101 | 24 | Policy::PolicyType::Agnostic)}; // TU |
1102 | 39 | if (!HasTailPolicy && HasMaskPolicy) |
1103 | 39 | return {Policy(Policy::PolicyType::Agnostic, |
1104 | 39 | Policy::PolicyType::Undisturbed)}; // MU |
1105 | 0 | llvm_unreachable("An RVV instruction should not be without both tail policy " |
1106 | 0 | "and mask policy"); |
1107 | 0 | } |
1108 | | |
1109 | | void RVVIntrinsic::updateNamesAndPolicy( |
1110 | | bool IsMasked, bool HasPolicy, std::string &Name, std::string &BuiltinName, |
1111 | 53.6k | std::string &OverloadedName, Policy &PolicyAttrs, bool HasFRMRoundModeOp) { |
1112 | | |
1113 | 53.6k | auto appendPolicySuffix = [&](const std::string &suffix) { |
1114 | 30.1k | Name += suffix; |
1115 | 30.1k | BuiltinName += suffix; |
1116 | 30.1k | OverloadedName += suffix; |
1117 | 30.1k | }; |
1118 | | |
1119 | | // This follows the naming guideline under riscv-c-api-doc to add the |
1120 | | // `__riscv_` suffix for all RVV intrinsics. |
1121 | 53.6k | Name = "__riscv_" + Name; |
1122 | 53.6k | OverloadedName = "__riscv_" + OverloadedName; |
1123 | | |
1124 | 53.6k | if (HasFRMRoundModeOp) { |
1125 | 2.70k | Name += "_rm"; |
1126 | 2.70k | BuiltinName += "_rm"; |
1127 | 2.70k | } |
1128 | | |
1129 | 53.6k | if (IsMasked) { |
1130 | 32.7k | if (PolicyAttrs.isTUMUPolicy()) |
1131 | 7.05k | appendPolicySuffix("_tumu"); |
1132 | 25.7k | else if (PolicyAttrs.isTUMAPolicy()) |
1133 | 7.42k | appendPolicySuffix("_tum"); |
1134 | 18.2k | else if (PolicyAttrs.isTAMUPolicy()) |
1135 | 7.70k | appendPolicySuffix("_mu"); |
1136 | 10.5k | else if (PolicyAttrs.isTAMAPolicy()) { |
1137 | 10.5k | Name += "_m"; |
1138 | 10.5k | BuiltinName += "_m"; |
1139 | 10.5k | } else |
1140 | 0 | llvm_unreachable("Unhandled policy condition"); |
1141 | 32.7k | } else { |
1142 | 20.8k | if (PolicyAttrs.isTUPolicy()) |
1143 | 7.92k | appendPolicySuffix("_tu"); |
1144 | 12.9k | else if (PolicyAttrs.isTAPolicy()) // no suffix needed |
1145 | 12.9k | return; |
1146 | 0 | else |
1147 | 0 | llvm_unreachable("Unhandled policy condition"); |
1148 | 20.8k | } |
1149 | 53.6k | } |
1150 | | |
1151 | 0 | SmallVector<PrototypeDescriptor> parsePrototypes(StringRef Prototypes) { |
1152 | 0 | SmallVector<PrototypeDescriptor> PrototypeDescriptors; |
1153 | 0 | const StringRef Primaries("evwqom0ztul"); |
1154 | 0 | while (!Prototypes.empty()) { |
1155 | 0 | size_t Idx = 0; |
1156 | | // Skip over complex prototype because it could contain primitive type |
1157 | | // character. |
1158 | 0 | if (Prototypes[0] == '(') |
1159 | 0 | Idx = Prototypes.find_first_of(')'); |
1160 | 0 | Idx = Prototypes.find_first_of(Primaries, Idx); |
1161 | 0 | assert(Idx != StringRef::npos); |
1162 | 0 | auto PD = PrototypeDescriptor::parsePrototypeDescriptor( |
1163 | 0 | Prototypes.slice(0, Idx + 1)); |
1164 | 0 | if (!PD) |
1165 | 0 | llvm_unreachable("Error during parsing prototype."); |
1166 | 0 | PrototypeDescriptors.push_back(*PD); |
1167 | 0 | Prototypes = Prototypes.drop_front(Idx + 1); |
1168 | 0 | } |
1169 | 0 | return PrototypeDescriptors; |
1170 | 0 | } |
1171 | | |
1172 | 0 | raw_ostream &operator<<(raw_ostream &OS, const RVVIntrinsicRecord &Record) { |
1173 | 0 | OS << "{"; |
1174 | 0 | OS << "\"" << Record.Name << "\","; |
1175 | 0 | if (Record.OverloadedName == nullptr || |
1176 | 0 | StringRef(Record.OverloadedName).empty()) |
1177 | 0 | OS << "nullptr,"; |
1178 | 0 | else |
1179 | 0 | OS << "\"" << Record.OverloadedName << "\","; |
1180 | 0 | OS << Record.PrototypeIndex << ","; |
1181 | 0 | OS << Record.SuffixIndex << ","; |
1182 | 0 | OS << Record.OverloadedSuffixIndex << ","; |
1183 | 0 | OS << (int)Record.PrototypeLength << ","; |
1184 | 0 | OS << (int)Record.SuffixLength << ","; |
1185 | 0 | OS << (int)Record.OverloadedSuffixSize << ","; |
1186 | 0 | OS << (int)Record.RequiredExtensions << ","; |
1187 | 0 | OS << (int)Record.TypeRangeMask << ","; |
1188 | 0 | OS << (int)Record.Log2LMULMask << ","; |
1189 | 0 | OS << (int)Record.NF << ","; |
1190 | 0 | OS << (int)Record.HasMasked << ","; |
1191 | 0 | OS << (int)Record.HasVL << ","; |
1192 | 0 | OS << (int)Record.HasMaskedOffOperand << ","; |
1193 | 0 | OS << (int)Record.HasTailPolicy << ","; |
1194 | 0 | OS << (int)Record.HasMaskPolicy << ","; |
1195 | 0 | OS << (int)Record.HasFRMRoundModeOp << ","; |
1196 | 0 | OS << (int)Record.IsTuple << ","; |
1197 | 0 | OS << (int)Record.UnMaskedPolicyScheme << ","; |
1198 | 0 | OS << (int)Record.MaskedPolicyScheme << ","; |
1199 | 0 | OS << "},\n"; |
1200 | 0 | return OS; |
1201 | 0 | } |
1202 | | |
1203 | | } // end namespace RISCV |
1204 | | } // end namespace clang |