/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // Define several functions to decode x86 specific shuffle semantics using |
10 | | // constants from the constant pool. |
11 | | // |
12 | | //===----------------------------------------------------------------------===// |
13 | | |
14 | | #include "Utils/X86ShuffleDecode.h" |
15 | | #include "llvm/ADT/APInt.h" |
16 | | #include "llvm/IR/Constants.h" |
17 | | |
18 | | //===----------------------------------------------------------------------===// |
19 | | // Vector Mask Decoding |
20 | | //===----------------------------------------------------------------------===// |
21 | | |
22 | | namespace llvm { |
23 | | |
24 | | static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits, |
25 | | APInt &UndefElts, |
26 | 3.04k | SmallVectorImpl<uint64_t> &RawMask) { |
27 | 3.04k | // It is not an error for shuffle masks to not be a vector of |
28 | 3.04k | // MaskEltSizeInBits because the constant pool uniques constants by their |
29 | 3.04k | // bit representation. |
30 | 3.04k | // e.g. the following take up the same space in the constant pool: |
31 | 3.04k | // i128 -170141183420855150465331762880109871104 |
32 | 3.04k | // |
33 | 3.04k | // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160> |
34 | 3.04k | // |
35 | 3.04k | // <4 x i32> <i32 -2147483648, i32 -2147483648, |
36 | 3.04k | // i32 -2147483648, i32 -2147483648> |
37 | 3.04k | Type *CstTy = C->getType(); |
38 | 3.04k | if (!CstTy->isVectorTy()) |
39 | 0 | return false; |
40 | 3.04k | |
41 | 3.04k | Type *CstEltTy = CstTy->getVectorElementType(); |
42 | 3.04k | if (!CstEltTy->isIntegerTy()) |
43 | 0 | return false; |
44 | 3.04k | |
45 | 3.04k | unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits(); |
46 | 3.04k | unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits(); |
47 | 3.04k | unsigned NumCstElts = CstTy->getVectorNumElements(); |
48 | 3.04k | |
49 | 3.04k | assert((CstSizeInBits % MaskEltSizeInBits) == 0 && |
50 | 3.04k | "Unaligned shuffle mask size"); |
51 | 3.04k | |
52 | 3.04k | unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits; |
53 | 3.04k | UndefElts = APInt(NumMaskElts, 0); |
54 | 3.04k | RawMask.resize(NumMaskElts, 0); |
55 | 3.04k | |
56 | 3.04k | // Fast path - if the constants match the mask size then copy direct. |
57 | 3.04k | if (MaskEltSizeInBits == CstEltSizeInBits) { |
58 | 3.03k | assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size"); |
59 | 65.6k | for (unsigned i = 0; i != NumMaskElts; ++i62.6k ) { |
60 | 62.6k | Constant *COp = C->getAggregateElement(i); |
61 | 62.6k | if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)54.7k )) |
62 | 0 | return false; |
63 | 62.6k | |
64 | 62.6k | if (isa<UndefValue>(COp)) { |
65 | 7.87k | UndefElts.setBit(i); |
66 | 7.87k | RawMask[i] = 0; |
67 | 7.87k | continue; |
68 | 7.87k | } |
69 | 54.7k | |
70 | 54.7k | auto *Elt = cast<ConstantInt>(COp); |
71 | 54.7k | RawMask[i] = Elt->getValue().getZExtValue(); |
72 | 54.7k | } |
73 | 3.03k | return true; |
74 | 4 | } |
75 | 4 | |
76 | 4 | // Extract all the undef/constant element data and pack into single bitsets. |
77 | 4 | APInt UndefBits(CstSizeInBits, 0); |
78 | 4 | APInt MaskBits(CstSizeInBits, 0); |
79 | 36 | for (unsigned i = 0; i != NumCstElts; ++i32 ) { |
80 | 32 | Constant *COp = C->getAggregateElement(i); |
81 | 32 | if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp))) |
82 | 0 | return false; |
83 | 32 | |
84 | 32 | unsigned BitOffset = i * CstEltSizeInBits; |
85 | 32 | |
86 | 32 | if (isa<UndefValue>(COp)) { |
87 | 0 | UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits); |
88 | 0 | continue; |
89 | 0 | } |
90 | 32 | |
91 | 32 | MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset); |
92 | 32 | } |
93 | 4 | |
94 | 4 | // Now extract the undef/constant bit data into the raw shuffle masks. |
95 | 20 | for (unsigned i = 0; 4 i != NumMaskElts; ++i16 ) { |
96 | 16 | unsigned BitOffset = i * MaskEltSizeInBits; |
97 | 16 | APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset); |
98 | 16 | |
99 | 16 | // Only treat the element as UNDEF if all bits are UNDEF, otherwise |
100 | 16 | // treat it as zero. |
101 | 16 | if (EltUndef.isAllOnesValue()) { |
102 | 0 | UndefElts.setBit(i); |
103 | 0 | RawMask[i] = 0; |
104 | 0 | continue; |
105 | 0 | } |
106 | 16 | |
107 | 16 | APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset); |
108 | 16 | RawMask[i] = EltBits.getZExtValue(); |
109 | 16 | } |
110 | 4 | |
111 | 4 | return true; |
112 | 4 | } |
113 | | |
114 | | void DecodePSHUFBMask(const Constant *C, unsigned Width, |
115 | 2.75k | SmallVectorImpl<int> &ShuffleMask) { |
116 | 2.75k | assert((Width == 128 || Width == 256 || Width == 512) && |
117 | 2.75k | C->getType()->getPrimitiveSizeInBits() >= Width && |
118 | 2.75k | "Unexpected vector size."); |
119 | 2.75k | |
120 | 2.75k | // The shuffle mask requires a byte vector. |
121 | 2.75k | APInt UndefElts; |
122 | 2.75k | SmallVector<uint64_t, 64> RawMask; |
123 | 2.75k | if (!extractConstantMask(C, 8, UndefElts, RawMask)) |
124 | 0 | return; |
125 | 2.75k | |
126 | 2.75k | unsigned NumElts = Width / 8; |
127 | 2.75k | assert((NumElts == 16 || NumElts == 32 || NumElts == 64) && |
128 | 2.75k | "Unexpected number of vector elements."); |
129 | 2.75k | |
130 | 62.3k | for (unsigned i = 0; i != NumElts; ++i59.6k ) { |
131 | 59.6k | if (UndefElts[i]) { |
132 | 7.70k | ShuffleMask.push_back(SM_SentinelUndef); |
133 | 7.70k | continue; |
134 | 7.70k | } |
135 | 51.8k | |
136 | 51.8k | uint64_t Element = RawMask[i]; |
137 | 51.8k | // If the high bit (7) of the byte is set, the element is zeroed. |
138 | 51.8k | if (Element & (1 << 7)) |
139 | 5.88k | ShuffleMask.push_back(SM_SentinelZero); |
140 | 46.0k | else { |
141 | 46.0k | // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte |
142 | 46.0k | // lane of the vector we're inside. |
143 | 46.0k | unsigned Base = i & ~0xf; |
144 | 46.0k | |
145 | 46.0k | // Only the least significant 4 bits of the byte are used. |
146 | 46.0k | int Index = Base + (Element & 0xf); |
147 | 46.0k | ShuffleMask.push_back(Index); |
148 | 46.0k | } |
149 | 51.8k | } |
150 | 2.75k | } |
151 | | |
152 | | void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width, |
153 | 191 | SmallVectorImpl<int> &ShuffleMask) { |
154 | 191 | assert((Width == 128 || Width == 256 || Width == 512) && |
155 | 191 | C->getType()->getPrimitiveSizeInBits() >= Width && |
156 | 191 | "Unexpected vector size."); |
157 | 191 | assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size."); |
158 | 191 | |
159 | 191 | // The shuffle mask requires elements the same size as the target. |
160 | 191 | APInt UndefElts; |
161 | 191 | SmallVector<uint64_t, 16> RawMask; |
162 | 191 | if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) |
163 | 0 | return; |
164 | 191 | |
165 | 191 | unsigned NumElts = Width / ElSize; |
166 | 191 | unsigned NumEltsPerLane = 128 / ElSize; |
167 | 191 | assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) && |
168 | 191 | "Unexpected number of vector elements."); |
169 | 191 | |
170 | 2.00k | for (unsigned i = 0; i != NumElts; ++i1.81k ) { |
171 | 1.81k | if (UndefElts[i]) { |
172 | 140 | ShuffleMask.push_back(SM_SentinelUndef); |
173 | 140 | continue; |
174 | 140 | } |
175 | 1.67k | |
176 | 1.67k | int Index = i & ~(NumEltsPerLane - 1); |
177 | 1.67k | uint64_t Element = RawMask[i]; |
178 | 1.67k | if (ElSize == 64) |
179 | 0 | Index += (Element >> 1) & 0x1; |
180 | 1.67k | else |
181 | 1.67k | Index += Element & 0x3; |
182 | 1.67k | |
183 | 1.67k | ShuffleMask.push_back(Index); |
184 | 1.67k | } |
185 | 191 | } |
186 | | |
187 | | void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize, |
188 | | unsigned Width, |
189 | 27 | SmallVectorImpl<int> &ShuffleMask) { |
190 | 27 | Type *MaskTy = C->getType(); |
191 | 27 | unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); |
192 | 27 | (void)MaskTySize; |
193 | 27 | assert((MaskTySize == 128 || MaskTySize == 256) && |
194 | 27 | Width >= MaskTySize && "Unexpected vector size."); |
195 | 27 | |
196 | 27 | // The shuffle mask requires elements the same size as the target. |
197 | 27 | APInt UndefElts; |
198 | 27 | SmallVector<uint64_t, 8> RawMask; |
199 | 27 | if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) |
200 | 0 | return; |
201 | 27 | |
202 | 27 | unsigned NumElts = Width / ElSize; |
203 | 27 | unsigned NumEltsPerLane = 128 / ElSize; |
204 | 27 | assert((NumElts == 2 || NumElts == 4 || NumElts == 8) && |
205 | 27 | "Unexpected number of vector elements."); |
206 | 27 | |
207 | 175 | for (unsigned i = 0; i != NumElts; ++i148 ) { |
208 | 148 | if (UndefElts[i]) { |
209 | 2 | ShuffleMask.push_back(SM_SentinelUndef); |
210 | 2 | continue; |
211 | 2 | } |
212 | 146 | |
213 | 146 | // VPERMIL2 Operation. |
214 | 146 | // Bits[3] - Match Bit. |
215 | 146 | // Bits[2:1] - (Per Lane) PD Shuffle Mask. |
216 | 146 | // Bits[2:0] - (Per Lane) PS Shuffle Mask. |
217 | 146 | uint64_t Selector = RawMask[i]; |
218 | 146 | unsigned MatchBit = (Selector >> 3) & 0x1; |
219 | 146 | |
220 | 146 | // M2Z[0:1] MatchBit |
221 | 146 | // 0Xb X Source selected by Selector index. |
222 | 146 | // 10b 0 Source selected by Selector index. |
223 | 146 | // 10b 1 Zero. |
224 | 146 | // 11b 0 Zero. |
225 | 146 | // 11b 1 Source selected by Selector index. |
226 | 146 | if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)88 ) { |
227 | 28 | ShuffleMask.push_back(SM_SentinelZero); |
228 | 28 | continue; |
229 | 28 | } |
230 | 118 | |
231 | 118 | int Index = i & ~(NumEltsPerLane - 1); |
232 | 118 | if (ElSize == 64) |
233 | 32 | Index += (Selector >> 1) & 0x1; |
234 | 86 | else |
235 | 86 | Index += Selector & 0x3; |
236 | 118 | |
237 | 118 | int Src = (Selector >> 2) & 0x1; |
238 | 118 | Index += Src * NumElts; |
239 | 118 | ShuffleMask.push_back(Index); |
240 | 118 | } |
241 | 27 | } |
242 | | |
243 | | void DecodeVPPERMMask(const Constant *C, unsigned Width, |
244 | 68 | SmallVectorImpl<int> &ShuffleMask) { |
245 | 68 | Type *MaskTy = C->getType(); |
246 | 68 | unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); |
247 | 68 | (void)MaskTySize; |
248 | 68 | assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size."); |
249 | 68 | |
250 | 68 | // The shuffle mask requires a byte vector. |
251 | 68 | APInt UndefElts; |
252 | 68 | SmallVector<uint64_t, 16> RawMask; |
253 | 68 | if (!extractConstantMask(C, 8, UndefElts, RawMask)) |
254 | 0 | return; |
255 | 68 | |
256 | 68 | unsigned NumElts = Width / 8; |
257 | 68 | assert(NumElts == 16 && "Unexpected number of vector elements."); |
258 | 68 | |
259 | 764 | for (unsigned i = 0; i != NumElts; ++i696 ) { |
260 | 722 | if (UndefElts[i]) { |
261 | 27 | ShuffleMask.push_back(SM_SentinelUndef); |
262 | 27 | continue; |
263 | 27 | } |
264 | 695 | |
265 | 695 | // VPPERM Operation |
266 | 695 | // Bits[4:0] - Byte Index (0 - 31) |
267 | 695 | // Bits[7:5] - Permute Operation |
268 | 695 | // |
269 | 695 | // Permute Operation: |
270 | 695 | // 0 - Source byte (no logical operation). |
271 | 695 | // 1 - Invert source byte. |
272 | 695 | // 2 - Bit reverse of source byte. |
273 | 695 | // 3 - Bit reverse of inverted source byte. |
274 | 695 | // 4 - 00h (zero - fill). |
275 | 695 | // 5 - FFh (ones - fill). |
276 | 695 | // 6 - Most significant bit of source byte replicated in all bit positions. |
277 | 695 | // 7 - Invert most significant bit of source byte and replicate in all bit |
278 | 695 | // positions. |
279 | 695 | uint64_t Element = RawMask[i]; |
280 | 695 | uint64_t Index = Element & 0x1F; |
281 | 695 | uint64_t PermuteOp = (Element >> 5) & 0x7; |
282 | 695 | |
283 | 695 | if (PermuteOp == 4) { |
284 | 28 | ShuffleMask.push_back(SM_SentinelZero); |
285 | 28 | continue; |
286 | 28 | } |
287 | 667 | if (PermuteOp != 0) { |
288 | 26 | ShuffleMask.clear(); |
289 | 26 | return; |
290 | 26 | } |
291 | 641 | ShuffleMask.push_back((int)Index); |
292 | 641 | } |
293 | 68 | } |
294 | | |
295 | | void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width, |
296 | 0 | SmallVectorImpl<int> &ShuffleMask) { |
297 | 0 | assert((Width == 128 || Width == 256 || Width == 512) && |
298 | 0 | C->getType()->getPrimitiveSizeInBits() >= Width && |
299 | 0 | "Unexpected vector size."); |
300 | 0 | assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && |
301 | 0 | "Unexpected vector element size."); |
302 | 0 |
|
303 | 0 | // The shuffle mask requires elements the same size as the target. |
304 | 0 | APInt UndefElts; |
305 | 0 | SmallVector<uint64_t, 64> RawMask; |
306 | 0 | if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) |
307 | 0 | return; |
308 | 0 | |
309 | 0 | unsigned NumElts = Width / ElSize; |
310 | 0 |
|
311 | 0 | for (unsigned i = 0; i != NumElts; ++i) { |
312 | 0 | if (UndefElts[i]) { |
313 | 0 | ShuffleMask.push_back(SM_SentinelUndef); |
314 | 0 | continue; |
315 | 0 | } |
316 | 0 | int Index = RawMask[i] & (NumElts - 1); |
317 | 0 | ShuffleMask.push_back(Index); |
318 | 0 | } |
319 | 0 | } |
320 | | |
321 | | void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width, |
322 | 0 | SmallVectorImpl<int> &ShuffleMask) { |
323 | 0 | assert((Width == 128 || Width == 256 || Width == 512) && |
324 | 0 | C->getType()->getPrimitiveSizeInBits() >= Width && |
325 | 0 | "Unexpected vector size."); |
326 | 0 | assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && |
327 | 0 | "Unexpected vector element size."); |
328 | 0 |
|
329 | 0 | // The shuffle mask requires elements the same size as the target. |
330 | 0 | APInt UndefElts; |
331 | 0 | SmallVector<uint64_t, 64> RawMask; |
332 | 0 | if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) |
333 | 0 | return; |
334 | 0 | |
335 | 0 | unsigned NumElts = Width / ElSize; |
336 | 0 |
|
337 | 0 | for (unsigned i = 0; i != NumElts; ++i) { |
338 | 0 | if (UndefElts[i]) { |
339 | 0 | ShuffleMask.push_back(SM_SentinelUndef); |
340 | 0 | continue; |
341 | 0 | } |
342 | 0 | int Index = RawMask[i] & (NumElts*2 - 1); |
343 | 0 | ShuffleMask.push_back(Index); |
344 | 0 | } |
345 | 0 | } |
346 | | } // llvm namespace |