/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | // |
10 | | // Define several functions to decode x86 specific shuffle semantics using |
11 | | // constants from the constant pool. |
12 | | // |
13 | | //===----------------------------------------------------------------------===// |
14 | | |
15 | | #include "X86ShuffleDecodeConstantPool.h" |
16 | | #include "Utils/X86ShuffleDecode.h" |
17 | | #include "llvm/ADT/APInt.h" |
18 | | #include "llvm/CodeGen/MachineValueType.h" |
19 | | #include "llvm/IR/Constants.h" |
20 | | |
21 | | //===----------------------------------------------------------------------===// |
22 | | // Vector Mask Decoding |
23 | | //===----------------------------------------------------------------------===// |
24 | | |
25 | | namespace llvm { |
26 | | |
27 | | static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits, |
28 | | APInt &UndefElts, |
29 | 37.4k | SmallVectorImpl<uint64_t> &RawMask) { |
30 | 37.4k | // It is not an error for shuffle masks to not be a vector of |
31 | 37.4k | // MaskEltSizeInBits because the constant pool uniques constants by their |
32 | 37.4k | // bit representation. |
33 | 37.4k | // e.g. the following take up the same space in the constant pool: |
34 | 37.4k | // i128 -170141183420855150465331762880109871104 |
35 | 37.4k | // |
36 | 37.4k | // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160> |
37 | 37.4k | // |
38 | 37.4k | // <4 x i32> <i32 -2147483648, i32 -2147483648, |
39 | 37.4k | // i32 -2147483648, i32 -2147483648> |
40 | 37.4k | Type *CstTy = C->getType(); |
41 | 37.4k | if (!CstTy->isVectorTy()) |
42 | 0 | return false; |
43 | 37.4k | |
44 | 37.4k | Type *CstEltTy = CstTy->getVectorElementType(); |
45 | 37.4k | if (!CstEltTy->isIntegerTy()) |
46 | 0 | return false; |
47 | 37.4k | |
48 | 37.4k | unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits(); |
49 | 37.4k | unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits(); |
50 | 37.4k | unsigned NumCstElts = CstTy->getVectorNumElements(); |
51 | 37.4k | |
52 | 37.4k | assert((CstSizeInBits % MaskEltSizeInBits) == 0 && |
53 | 37.4k | "Unaligned shuffle mask size"); |
54 | 37.4k | |
55 | 37.4k | unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits; |
56 | 37.4k | UndefElts = APInt(NumMaskElts, 0); |
57 | 37.4k | RawMask.resize(NumMaskElts, 0); |
58 | 37.4k | |
59 | 37.4k | // Fast path - if the constants match the mask size then copy direct. |
60 | 37.4k | if (MaskEltSizeInBits == CstEltSizeInBits37.4k ) { |
61 | 37.1k | assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size"); |
62 | 636k | for (unsigned i = 0; i != NumMaskElts636k ; ++i599k ) { |
63 | 599k | Constant *COp = C->getAggregateElement(i); |
64 | 599k | if (!COp || 599k (!isa<UndefValue>(COp) && 599k !isa<ConstantInt>(COp)212k )) |
65 | 0 | return false; |
66 | 599k | |
67 | 599k | if (599k isa<UndefValue>(COp)599k ) { |
68 | 386k | UndefElts.setBit(i); |
69 | 386k | RawMask[i] = 0; |
70 | 386k | continue; |
71 | 386k | } |
72 | 212k | |
73 | 212k | auto *Elt = cast<ConstantInt>(COp); |
74 | 212k | RawMask[i] = Elt->getValue().getZExtValue(); |
75 | 212k | } |
76 | 37.1k | return true; |
77 | 232 | } |
78 | 232 | |
79 | 232 | // Extract all the undef/constant element data and pack into single bitsets. |
80 | 232 | APInt UndefBits(CstSizeInBits, 0); |
81 | 232 | APInt MaskBits(CstSizeInBits, 0); |
82 | 3.91k | for (unsigned i = 0; i != NumCstElts3.91k ; ++i3.68k ) { |
83 | 3.68k | Constant *COp = C->getAggregateElement(i); |
84 | 3.68k | if (!COp || 3.68k (!isa<UndefValue>(COp) && 3.68k !isa<ConstantInt>(COp)3.34k )) |
85 | 0 | return false; |
86 | 3.68k | |
87 | 3.68k | unsigned BitOffset = i * CstEltSizeInBits; |
88 | 3.68k | |
89 | 3.68k | if (isa<UndefValue>(COp)3.68k ) { |
90 | 332 | UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits); |
91 | 332 | continue; |
92 | 332 | } |
93 | 3.34k | |
94 | 3.34k | MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset); |
95 | 3.34k | } |
96 | 232 | |
97 | 232 | // Now extract the undef/constant bit data into the raw shuffle masks. |
98 | 2.07k | for (unsigned i = 0; 232 i != NumMaskElts2.07k ; ++i1.84k ) { |
99 | 1.84k | unsigned BitOffset = i * MaskEltSizeInBits; |
100 | 1.84k | APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset); |
101 | 1.84k | |
102 | 1.84k | // Only treat the element as UNDEF if all bits are UNDEF, otherwise |
103 | 1.84k | // treat it as zero. |
104 | 1.84k | if (EltUndef.isAllOnesValue()1.84k ) { |
105 | 166 | UndefElts.setBit(i); |
106 | 166 | RawMask[i] = 0; |
107 | 166 | continue; |
108 | 166 | } |
109 | 1.67k | |
110 | 1.67k | APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset); |
111 | 1.67k | RawMask[i] = EltBits.getZExtValue(); |
112 | 1.67k | } |
113 | 232 | |
114 | 232 | return true; |
115 | 37.4k | } |
116 | | |
117 | 33.4k | void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) { |
118 | 33.4k | Type *MaskTy = C->getType(); |
119 | 33.4k | unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); |
120 | 33.4k | (void)MaskTySize; |
121 | 33.4k | assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) && |
122 | 33.4k | "Unexpected vector size."); |
123 | 33.4k | |
124 | 33.4k | // The shuffle mask requires a byte vector. |
125 | 33.4k | APInt UndefElts; |
126 | 33.4k | SmallVector<uint64_t, 64> RawMask; |
127 | 33.4k | if (!extractConstantMask(C, 8, UndefElts, RawMask)) |
128 | 0 | return; |
129 | 33.4k | |
130 | 33.4k | unsigned NumElts = RawMask.size(); |
131 | 33.4k | assert((NumElts == 16 || NumElts == 32 || NumElts == 64) && |
132 | 33.4k | "Unexpected number of vector elements."); |
133 | 33.4k | |
134 | 588k | for (unsigned i = 0; i != NumElts588k ; ++i554k ) { |
135 | 554k | if (UndefElts[i]554k ) { |
136 | 382k | ShuffleMask.push_back(SM_SentinelUndef); |
137 | 382k | continue; |
138 | 382k | } |
139 | 172k | |
140 | 172k | uint64_t Element = RawMask[i]; |
141 | 172k | // If the high bit (7) of the byte is set, the element is zeroed. |
142 | 172k | if (Element & (1 << 7)) |
143 | 14.9k | ShuffleMask.push_back(SM_SentinelZero); |
144 | 157k | else { |
145 | 157k | // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte |
146 | 157k | // lane of the vector we're inside. |
147 | 157k | unsigned Base = i & ~0xf; |
148 | 157k | |
149 | 157k | // Only the least significant 4 bits of the byte are used. |
150 | 157k | int Index = Base + (Element & 0xf); |
151 | 157k | ShuffleMask.push_back(Index); |
152 | 157k | } |
153 | 554k | } |
154 | 33.4k | } |
155 | | |
156 | | void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, |
157 | 361 | SmallVectorImpl<int> &ShuffleMask) { |
158 | 361 | Type *MaskTy = C->getType(); |
159 | 361 | unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); |
160 | 361 | (void)MaskTySize; |
161 | 361 | assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) && |
162 | 361 | "Unexpected vector size."); |
163 | 361 | assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size."); |
164 | 361 | |
165 | 361 | // The shuffle mask requires elements the same size as the target. |
166 | 361 | APInt UndefElts; |
167 | 361 | SmallVector<uint64_t, 16> RawMask; |
168 | 361 | if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) |
169 | 0 | return; |
170 | 361 | |
171 | 361 | unsigned NumElts = RawMask.size(); |
172 | 361 | unsigned NumEltsPerLane = 128 / ElSize; |
173 | 361 | assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) && |
174 | 361 | "Unexpected number of vector elements."); |
175 | 361 | |
176 | 3.36k | for (unsigned i = 0; i != NumElts3.36k ; ++i3.00k ) { |
177 | 3.00k | if (UndefElts[i]3.00k ) { |
178 | 568 | ShuffleMask.push_back(SM_SentinelUndef); |
179 | 568 | continue; |
180 | 568 | } |
181 | 2.44k | |
182 | 2.44k | int Index = i & ~(NumEltsPerLane - 1); |
183 | 2.44k | uint64_t Element = RawMask[i]; |
184 | 2.44k | if (ElSize == 64) |
185 | 0 | Index += (Element >> 1) & 0x1; |
186 | 2.44k | else |
187 | 2.44k | Index += Element & 0x3; |
188 | 3.00k | |
189 | 3.00k | ShuffleMask.push_back(Index); |
190 | 3.00k | } |
191 | 361 | } |
192 | | |
193 | | void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize, |
194 | 68 | SmallVectorImpl<int> &ShuffleMask) { |
195 | 68 | Type *MaskTy = C->getType(); |
196 | 68 | unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); |
197 | 68 | (void)MaskTySize; |
198 | 68 | assert((MaskTySize == 128 || MaskTySize == 256) && "Unexpected vector size."); |
199 | 68 | |
200 | 68 | // The shuffle mask requires elements the same size as the target. |
201 | 68 | APInt UndefElts; |
202 | 68 | SmallVector<uint64_t, 8> RawMask; |
203 | 68 | if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) |
204 | 0 | return; |
205 | 68 | |
206 | 68 | unsigned NumElts = RawMask.size(); |
207 | 68 | unsigned NumEltsPerLane = 128 / ElSize; |
208 | 68 | assert((NumElts == 2 || NumElts == 4 || NumElts == 8) && |
209 | 68 | "Unexpected number of vector elements."); |
210 | 68 | |
211 | 372 | for (unsigned i = 0; i != NumElts372 ; ++i304 ) { |
212 | 304 | if (UndefElts[i]304 ) { |
213 | 44 | ShuffleMask.push_back(SM_SentinelUndef); |
214 | 44 | continue; |
215 | 44 | } |
216 | 260 | |
217 | 260 | // VPERMIL2 Operation. |
218 | 260 | // Bits[3] - Match Bit. |
219 | 260 | // Bits[2:1] - (Per Lane) PD Shuffle Mask. |
220 | 260 | // Bits[2:0] - (Per Lane) PS Shuffle Mask. |
221 | 260 | uint64_t Selector = RawMask[i]; |
222 | 260 | unsigned MatchBit = (Selector >> 3) & 0x1; |
223 | 260 | |
224 | 260 | // M2Z[0:1] MatchBit |
225 | 260 | // 0Xb X Source selected by Selector index. |
226 | 260 | // 10b 0 Source selected by Selector index. |
227 | 260 | // 10b 1 Zero. |
228 | 260 | // 11b 0 Zero. |
229 | 260 | // 11b 1 Source selected by Selector index. |
230 | 260 | if ((M2Z & 0x2) != 0u && 260 MatchBit != (M2Z & 0x1)88 ) { |
231 | 28 | ShuffleMask.push_back(SM_SentinelZero); |
232 | 28 | continue; |
233 | 28 | } |
234 | 232 | |
235 | 232 | int Index = i & ~(NumEltsPerLane - 1); |
236 | 232 | if (ElSize == 64) |
237 | 24 | Index += (Selector >> 1) & 0x1; |
238 | 232 | else |
239 | 208 | Index += Selector & 0x3; |
240 | 304 | |
241 | 304 | int Src = (Selector >> 2) & 0x1; |
242 | 304 | Index += Src * NumElts; |
243 | 304 | ShuffleMask.push_back(Index); |
244 | 304 | } |
245 | 68 | } |
246 | | |
247 | 40 | void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) { |
248 | 40 | assert(C->getType()->getPrimitiveSizeInBits() == 128 && |
249 | 40 | "Unexpected vector size."); |
250 | 40 | |
251 | 40 | // The shuffle mask requires a byte vector. |
252 | 40 | APInt UndefElts; |
253 | 40 | SmallVector<uint64_t, 16> RawMask; |
254 | 40 | if (!extractConstantMask(C, 8, UndefElts, RawMask)) |
255 | 0 | return; |
256 | 40 | |
257 | 40 | unsigned NumElts = RawMask.size(); |
258 | 40 | assert(NumElts == 16 && "Unexpected number of vector elements."); |
259 | 40 | |
260 | 416 | for (unsigned i = 0; i != NumElts416 ; ++i376 ) { |
261 | 394 | if (UndefElts[i]394 ) { |
262 | 8 | ShuffleMask.push_back(SM_SentinelUndef); |
263 | 8 | continue; |
264 | 8 | } |
265 | 386 | |
266 | 386 | // VPPERM Operation |
267 | 386 | // Bits[4:0] - Byte Index (0 - 31) |
268 | 386 | // Bits[7:5] - Permute Operation |
269 | 386 | // |
270 | 386 | // Permute Operation: |
271 | 386 | // 0 - Source byte (no logical operation). |
272 | 386 | // 1 - Invert source byte. |
273 | 386 | // 2 - Bit reverse of source byte. |
274 | 386 | // 3 - Bit reverse of inverted source byte. |
275 | 386 | // 4 - 00h (zero - fill). |
276 | 386 | // 5 - FFh (ones - fill). |
277 | 386 | // 6 - Most significant bit of source byte replicated in all bit positions. |
278 | 386 | // 7 - Invert most significant bit of source byte and replicate in all bit |
279 | 386 | // positions. |
280 | 386 | uint64_t Element = RawMask[i]; |
281 | 386 | uint64_t Index = Element & 0x1F; |
282 | 386 | uint64_t PermuteOp = (Element >> 5) & 0x7; |
283 | 386 | |
284 | 386 | if (PermuteOp == 4386 ) { |
285 | 28 | ShuffleMask.push_back(SM_SentinelZero); |
286 | 28 | continue; |
287 | 28 | } |
288 | 358 | if (358 PermuteOp != 0358 ) { |
289 | 18 | ShuffleMask.clear(); |
290 | 18 | return; |
291 | 18 | } |
292 | 340 | ShuffleMask.push_back((int)Index); |
293 | 340 | } |
294 | 40 | } |
295 | | |
296 | | void DecodeVPERMVMask(const Constant *C, unsigned ElSize, |
297 | 729 | SmallVectorImpl<int> &ShuffleMask) { |
298 | 729 | Type *MaskTy = C->getType(); |
299 | 729 | unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); |
300 | 729 | (void)MaskTySize; |
301 | 729 | assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) && |
302 | 729 | "Unexpected vector size."); |
303 | 729 | assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && |
304 | 729 | "Unexpected vector element size."); |
305 | 729 | |
306 | 729 | // The shuffle mask requires elements the same size as the target. |
307 | 729 | APInt UndefElts; |
308 | 729 | SmallVector<uint64_t, 64> RawMask; |
309 | 729 | if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) |
310 | 0 | return; |
311 | 729 | |
312 | 729 | unsigned NumElts = RawMask.size(); |
313 | 729 | |
314 | 7.12k | for (unsigned i = 0; i != NumElts7.12k ; ++i6.39k ) { |
315 | 6.39k | if (UndefElts[i]6.39k ) { |
316 | 2.85k | ShuffleMask.push_back(SM_SentinelUndef); |
317 | 2.85k | continue; |
318 | 2.85k | } |
319 | 3.53k | int Index = RawMask[i] & (NumElts - 1); |
320 | 3.53k | ShuffleMask.push_back(Index); |
321 | 3.53k | } |
322 | 729 | } |
323 | | |
324 | | void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, |
325 | 2.74k | SmallVectorImpl<int> &ShuffleMask) { |
326 | 2.74k | Type *MaskTy = C->getType(); |
327 | 2.74k | unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); |
328 | 2.74k | (void)MaskTySize; |
329 | 2.74k | assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) && |
330 | 2.74k | "Unexpected vector size."); |
331 | 2.74k | assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && |
332 | 2.74k | "Unexpected vector element size."); |
333 | 2.74k | |
334 | 2.74k | // The shuffle mask requires elements the same size as the target. |
335 | 2.74k | APInt UndefElts; |
336 | 2.74k | SmallVector<uint64_t, 64> RawMask; |
337 | 2.74k | if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) |
338 | 0 | return; |
339 | 2.74k | |
340 | 2.74k | unsigned NumElts = RawMask.size(); |
341 | 2.74k | |
342 | 38.9k | for (unsigned i = 0; i != NumElts38.9k ; ++i36.2k ) { |
343 | 36.2k | if (UndefElts[i]36.2k ) { |
344 | 1.06k | ShuffleMask.push_back(SM_SentinelUndef); |
345 | 1.06k | continue; |
346 | 1.06k | } |
347 | 35.1k | int Index = RawMask[i] & (NumElts*2 - 1); |
348 | 35.1k | ShuffleMask.push_back(Index); |
349 | 35.1k | } |
350 | 2.74k | } |
351 | | } // llvm namespace |