Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// Define several functions to decode x86 specific shuffle semantics using
11
// constants from the constant pool.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#include "X86ShuffleDecodeConstantPool.h"
16
#include "Utils/X86ShuffleDecode.h"
17
#include "llvm/ADT/APInt.h"
18
#include "llvm/CodeGen/MachineValueType.h"
19
#include "llvm/IR/Constants.h"
20
21
//===----------------------------------------------------------------------===//
22
//  Vector Mask Decoding
23
//===----------------------------------------------------------------------===//
24
25
namespace llvm {
26
27
static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
28
                                APInt &UndefElts,
29
37.4k
                                SmallVectorImpl<uint64_t> &RawMask) {
30
37.4k
  // It is not an error for shuffle masks to not be a vector of
31
37.4k
  // MaskEltSizeInBits because the constant pool uniques constants by their
32
37.4k
  // bit representation.
33
37.4k
  // e.g. the following take up the same space in the constant pool:
34
37.4k
  //   i128 -170141183420855150465331762880109871104
35
37.4k
  //
36
37.4k
  //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
37
37.4k
  //
38
37.4k
  //   <4 x i32> <i32 -2147483648, i32 -2147483648,
39
37.4k
  //              i32 -2147483648, i32 -2147483648>
40
37.4k
  Type *CstTy = C->getType();
41
37.4k
  if (!CstTy->isVectorTy())
42
0
    return false;
43
37.4k
44
37.4k
  Type *CstEltTy = CstTy->getVectorElementType();
45
37.4k
  if (!CstEltTy->isIntegerTy())
46
0
    return false;
47
37.4k
48
37.4k
  unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
49
37.4k
  unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
50
37.4k
  unsigned NumCstElts = CstTy->getVectorNumElements();
51
37.4k
52
37.4k
  assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
53
37.4k
         "Unaligned shuffle mask size");
54
37.4k
55
37.4k
  unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
56
37.4k
  UndefElts = APInt(NumMaskElts, 0);
57
37.4k
  RawMask.resize(NumMaskElts, 0);
58
37.4k
59
37.4k
  // Fast path - if the constants match the mask size then copy direct.
60
37.4k
  if (
MaskEltSizeInBits == CstEltSizeInBits37.4k
) {
61
37.1k
    assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
62
636k
    for (unsigned i = 0; 
i != NumMaskElts636k
;
++i599k
) {
63
599k
      Constant *COp = C->getAggregateElement(i);
64
599k
      if (
!COp || 599k
(!isa<UndefValue>(COp) && 599k
!isa<ConstantInt>(COp)212k
))
65
0
        return false;
66
599k
67
599k
      
if (599k
isa<UndefValue>(COp)599k
) {
68
386k
        UndefElts.setBit(i);
69
386k
        RawMask[i] = 0;
70
386k
        continue;
71
386k
      }
72
212k
73
212k
      auto *Elt = cast<ConstantInt>(COp);
74
212k
      RawMask[i] = Elt->getValue().getZExtValue();
75
212k
    }
76
37.1k
    return true;
77
232
  }
78
232
79
232
  // Extract all the undef/constant element data and pack into single bitsets.
80
232
  APInt UndefBits(CstSizeInBits, 0);
81
232
  APInt MaskBits(CstSizeInBits, 0);
82
3.91k
  for (unsigned i = 0; 
i != NumCstElts3.91k
;
++i3.68k
) {
83
3.68k
    Constant *COp = C->getAggregateElement(i);
84
3.68k
    if (
!COp || 3.68k
(!isa<UndefValue>(COp) && 3.68k
!isa<ConstantInt>(COp)3.34k
))
85
0
      return false;
86
3.68k
87
3.68k
    unsigned BitOffset = i * CstEltSizeInBits;
88
3.68k
89
3.68k
    if (
isa<UndefValue>(COp)3.68k
) {
90
332
      UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
91
332
      continue;
92
332
    }
93
3.34k
94
3.34k
    MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
95
3.34k
  }
96
232
97
232
  // Now extract the undef/constant bit data into the raw shuffle masks.
98
2.07k
  
for (unsigned i = 0; 232
i != NumMaskElts2.07k
;
++i1.84k
) {
99
1.84k
    unsigned BitOffset = i * MaskEltSizeInBits;
100
1.84k
    APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
101
1.84k
102
1.84k
    // Only treat the element as UNDEF if all bits are UNDEF, otherwise
103
1.84k
    // treat it as zero.
104
1.84k
    if (
EltUndef.isAllOnesValue()1.84k
) {
105
166
      UndefElts.setBit(i);
106
166
      RawMask[i] = 0;
107
166
      continue;
108
166
    }
109
1.67k
110
1.67k
    APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
111
1.67k
    RawMask[i] = EltBits.getZExtValue();
112
1.67k
  }
113
232
114
232
  return true;
115
37.4k
}
116
117
33.4k
void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
118
33.4k
  Type *MaskTy = C->getType();
119
33.4k
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
120
33.4k
  (void)MaskTySize;
121
33.4k
  assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
122
33.4k
         "Unexpected vector size.");
123
33.4k
124
33.4k
  // The shuffle mask requires a byte vector.
125
33.4k
  APInt UndefElts;
126
33.4k
  SmallVector<uint64_t, 64> RawMask;
127
33.4k
  if (!extractConstantMask(C, 8, UndefElts, RawMask))
128
0
    return;
129
33.4k
130
33.4k
  unsigned NumElts = RawMask.size();
131
33.4k
  assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
132
33.4k
         "Unexpected number of vector elements.");
133
33.4k
134
588k
  for (unsigned i = 0; 
i != NumElts588k
;
++i554k
) {
135
554k
    if (
UndefElts[i]554k
) {
136
382k
      ShuffleMask.push_back(SM_SentinelUndef);
137
382k
      continue;
138
382k
    }
139
172k
140
172k
    uint64_t Element = RawMask[i];
141
172k
    // If the high bit (7) of the byte is set, the element is zeroed.
142
172k
    if (Element & (1 << 7))
143
14.9k
      ShuffleMask.push_back(SM_SentinelZero);
144
157k
    else {
145
157k
      // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
146
157k
      // lane of the vector we're inside.
147
157k
      unsigned Base = i & ~0xf;
148
157k
149
157k
      // Only the least significant 4 bits of the byte are used.
150
157k
      int Index = Base + (Element & 0xf);
151
157k
      ShuffleMask.push_back(Index);
152
157k
    }
153
554k
  }
154
33.4k
}
155
156
void DecodeVPERMILPMask(const Constant *C, unsigned ElSize,
157
361
                        SmallVectorImpl<int> &ShuffleMask) {
158
361
  Type *MaskTy = C->getType();
159
361
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
160
361
  (void)MaskTySize;
161
361
  assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
162
361
         "Unexpected vector size.");
163
361
  assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
164
361
165
361
  // The shuffle mask requires elements the same size as the target.
166
361
  APInt UndefElts;
167
361
  SmallVector<uint64_t, 16> RawMask;
168
361
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
169
0
    return;
170
361
171
361
  unsigned NumElts = RawMask.size();
172
361
  unsigned NumEltsPerLane = 128 / ElSize;
173
361
  assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
174
361
         "Unexpected number of vector elements.");
175
361
176
3.36k
  for (unsigned i = 0; 
i != NumElts3.36k
;
++i3.00k
) {
177
3.00k
    if (
UndefElts[i]3.00k
) {
178
568
      ShuffleMask.push_back(SM_SentinelUndef);
179
568
      continue;
180
568
    }
181
2.44k
182
2.44k
    int Index = i & ~(NumEltsPerLane - 1);
183
2.44k
    uint64_t Element = RawMask[i];
184
2.44k
    if (ElSize == 64)
185
0
      Index += (Element >> 1) & 0x1;
186
2.44k
    else
187
2.44k
      Index += Element & 0x3;
188
3.00k
189
3.00k
    ShuffleMask.push_back(Index);
190
3.00k
  }
191
361
}
192
193
void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
194
68
                         SmallVectorImpl<int> &ShuffleMask) {
195
68
  Type *MaskTy = C->getType();
196
68
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
197
68
  (void)MaskTySize;
198
68
  assert((MaskTySize == 128 || MaskTySize == 256) && "Unexpected vector size.");
199
68
200
68
  // The shuffle mask requires elements the same size as the target.
201
68
  APInt UndefElts;
202
68
  SmallVector<uint64_t, 8> RawMask;
203
68
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
204
0
    return;
205
68
206
68
  unsigned NumElts = RawMask.size();
207
68
  unsigned NumEltsPerLane = 128 / ElSize;
208
68
  assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
209
68
         "Unexpected number of vector elements.");
210
68
211
372
  for (unsigned i = 0; 
i != NumElts372
;
++i304
) {
212
304
    if (
UndefElts[i]304
) {
213
44
      ShuffleMask.push_back(SM_SentinelUndef);
214
44
      continue;
215
44
    }
216
260
217
260
    // VPERMIL2 Operation.
218
260
    // Bits[3] - Match Bit.
219
260
    // Bits[2:1] - (Per Lane) PD Shuffle Mask.
220
260
    // Bits[2:0] - (Per Lane) PS Shuffle Mask.
221
260
    uint64_t Selector = RawMask[i];
222
260
    unsigned MatchBit = (Selector >> 3) & 0x1;
223
260
224
260
    // M2Z[0:1]     MatchBit
225
260
    //   0Xb           X        Source selected by Selector index.
226
260
    //   10b           0        Source selected by Selector index.
227
260
    //   10b           1        Zero.
228
260
    //   11b           0        Zero.
229
260
    //   11b           1        Source selected by Selector index.
230
260
    if (
(M2Z & 0x2) != 0u && 260
MatchBit != (M2Z & 0x1)88
) {
231
28
      ShuffleMask.push_back(SM_SentinelZero);
232
28
      continue;
233
28
    }
234
232
235
232
    int Index = i & ~(NumEltsPerLane - 1);
236
232
    if (ElSize == 64)
237
24
      Index += (Selector >> 1) & 0x1;
238
232
    else
239
208
      Index += Selector & 0x3;
240
304
241
304
    int Src = (Selector >> 2) & 0x1;
242
304
    Index += Src * NumElts;
243
304
    ShuffleMask.push_back(Index);
244
304
  }
245
68
}
246
247
40
void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
248
40
  assert(C->getType()->getPrimitiveSizeInBits() == 128 &&
249
40
         "Unexpected vector size.");
250
40
251
40
  // The shuffle mask requires a byte vector.
252
40
  APInt UndefElts;
253
40
  SmallVector<uint64_t, 16> RawMask;
254
40
  if (!extractConstantMask(C, 8, UndefElts, RawMask))
255
0
    return;
256
40
257
40
  unsigned NumElts = RawMask.size();
258
40
  assert(NumElts == 16 && "Unexpected number of vector elements.");
259
40
260
416
  for (unsigned i = 0; 
i != NumElts416
;
++i376
) {
261
394
    if (
UndefElts[i]394
) {
262
8
      ShuffleMask.push_back(SM_SentinelUndef);
263
8
      continue;
264
8
    }
265
386
266
386
    // VPPERM Operation
267
386
    // Bits[4:0] - Byte Index (0 - 31)
268
386
    // Bits[7:5] - Permute Operation
269
386
    //
270
386
    // Permute Operation:
271
386
    // 0 - Source byte (no logical operation).
272
386
    // 1 - Invert source byte.
273
386
    // 2 - Bit reverse of source byte.
274
386
    // 3 - Bit reverse of inverted source byte.
275
386
    // 4 - 00h (zero - fill).
276
386
    // 5 - FFh (ones - fill).
277
386
    // 6 - Most significant bit of source byte replicated in all bit positions.
278
386
    // 7 - Invert most significant bit of source byte and replicate in all bit
279
386
    // positions.
280
386
    uint64_t Element = RawMask[i];
281
386
    uint64_t Index = Element & 0x1F;
282
386
    uint64_t PermuteOp = (Element >> 5) & 0x7;
283
386
284
386
    if (
PermuteOp == 4386
) {
285
28
      ShuffleMask.push_back(SM_SentinelZero);
286
28
      continue;
287
28
    }
288
358
    
if (358
PermuteOp != 0358
) {
289
18
      ShuffleMask.clear();
290
18
      return;
291
18
    }
292
340
    ShuffleMask.push_back((int)Index);
293
340
  }
294
40
}
295
296
void DecodeVPERMVMask(const Constant *C, unsigned ElSize,
297
729
                      SmallVectorImpl<int> &ShuffleMask) {
298
729
  Type *MaskTy = C->getType();
299
729
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
300
729
  (void)MaskTySize;
301
729
  assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
302
729
         "Unexpected vector size.");
303
729
  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
304
729
         "Unexpected vector element size.");
305
729
306
729
  // The shuffle mask requires elements the same size as the target.
307
729
  APInt UndefElts;
308
729
  SmallVector<uint64_t, 64> RawMask;
309
729
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
310
0
    return;
311
729
312
729
  unsigned NumElts = RawMask.size();
313
729
314
7.12k
  for (unsigned i = 0; 
i != NumElts7.12k
;
++i6.39k
) {
315
6.39k
    if (
UndefElts[i]6.39k
) {
316
2.85k
      ShuffleMask.push_back(SM_SentinelUndef);
317
2.85k
      continue;
318
2.85k
    }
319
3.53k
    int Index = RawMask[i] & (NumElts - 1);
320
3.53k
    ShuffleMask.push_back(Index);
321
3.53k
  }
322
729
}
323
324
void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize,
325
2.74k
                       SmallVectorImpl<int> &ShuffleMask) {
326
2.74k
  Type *MaskTy = C->getType();
327
2.74k
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
328
2.74k
  (void)MaskTySize;
329
2.74k
  assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) &&
330
2.74k
         "Unexpected vector size.");
331
2.74k
  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
332
2.74k
         "Unexpected vector element size.");
333
2.74k
334
2.74k
  // The shuffle mask requires elements the same size as the target.
335
2.74k
  APInt UndefElts;
336
2.74k
  SmallVector<uint64_t, 64> RawMask;
337
2.74k
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
338
0
    return;
339
2.74k
340
2.74k
  unsigned NumElts = RawMask.size();
341
2.74k
342
38.9k
  for (unsigned i = 0; 
i != NumElts38.9k
;
++i36.2k
) {
343
36.2k
    if (
UndefElts[i]36.2k
) {
344
1.06k
      ShuffleMask.push_back(SM_SentinelUndef);
345
1.06k
      continue;
346
1.06k
    }
347
35.1k
    int Index = RawMask[i] & (NumElts*2 - 1);
348
35.1k
    ShuffleMask.push_back(Index);
349
35.1k
  }
350
2.74k
}
351
} // llvm namespace