Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// Define several functions to decode x86 specific shuffle semantics using
10
// constants from the constant pool.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "Utils/X86ShuffleDecode.h"
15
#include "llvm/ADT/APInt.h"
16
#include "llvm/IR/Constants.h"
17
18
//===----------------------------------------------------------------------===//
19
//  Vector Mask Decoding
20
//===----------------------------------------------------------------------===//
21
22
namespace llvm {
23
24
static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
25
                                APInt &UndefElts,
26
3.04k
                                SmallVectorImpl<uint64_t> &RawMask) {
27
3.04k
  // It is not an error for shuffle masks to not be a vector of
28
3.04k
  // MaskEltSizeInBits because the constant pool uniques constants by their
29
3.04k
  // bit representation.
30
3.04k
  // e.g. the following take up the same space in the constant pool:
31
3.04k
  //   i128 -170141183420855150465331762880109871104
32
3.04k
  //
33
3.04k
  //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
34
3.04k
  //
35
3.04k
  //   <4 x i32> <i32 -2147483648, i32 -2147483648,
36
3.04k
  //              i32 -2147483648, i32 -2147483648>
37
3.04k
  Type *CstTy = C->getType();
38
3.04k
  if (!CstTy->isVectorTy())
39
0
    return false;
40
3.04k
41
3.04k
  Type *CstEltTy = CstTy->getVectorElementType();
42
3.04k
  if (!CstEltTy->isIntegerTy())
43
0
    return false;
44
3.04k
45
3.04k
  unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
46
3.04k
  unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
47
3.04k
  unsigned NumCstElts = CstTy->getVectorNumElements();
48
3.04k
49
3.04k
  assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
50
3.04k
         "Unaligned shuffle mask size");
51
3.04k
52
3.04k
  unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
53
3.04k
  UndefElts = APInt(NumMaskElts, 0);
54
3.04k
  RawMask.resize(NumMaskElts, 0);
55
3.04k
56
3.04k
  // Fast path - if the constants match the mask size then copy direct.
57
3.04k
  if (MaskEltSizeInBits == CstEltSizeInBits) {
58
3.03k
    assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
59
65.6k
    for (unsigned i = 0; i != NumMaskElts; 
++i62.6k
) {
60
62.6k
      Constant *COp = C->getAggregateElement(i);
61
62.6k
      if (!COp || (!isa<UndefValue>(COp) && 
!isa<ConstantInt>(COp)54.7k
))
62
0
        return false;
63
62.6k
64
62.6k
      if (isa<UndefValue>(COp)) {
65
7.87k
        UndefElts.setBit(i);
66
7.87k
        RawMask[i] = 0;
67
7.87k
        continue;
68
7.87k
      }
69
54.7k
70
54.7k
      auto *Elt = cast<ConstantInt>(COp);
71
54.7k
      RawMask[i] = Elt->getValue().getZExtValue();
72
54.7k
    }
73
3.03k
    return true;
74
4
  }
75
4
76
4
  // Extract all the undef/constant element data and pack into single bitsets.
77
4
  APInt UndefBits(CstSizeInBits, 0);
78
4
  APInt MaskBits(CstSizeInBits, 0);
79
36
  for (unsigned i = 0; i != NumCstElts; 
++i32
) {
80
32
    Constant *COp = C->getAggregateElement(i);
81
32
    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
82
0
      return false;
83
32
84
32
    unsigned BitOffset = i * CstEltSizeInBits;
85
32
86
32
    if (isa<UndefValue>(COp)) {
87
0
      UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
88
0
      continue;
89
0
    }
90
32
91
32
    MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
92
32
  }
93
4
94
4
  // Now extract the undef/constant bit data into the raw shuffle masks.
95
20
  
for (unsigned i = 0; 4
i != NumMaskElts;
++i16
) {
96
16
    unsigned BitOffset = i * MaskEltSizeInBits;
97
16
    APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
98
16
99
16
    // Only treat the element as UNDEF if all bits are UNDEF, otherwise
100
16
    // treat it as zero.
101
16
    if (EltUndef.isAllOnesValue()) {
102
0
      UndefElts.setBit(i);
103
0
      RawMask[i] = 0;
104
0
      continue;
105
0
    }
106
16
107
16
    APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
108
16
    RawMask[i] = EltBits.getZExtValue();
109
16
  }
110
4
111
4
  return true;
112
4
}
113
114
void DecodePSHUFBMask(const Constant *C, unsigned Width,
115
2.75k
                      SmallVectorImpl<int> &ShuffleMask) {
116
2.75k
  assert((Width == 128 || Width == 256 || Width == 512) &&
117
2.75k
         C->getType()->getPrimitiveSizeInBits() >= Width &&
118
2.75k
         "Unexpected vector size.");
119
2.75k
120
2.75k
  // The shuffle mask requires a byte vector.
121
2.75k
  APInt UndefElts;
122
2.75k
  SmallVector<uint64_t, 64> RawMask;
123
2.75k
  if (!extractConstantMask(C, 8, UndefElts, RawMask))
124
0
    return;
125
2.75k
126
2.75k
  unsigned NumElts = Width / 8;
127
2.75k
  assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
128
2.75k
         "Unexpected number of vector elements.");
129
2.75k
130
62.3k
  for (unsigned i = 0; i != NumElts; 
++i59.6k
) {
131
59.6k
    if (UndefElts[i]) {
132
7.70k
      ShuffleMask.push_back(SM_SentinelUndef);
133
7.70k
      continue;
134
7.70k
    }
135
51.8k
136
51.8k
    uint64_t Element = RawMask[i];
137
51.8k
    // If the high bit (7) of the byte is set, the element is zeroed.
138
51.8k
    if (Element & (1 << 7))
139
5.88k
      ShuffleMask.push_back(SM_SentinelZero);
140
46.0k
    else {
141
46.0k
      // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
142
46.0k
      // lane of the vector we're inside.
143
46.0k
      unsigned Base = i & ~0xf;
144
46.0k
145
46.0k
      // Only the least significant 4 bits of the byte are used.
146
46.0k
      int Index = Base + (Element & 0xf);
147
46.0k
      ShuffleMask.push_back(Index);
148
46.0k
    }
149
51.8k
  }
150
2.75k
}
151
152
void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,
153
191
                        SmallVectorImpl<int> &ShuffleMask) {
154
191
  assert((Width == 128 || Width == 256 || Width == 512) &&
155
191
         C->getType()->getPrimitiveSizeInBits() >= Width &&
156
191
         "Unexpected vector size.");
157
191
  assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
158
191
159
191
  // The shuffle mask requires elements the same size as the target.
160
191
  APInt UndefElts;
161
191
  SmallVector<uint64_t, 16> RawMask;
162
191
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
163
0
    return;
164
191
165
191
  unsigned NumElts = Width / ElSize;
166
191
  unsigned NumEltsPerLane = 128 / ElSize;
167
191
  assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
168
191
         "Unexpected number of vector elements.");
169
191
170
2.00k
  for (unsigned i = 0; i != NumElts; 
++i1.81k
) {
171
1.81k
    if (UndefElts[i]) {
172
140
      ShuffleMask.push_back(SM_SentinelUndef);
173
140
      continue;
174
140
    }
175
1.67k
176
1.67k
    int Index = i & ~(NumEltsPerLane - 1);
177
1.67k
    uint64_t Element = RawMask[i];
178
1.67k
    if (ElSize == 64)
179
0
      Index += (Element >> 1) & 0x1;
180
1.67k
    else
181
1.67k
      Index += Element & 0x3;
182
1.67k
183
1.67k
    ShuffleMask.push_back(Index);
184
1.67k
  }
185
191
}
186
187
void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
188
                         unsigned Width,
189
27
                         SmallVectorImpl<int> &ShuffleMask) {
190
27
  Type *MaskTy = C->getType();
191
27
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
192
27
  (void)MaskTySize;
193
27
  assert((MaskTySize == 128 || MaskTySize == 256) &&
194
27
         Width >= MaskTySize && "Unexpected vector size.");
195
27
196
27
  // The shuffle mask requires elements the same size as the target.
197
27
  APInt UndefElts;
198
27
  SmallVector<uint64_t, 8> RawMask;
199
27
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
200
0
    return;
201
27
202
27
  unsigned NumElts = Width / ElSize;
203
27
  unsigned NumEltsPerLane = 128 / ElSize;
204
27
  assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
205
27
         "Unexpected number of vector elements.");
206
27
207
175
  for (unsigned i = 0; i != NumElts; 
++i148
) {
208
148
    if (UndefElts[i]) {
209
2
      ShuffleMask.push_back(SM_SentinelUndef);
210
2
      continue;
211
2
    }
212
146
213
146
    // VPERMIL2 Operation.
214
146
    // Bits[3] - Match Bit.
215
146
    // Bits[2:1] - (Per Lane) PD Shuffle Mask.
216
146
    // Bits[2:0] - (Per Lane) PS Shuffle Mask.
217
146
    uint64_t Selector = RawMask[i];
218
146
    unsigned MatchBit = (Selector >> 3) & 0x1;
219
146
220
146
    // M2Z[0:1]     MatchBit
221
146
    //   0Xb           X        Source selected by Selector index.
222
146
    //   10b           0        Source selected by Selector index.
223
146
    //   10b           1        Zero.
224
146
    //   11b           0        Zero.
225
146
    //   11b           1        Source selected by Selector index.
226
146
    if ((M2Z & 0x2) != 0u && 
MatchBit != (M2Z & 0x1)88
) {
227
28
      ShuffleMask.push_back(SM_SentinelZero);
228
28
      continue;
229
28
    }
230
118
231
118
    int Index = i & ~(NumEltsPerLane - 1);
232
118
    if (ElSize == 64)
233
32
      Index += (Selector >> 1) & 0x1;
234
86
    else
235
86
      Index += Selector & 0x3;
236
118
237
118
    int Src = (Selector >> 2) & 0x1;
238
118
    Index += Src * NumElts;
239
118
    ShuffleMask.push_back(Index);
240
118
  }
241
27
}
242
243
void DecodeVPPERMMask(const Constant *C, unsigned Width,
244
68
                      SmallVectorImpl<int> &ShuffleMask) {
245
68
  Type *MaskTy = C->getType();
246
68
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
247
68
  (void)MaskTySize;
248
68
  assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");
249
68
250
68
  // The shuffle mask requires a byte vector.
251
68
  APInt UndefElts;
252
68
  SmallVector<uint64_t, 16> RawMask;
253
68
  if (!extractConstantMask(C, 8, UndefElts, RawMask))
254
0
    return;
255
68
256
68
  unsigned NumElts = Width / 8;
257
68
  assert(NumElts == 16 && "Unexpected number of vector elements.");
258
68
259
764
  for (unsigned i = 0; i != NumElts; 
++i696
) {
260
722
    if (UndefElts[i]) {
261
27
      ShuffleMask.push_back(SM_SentinelUndef);
262
27
      continue;
263
27
    }
264
695
265
695
    // VPPERM Operation
266
695
    // Bits[4:0] - Byte Index (0 - 31)
267
695
    // Bits[7:5] - Permute Operation
268
695
    //
269
695
    // Permute Operation:
270
695
    // 0 - Source byte (no logical operation).
271
695
    // 1 - Invert source byte.
272
695
    // 2 - Bit reverse of source byte.
273
695
    // 3 - Bit reverse of inverted source byte.
274
695
    // 4 - 00h (zero - fill).
275
695
    // 5 - FFh (ones - fill).
276
695
    // 6 - Most significant bit of source byte replicated in all bit positions.
277
695
    // 7 - Invert most significant bit of source byte and replicate in all bit
278
695
    // positions.
279
695
    uint64_t Element = RawMask[i];
280
695
    uint64_t Index = Element & 0x1F;
281
695
    uint64_t PermuteOp = (Element >> 5) & 0x7;
282
695
283
695
    if (PermuteOp == 4) {
284
28
      ShuffleMask.push_back(SM_SentinelZero);
285
28
      continue;
286
28
    }
287
667
    if (PermuteOp != 0) {
288
26
      ShuffleMask.clear();
289
26
      return;
290
26
    }
291
641
    ShuffleMask.push_back((int)Index);
292
641
  }
293
68
}
294
295
void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
296
0
                      SmallVectorImpl<int> &ShuffleMask) {
297
0
  assert((Width == 128 || Width == 256 || Width == 512) &&
298
0
         C->getType()->getPrimitiveSizeInBits() >= Width &&
299
0
         "Unexpected vector size.");
300
0
  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
301
0
         "Unexpected vector element size.");
302
0
303
0
  // The shuffle mask requires elements the same size as the target.
304
0
  APInt UndefElts;
305
0
  SmallVector<uint64_t, 64> RawMask;
306
0
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
307
0
    return;
308
0
309
0
  unsigned NumElts = Width / ElSize;
310
0
311
0
  for (unsigned i = 0; i != NumElts; ++i) {
312
0
    if (UndefElts[i]) {
313
0
      ShuffleMask.push_back(SM_SentinelUndef);
314
0
      continue;
315
0
    }
316
0
    int Index = RawMask[i] & (NumElts - 1);
317
0
    ShuffleMask.push_back(Index);
318
0
  }
319
0
}
320
321
void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
322
0
                       SmallVectorImpl<int> &ShuffleMask) {
323
0
  assert((Width == 128 || Width == 256 || Width == 512) &&
324
0
         C->getType()->getPrimitiveSizeInBits() >= Width &&
325
0
         "Unexpected vector size.");
326
0
  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
327
0
         "Unexpected vector element size.");
328
0
329
0
  // The shuffle mask requires elements the same size as the target.
330
0
  APInt UndefElts;
331
0
  SmallVector<uint64_t, 64> RawMask;
332
0
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
333
0
    return;
334
0
335
0
  unsigned NumElts = Width / ElSize;
336
0
337
0
  for (unsigned i = 0; i != NumElts; ++i) {
338
0
    if (UndefElts[i]) {
339
0
      ShuffleMask.push_back(SM_SentinelUndef);
340
0
      continue;
341
0
    }
342
0
    int Index = RawMask[i] & (NumElts*2 - 1);
343
0
    ShuffleMask.push_back(Index);
344
0
  }
345
0
}
346
} // llvm namespace