Coverage Report

Created: 2018-07-12 09:57

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/clang-build/lib/Target/X86/X86GenFastISel.inc
Line
Count
Source (jump to first uncovered line)
1
/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2
|*                                                                            *|
3
|* "Fast" Instruction Selector for the X86 target                             *|
4
|*                                                                            *|
5
|* Automatically generated file, do not edit!                                 *|
6
|*                                                                            *|
7
\*===----------------------------------------------------------------------===*/
8
9
10
// FastEmit Immediate Predicate functions.
11
16
static bool Predicate_i16immSExt8(int64_t Imm) {
12
16
 return isInt<8>(Imm); 
13
16
}
14
288
static bool Predicate_i32immSExt8(int64_t Imm) {
15
288
 return isInt<8>(Imm); 
16
288
}
17
104
static bool Predicate_i64immSExt8(int64_t Imm) {
18
104
 return isInt<8>(Imm); 
19
104
}
20
34
static bool Predicate_i64immSExt32(int64_t Imm) {
21
34
 return isInt<32>(Imm); 
22
34
}
23
0
static bool Predicate_AndMask64(int64_t Imm) {
24
0
25
0
  return isMask_64(Imm) && !isUInt<32>(Imm);
26
0
27
0
}
28
0
static bool Predicate_BTRMask64(int64_t Imm) {
29
0
30
0
  return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm);
31
0
32
0
}
33
0
static bool Predicate_BTCBTSMask64(int64_t Imm) {
34
0
35
0
  return !isInt<32>(Imm) && isPowerOf2_64(Imm);
36
0
37
0
}
38
39
40
// FastEmit functions for ISD::ABS.
41
42
0
unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
43
0
  if (RetVT.SimpleTy != MVT::v16i8)
44
0
    return 0;
45
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
46
0
    return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
47
0
  }
48
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
49
0
    return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
50
0
  }
51
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
52
0
    return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
53
0
  }
54
0
  return 0;
55
0
}
56
57
0
unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
58
0
  if (RetVT.SimpleTy != MVT::v32i8)
59
0
    return 0;
60
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
61
0
    return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
62
0
  }
63
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
64
0
    return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
65
0
  }
66
0
  return 0;
67
0
}
68
69
0
unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
70
0
  if (RetVT.SimpleTy != MVT::v64i8)
71
0
    return 0;
72
0
  if ((Subtarget->hasBWI())) {
73
0
    return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
74
0
  }
75
0
  return 0;
76
0
}
77
78
0
unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
79
0
  if (RetVT.SimpleTy != MVT::v8i16)
80
0
    return 0;
81
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
82
0
    return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
83
0
  }
84
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
85
0
    return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
86
0
  }
87
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
88
0
    return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
89
0
  }
90
0
  return 0;
91
0
}
92
93
0
unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
94
0
  if (RetVT.SimpleTy != MVT::v16i16)
95
0
    return 0;
96
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
97
0
    return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
98
0
  }
99
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
100
0
    return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
101
0
  }
102
0
  return 0;
103
0
}
104
105
0
unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
106
0
  if (RetVT.SimpleTy != MVT::v32i16)
107
0
    return 0;
108
0
  if ((Subtarget->hasBWI())) {
109
0
    return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
110
0
  }
111
0
  return 0;
112
0
}
113
114
0
unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
115
0
  if (RetVT.SimpleTy != MVT::v4i32)
116
0
    return 0;
117
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
118
0
    return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
119
0
  }
120
0
  if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
121
0
    return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
122
0
  }
123
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
124
0
    return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
125
0
  }
126
0
  return 0;
127
0
}
128
129
0
unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
130
0
  if (RetVT.SimpleTy != MVT::v8i32)
131
0
    return 0;
132
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
133
0
    return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
134
0
  }
135
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
136
0
    return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
137
0
  }
138
0
  return 0;
139
0
}
140
141
0
unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
142
0
  if (RetVT.SimpleTy != MVT::v16i32)
143
0
    return 0;
144
0
  if ((Subtarget->hasAVX512())) {
145
0
    return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
146
0
  }
147
0
  return 0;
148
0
}
149
150
0
unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
151
0
  if (RetVT.SimpleTy != MVT::v2i64)
152
0
    return 0;
153
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154
0
    return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
155
0
  }
156
0
  return 0;
157
0
}
158
159
0
unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
160
0
  if (RetVT.SimpleTy != MVT::v4i64)
161
0
    return 0;
162
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
163
0
    return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
164
0
  }
165
0
  return 0;
166
0
}
167
168
0
unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
169
0
  if (RetVT.SimpleTy != MVT::v8i64)
170
0
    return 0;
171
0
  if ((Subtarget->hasAVX512())) {
172
0
    return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
173
0
  }
174
0
  return 0;
175
0
}
176
177
unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
178
  switch (VT.SimpleTy) {
179
  case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
180
  case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
181
  case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
182
  case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
183
  case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
184
  case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
185
  case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
186
  case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
187
  case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
188
  case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
189
  case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
190
  case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
191
  default: return 0;
192
  }
193
}
194
195
// FastEmit functions for ISD::ANY_EXTEND.
196
197
3
unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
198
3
  if (RetVT.SimpleTy != MVT::i32)
199
0
    return 0;
200
3
  return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
201
3
}
202
203
6
unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
204
6
  switch (VT.SimpleTy) {
205
6
  
case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill)3
;
206
6
  
default: return 03
;
207
6
  }
208
6
}
209
210
// FastEmit functions for ISD::BITCAST.
211
212
13
unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
213
13
  if (RetVT.SimpleTy != MVT::f32)
214
12
    return 0;
215
1
  if ((Subtarget->hasAVX512())) {
216
0
    return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill);
217
0
  }
218
1
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
219
1
    return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
220
1
  }
221
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
222
0
    return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
223
0
  }
224
0
  return 0;
225
0
}
226
227
1
unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
228
1
  if ((Subtarget->hasAVX512())) {
229
0
    return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill);
230
0
  }
231
1
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
232
1
    return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
233
1
  }
234
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
235
0
    return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
236
0
  }
237
0
  return 0;
238
0
}
239
240
0
unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
241
0
  if ((Subtarget->hasMMX())) {
242
0
    return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill);
243
0
  }
244
0
  return 0;
245
0
}
246
247
4
unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
248
4
switch (RetVT.SimpleTy) {
249
4
  
case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill)1
;
250
4
  
case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill)0
;
251
4
  
default: return 03
;
252
4
}
253
4
}
254
255
7
unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
256
7
  if (RetVT.SimpleTy != MVT::i32)
257
0
    return 0;
258
7
  if ((Subtarget->hasAVX512())) {
259
2
    return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
260
2
  }
261
5
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
262
3
    return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
263
3
  }
264
2
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
265
2
    return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
266
2
  }
267
0
  return 0;
268
0
}
269
270
2
unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
271
2
  if ((Subtarget->hasAVX512())) {
272
0
    return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
273
0
  }
274
2
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
275
1
    return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
276
1
  }
277
1
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
278
1
    return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
279
1
  }
280
0
  return 0;
281
0
}
282
283
0
unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
284
0
  if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
285
0
    return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
286
0
  }
287
0
  return 0;
288
0
}
289
290
2
unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
291
2
switch (RetVT.SimpleTy) {
292
2
  case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
293
2
  
case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill)0
;
294
2
  
default: return 00
;
295
2
}
296
2
}
297
298
0
unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
299
0
  if ((Subtarget->hasMMX())) {
300
0
    return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill);
301
0
  }
302
0
  return 0;
303
0
}
304
305
0
unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
306
0
  if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
307
0
    return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill);
308
0
  }
309
0
  return 0;
310
0
}
311
312
unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
313
switch (RetVT.SimpleTy) {
314
  case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill);
315
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill);
316
  default: return 0;
317
}
318
}
319
320
4.02k
unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
321
4.02k
  switch (VT.SimpleTy) {
322
4.02k
  
case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill)13
;
323
4.02k
  
case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill)4
;
324
4.02k
  
case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill)7
;
325
4.02k
  
case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill)2
;
326
4.02k
  
case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill)0
;
327
4.02k
  
default: return 03.99k
;
328
4.02k
  }
329
4.02k
}
330
331
// FastEmit functions for ISD::BRIND.
332
333
0
unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
334
0
  if (RetVT.SimpleTy != MVT::isVoid)
335
0
    return 0;
336
0
  if ((!Subtarget->is64Bit())) {
337
0
    return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill);
338
0
  }
339
0
  return 0;
340
0
}
341
342
0
unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
343
0
  if (RetVT.SimpleTy != MVT::isVoid)
344
0
    return 0;
345
0
  if ((!Subtarget->is64Bit())) {
346
0
    return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill);
347
0
  }
348
0
  return 0;
349
0
}
350
351
0
unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
352
0
  if (RetVT.SimpleTy != MVT::isVoid)
353
0
    return 0;
354
0
  if ((Subtarget->is64Bit())) {
355
0
    return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill);
356
0
  }
357
0
  return 0;
358
0
}
359
360
unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
361
  switch (VT.SimpleTy) {
362
  case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
363
  case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
364
  case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
365
  default: return 0;
366
  }
367
}
368
369
// FastEmit functions for ISD::BSWAP.
370
371
0
unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
372
0
  if (RetVT.SimpleTy != MVT::i32)
373
0
    return 0;
374
0
  return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill);
375
0
}
376
377
0
unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
378
0
  if (RetVT.SimpleTy != MVT::i64)
379
0
    return 0;
380
0
  return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill);
381
0
}
382
383
unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
384
  switch (VT.SimpleTy) {
385
  case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
386
  case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
387
  default: return 0;
388
  }
389
}
390
391
// FastEmit functions for ISD::CTLZ.
392
393
0
unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
394
0
  if (RetVT.SimpleTy != MVT::i16)
395
0
    return 0;
396
0
  if ((Subtarget->hasLZCNT())) {
397
0
    return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
398
0
  }
399
0
  return 0;
400
0
}
401
402
0
unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
403
0
  if (RetVT.SimpleTy != MVT::i32)
404
0
    return 0;
405
0
  if ((Subtarget->hasLZCNT())) {
406
0
    return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
407
0
  }
408
0
  return 0;
409
0
}
410
411
0
unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
412
0
  if (RetVT.SimpleTy != MVT::i64)
413
0
    return 0;
414
0
  if ((Subtarget->hasLZCNT())) {
415
0
    return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
416
0
  }
417
0
  return 0;
418
0
}
419
420
0
unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
421
0
  if (RetVT.SimpleTy != MVT::v4i32)
422
0
    return 0;
423
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
424
0
    return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
425
0
  }
426
0
  return 0;
427
0
}
428
429
0
unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
430
0
  if (RetVT.SimpleTy != MVT::v8i32)
431
0
    return 0;
432
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
433
0
    return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
434
0
  }
435
0
  return 0;
436
0
}
437
438
0
unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
439
0
  if (RetVT.SimpleTy != MVT::v16i32)
440
0
    return 0;
441
0
  if ((Subtarget->hasCDI())) {
442
0
    return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
443
0
  }
444
0
  return 0;
445
0
}
446
447
0
unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
448
0
  if (RetVT.SimpleTy != MVT::v2i64)
449
0
    return 0;
450
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
451
0
    return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
452
0
  }
453
0
  return 0;
454
0
}
455
456
0
unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
457
0
  if (RetVT.SimpleTy != MVT::v4i64)
458
0
    return 0;
459
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
460
0
    return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
461
0
  }
462
0
  return 0;
463
0
}
464
465
0
unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
466
0
  if (RetVT.SimpleTy != MVT::v8i64)
467
0
    return 0;
468
0
  if ((Subtarget->hasCDI())) {
469
0
    return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
470
0
  }
471
0
  return 0;
472
0
}
473
474
unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
475
  switch (VT.SimpleTy) {
476
  case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
477
  case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
478
  case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
479
  case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
480
  case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
481
  case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
482
  case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
483
  case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
484
  case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
485
  default: return 0;
486
  }
487
}
488
489
// FastEmit functions for ISD::CTPOP.
490
491
0
unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
492
0
  if (RetVT.SimpleTy != MVT::i16)
493
0
    return 0;
494
0
  if ((Subtarget->hasPOPCNT())) {
495
0
    return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
496
0
  }
497
0
  return 0;
498
0
}
499
500
0
unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
501
0
  if (RetVT.SimpleTy != MVT::i32)
502
0
    return 0;
503
0
  if ((Subtarget->hasPOPCNT())) {
504
0
    return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
505
0
  }
506
0
  return 0;
507
0
}
508
509
0
unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
510
0
  if (RetVT.SimpleTy != MVT::i64)
511
0
    return 0;
512
0
  if ((Subtarget->hasPOPCNT())) {
513
0
    return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
514
0
  }
515
0
  return 0;
516
0
}
517
518
0
unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
519
0
  if (RetVT.SimpleTy != MVT::v16i8)
520
0
    return 0;
521
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
522
0
    return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
523
0
  }
524
0
  return 0;
525
0
}
526
527
0
unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
528
0
  if (RetVT.SimpleTy != MVT::v32i8)
529
0
    return 0;
530
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
531
0
    return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
532
0
  }
533
0
  return 0;
534
0
}
535
536
0
unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
537
0
  if (RetVT.SimpleTy != MVT::v64i8)
538
0
    return 0;
539
0
  if ((Subtarget->hasBITALG())) {
540
0
    return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
541
0
  }
542
0
  return 0;
543
0
}
544
545
0
unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
546
0
  if (RetVT.SimpleTy != MVT::v8i16)
547
0
    return 0;
548
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
549
0
    return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
550
0
  }
551
0
  return 0;
552
0
}
553
554
0
unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
555
0
  if (RetVT.SimpleTy != MVT::v16i16)
556
0
    return 0;
557
0
  if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
558
0
    return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
559
0
  }
560
0
  return 0;
561
0
}
562
563
0
unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
564
0
  if (RetVT.SimpleTy != MVT::v32i16)
565
0
    return 0;
566
0
  if ((Subtarget->hasBITALG())) {
567
0
    return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
568
0
  }
569
0
  return 0;
570
0
}
571
572
0
unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
573
0
  if (RetVT.SimpleTy != MVT::v4i32)
574
0
    return 0;
575
0
  if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
576
0
    return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
577
0
  }
578
0
  return 0;
579
0
}
580
581
0
unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
582
0
  if (RetVT.SimpleTy != MVT::v8i32)
583
0
    return 0;
584
0
  if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
585
0
    return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
586
0
  }
587
0
  return 0;
588
0
}
589
590
0
unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
591
0
  if (RetVT.SimpleTy != MVT::v16i32)
592
0
    return 0;
593
0
  if ((Subtarget->hasVPOPCNTDQ())) {
594
0
    return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
595
0
  }
596
0
  return 0;
597
0
}
598
599
0
unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
600
0
  if (RetVT.SimpleTy != MVT::v2i64)
601
0
    return 0;
602
0
  if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
603
0
    return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
604
0
  }
605
0
  return 0;
606
0
}
607
608
0
unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
609
0
  if (RetVT.SimpleTy != MVT::v4i64)
610
0
    return 0;
611
0
  if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
612
0
    return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
613
0
  }
614
0
  return 0;
615
0
}
616
617
0
unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
618
0
  if (RetVT.SimpleTy != MVT::v8i64)
619
0
    return 0;
620
0
  if ((Subtarget->hasVPOPCNTDQ())) {
621
0
    return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
622
0
  }
623
0
  return 0;
624
0
}
625
626
unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
627
  switch (VT.SimpleTy) {
628
  case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill);
629
  case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill);
630
  case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill);
631
  case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
632
  case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
633
  case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
634
  case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
635
  case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
636
  case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
637
  case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
638
  case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
639
  case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
640
  case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
641
  case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
642
  case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
643
  default: return 0;
644
  }
645
}
646
647
// FastEmit functions for ISD::CTTZ.
648
649
0
unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
650
0
  if (RetVT.SimpleTy != MVT::i16)
651
0
    return 0;
652
0
  if ((Subtarget->hasBMI())) {
653
0
    return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
654
0
  }
655
0
  return 0;
656
0
}
657
658
0
unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
659
0
  if (RetVT.SimpleTy != MVT::i32)
660
0
    return 0;
661
0
  if ((Subtarget->hasBMI())) {
662
0
    return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
663
0
  }
664
0
  return 0;
665
0
}
666
667
0
unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
668
0
  if (RetVT.SimpleTy != MVT::i64)
669
0
    return 0;
670
0
  if ((Subtarget->hasBMI())) {
671
0
    return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
672
0
  }
673
0
  return 0;
674
0
}
675
676
unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
677
  switch (VT.SimpleTy) {
678
  case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
679
  case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
680
  case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
681
  default: return 0;
682
  }
683
}
684
685
// FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
686
687
0
unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
688
0
  if (RetVT.SimpleTy != MVT::i16)
689
0
    return 0;
690
0
  return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill);
691
0
}
692
693
0
unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
694
0
  if (RetVT.SimpleTy != MVT::i32)
695
0
    return 0;
696
0
  return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill);
697
0
}
698
699
0
unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
700
0
  if (RetVT.SimpleTy != MVT::i64)
701
0
    return 0;
702
0
  return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill);
703
0
}
704
705
unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
706
  switch (VT.SimpleTy) {
707
  case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill);
708
  case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill);
709
  case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill);
710
  default: return 0;
711
  }
712
}
713
714
// FastEmit functions for ISD::FABS.
715
716
0
unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
717
0
  if (RetVT.SimpleTy != MVT::f32)
718
0
    return 0;
719
0
  if ((!Subtarget->hasSSE1())) {
720
0
    return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
721
0
  }
722
0
  return 0;
723
0
}
724
725
0
unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
726
0
  if (RetVT.SimpleTy != MVT::f64)
727
0
    return 0;
728
0
  if ((!Subtarget->hasSSE2())) {
729
0
    return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
730
0
  }
731
0
  return 0;
732
0
}
733
734
0
unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
735
0
  if (RetVT.SimpleTy != MVT::f80)
736
0
    return 0;
737
0
  return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
738
0
}
739
740
unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
741
  switch (VT.SimpleTy) {
742
  case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
743
  case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
744
  case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill);
745
  default: return 0;
746
  }
747
}
748
749
// FastEmit functions for ISD::FCOS.
750
751
0
unsigned fastEmit_ISD_FCOS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
752
0
  if (RetVT.SimpleTy != MVT::f32)
753
0
    return 0;
754
0
  if ((!Subtarget->hasSSE1())) {
755
0
    return fastEmitInst_r(X86::COS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
756
0
  }
757
0
  return 0;
758
0
}
759
760
0
unsigned fastEmit_ISD_FCOS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
761
0
  if (RetVT.SimpleTy != MVT::f64)
762
0
    return 0;
763
0
  if ((!Subtarget->hasSSE2())) {
764
0
    return fastEmitInst_r(X86::COS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
765
0
  }
766
0
  return 0;
767
0
}
768
769
0
unsigned fastEmit_ISD_FCOS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
770
0
  if (RetVT.SimpleTy != MVT::f80)
771
0
    return 0;
772
0
  return fastEmitInst_r(X86::COS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
773
0
}
774
775
unsigned fastEmit_ISD_FCOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
776
  switch (VT.SimpleTy) {
777
  case MVT::f32: return fastEmit_ISD_FCOS_MVT_f32_r(RetVT, Op0, Op0IsKill);
778
  case MVT::f64: return fastEmit_ISD_FCOS_MVT_f64_r(RetVT, Op0, Op0IsKill);
779
  case MVT::f80: return fastEmit_ISD_FCOS_MVT_f80_r(RetVT, Op0, Op0IsKill);
780
  default: return 0;
781
  }
782
}
783
784
// FastEmit functions for ISD::FNEG.
785
786
1
unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
787
1
  if (RetVT.SimpleTy != MVT::f32)
788
0
    return 0;
789
1
  if ((!Subtarget->hasSSE1())) {
790
0
    return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
791
0
  }
792
1
  return 0;
793
1
}
794
795
1
unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
796
1
  if (RetVT.SimpleTy != MVT::f64)
797
0
    return 0;
798
1
  if ((!Subtarget->hasSSE2())) {
799
0
    return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
800
0
  }
801
1
  return 0;
802
1
}
803
804
0
unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
805
0
  if (RetVT.SimpleTy != MVT::f80)
806
0
    return 0;
807
0
  return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
808
0
}
809
810
42
unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
811
42
  switch (VT.SimpleTy) {
812
42
  
case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill)1
;
813
42
  
case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill)1
;
814
42
  
case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill)0
;
815
42
  
default: return 040
;
816
42
  }
817
42
}
818
819
// FastEmit functions for ISD::FP_EXTEND.
820
821
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
822
0
  if (RetVT.SimpleTy != MVT::f64)
823
0
    return 0;
824
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
825
0
    return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
826
0
  }
827
0
  return 0;
828
0
}
829
830
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
831
0
  if (RetVT.SimpleTy != MVT::v4f64)
832
0
    return 0;
833
0
  if ((Subtarget->hasVLX())) {
834
0
    return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
835
0
  }
836
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
837
0
    return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
838
0
  }
839
0
  return 0;
840
0
}
841
842
0
unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
843
0
  if (RetVT.SimpleTy != MVT::v8f64)
844
0
    return 0;
845
0
  if ((Subtarget->hasAVX512())) {
846
0
    return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
847
0
  }
848
0
  return 0;
849
0
}
850
851
unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
852
  switch (VT.SimpleTy) {
853
  case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
854
  case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
855
  case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
856
  default: return 0;
857
  }
858
}
859
860
// FastEmit functions for ISD::FP_ROUND.
861
862
0
unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
863
0
  if (RetVT.SimpleTy != MVT::f32)
864
0
    return 0;
865
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
866
0
    return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
867
0
  }
868
0
  return 0;
869
0
}
870
871
0
unsigned fastEmit_ISD_FP_ROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
872
0
  if (RetVT.SimpleTy != MVT::v4f32)
873
0
    return 0;
874
0
  if ((Subtarget->hasVLX())) {
875
0
    return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
876
0
  }
877
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
878
0
    return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
879
0
  }
880
0
  return 0;
881
0
}
882
883
0
unsigned fastEmit_ISD_FP_ROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
884
0
  if (RetVT.SimpleTy != MVT::v8f32)
885
0
    return 0;
886
0
  if ((Subtarget->hasAVX512())) {
887
0
    return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
888
0
  }
889
0
  return 0;
890
0
}
891
892
unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
893
  switch (VT.SimpleTy) {
894
  case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
895
  case MVT::v4f64: return fastEmit_ISD_FP_ROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
896
  case MVT::v8f64: return fastEmit_ISD_FP_ROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
897
  default: return 0;
898
  }
899
}
900
901
// FastEmit functions for ISD::FP_TO_SINT.
902
903
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
904
0
  if ((Subtarget->hasAVX512())) {
905
0
    return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
906
0
  }
907
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
908
0
    return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
909
0
  }
910
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
911
0
    return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
912
0
  }
913
0
  return 0;
914
0
}
915
916
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
917
0
  if ((Subtarget->hasAVX512())) {
918
0
    return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
919
0
  }
920
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
921
0
    return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
922
0
  }
923
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
924
0
    return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
925
0
  }
926
0
  return 0;
927
0
}
928
929
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
930
switch (RetVT.SimpleTy) {
931
  case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
932
  case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
933
  default: return 0;
934
}
935
}
936
937
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
938
0
  if ((Subtarget->hasAVX512())) {
939
0
    return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
940
0
  }
941
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
942
0
    return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
943
0
  }
944
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
945
0
    return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
946
0
  }
947
0
  return 0;
948
0
}
949
950
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
951
0
  if ((Subtarget->hasAVX512())) {
952
0
    return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
953
0
  }
954
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
955
0
    return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
956
0
  }
957
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
958
0
    return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
959
0
  }
960
0
  return 0;
961
0
}
962
963
unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
964
switch (RetVT.SimpleTy) {
965
  case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
966
  case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
967
  default: return 0;
968
}
969
}
970
971
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
972
0
  if ((Subtarget->hasVLX())) {
973
0
    return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
974
0
  }
975
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
976
0
    return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
977
0
  }
978
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
979
0
    return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
980
0
  }
981
0
  return 0;
982
0
}
983
984
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
985
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
986
0
    return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
987
0
  }
988
0
  return 0;
989
0
}
990
991
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
992
switch (RetVT.SimpleTy) {
993
  case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
994
  case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
995
  default: return 0;
996
}
997
}
998
999
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1000
0
  if ((Subtarget->hasVLX())) {
1001
0
    return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1002
0
  }
1003
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1004
0
    return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1005
0
  }
1006
0
  return 0;
1007
0
}
1008
1009
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1010
0
  if ((Subtarget->hasDQI())) {
1011
0
    return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1012
0
  }
1013
0
  return 0;
1014
0
}
1015
1016
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1017
switch (RetVT.SimpleTy) {
1018
  case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
1019
  case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
1020
  default: return 0;
1021
}
1022
}
1023
1024
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1025
0
  if (RetVT.SimpleTy != MVT::v16i32)
1026
0
    return 0;
1027
0
  if ((Subtarget->hasAVX512())) {
1028
0
    return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1029
0
  }
1030
0
  return 0;
1031
0
}
1032
1033
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1034
0
  if (RetVT.SimpleTy != MVT::v2i64)
1035
0
    return 0;
1036
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1037
0
    return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1038
0
  }
1039
0
  return 0;
1040
0
}
1041
1042
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1043
0
  if ((Subtarget->hasVLX())) {
1044
0
    return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1045
0
  }
1046
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1047
0
    return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
1048
0
  }
1049
0
  return 0;
1050
0
}
1051
1052
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1053
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1054
0
    return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1055
0
  }
1056
0
  return 0;
1057
0
}
1058
1059
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1060
switch (RetVT.SimpleTy) {
1061
  case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
1062
  case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
1063
  default: return 0;
1064
}
1065
}
1066
1067
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1068
0
  if ((Subtarget->hasAVX512())) {
1069
0
    return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
1070
0
  }
1071
0
  return 0;
1072
0
}
1073
1074
0
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1075
0
  if ((Subtarget->hasDQI())) {
1076
0
    return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1077
0
  }
1078
0
  return 0;
1079
0
}
1080
1081
unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1082
switch (RetVT.SimpleTy) {
1083
  case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
1084
  case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
1085
  default: return 0;
1086
}
1087
}
1088
1089
unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1090
  switch (VT.SimpleTy) {
1091
  case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1092
  case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1093
  case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1094
  case MVT::v8f32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
1095
  case MVT::v16f32: return fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
1096
  case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1097
  case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
1098
  case MVT::v8f64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
1099
  default: return 0;
1100
  }
1101
}
1102
1103
// FastEmit functions for ISD::FP_TO_UINT.
1104
1105
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1106
0
  if ((Subtarget->hasAVX512())) {
1107
0
    return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
1108
0
  }
1109
0
  return 0;
1110
0
}
1111
1112
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1113
0
  if ((Subtarget->hasAVX512())) {
1114
0
    return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1115
0
  }
1116
0
  return 0;
1117
0
}
1118
1119
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1120
switch (RetVT.SimpleTy) {
1121
  case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
1122
  case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
1123
  default: return 0;
1124
}
1125
}
1126
1127
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1128
0
  if ((Subtarget->hasAVX512())) {
1129
0
    return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
1130
0
  }
1131
0
  return 0;
1132
0
}
1133
1134
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1135
0
  if ((Subtarget->hasAVX512())) {
1136
0
    return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1137
0
  }
1138
0
  return 0;
1139
0
}
1140
1141
unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1142
switch (RetVT.SimpleTy) {
1143
  case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
1144
  case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
1145
  default: return 0;
1146
}
1147
}
1148
1149
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1150
0
  if ((Subtarget->hasVLX())) {
1151
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1152
0
  }
1153
0
  return 0;
1154
0
}
1155
1156
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1157
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1158
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1159
0
  }
1160
0
  return 0;
1161
0
}
1162
1163
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1164
switch (RetVT.SimpleTy) {
1165
  case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
1166
  case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
1167
  default: return 0;
1168
}
1169
}
1170
1171
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1172
0
  if ((Subtarget->hasVLX())) {
1173
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1174
0
  }
1175
0
  return 0;
1176
0
}
1177
1178
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1179
0
  if ((Subtarget->hasDQI())) {
1180
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1181
0
  }
1182
0
  return 0;
1183
0
}
1184
1185
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1186
switch (RetVT.SimpleTy) {
1187
  case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
1188
  case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
1189
  default: return 0;
1190
}
1191
}
1192
1193
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1194
0
  if (RetVT.SimpleTy != MVT::v16i32)
1195
0
    return 0;
1196
0
  if ((Subtarget->hasAVX512())) {
1197
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1198
0
  }
1199
0
  return 0;
1200
0
}
1201
1202
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1203
0
  if (RetVT.SimpleTy != MVT::v2i64)
1204
0
    return 0;
1205
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1206
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1207
0
  }
1208
0
  return 0;
1209
0
}
1210
1211
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1212
0
  if ((Subtarget->hasVLX())) {
1213
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1214
0
  }
1215
0
  return 0;
1216
0
}
1217
1218
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1219
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1220
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1221
0
  }
1222
0
  return 0;
1223
0
}
1224
1225
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1226
switch (RetVT.SimpleTy) {
1227
  case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
1228
  case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
1229
  default: return 0;
1230
}
1231
}
1232
1233
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1234
0
  if ((Subtarget->hasAVX512())) {
1235
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
1236
0
  }
1237
0
  return 0;
1238
0
}
1239
1240
0
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1241
0
  if ((Subtarget->hasDQI())) {
1242
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1243
0
  }
1244
0
  return 0;
1245
0
}
1246
1247
unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1248
switch (RetVT.SimpleTy) {
1249
  case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
1250
  case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
1251
  default: return 0;
1252
}
1253
}
1254
1255
unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1256
  switch (VT.SimpleTy) {
1257
  case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1258
  case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1259
  case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1260
  case MVT::v8f32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
1261
  case MVT::v16f32: return fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
1262
  case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1263
  case MVT::v4f64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
1264
  case MVT::v8f64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
1265
  default: return 0;
1266
  }
1267
}
1268
1269
// FastEmit functions for ISD::FSIN.
1270
1271
0
unsigned fastEmit_ISD_FSIN_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1272
0
  if (RetVT.SimpleTy != MVT::f32)
1273
0
    return 0;
1274
0
  if ((!Subtarget->hasSSE1())) {
1275
0
    return fastEmitInst_r(X86::SIN_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
1276
0
  }
1277
0
  return 0;
1278
0
}
1279
1280
0
unsigned fastEmit_ISD_FSIN_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1281
0
  if (RetVT.SimpleTy != MVT::f64)
1282
0
    return 0;
1283
0
  if ((!Subtarget->hasSSE2())) {
1284
0
    return fastEmitInst_r(X86::SIN_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
1285
0
  }
1286
0
  return 0;
1287
0
}
1288
1289
0
unsigned fastEmit_ISD_FSIN_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1290
0
  if (RetVT.SimpleTy != MVT::f80)
1291
0
    return 0;
1292
0
  return fastEmitInst_r(X86::SIN_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
1293
0
}
1294
1295
unsigned fastEmit_ISD_FSIN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1296
  switch (VT.SimpleTy) {
1297
  case MVT::f32: return fastEmit_ISD_FSIN_MVT_f32_r(RetVT, Op0, Op0IsKill);
1298
  case MVT::f64: return fastEmit_ISD_FSIN_MVT_f64_r(RetVT, Op0, Op0IsKill);
1299
  case MVT::f80: return fastEmit_ISD_FSIN_MVT_f80_r(RetVT, Op0, Op0IsKill);
1300
  default: return 0;
1301
  }
1302
}
1303
1304
// FastEmit functions for ISD::FSQRT.
1305
1306
0
unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1307
0
  if (RetVT.SimpleTy != MVT::f32)
1308
0
    return 0;
1309
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1310
0
    return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
1311
0
  }
1312
0
  if ((!Subtarget->hasSSE1())) {
1313
0
    return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
1314
0
  }
1315
0
  return 0;
1316
0
}
1317
1318
0
unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1319
0
  if (RetVT.SimpleTy != MVT::f64)
1320
0
    return 0;
1321
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1322
0
    return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
1323
0
  }
1324
0
  if ((!Subtarget->hasSSE2())) {
1325
0
    return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
1326
0
  }
1327
0
  return 0;
1328
0
}
1329
1330
0
unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1331
0
  if (RetVT.SimpleTy != MVT::f80)
1332
0
    return 0;
1333
0
  return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
1334
0
}
1335
1336
0
unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1337
0
  if (RetVT.SimpleTy != MVT::v4f32)
1338
0
    return 0;
1339
0
  if ((Subtarget->hasVLX())) {
1340
0
    return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
1341
0
  }
1342
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1343
0
    return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
1344
0
  }
1345
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1346
0
    return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
1347
0
  }
1348
0
  return 0;
1349
0
}
1350
1351
0
unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1352
0
  if (RetVT.SimpleTy != MVT::v8f32)
1353
0
    return 0;
1354
0
  if ((Subtarget->hasVLX())) {
1355
0
    return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
1356
0
  }
1357
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1358
0
    return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
1359
0
  }
1360
0
  return 0;
1361
0
}
1362
1363
0
unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1364
0
  if (RetVT.SimpleTy != MVT::v16f32)
1365
0
    return 0;
1366
0
  if ((Subtarget->hasAVX512())) {
1367
0
    return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
1368
0
  }
1369
0
  return 0;
1370
0
}
1371
1372
0
unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1373
0
  if (RetVT.SimpleTy != MVT::v2f64)
1374
0
    return 0;
1375
0
  if ((Subtarget->hasVLX())) {
1376
0
    return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
1377
0
  }
1378
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1379
0
    return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
1380
0
  }
1381
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1382
0
    return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
1383
0
  }
1384
0
  return 0;
1385
0
}
1386
1387
0
unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1388
0
  if (RetVT.SimpleTy != MVT::v4f64)
1389
0
    return 0;
1390
0
  if ((Subtarget->hasVLX())) {
1391
0
    return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
1392
0
  }
1393
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1394
0
    return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
1395
0
  }
1396
0
  return 0;
1397
0
}
1398
1399
0
unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1400
0
  if (RetVT.SimpleTy != MVT::v8f64)
1401
0
    return 0;
1402
0
  if ((Subtarget->hasAVX512())) {
1403
0
    return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
1404
0
  }
1405
0
  return 0;
1406
0
}
1407
1408
unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1409
  switch (VT.SimpleTy) {
1410
  case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1411
  case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1412
  case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
1413
  case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1414
  case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
1415
  case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
1416
  case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1417
  case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
1418
  case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
1419
  default: return 0;
1420
  }
1421
}
1422
1423
// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1424
1425
1
unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1426
1
  if ((Subtarget->hasAVX512())) {
1427
0
    return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
1428
0
  }
1429
1
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1430
0
    return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1431
0
  }
1432
1
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1433
1
    return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1434
1
  }
1435
0
  return 0;
1436
0
}
1437
1438
0
unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
1439
0
  if ((Subtarget->hasMMX())) {
1440
0
    return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
1441
0
  }
1442
0
  return 0;
1443
0
}
1444
1445
1
unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1446
1
switch (RetVT.SimpleTy) {
1447
1
  case MVT::v4i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
1448
1
  
case MVT::x86mmx: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(Op0, Op0IsKill)0
;
1449
1
  
default: return 00
;
1450
1
}
1451
1
}
1452
1453
0
unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1454
0
  if (RetVT.SimpleTy != MVT::v2i64)
1455
0
    return 0;
1456
0
  if ((Subtarget->hasAVX512())) {
1457
0
    return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
1458
0
  }
1459
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1460
0
    return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1461
0
  }
1462
0
  if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1463
0
    return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1464
0
  }
1465
0
  return 0;
1466
0
}
1467
1468
1
unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1469
1
  switch (VT.SimpleTy) {
1470
1
  case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill);
1471
1
  
case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill)0
;
1472
1
  
default: return 00
;
1473
1
  }
1474
1
}
1475
1476
// FastEmit functions for ISD::SIGN_EXTEND.
1477
1478
38
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1479
38
  return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
1480
38
}
1481
1482
3
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1483
3
  return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill);
1484
3
}
1485
1486
49
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1487
49
switch (RetVT.SimpleTy) {
1488
49
  
case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill)38
;
1489
49
  
case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill)3
;
1490
49
  
default: return 08
;
1491
49
}
1492
49
}
1493
1494
7
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1495
7
  return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
1496
7
}
1497
1498
1
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1499
1
  return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill);
1500
1
}
1501
1502
8
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1503
8
switch (RetVT.SimpleTy) {
1504
8
  
case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill)7
;
1505
8
  
case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill)1
;
1506
8
  
default: return 00
;
1507
8
}
1508
8
}
1509
1510
58
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1511
58
  if (RetVT.SimpleTy != MVT::i64)
1512
0
    return 0;
1513
58
  if ((Subtarget->is64Bit())) {
1514
58
    return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill);
1515
58
  }
1516
0
  return 0;
1517
0
}
1518
1519
28
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1520
28
  if (RetVT.SimpleTy != MVT::v2i64)
1521
0
    return 0;
1522
28
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1523
28
    return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1524
28
  }
1525
0
  return 0;
1526
0
}
1527
1528
30
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1529
30
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1530
30
    return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1531
30
  }
1532
0
  return 0;
1533
0
}
1534
1535
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1536
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1537
0
    return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1538
0
  }
1539
0
  return 0;
1540
0
}
1541
1542
30
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1543
30
switch (RetVT.SimpleTy) {
1544
30
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill);
1545
30
  
case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill)0
;
1546
30
  
default: return 00
;
1547
30
}
1548
30
}
1549
1550
6
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1551
6
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1552
6
    return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1553
6
  }
1554
0
  return 0;
1555
0
}
1556
1557
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1558
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1559
0
    return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1560
0
  }
1561
0
  return 0;
1562
0
}
1563
1564
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1565
0
  if ((Subtarget->hasDQI())) {
1566
0
    return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1567
0
  }
1568
0
  return 0;
1569
0
}
1570
1571
6
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1572
6
switch (RetVT.SimpleTy) {
1573
6
  case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill);
1574
6
  
case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill)0
;
1575
6
  
case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill)0
;
1576
6
  
default: return 00
;
1577
6
}
1578
6
}
1579
1580
6
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1581
6
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1582
6
    return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1583
6
  }
1584
0
  return 0;
1585
0
}
1586
1587
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
1588
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1589
0
    return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1590
0
  }
1591
0
  return 0;
1592
0
}
1593
1594
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
1595
0
  if ((Subtarget->hasDQI())) {
1596
0
    return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1597
0
  }
1598
0
  return 0;
1599
0
}
1600
1601
6
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1602
6
switch (RetVT.SimpleTy) {
1603
6
  case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill);
1604
6
  
case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill)0
;
1605
6
  
case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill)0
;
1606
6
  
default: return 00
;
1607
6
}
1608
6
}
1609
1610
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
1611
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1612
0
    return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1613
0
  }
1614
0
  return 0;
1615
0
}
1616
1617
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
1618
0
  if ((Subtarget->hasBWI())) {
1619
0
    return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1620
0
  }
1621
0
  return 0;
1622
0
}
1623
1624
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1625
switch (RetVT.SimpleTy) {
1626
  case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill);
1627
  case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill);
1628
  default: return 0;
1629
}
1630
}
1631
1632
0
unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1633
0
  if (RetVT.SimpleTy != MVT::v64i8)
1634
0
    return 0;
1635
0
  if ((Subtarget->hasBWI())) {
1636
0
    return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1637
0
  }
1638
0
  return 0;
1639
0
}
1640
1641
191
unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1642
191
  switch (VT.SimpleTy) {
1643
191
  
case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill)49
;
1644
191
  
case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill)8
;
1645
191
  
case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill)58
;
1646
191
  
case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill)28
;
1647
191
  
case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill)30
;
1648
191
  
case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill)6
;
1649
191
  
case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill)6
;
1650
191
  
case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill)0
;
1651
191
  
case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill)0
;
1652
191
  
default: return 06
;
1653
191
  }
1654
191
}
1655
1656
// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1657
1658
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1659
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1660
0
    return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
1661
0
  }
1662
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1663
0
    return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
1664
0
  }
1665
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1666
0
    return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1667
0
  }
1668
0
  return 0;
1669
0
}
1670
1671
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1672
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1673
0
    return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1674
0
  }
1675
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1676
0
    return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1677
0
  }
1678
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1679
0
    return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1680
0
  }
1681
0
  return 0;
1682
0
}
1683
1684
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1685
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1686
0
    return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1687
0
  }
1688
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1689
0
    return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1690
0
  }
1691
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1692
0
    return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1693
0
  }
1694
0
  return 0;
1695
0
}
1696
1697
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1698
switch (RetVT.SimpleTy) {
1699
  case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
1700
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
1701
  case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
1702
  default: return 0;
1703
}
1704
}
1705
1706
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1707
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1708
0
    return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1709
0
  }
1710
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1711
0
    return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1712
0
  }
1713
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1714
0
    return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1715
0
  }
1716
0
  return 0;
1717
0
}
1718
1719
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1720
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1721
0
    return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1722
0
  }
1723
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1724
0
    return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1725
0
  }
1726
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1727
0
    return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1728
0
  }
1729
0
  return 0;
1730
0
}
1731
1732
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1733
switch (RetVT.SimpleTy) {
1734
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
1735
  case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
1736
  default: return 0;
1737
}
1738
}
1739
1740
0
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1741
0
  if (RetVT.SimpleTy != MVT::v2i64)
1742
0
    return 0;
1743
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1744
0
    return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1745
0
  }
1746
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1747
0
    return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1748
0
  }
1749
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1750
0
    return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1751
0
  }
1752
0
  return 0;
1753
0
}
1754
1755
unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1756
  switch (VT.SimpleTy) {
1757
  case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1758
  case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1759
  case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1760
  default: return 0;
1761
  }
1762
}
1763
1764
// FastEmit functions for ISD::SINT_TO_FP.
1765
1766
6
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
1767
6
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1768
3
    return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
1769
3
  }
1770
3
  return 0;
1771
3
}
1772
1773
8
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1774
8
  if ((Subtarget->hasSSE2() && 
!Subtarget->hasAVX()6
)) {
1775
3
    return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
1776
3
  }
1777
5
  return 0;
1778
5
}
1779
1780
14
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1781
14
switch (RetVT.SimpleTy) {
1782
14
  
case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill)6
;
1783
14
  
case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill)8
;
1784
14
  
default: return 00
;
1785
14
}
1786
14
}
1787
1788
6
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
1789
6
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1790
3
    return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
1791
3
  }
1792
3
  return 0;
1793
3
}
1794
1795
6
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1796
6
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1797
3
    return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
1798
3
  }
1799
3
  return 0;
1800
3
}
1801
1802
12
unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1803
12
switch (RetVT.SimpleTy) {
1804
12
  
case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill)6
;
1805
12
  
case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill)6
;
1806
12
  
default: return 00
;
1807
12
}
1808
12
}
1809
1810
6
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1811
6
  if ((Subtarget->hasVLX())) {
1812
2
    return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1813
2
  }
1814
4
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1815
2
    return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
1816
2
  }
1817
2
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1818
2
    return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
1819
2
  }
1820
0
  return 0;
1821
0
}
1822
1823
2
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
1824
2
  if ((Subtarget->hasVLX())) {
1825
0
    return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1826
0
  }
1827
2
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1828
2
    return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1829
2
  }
1830
0
  return 0;
1831
0
}
1832
1833
8
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1834
8
switch (RetVT.SimpleTy) {
1835
8
  
case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill)6
;
1836
8
  
case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill)2
;
1837
8
  
default: return 00
;
1838
8
}
1839
8
}
1840
1841
2
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
1842
2
  if ((Subtarget->hasVLX())) {
1843
0
    return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1844
0
  }
1845
2
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1846
2
    return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1847
2
  }
1848
0
  return 0;
1849
0
}
1850
1851
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
1852
0
  if ((Subtarget->hasAVX512())) {
1853
0
    return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1854
0
  }
1855
0
  return 0;
1856
0
}
1857
1858
2
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1859
2
switch (RetVT.SimpleTy) {
1860
2
  case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
1861
2
  
case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill)0
;
1862
2
  
default: return 00
;
1863
2
}
1864
2
}
1865
1866
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1867
0
  if (RetVT.SimpleTy != MVT::v16f32)
1868
0
    return 0;
1869
0
  if ((Subtarget->hasAVX512())) {
1870
0
    return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1871
0
  }
1872
0
  return 0;
1873
0
}
1874
1875
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1876
0
  if (RetVT.SimpleTy != MVT::v2f64)
1877
0
    return 0;
1878
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1879
0
    return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1880
0
  }
1881
0
  return 0;
1882
0
}
1883
1884
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1885
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1886
0
    return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1887
0
  }
1888
0
  return 0;
1889
0
}
1890
1891
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
1892
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1893
0
    return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1894
0
  }
1895
0
  return 0;
1896
0
}
1897
1898
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1899
switch (RetVT.SimpleTy) {
1900
  case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
1901
  case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
1902
  default: return 0;
1903
}
1904
}
1905
1906
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
1907
0
  if ((Subtarget->hasDQI())) {
1908
0
    return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
1909
0
  }
1910
0
  return 0;
1911
0
}
1912
1913
0
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
1914
0
  if ((Subtarget->hasDQI())) {
1915
0
    return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1916
0
  }
1917
0
  return 0;
1918
0
}
1919
1920
unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1921
switch (RetVT.SimpleTy) {
1922
  case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
1923
  case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
1924
  default: return 0;
1925
}
1926
}
1927
1928
36
unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1929
36
  switch (VT.SimpleTy) {
1930
36
  
case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill)14
;
1931
36
  
case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill)12
;
1932
36
  
case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill)8
;
1933
36
  
case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill)2
;
1934
36
  
case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill)0
;
1935
36
  
case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill)0
;
1936
36
  
case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill)0
;
1937
36
  
case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill)0
;
1938
36
  
default: return 00
;
1939
36
  }
1940
36
}
1941
1942
// FastEmit functions for ISD::TRUNCATE.
1943
1944
2
unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1945
2
  if (RetVT.SimpleTy != MVT::i8)
1946
0
    return 0;
1947
2
  if ((Subtarget->is64Bit())) {
1948
1
    return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit);
1949
1
  }
1950
1
  return 0;
1951
1
}
1952
1953
8
unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
1954
8
  if ((Subtarget->is64Bit())) {
1955
4
    return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
1956
4
  }
1957
4
  return 0;
1958
4
}
1959
1960
13
unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
1961
13
  return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
1962
13
}
1963
1964
21
unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1965
21
switch (RetVT.SimpleTy) {
1966
21
  
case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill)8
;
1967
21
  
case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill)13
;
1968
21
  
default: return 00
;
1969
21
}
1970
21
}
1971
1972
9
unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
1973
9
  return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
1974
9
}
1975
1976
2
unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
1977
2
  return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
1978
2
}
1979
1980
12
unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1981
12
  return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit);
1982
12
}
1983
1984
23
unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1985
23
switch (RetVT.SimpleTy) {
1986
23
  
case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill)9
;
1987
23
  
case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill)2
;
1988
23
  
case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill)12
;
1989
23
  
default: return 00
;
1990
23
}
1991
23
}
1992
1993
6
unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1994
6
  if (RetVT.SimpleTy != MVT::v16i8)
1995
0
    return 0;
1996
6
  if ((Subtarget->hasBWI()) && 
(Subtarget->hasVLX())4
) {
1997
4
    return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1998
4
  }
1999
2
  return 0;
2000
2
}
2001
2002
2
unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2003
2
  if (RetVT.SimpleTy != MVT::v32i8)
2004
0
    return 0;
2005
2
  if ((Subtarget->hasBWI())) {
2006
2
    return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2007
2
  }
2008
0
  return 0;
2009
0
}
2010
2011
6
unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2012
6
  if (RetVT.SimpleTy != MVT::v8i16)
2013
0
    return 0;
2014
6
  if ((Subtarget->hasAVX512()) && 
(Subtarget->hasVLX())4
) {
2015
4
    return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2016
4
  }
2017
2
  return 0;
2018
2
}
2019
2020
2
unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2021
2
  if ((Subtarget->hasAVX512())) {
2022
2
    return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
2023
2
  }
2024
0
  return 0;
2025
0
}
2026
2027
0
unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
2028
0
  if ((Subtarget->hasAVX512())) {
2029
0
    return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2030
0
  }
2031
0
  return 0;
2032
0
}
2033
2034
2
unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2035
2
switch (RetVT.SimpleTy) {
2036
2
  case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
2037
2
  
case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill)0
;
2038
2
  
default: return 00
;
2039
2
}
2040
2
}
2041
2042
4
unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2043
4
  if (RetVT.SimpleTy != MVT::v4i32)
2044
2
    return 0;
2045
2
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2046
2
    return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2047
2
  }
2048
0
  return 0;
2049
0
}
2050
2051
2
unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2052
2
  if ((Subtarget->hasAVX512())) {
2053
2
    return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
2054
2
  }
2055
0
  return 0;
2056
0
}
2057
2058
2
unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2059
2
  if ((Subtarget->hasAVX512())) {
2060
2
    return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2061
2
  }
2062
0
  return 0;
2063
0
}
2064
2065
8
unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2066
8
switch (RetVT.SimpleTy) {
2067
8
  
case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill)2
;
2068
8
  
case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill)2
;
2069
8
  
default: return 04
;
2070
8
}
2071
8
}
2072
2073
77
unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2074
77
  switch (VT.SimpleTy) {
2075
77
  
case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill)2
;
2076
77
  
case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill)21
;
2077
77
  
case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill)23
;
2078
77
  
case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0, Op0IsKill)6
;
2079
77
  
case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0, Op0IsKill)2
;
2080
77
  
case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0, Op0IsKill)6
;
2081
77
  
case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0, Op0IsKill)2
;
2082
77
  
case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0, Op0IsKill)4
;
2083
77
  
case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0, Op0IsKill)8
;
2084
77
  
default: return 03
;
2085
77
  }
2086
77
}
2087
2088
// FastEmit functions for ISD::UINT_TO_FP.
2089
2090
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2091
0
  if ((Subtarget->hasVLX())) {
2092
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2093
0
  }
2094
0
  return 0;
2095
0
}
2096
2097
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2098
0
  if ((Subtarget->hasVLX())) {
2099
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2100
0
  }
2101
0
  return 0;
2102
0
}
2103
2104
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2105
switch (RetVT.SimpleTy) {
2106
  case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
2107
  case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
2108
  default: return 0;
2109
}
2110
}
2111
2112
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2113
0
  if ((Subtarget->hasVLX())) {
2114
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2115
0
  }
2116
0
  return 0;
2117
0
}
2118
2119
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2120
0
  if ((Subtarget->hasAVX512())) {
2121
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2122
0
  }
2123
0
  return 0;
2124
0
}
2125
2126
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2127
switch (RetVT.SimpleTy) {
2128
  case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
2129
  case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
2130
  default: return 0;
2131
}
2132
}
2133
2134
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2135
0
  if (RetVT.SimpleTy != MVT::v16f32)
2136
0
    return 0;
2137
0
  if ((Subtarget->hasAVX512())) {
2138
0
    return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2139
0
  }
2140
0
  return 0;
2141
0
}
2142
2143
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2144
0
  if (RetVT.SimpleTy != MVT::v2f64)
2145
0
    return 0;
2146
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2147
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2148
0
  }
2149
0
  return 0;
2150
0
}
2151
2152
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2153
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2154
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2155
0
  }
2156
0
  return 0;
2157
0
}
2158
2159
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2160
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2161
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2162
0
  }
2163
0
  return 0;
2164
0
}
2165
2166
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2167
switch (RetVT.SimpleTy) {
2168
  case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
2169
  case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
2170
  default: return 0;
2171
}
2172
}
2173
2174
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2175
0
  if ((Subtarget->hasDQI())) {
2176
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2177
0
  }
2178
0
  return 0;
2179
0
}
2180
2181
0
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2182
0
  if ((Subtarget->hasDQI())) {
2183
0
    return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2184
0
  }
2185
0
  return 0;
2186
0
}
2187
2188
unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2189
switch (RetVT.SimpleTy) {
2190
  case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
2191
  case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
2192
  default: return 0;
2193
}
2194
}
2195
2196
unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2197
  switch (VT.SimpleTy) {
2198
  case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2199
  case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2200
  case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2201
  case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2202
  case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2203
  case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2204
  default: return 0;
2205
  }
2206
}
2207
2208
// FastEmit functions for ISD::ZERO_EXTEND.
2209
2210
390
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2211
390
  if (RetVT.SimpleTy != MVT::i32)
2212
14
    return 0;
2213
376
  return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
2214
376
}
2215
2216
65
unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2217
65
  if (RetVT.SimpleTy != MVT::i32)
2218
2
    return 0;
2219
63
  return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
2220
63
}
2221
2222
473
unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2223
473
  switch (VT.SimpleTy) {
2224
473
  
case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill)390
;
2225
473
  
case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill)65
;
2226
473
  
default: return 018
;
2227
473
  }
2228
473
}
2229
2230
// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
2231
2232
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2233
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2234
0
    return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
2235
0
  }
2236
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
2237
0
    return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
2238
0
  }
2239
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
2240
0
    return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2241
0
  }
2242
0
  return 0;
2243
0
}
2244
2245
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2246
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2247
0
    return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2248
0
  }
2249
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2250
0
    return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2251
0
  }
2252
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2253
0
    return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2254
0
  }
2255
0
  return 0;
2256
0
}
2257
2258
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2259
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2260
0
    return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2261
0
  }
2262
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2263
0
    return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2264
0
  }
2265
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2266
0
    return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2267
0
  }
2268
0
  return 0;
2269
0
}
2270
2271
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2272
switch (RetVT.SimpleTy) {
2273
  case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
2274
  case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
2275
  case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
2276
  default: return 0;
2277
}
2278
}
2279
2280
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2281
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2282
0
    return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2283
0
  }
2284
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2285
0
    return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2286
0
  }
2287
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2288
0
    return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2289
0
  }
2290
0
  return 0;
2291
0
}
2292
2293
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2294
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2295
0
    return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2296
0
  }
2297
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2298
0
    return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2299
0
  }
2300
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2301
0
    return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2302
0
  }
2303
0
  return 0;
2304
0
}
2305
2306
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2307
switch (RetVT.SimpleTy) {
2308
  case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
2309
  case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
2310
  default: return 0;
2311
}
2312
}
2313
2314
0
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2315
0
  if (RetVT.SimpleTy != MVT::v2i64)
2316
0
    return 0;
2317
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2318
0
    return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2319
0
  }
2320
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2321
0
    return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2322
0
  }
2323
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2324
0
    return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2325
0
  }
2326
0
  return 0;
2327
0
}
2328
2329
unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2330
  switch (VT.SimpleTy) {
2331
  case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2332
  case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2333
  case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2334
  default: return 0;
2335
  }
2336
}
2337
2338
// FastEmit functions for X86ISD::CALL.
2339
2340
0
unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2341
0
  if (RetVT.SimpleTy != MVT::isVoid)
2342
0
    return 0;
2343
0
  if ((!Subtarget->is64Bit())) {
2344
0
    return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill);
2345
0
  }
2346
0
  return 0;
2347
0
}
2348
2349
0
unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2350
0
  if (RetVT.SimpleTy != MVT::isVoid)
2351
0
    return 0;
2352
0
  if ((!Subtarget->is64Bit()) && (Subtarget->useRetpoline())) {
2353
0
    return fastEmitInst_r(X86::RETPOLINE_CALL32, &X86::GR32RegClass, Op0, Op0IsKill);
2354
0
  }
2355
0
  if ((!Subtarget->is64Bit()) && (!Subtarget->useRetpoline())) {
2356
0
    return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill);
2357
0
  }
2358
0
  return 0;
2359
0
}
2360
2361
0
unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2362
0
  if (RetVT.SimpleTy != MVT::isVoid)
2363
0
    return 0;
2364
0
  if ((Subtarget->is64Bit()) && (Subtarget->useRetpoline())) {
2365
0
    return fastEmitInst_r(X86::RETPOLINE_CALL64, &X86::GR64RegClass, Op0, Op0IsKill);
2366
0
  }
2367
0
  if ((Subtarget->is64Bit()) && (!Subtarget->useRetpoline())) {
2368
0
    return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill);
2369
0
  }
2370
0
  return 0;
2371
0
}
2372
2373
unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2374
  switch (VT.SimpleTy) {
2375
  case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
2376
  case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
2377
  case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
2378
  default: return 0;
2379
  }
2380
}
2381
2382
// FastEmit functions for X86ISD::COMPRESS.
2383
2384
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2385
0
  if (RetVT.SimpleTy != MVT::v16i8)
2386
0
    return 0;
2387
0
  if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
2388
0
    return fastEmitInst_r(X86::VPCOMPRESSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2389
0
  }
2390
0
  return 0;
2391
0
}
2392
2393
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2394
0
  if (RetVT.SimpleTy != MVT::v32i8)
2395
0
    return 0;
2396
0
  if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
2397
0
    return fastEmitInst_r(X86::VPCOMPRESSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2398
0
  }
2399
0
  return 0;
2400
0
}
2401
2402
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2403
0
  if (RetVT.SimpleTy != MVT::v64i8)
2404
0
    return 0;
2405
0
  if ((Subtarget->hasVBMI2())) {
2406
0
    return fastEmitInst_r(X86::VPCOMPRESSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2407
0
  }
2408
0
  return 0;
2409
0
}
2410
2411
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2412
0
  if (RetVT.SimpleTy != MVT::v8i16)
2413
0
    return 0;
2414
0
  if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
2415
0
    return fastEmitInst_r(X86::VPCOMPRESSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2416
0
  }
2417
0
  return 0;
2418
0
}
2419
2420
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2421
0
  if (RetVT.SimpleTy != MVT::v16i16)
2422
0
    return 0;
2423
0
  if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
2424
0
    return fastEmitInst_r(X86::VPCOMPRESSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2425
0
  }
2426
0
  return 0;
2427
0
}
2428
2429
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2430
0
  if (RetVT.SimpleTy != MVT::v32i16)
2431
0
    return 0;
2432
0
  if ((Subtarget->hasVBMI2())) {
2433
0
    return fastEmitInst_r(X86::VPCOMPRESSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2434
0
  }
2435
0
  return 0;
2436
0
}
2437
2438
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2439
0
  if (RetVT.SimpleTy != MVT::v4i32)
2440
0
    return 0;
2441
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2442
0
    return fastEmitInst_r(X86::VPCOMPRESSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2443
0
  }
2444
0
  return 0;
2445
0
}
2446
2447
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2448
0
  if (RetVT.SimpleTy != MVT::v8i32)
2449
0
    return 0;
2450
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2451
0
    return fastEmitInst_r(X86::VPCOMPRESSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2452
0
  }
2453
0
  return 0;
2454
0
}
2455
2456
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2457
0
  if (RetVT.SimpleTy != MVT::v16i32)
2458
0
    return 0;
2459
0
  if ((Subtarget->hasAVX512())) {
2460
0
    return fastEmitInst_r(X86::VPCOMPRESSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2461
0
  }
2462
0
  return 0;
2463
0
}
2464
2465
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2466
0
  if (RetVT.SimpleTy != MVT::v2i64)
2467
0
    return 0;
2468
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2469
0
    return fastEmitInst_r(X86::VPCOMPRESSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2470
0
  }
2471
0
  return 0;
2472
0
}
2473
2474
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2475
0
  if (RetVT.SimpleTy != MVT::v4i64)
2476
0
    return 0;
2477
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2478
0
    return fastEmitInst_r(X86::VPCOMPRESSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2479
0
  }
2480
0
  return 0;
2481
0
}
2482
2483
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2484
0
  if (RetVT.SimpleTy != MVT::v8i64)
2485
0
    return 0;
2486
0
  if ((Subtarget->hasAVX512())) {
2487
0
    return fastEmitInst_r(X86::VPCOMPRESSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2488
0
  }
2489
0
  return 0;
2490
0
}
2491
2492
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2493
0
  if (RetVT.SimpleTy != MVT::v4f32)
2494
0
    return 0;
2495
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2496
0
    return fastEmitInst_r(X86::VCOMPRESSPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2497
0
  }
2498
0
  return 0;
2499
0
}
2500
2501
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2502
0
  if (RetVT.SimpleTy != MVT::v8f32)
2503
0
    return 0;
2504
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2505
0
    return fastEmitInst_r(X86::VCOMPRESSPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2506
0
  }
2507
0
  return 0;
2508
0
}
2509
2510
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2511
0
  if (RetVT.SimpleTy != MVT::v16f32)
2512
0
    return 0;
2513
0
  if ((Subtarget->hasAVX512())) {
2514
0
    return fastEmitInst_r(X86::VCOMPRESSPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2515
0
  }
2516
0
  return 0;
2517
0
}
2518
2519
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2520
0
  if (RetVT.SimpleTy != MVT::v2f64)
2521
0
    return 0;
2522
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2523
0
    return fastEmitInst_r(X86::VCOMPRESSPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2524
0
  }
2525
0
  return 0;
2526
0
}
2527
2528
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2529
0
  if (RetVT.SimpleTy != MVT::v4f64)
2530
0
    return 0;
2531
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2532
0
    return fastEmitInst_r(X86::VCOMPRESSPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2533
0
  }
2534
0
  return 0;
2535
0
}
2536
2537
0
unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2538
0
  if (RetVT.SimpleTy != MVT::v8f64)
2539
0
    return 0;
2540
0
  if ((Subtarget->hasAVX512())) {
2541
0
    return fastEmitInst_r(X86::VCOMPRESSPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2542
0
  }
2543
0
  return 0;
2544
0
}
2545
2546
unsigned fastEmit_X86ISD_COMPRESS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2547
  switch (VT.SimpleTy) {
2548
  case MVT::v16i8: return fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2549
  case MVT::v32i8: return fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
2550
  case MVT::v64i8: return fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
2551
  case MVT::v8i16: return fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2552
  case MVT::v16i16: return fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
2553
  case MVT::v32i16: return fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
2554
  case MVT::v4i32: return fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2555
  case MVT::v8i32: return fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2556
  case MVT::v16i32: return fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2557
  case MVT::v2i64: return fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2558
  case MVT::v4i64: return fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2559
  case MVT::v8i64: return fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2560
  case MVT::v4f32: return fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2561
  case MVT::v8f32: return fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
2562
  case MVT::v16f32: return fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
2563
  case MVT::v2f64: return fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
2564
  case MVT::v4f64: return fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
2565
  case MVT::v8f64: return fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
2566
  default: return 0;
2567
  }
2568
}
2569
2570
// FastEmit functions for X86ISD::CONFLICT.
2571
2572
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2573
0
  if (RetVT.SimpleTy != MVT::v4i32)
2574
0
    return 0;
2575
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
2576
0
    return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2577
0
  }
2578
0
  return 0;
2579
0
}
2580
2581
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2582
0
  if (RetVT.SimpleTy != MVT::v8i32)
2583
0
    return 0;
2584
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
2585
0
    return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2586
0
  }
2587
0
  return 0;
2588
0
}
2589
2590
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2591
0
  if (RetVT.SimpleTy != MVT::v16i32)
2592
0
    return 0;
2593
0
  if ((Subtarget->hasCDI())) {
2594
0
    return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2595
0
  }
2596
0
  return 0;
2597
0
}
2598
2599
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2600
0
  if (RetVT.SimpleTy != MVT::v2i64)
2601
0
    return 0;
2602
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
2603
0
    return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2604
0
  }
2605
0
  return 0;
2606
0
}
2607
2608
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2609
0
  if (RetVT.SimpleTy != MVT::v4i64)
2610
0
    return 0;
2611
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
2612
0
    return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2613
0
  }
2614
0
  return 0;
2615
0
}
2616
2617
0
unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2618
0
  if (RetVT.SimpleTy != MVT::v8i64)
2619
0
    return 0;
2620
0
  if ((Subtarget->hasCDI())) {
2621
0
    return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2622
0
  }
2623
0
  return 0;
2624
0
}
2625
2626
unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2627
  switch (VT.SimpleTy) {
2628
  case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2629
  case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2630
  case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2631
  case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2632
  case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2633
  case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2634
  default: return 0;
2635
  }
2636
}
2637
2638
// FastEmit functions for X86ISD::CVTP2SI.
2639
2640
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2641
0
  if ((Subtarget->hasVLX())) {
2642
0
    return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2643
0
  }
2644
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2645
0
    return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2646
0
  }
2647
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2648
0
    return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2649
0
  }
2650
0
  return 0;
2651
0
}
2652
2653
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2654
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2655
0
    return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2656
0
  }
2657
0
  return 0;
2658
0
}
2659
2660
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2661
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2662
0
    return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2663
0
  }
2664
0
  return 0;
2665
0
}
2666
2667
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2668
switch (RetVT.SimpleTy) {
2669
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
2670
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
2671
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
2672
  default: return 0;
2673
}
2674
}
2675
2676
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2677
0
  if ((Subtarget->hasVLX())) {
2678
0
    return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2679
0
  }
2680
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2681
0
    return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2682
0
  }
2683
0
  return 0;
2684
0
}
2685
2686
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2687
0
  if ((Subtarget->hasDQI())) {
2688
0
    return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2689
0
  }
2690
0
  return 0;
2691
0
}
2692
2693
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2694
switch (RetVT.SimpleTy) {
2695
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
2696
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
2697
  default: return 0;
2698
}
2699
}
2700
2701
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2702
0
  if (RetVT.SimpleTy != MVT::v16i32)
2703
0
    return 0;
2704
0
  if ((Subtarget->hasAVX512())) {
2705
0
    return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2706
0
  }
2707
0
  return 0;
2708
0
}
2709
2710
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2711
0
  if ((Subtarget->hasVLX())) {
2712
0
    return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2713
0
  }
2714
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2715
0
    return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2716
0
  }
2717
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2718
0
    return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2719
0
  }
2720
0
  return 0;
2721
0
}
2722
2723
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2724
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2725
0
    return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2726
0
  }
2727
0
  return 0;
2728
0
}
2729
2730
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2731
switch (RetVT.SimpleTy) {
2732
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
2733
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
2734
  default: return 0;
2735
}
2736
}
2737
2738
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2739
0
  if ((Subtarget->hasVLX())) {
2740
0
    return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2741
0
  }
2742
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2743
0
    return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
2744
0
  }
2745
0
  return 0;
2746
0
}
2747
2748
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2749
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2750
0
    return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2751
0
  }
2752
0
  return 0;
2753
0
}
2754
2755
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2756
switch (RetVT.SimpleTy) {
2757
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
2758
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
2759
  default: return 0;
2760
}
2761
}
2762
2763
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2764
0
  if ((Subtarget->hasAVX512())) {
2765
0
    return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2766
0
  }
2767
0
  return 0;
2768
0
}
2769
2770
0
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2771
0
  if ((Subtarget->hasDQI())) {
2772
0
    return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2773
0
  }
2774
0
  return 0;
2775
0
}
2776
2777
unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2778
switch (RetVT.SimpleTy) {
2779
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
2780
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
2781
  default: return 0;
2782
}
2783
}
2784
2785
unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2786
  switch (VT.SimpleTy) {
2787
  case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2788
  case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
2789
  case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
2790
  case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
2791
  case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
2792
  case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
2793
  default: return 0;
2794
  }
2795
}
2796
2797
// FastEmit functions for X86ISD::CVTP2UI.
2798
2799
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2800
0
  if ((Subtarget->hasVLX())) {
2801
0
    return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2802
0
  }
2803
0
  return 0;
2804
0
}
2805
2806
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2807
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2808
0
    return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2809
0
  }
2810
0
  return 0;
2811
0
}
2812
2813
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2814
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2815
0
    return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2816
0
  }
2817
0
  return 0;
2818
0
}
2819
2820
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2821
switch (RetVT.SimpleTy) {
2822
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
2823
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
2824
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
2825
  default: return 0;
2826
}
2827
}
2828
2829
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2830
0
  if ((Subtarget->hasVLX())) {
2831
0
    return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2832
0
  }
2833
0
  return 0;
2834
0
}
2835
2836
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2837
0
  if ((Subtarget->hasDQI())) {
2838
0
    return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2839
0
  }
2840
0
  return 0;
2841
0
}
2842
2843
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2844
switch (RetVT.SimpleTy) {
2845
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
2846
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
2847
  default: return 0;
2848
}
2849
}
2850
2851
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2852
0
  if (RetVT.SimpleTy != MVT::v16i32)
2853
0
    return 0;
2854
0
  if ((Subtarget->hasAVX512())) {
2855
0
    return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2856
0
  }
2857
0
  return 0;
2858
0
}
2859
2860
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2861
0
  if ((Subtarget->hasVLX())) {
2862
0
    return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2863
0
  }
2864
0
  return 0;
2865
0
}
2866
2867
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2868
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2869
0
    return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2870
0
  }
2871
0
  return 0;
2872
0
}
2873
2874
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2875
switch (RetVT.SimpleTy) {
2876
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
2877
  case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
2878
  default: return 0;
2879
}
2880
}
2881
2882
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2883
0
  if ((Subtarget->hasVLX())) {
2884
0
    return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2885
0
  }
2886
0
  return 0;
2887
0
}
2888
2889
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2890
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2891
0
    return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2892
0
  }
2893
0
  return 0;
2894
0
}
2895
2896
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2897
switch (RetVT.SimpleTy) {
2898
  case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
2899
  case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
2900
  default: return 0;
2901
}
2902
}
2903
2904
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2905
0
  if ((Subtarget->hasAVX512())) {
2906
0
    return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2907
0
  }
2908
0
  return 0;
2909
0
}
2910
2911
0
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2912
0
  if ((Subtarget->hasDQI())) {
2913
0
    return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2914
0
  }
2915
0
  return 0;
2916
0
}
2917
2918
unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2919
switch (RetVT.SimpleTy) {
2920
  case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
2921
  case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
2922
  default: return 0;
2923
}
2924
}
2925
2926
unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2927
  switch (VT.SimpleTy) {
2928
  case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2929
  case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
2930
  case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
2931
  case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
2932
  case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
2933
  case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
2934
  default: return 0;
2935
  }
2936
}
2937
2938
// FastEmit functions for X86ISD::CVTPH2PS.
2939
2940
0
unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2941
0
  if ((Subtarget->hasVLX())) {
2942
0
    return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2943
0
  }
2944
0
  if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
2945
0
    return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
2946
0
  }
2947
0
  return 0;
2948
0
}
2949
2950
0
unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2951
0
  if ((Subtarget->hasVLX())) {
2952
0
    return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2953
0
  }
2954
0
  if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
2955
0
    return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2956
0
  }
2957
0
  return 0;
2958
0
}
2959
2960
unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2961
switch (RetVT.SimpleTy) {
2962
  case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
2963
  case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill);
2964
  default: return 0;
2965
}
2966
}
2967
2968
0
unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2969
0
  if (RetVT.SimpleTy != MVT::v16f32)
2970
0
    return 0;
2971
0
  if ((Subtarget->hasAVX512())) {
2972
0
    return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2973
0
  }
2974
0
  return 0;
2975
0
}
2976
2977
unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2978
  switch (VT.SimpleTy) {
2979
  case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2980
  case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
2981
  default: return 0;
2982
  }
2983
}
2984
2985
// FastEmit functions for X86ISD::CVTSI2P.
2986
2987
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2988
0
  if (RetVT.SimpleTy != MVT::v2f64)
2989
0
    return 0;
2990
0
  if ((Subtarget->hasVLX())) {
2991
0
    return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2992
0
  }
2993
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2994
0
    return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2995
0
  }
2996
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2997
0
    return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2998
0
  }
2999
0
  return 0;
3000
0
}
3001
3002
0
unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3003
0
  if (RetVT.SimpleTy != MVT::v4f32)
3004
0
    return 0;
3005
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3006
0
    return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3007
0
  }
3008
0
  return 0;
3009
0
}
3010
3011
unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3012
  switch (VT.SimpleTy) {
3013
  case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3014
  case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3015
  default: return 0;
3016
  }
3017
}
3018
3019
// FastEmit functions for X86ISD::CVTTP2SI.
3020
3021
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3022
0
  if ((Subtarget->hasVLX())) {
3023
0
    return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3024
0
  }
3025
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3026
0
    return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3027
0
  }
3028
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3029
0
    return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3030
0
  }
3031
0
  return 0;
3032
0
}
3033
3034
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3035
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3036
0
    return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3037
0
  }
3038
0
  return 0;
3039
0
}
3040
3041
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3042
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3043
0
    return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3044
0
  }
3045
0
  return 0;
3046
0
}
3047
3048
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3049
switch (RetVT.SimpleTy) {
3050
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
3051
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
3052
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
3053
  default: return 0;
3054
}
3055
}
3056
3057
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3058
0
  if ((Subtarget->hasVLX())) {
3059
0
    return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3060
0
  }
3061
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3062
0
    return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3063
0
  }
3064
0
  return 0;
3065
0
}
3066
3067
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3068
0
  if ((Subtarget->hasDQI())) {
3069
0
    return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3070
0
  }
3071
0
  return 0;
3072
0
}
3073
3074
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3075
switch (RetVT.SimpleTy) {
3076
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
3077
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
3078
  default: return 0;
3079
}
3080
}
3081
3082
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3083
0
  if (RetVT.SimpleTy != MVT::v16i32)
3084
0
    return 0;
3085
0
  if ((Subtarget->hasAVX512())) {
3086
0
    return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3087
0
  }
3088
0
  return 0;
3089
0
}
3090
3091
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3092
0
  if ((Subtarget->hasVLX())) {
3093
0
    return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3094
0
  }
3095
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3096
0
    return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3097
0
  }
3098
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3099
0
    return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3100
0
  }
3101
0
  return 0;
3102
0
}
3103
3104
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3105
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3106
0
    return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3107
0
  }
3108
0
  return 0;
3109
0
}
3110
3111
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3112
switch (RetVT.SimpleTy) {
3113
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
3114
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
3115
  default: return 0;
3116
}
3117
}
3118
3119
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3120
0
  if ((Subtarget->hasVLX())) {
3121
0
    return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3122
0
  }
3123
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3124
0
    return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
3125
0
  }
3126
0
  return 0;
3127
0
}
3128
3129
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3130
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3131
0
    return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3132
0
  }
3133
0
  return 0;
3134
0
}
3135
3136
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3137
switch (RetVT.SimpleTy) {
3138
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
3139
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
3140
  default: return 0;
3141
}
3142
}
3143
3144
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3145
0
  if ((Subtarget->hasAVX512())) {
3146
0
    return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
3147
0
  }
3148
0
  return 0;
3149
0
}
3150
3151
0
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3152
0
  if ((Subtarget->hasDQI())) {
3153
0
    return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3154
0
  }
3155
0
  return 0;
3156
0
}
3157
3158
unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3159
switch (RetVT.SimpleTy) {
3160
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
3161
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
3162
  default: return 0;
3163
}
3164
}
3165
3166
unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3167
  switch (VT.SimpleTy) {
3168
  case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3169
  case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3170
  case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3171
  case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3172
  case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3173
  case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3174
  default: return 0;
3175
  }
3176
}
3177
3178
// FastEmit functions for X86ISD::CVTTP2UI.
3179
3180
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3181
0
  if ((Subtarget->hasVLX())) {
3182
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3183
0
  }
3184
0
  return 0;
3185
0
}
3186
3187
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3188
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3189
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3190
0
  }
3191
0
  return 0;
3192
0
}
3193
3194
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3195
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3196
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3197
0
  }
3198
0
  return 0;
3199
0
}
3200
3201
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3202
switch (RetVT.SimpleTy) {
3203
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
3204
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
3205
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
3206
  default: return 0;
3207
}
3208
}
3209
3210
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3211
0
  if ((Subtarget->hasVLX())) {
3212
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3213
0
  }
3214
0
  return 0;
3215
0
}
3216
3217
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3218
0
  if ((Subtarget->hasDQI())) {
3219
0
    return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3220
0
  }
3221
0
  return 0;
3222
0
}
3223
3224
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3225
switch (RetVT.SimpleTy) {
3226
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
3227
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
3228
  default: return 0;
3229
}
3230
}
3231
3232
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3233
0
  if (RetVT.SimpleTy != MVT::v16i32)
3234
0
    return 0;
3235
0
  if ((Subtarget->hasAVX512())) {
3236
0
    return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3237
0
  }
3238
0
  return 0;
3239
0
}
3240
3241
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3242
0
  if ((Subtarget->hasVLX())) {
3243
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3244
0
  }
3245
0
  return 0;
3246
0
}
3247
3248
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3249
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3250
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3251
0
  }
3252
0
  return 0;
3253
0
}
3254
3255
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3256
switch (RetVT.SimpleTy) {
3257
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
3258
  case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
3259
  default: return 0;
3260
}
3261
}
3262
3263
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3264
0
  if ((Subtarget->hasVLX())) {
3265
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3266
0
  }
3267
0
  return 0;
3268
0
}
3269
3270
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3271
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3272
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3273
0
  }
3274
0
  return 0;
3275
0
}
3276
3277
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3278
switch (RetVT.SimpleTy) {
3279
  case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
3280
  case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
3281
  default: return 0;
3282
}
3283
}
3284
3285
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3286
0
  if ((Subtarget->hasAVX512())) {
3287
0
    return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
3288
0
  }
3289
0
  return 0;
3290
0
}
3291
3292
0
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3293
0
  if ((Subtarget->hasDQI())) {
3294
0
    return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3295
0
  }
3296
0
  return 0;
3297
0
}
3298
3299
unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3300
switch (RetVT.SimpleTy) {
3301
  case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
3302
  case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
3303
  default: return 0;
3304
}
3305
}
3306
3307
unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3308
  switch (VT.SimpleTy) {
3309
  case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3310
  case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3311
  case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3312
  case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3313
  case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3314
  case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3315
  default: return 0;
3316
  }
3317
}
3318
3319
// FastEmit functions for X86ISD::CVTUI2P.
3320
3321
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3322
0
  if (RetVT.SimpleTy != MVT::v2f64)
3323
0
    return 0;
3324
0
  if ((Subtarget->hasVLX())) {
3325
0
    return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3326
0
  }
3327
0
  return 0;
3328
0
}
3329
3330
0
unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3331
0
  if (RetVT.SimpleTy != MVT::v4f32)
3332
0
    return 0;
3333
0
  if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3334
0
    return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3335
0
  }
3336
0
  return 0;
3337
0
}
3338
3339
unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3340
  switch (VT.SimpleTy) {
3341
  case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3342
  case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3343
  default: return 0;
3344
  }
3345
}
3346
3347
// FastEmit functions for X86ISD::EH_RETURN.
3348
3349
0
unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3350
0
  if (RetVT.SimpleTy != MVT::isVoid)
3351
0
    return 0;
3352
0
  return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill);
3353
0
}
3354
3355
0
unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3356
0
  if (RetVT.SimpleTy != MVT::isVoid)
3357
0
    return 0;
3358
0
  return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill);
3359
0
}
3360
3361
unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3362
  switch (VT.SimpleTy) {
3363
  case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
3364
  case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill);
3365
  default: return 0;
3366
  }
3367
}
3368
3369
// FastEmit functions for X86ISD::EXPAND.
3370
3371
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3372
0
  if (RetVT.SimpleTy != MVT::v16i8)
3373
0
    return 0;
3374
0
  if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
3375
0
    return fastEmitInst_r(X86::VPEXPANDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3376
0
  }
3377
0
  return 0;
3378
0
}
3379
3380
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3381
0
  if (RetVT.SimpleTy != MVT::v32i8)
3382
0
    return 0;
3383
0
  if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
3384
0
    return fastEmitInst_r(X86::VPEXPANDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3385
0
  }
3386
0
  return 0;
3387
0
}
3388
3389
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3390
0
  if (RetVT.SimpleTy != MVT::v64i8)
3391
0
    return 0;
3392
0
  if ((Subtarget->hasVBMI2())) {
3393
0
    return fastEmitInst_r(X86::VPEXPANDBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3394
0
  }
3395
0
  return 0;
3396
0
}
3397
3398
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3399
0
  if (RetVT.SimpleTy != MVT::v8i16)
3400
0
    return 0;
3401
0
  if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
3402
0
    return fastEmitInst_r(X86::VPEXPANDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3403
0
  }
3404
0
  return 0;
3405
0
}
3406
3407
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3408
0
  if (RetVT.SimpleTy != MVT::v16i16)
3409
0
    return 0;
3410
0
  if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
3411
0
    return fastEmitInst_r(X86::VPEXPANDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3412
0
  }
3413
0
  return 0;
3414
0
}
3415
3416
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3417
0
  if (RetVT.SimpleTy != MVT::v32i16)
3418
0
    return 0;
3419
0
  if ((Subtarget->hasVBMI2())) {
3420
0
    return fastEmitInst_r(X86::VPEXPANDWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3421
0
  }
3422
0
  return 0;
3423
0
}
3424
3425
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3426
0
  if (RetVT.SimpleTy != MVT::v4i32)
3427
0
    return 0;
3428
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3429
0
    return fastEmitInst_r(X86::VPEXPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3430
0
  }
3431
0
  return 0;
3432
0
}
3433
3434
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3435
0
  if (RetVT.SimpleTy != MVT::v8i32)
3436
0
    return 0;
3437
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3438
0
    return fastEmitInst_r(X86::VPEXPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3439
0
  }
3440
0
  return 0;
3441
0
}
3442
3443
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3444
0
  if (RetVT.SimpleTy != MVT::v16i32)
3445
0
    return 0;
3446
0
  if ((Subtarget->hasAVX512())) {
3447
0
    return fastEmitInst_r(X86::VPEXPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3448
0
  }
3449
0
  return 0;
3450
0
}
3451
3452
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3453
0
  if (RetVT.SimpleTy != MVT::v2i64)
3454
0
    return 0;
3455
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3456
0
    return fastEmitInst_r(X86::VPEXPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3457
0
  }
3458
0
  return 0;
3459
0
}
3460
3461
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3462
0
  if (RetVT.SimpleTy != MVT::v4i64)
3463
0
    return 0;
3464
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3465
0
    return fastEmitInst_r(X86::VPEXPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3466
0
  }
3467
0
  return 0;
3468
0
}
3469
3470
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3471
0
  if (RetVT.SimpleTy != MVT::v8i64)
3472
0
    return 0;
3473
0
  if ((Subtarget->hasAVX512())) {
3474
0
    return fastEmitInst_r(X86::VPEXPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3475
0
  }
3476
0
  return 0;
3477
0
}
3478
3479
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3480
0
  if (RetVT.SimpleTy != MVT::v4f32)
3481
0
    return 0;
3482
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3483
0
    return fastEmitInst_r(X86::VEXPANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3484
0
  }
3485
0
  return 0;
3486
0
}
3487
3488
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3489
0
  if (RetVT.SimpleTy != MVT::v8f32)
3490
0
    return 0;
3491
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3492
0
    return fastEmitInst_r(X86::VEXPANDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3493
0
  }
3494
0
  return 0;
3495
0
}
3496
3497
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3498
0
  if (RetVT.SimpleTy != MVT::v16f32)
3499
0
    return 0;
3500
0
  if ((Subtarget->hasAVX512())) {
3501
0
    return fastEmitInst_r(X86::VEXPANDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3502
0
  }
3503
0
  return 0;
3504
0
}
3505
3506
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3507
0
  if (RetVT.SimpleTy != MVT::v2f64)
3508
0
    return 0;
3509
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3510
0
    return fastEmitInst_r(X86::VEXPANDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3511
0
  }
3512
0
  return 0;
3513
0
}
3514
3515
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3516
0
  if (RetVT.SimpleTy != MVT::v4f64)
3517
0
    return 0;
3518
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3519
0
    return fastEmitInst_r(X86::VEXPANDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3520
0
  }
3521
0
  return 0;
3522
0
}
3523
3524
0
unsigned fastEmit_X86ISD_EXPAND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3525
0
  if (RetVT.SimpleTy != MVT::v8f64)
3526
0
    return 0;
3527
0
  if ((Subtarget->hasAVX512())) {
3528
0
    return fastEmitInst_r(X86::VEXPANDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3529
0
  }
3530
0
  return 0;
3531
0
}
3532
3533
unsigned fastEmit_X86ISD_EXPAND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3534
  switch (VT.SimpleTy) {
3535
  case MVT::v16i8: return fastEmit_X86ISD_EXPAND_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
3536
  case MVT::v32i8: return fastEmit_X86ISD_EXPAND_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
3537
  case MVT::v64i8: return fastEmit_X86ISD_EXPAND_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
3538
  case MVT::v8i16: return fastEmit_X86ISD_EXPAND_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
3539
  case MVT::v16i16: return fastEmit_X86ISD_EXPAND_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
3540
  case MVT::v32i16: return fastEmit_X86ISD_EXPAND_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
3541
  case MVT::v4i32: return fastEmit_X86ISD_EXPAND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3542
  case MVT::v8i32: return fastEmit_X86ISD_EXPAND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
3543
  case MVT::v16i32: return fastEmit_X86ISD_EXPAND_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
3544
  case MVT::v2i64: return fastEmit_X86ISD_EXPAND_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3545
  case MVT::v4i64: return fastEmit_X86ISD_EXPAND_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
3546
  case MVT::v8i64: return fastEmit_X86ISD_EXPAND_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
3547
  case MVT::v4f32: return fastEmit_X86ISD_EXPAND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3548
  case MVT::v8f32: return fastEmit_X86ISD_EXPAND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3549
  case MVT::v16f32: return fastEmit_X86ISD_EXPAND_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3550
  case MVT::v2f64: return fastEmit_X86ISD_EXPAND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3551
  case MVT::v4f64: return fastEmit_X86ISD_EXPAND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3552
  case MVT::v8f64: return fastEmit_X86ISD_EXPAND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3553
  default: return 0;
3554
  }
3555
}
3556
3557
// FastEmit functions for X86ISD::FRCP.
3558
3559
0
unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3560
0
  if (RetVT.SimpleTy != MVT::f32)
3561
0
    return 0;
3562
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3563
0
    return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill);
3564
0
  }
3565
0
  return 0;
3566
0
}
3567
3568
0
unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3569
0
  if (RetVT.SimpleTy != MVT::v4f32)
3570
0
    return 0;
3571
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3572
0
    return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
3573
0
  }
3574
0
  if ((Subtarget->hasAVX())) {
3575
0
    return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
3576
0
  }
3577
0
  return 0;
3578
0
}
3579
3580
0
unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3581
0
  if (RetVT.SimpleTy != MVT::v8f32)
3582
0
    return 0;
3583
0
  if ((Subtarget->hasAVX())) {
3584
0
    return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
3585
0
  }
3586
0
  return 0;
3587
0
}
3588
3589
unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3590
  switch (VT.SimpleTy) {
3591
  case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill);
3592
  case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3593
  case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3594
  default: return 0;
3595
  }
3596
}
3597
3598
// FastEmit functions for X86ISD::FRSQRT.
3599
3600
0
unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3601
0
  if (RetVT.SimpleTy != MVT::f32)
3602
0
    return 0;
3603
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3604
0
    return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
3605
0
  }
3606
0
  return 0;
3607
0
}
3608
3609
0
unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3610
0
  if (RetVT.SimpleTy != MVT::v4f32)
3611
0
    return 0;
3612
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3613
0
    return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
3614
0
  }
3615
0
  if ((Subtarget->hasAVX())) {
3616
0
    return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
3617
0
  }
3618
0
  return 0;
3619
0
}
3620
3621
0
unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3622
0
  if (RetVT.SimpleTy != MVT::v8f32)
3623
0
    return 0;
3624
0
  if ((Subtarget->hasAVX())) {
3625
0
    return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
3626
0
  }
3627
0
  return 0;
3628
0
}
3629
3630
unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3631
  switch (VT.SimpleTy) {
3632
  case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
3633
  case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3634
  case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3635
  default: return 0;
3636
  }
3637
}
3638
3639
// FastEmit functions for X86ISD::MMX_MOVD2W.
3640
3641
0
unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3642
0
  if (RetVT.SimpleTy != MVT::i32)
3643
0
    return 0;
3644
0
  if ((Subtarget->hasMMX())) {
3645
0
    return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill);
3646
0
  }
3647
0
  return 0;
3648
0
}
3649
3650
unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3651
  switch (VT.SimpleTy) {
3652
  case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
3653
  default: return 0;
3654
  }
3655
}
3656
3657
// FastEmit functions for X86ISD::MMX_MOVW2D.
3658
3659
0
unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3660
0
  if (RetVT.SimpleTy != MVT::x86mmx)
3661
0
    return 0;
3662
0
  if ((Subtarget->hasMMX())) {
3663
0
    return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
3664
0
  }
3665
0
  return 0;
3666
0
}
3667
3668
unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3669
  switch (VT.SimpleTy) {
3670
  case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill);
3671
  default: return 0;
3672
  }
3673
}
3674
3675
// FastEmit functions for X86ISD::MOVDDUP.
3676
3677
0
unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3678
0
  if (RetVT.SimpleTy != MVT::v2f64)
3679
0
    return 0;
3680
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
3681
0
    return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3682
0
  }
3683
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3684
0
    return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3685
0
  }
3686
0
  return 0;
3687
0
}
3688
3689
0
unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3690
0
  if (RetVT.SimpleTy != MVT::v4f64)
3691
0
    return 0;
3692
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3693
0
    return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3694
0
  }
3695
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3696
0
    return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3697
0
  }
3698
0
  return 0;
3699
0
}
3700
3701
0
unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3702
0
  if (RetVT.SimpleTy != MVT::v8f64)
3703
0
    return 0;
3704
0
  if ((Subtarget->hasAVX512())) {
3705
0
    return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3706
0
  }
3707
0
  return 0;
3708
0
}
3709
3710
unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3711
  switch (VT.SimpleTy) {
3712
  case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3713
  case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3714
  case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3715
  default: return 0;
3716
  }
3717
}
3718
3719
// FastEmit functions for X86ISD::MOVDQ2Q.
3720
3721
0
unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3722
0
  if (RetVT.SimpleTy != MVT::x86mmx)
3723
0
    return 0;
3724
0
  return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
3725
0
}
3726
3727
unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3728
  switch (VT.SimpleTy) {
3729
  case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3730
  default: return 0;
3731
  }
3732
}
3733
3734
// FastEmit functions for X86ISD::MOVMSK.
3735
3736
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3737
0
  if (RetVT.SimpleTy != MVT::i32)
3738
0
    return 0;
3739
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3740
0
    return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
3741
0
  }
3742
0
  if ((Subtarget->hasAVX())) {
3743
0
    return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
3744
0
  }
3745
0
  return 0;
3746
0
}
3747
3748
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3749
0
  if (RetVT.SimpleTy != MVT::i32)
3750
0
    return 0;
3751
0
  if ((Subtarget->hasAVX2())) {
3752
0
    return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill);
3753
0
  }
3754
0
  return 0;
3755
0
}
3756
3757
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3758
0
  if (RetVT.SimpleTy != MVT::i32)
3759
0
    return 0;
3760
0
  if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3761
0
    return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
3762
0
  }
3763
0
  if ((Subtarget->hasAVX())) {
3764
0
    return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
3765
0
  }
3766
0
  return 0;
3767
0
}
3768
3769
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3770
0
  if (RetVT.SimpleTy != MVT::i32)
3771
0
    return 0;
3772
0
  if ((Subtarget->hasAVX())) {
3773
0
    return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
3774
0
  }
3775
0
  return 0;
3776
0
}
3777
3778
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3779
0
  if (RetVT.SimpleTy != MVT::i32)
3780
0
    return 0;
3781
0
  if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3782
0
    return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
3783
0
  }
3784
0
  if ((Subtarget->hasAVX())) {
3785
0
    return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
3786
0
  }
3787
0
  return 0;
3788
0
}
3789
3790
0
unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3791
0
  if (RetVT.SimpleTy != MVT::i32)
3792
0
    return 0;
3793
0
  if ((Subtarget->hasAVX())) {
3794
0
    return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
3795
0
  }
3796
0
  return 0;
3797
0
}
3798
3799
unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3800
  switch (VT.SimpleTy) {
3801
  case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
3802
  case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
3803
  case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3804
  case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3805
  case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3806
  case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3807
  default: return 0;
3808
  }
3809
}
3810
3811
// FastEmit functions for X86ISD::MOVSHDUP.
3812
3813
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3814
0
  if (RetVT.SimpleTy != MVT::v4i32)
3815
0
    return 0;
3816
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
3817
0
    return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3818
0
  }
3819
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3820
0
    return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3821
0
  }
3822
0
  return 0;
3823
0
}
3824
3825
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3826
0
  if (RetVT.SimpleTy != MVT::v8i32)
3827
0
    return 0;
3828
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3829
0
    return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3830
0
  }
3831
0
  return 0;
3832
0
}
3833
3834
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3835
0
  if (RetVT.SimpleTy != MVT::v4f32)
3836
0
    return 0;
3837
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3838
0
    return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3839
0
  }
3840
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
3841
0
    return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3842
0
  }
3843
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3844
0
    return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3845
0
  }
3846
0
  return 0;
3847
0
}
3848
3849
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3850
0
  if (RetVT.SimpleTy != MVT::v8f32)
3851
0
    return 0;
3852
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3853
0
    return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3854
0
  }
3855
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3856
0
    return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3857
0
  }
3858
0
  return 0;
3859
0
}
3860
3861
0
unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3862
0
  if (RetVT.SimpleTy != MVT::v16f32)
3863
0
    return 0;
3864
0
  if ((Subtarget->hasAVX512())) {
3865
0
    return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3866
0
  }
3867
0
  return 0;
3868
0
}
3869
3870
unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3871
  switch (VT.SimpleTy) {
3872
  case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3873
  case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
3874
  case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3875
  case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3876
  case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3877
  default: return 0;
3878
  }
3879
}
3880
3881
// FastEmit functions for X86ISD::MOVSLDUP.
3882
3883
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3884
0
  if (RetVT.SimpleTy != MVT::v4i32)
3885
0
    return 0;
3886
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
3887
0
    return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3888
0
  }
3889
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3890
0
    return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3891
0
  }
3892
0
  return 0;
3893
0
}
3894
3895
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3896
0
  if (RetVT.SimpleTy != MVT::v8i32)
3897
0
    return 0;
3898
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3899
0
    return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3900
0
  }
3901
0
  return 0;
3902
0
}
3903
3904
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3905
0
  if (RetVT.SimpleTy != MVT::v4f32)
3906
0
    return 0;
3907
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3908
0
    return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3909
0
  }
3910
0
  if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
3911
0
    return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3912
0
  }
3913
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3914
0
    return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
3915
0
  }
3916
0
  return 0;
3917
0
}
3918
3919
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3920
0
  if (RetVT.SimpleTy != MVT::v8f32)
3921
0
    return 0;
3922
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3923
0
    return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3924
0
  }
3925
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3926
0
    return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3927
0
  }
3928
0
  return 0;
3929
0
}
3930
3931
0
unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3932
0
  if (RetVT.SimpleTy != MVT::v16f32)
3933
0
    return 0;
3934
0
  if ((Subtarget->hasAVX512())) {
3935
0
    return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3936
0
  }
3937
0
  return 0;
3938
0
}
3939
3940
unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3941
  switch (VT.SimpleTy) {
3942
  case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3943
  case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
3944
  case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3945
  case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3946
  case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3947
  default: return 0;
3948
  }
3949
}
3950
3951
// FastEmit functions for X86ISD::NT_BRIND.
3952
3953
0
unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3954
0
  if (RetVT.SimpleTy != MVT::isVoid)
3955
0
    return 0;
3956
0
  if ((!Subtarget->is64Bit())) {
3957
0
    return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
3958
0
  }
3959
0
  return 0;
3960
0
}
3961
3962
0
unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3963
0
  if (RetVT.SimpleTy != MVT::isVoid)
3964
0
    return 0;
3965
0
  if ((!Subtarget->is64Bit())) {
3966
0
    return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
3967
0
  }
3968
0
  return 0;
3969
0
}
3970
3971
0
unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3972
0
  if (RetVT.SimpleTy != MVT::isVoid)
3973
0
    return 0;
3974
0
  if ((Subtarget->is64Bit())) {
3975
0
    return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
3976
0
  }
3977
0
  return 0;
3978
0
}
3979
3980
unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3981
  switch (VT.SimpleTy) {
3982
  case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
3983
  case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
3984
  case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
3985
  default: return 0;
3986
  }
3987
}
3988
3989
// FastEmit functions for X86ISD::NT_CALL.
3990
3991
0
unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3992
0
  if (RetVT.SimpleTy != MVT::isVoid)
3993
0
    return 0;
3994
0
  if ((!Subtarget->is64Bit())) {
3995
0
    return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
3996
0
  }
3997
0
  return 0;
3998
0
}
3999
4000
0
unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4001
0
  if (RetVT.SimpleTy != MVT::isVoid)
4002
0
    return 0;
4003
0
  if ((!Subtarget->is64Bit())) {
4004
0
    return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
4005
0
  }
4006
0
  return 0;
4007
0
}
4008
4009
0
unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4010
0
  if (RetVT.SimpleTy != MVT::isVoid)
4011
0
    return 0;
4012
0
  if ((Subtarget->is64Bit())) {
4013
0
    return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
4014
0
  }
4015
0
  return 0;
4016
0
}
4017
4018
unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4019
  switch (VT.SimpleTy) {
4020
  case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
4021
  case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
4022
  case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
4023
  default: return 0;
4024
  }
4025
}
4026
4027
// FastEmit functions for X86ISD::PHMINPOS.
4028
4029
0
unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4030
0
  if (RetVT.SimpleTy != MVT::v8i16)
4031
0
    return 0;
4032
0
  if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
4033
0
    return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
4034
0
  }
4035
0
  if ((Subtarget->hasAVX())) {
4036
0
    return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
4037
0
  }
4038
0
  return 0;
4039
0
}
4040
4041
unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4042
  switch (VT.SimpleTy) {
4043
  case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
4044
  default: return 0;
4045
  }
4046
}
4047
4048
// FastEmit functions for X86ISD::RCP14.
4049
4050
0
unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4051
0
  if (RetVT.SimpleTy != MVT::v4f32)
4052
0
    return 0;
4053
0
  if ((Subtarget->hasVLX())) {
4054
0
    return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4055
0
  }
4056
0
  return 0;
4057
0
}
4058
4059
0
unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4060
0
  if (RetVT.SimpleTy != MVT::v8f32)
4061
0
    return 0;
4062
0
  if ((Subtarget->hasVLX())) {
4063
0
    return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4064
0
  }
4065
0
  return 0;
4066
0
}
4067
4068
0
unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4069
0
  if (RetVT.SimpleTy != MVT::v16f32)
4070
0
    return 0;
4071
0
  if ((Subtarget->hasAVX512())) {
4072
0
    return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
4073
0
  }
4074
0
  return 0;
4075
0
}
4076
4077
0
unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4078
0
  if (RetVT.SimpleTy != MVT::v2f64)
4079
0
    return 0;
4080
0
  if ((Subtarget->hasVLX())) {
4081
0
    return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4082
0
  }
4083
0
  return 0;
4084
0
}
4085
4086
0
unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4087
0
  if (RetVT.SimpleTy != MVT::v4f64)
4088
0
    return 0;
4089
0
  if ((Subtarget->hasVLX())) {
4090
0
    return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4091
0
  }
4092
0
  return 0;
4093
0
}
4094
4095
0
unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4096
0
  if (RetVT.SimpleTy != MVT::v8f64)
4097
0
    return 0;
4098
0
  if ((Subtarget->hasAVX512())) {
4099
0
    return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4100
0
  }
4101
0
  return 0;
4102
0
}
4103
4104
unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4105
  switch (VT.SimpleTy) {
4106
  case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4107
  case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4108
  case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4109
  case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4110
  case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4111
  case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4112
  default: return 0;
4113
  }
4114
}
4115
4116
// FastEmit functions for X86ISD::RSQRT14.
4117
4118
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4119
0
  if (RetVT.SimpleTy != MVT::v4f32)
4120
0
    return 0;
4121
0
  if ((Subtarget->hasVLX())) {
4122
0
    return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4123
0
  }
4124
0
  return 0;
4125
0
}
4126
4127
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4128
0
  if (RetVT.SimpleTy != MVT::v8f32)
4129
0
    return 0;
4130
0
  if ((Subtarget->hasVLX())) {
4131
0
    return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4132
0
  }
4133
0
  return 0;
4134
0
}
4135
4136
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4137
0
  if (RetVT.SimpleTy != MVT::v16f32)
4138
0
    return 0;
4139
0
  if ((Subtarget->hasAVX512())) {
4140
0
    return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
4141
0
  }
4142
0
  return 0;
4143
0
}
4144
4145
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4146
0
  if (RetVT.SimpleTy != MVT::v2f64)
4147
0
    return 0;
4148
0
  if ((Subtarget->hasVLX())) {
4149
0
    return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4150
0
  }
4151
0
  return 0;
4152
0
}
4153
4154
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4155
0
  if (RetVT.SimpleTy != MVT::v4f64)
4156
0
    return 0;
4157
0
  if ((Subtarget->hasVLX())) {
4158
0
    return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4159
0
  }
4160
0
  return 0;
4161
0
}
4162
4163
0
unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4164
0
  if (RetVT.SimpleTy != MVT::v8f64)
4165
0
    return 0;
4166
0
  if ((Subtarget->hasAVX512())) {
4167
0
    return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4168
0
  }
4169
0
  return 0;
4170
0
}
4171
4172
unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4173
  switch (VT.SimpleTy) {
4174
  case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4175
  case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4176
  case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4177
  case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4178
  case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4179
  case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4180
  default: return 0;
4181
  }
4182
}
4183
4184
// FastEmit functions for X86ISD::SEG_ALLOCA.
4185
4186
0
unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4187
0
  if (RetVT.SimpleTy != MVT::i32)
4188
0
    return 0;
4189
0
  if ((!Subtarget->isTarget64BitLP64())) {
4190
0
    return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
4191
0
  }
4192
0
  return 0;
4193
0
}
4194
4195
0
unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4196
0
  if (RetVT.SimpleTy != MVT::i64)
4197
0
    return 0;
4198
0
  if ((Subtarget->is64Bit())) {
4199
0
    return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
4200
0
  }
4201
0
  return 0;
4202
0
}
4203
4204
unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4205
  switch (VT.SimpleTy) {
4206
  case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
4207
  case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
4208
  default: return 0;
4209
  }
4210
}
4211
4212
// FastEmit functions for X86ISD::VBROADCAST.
4213
4214
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
4215
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4216
0
    return fastEmitInst_r(X86::VPBROADCASTDrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4217
0
  }
4218
0
  return 0;
4219
0
}
4220
4221
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4222
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4223
0
    return fastEmitInst_r(X86::VPBROADCASTDrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4224
0
  }
4225
0
  return 0;
4226
0
}
4227
4228
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
4229
0
  if ((Subtarget->hasAVX512())) {
4230
0
    return fastEmitInst_r(X86::VPBROADCASTDrZr, &X86::VR512RegClass, Op0, Op0IsKill);
4231
0
  }
4232
0
  return 0;
4233
0
}
4234
4235
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4236
switch (RetVT.SimpleTy) {
4237
  case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
4238
  case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill);
4239
  case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill);
4240
  default: return 0;
4241
}
4242
}
4243
4244
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
4245
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4246
0
    return fastEmitInst_r(X86::VPBROADCASTQrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4247
0
  }
4248
0
  return 0;
4249
0
}
4250
4251
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
4252
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4253
0
    return fastEmitInst_r(X86::VPBROADCASTQrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4254
0
  }
4255
0
  return 0;
4256
0
}
4257
4258
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4259
0
  if ((Subtarget->hasAVX512())) {
4260
0
    return fastEmitInst_r(X86::VPBROADCASTQrZr, &X86::VR512RegClass, Op0, Op0IsKill);
4261
0
  }
4262
0
  return 0;
4263
0
}
4264
4265
unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4266
switch (RetVT.SimpleTy) {
4267
  case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
4268
  case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill);
4269
  case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill);
4270
  default: return 0;
4271
}
4272
}
4273
4274
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
4275
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4276
0
    return fastEmitInst_r(X86::VPBROADCASTBZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4277
0
  }
4278
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4279
0
    return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill);
4280
0
  }
4281
0
  return 0;
4282
0
}
4283
4284
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
4285
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4286
0
    return fastEmitInst_r(X86::VPBROADCASTBZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4287
0
  }
4288
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4289
0
    return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4290
0
  }
4291
0
  return 0;
4292
0
}
4293
4294
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) {
4295
0
  if ((Subtarget->hasBWI())) {
4296
0
    return fastEmitInst_r(X86::VPBROADCASTBZr, &X86::VR512RegClass, Op0, Op0IsKill);
4297
0
  }
4298
0
  return 0;
4299
0
}
4300
4301
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4302
switch (RetVT.SimpleTy) {
4303
  case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill);
4304
  case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill);
4305
  case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill);
4306
  default: return 0;
4307
}
4308
}
4309
4310
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
4311
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4312
0
    return fastEmitInst_r(X86::VPBROADCASTWZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4313
0
  }
4314
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4315
0
    return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill);
4316
0
  }
4317
0
  return 0;
4318
0
}
4319
4320
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
4321
0
  if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
4322
0
    return fastEmitInst_r(X86::VPBROADCASTWZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4323
0
  }
4324
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
4325
0
    return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4326
0
  }
4327
0
  return 0;
4328
0
}
4329
4330
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
4331
0
  if ((Subtarget->hasBWI())) {
4332
0
    return fastEmitInst_r(X86::VPBROADCASTWZr, &X86::VR512RegClass, Op0, Op0IsKill);
4333
0
  }
4334
0
  return 0;
4335
0
}
4336
4337
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4338
switch (RetVT.SimpleTy) {
4339
  case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill);
4340
  case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill);
4341
  case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill);
4342
  default: return 0;
4343
}
4344
}
4345
4346
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
4347
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4348
0
    return fastEmitInst_r(X86::VPBROADCASTDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4349
0
  }
4350
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4351
0
    return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill);
4352
0
  }
4353
0
  return 0;
4354
0
}
4355
4356
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4357
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4358
0
    return fastEmitInst_r(X86::VPBROADCASTDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4359
0
  }
4360
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4361
0
    return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4362
0
  }
4363
0
  return 0;
4364
0
}
4365
4366
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
4367
0
  if ((Subtarget->hasAVX512())) {
4368
0
    return fastEmitInst_r(X86::VPBROADCASTDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4369
0
  }
4370
0
  return 0;
4371
0
}
4372
4373
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4374
switch (RetVT.SimpleTy) {
4375
  case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill);
4376
  case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill);
4377
  case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill);
4378
  default: return 0;
4379
}
4380
}
4381
4382
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
4383
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4384
0
    return fastEmitInst_r(X86::VPBROADCASTQZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4385
0
  }
4386
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4387
0
    return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill);
4388
0
  }
4389
0
  return 0;
4390
0
}
4391
4392
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
4393
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4394
0
    return fastEmitInst_r(X86::VPBROADCASTQZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4395
0
  }
4396
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4397
0
    return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4398
0
  }
4399
0
  return 0;
4400
0
}
4401
4402
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4403
0
  if ((Subtarget->hasAVX512())) {
4404
0
    return fastEmitInst_r(X86::VPBROADCASTQZr, &X86::VR512RegClass, Op0, Op0IsKill);
4405
0
  }
4406
0
  return 0;
4407
0
}
4408
4409
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4410
switch (RetVT.SimpleTy) {
4411
  case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill);
4412
  case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill);
4413
  case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill);
4414
  default: return 0;
4415
}
4416
}
4417
4418
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
4419
0
  if ((Subtarget->hasVLX())) {
4420
0
    return fastEmitInst_r(X86::VBROADCASTSSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4421
0
  }
4422
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4423
0
    return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill);
4424
0
  }
4425
0
  return 0;
4426
0
}
4427
4428
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
4429
0
  if ((Subtarget->hasVLX())) {
4430
0
    return fastEmitInst_r(X86::VBROADCASTSSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4431
0
  }
4432
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4433
0
    return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4434
0
  }
4435
0
  return 0;
4436
0
}
4437
4438
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) {
4439
0
  if ((Subtarget->hasAVX512())) {
4440
0
    return fastEmitInst_r(X86::VBROADCASTSSZr, &X86::VR512RegClass, Op0, Op0IsKill);
4441
0
  }
4442
0
  return 0;
4443
0
}
4444
4445
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4446
switch (RetVT.SimpleTy) {
4447
  case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill);
4448
  case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill);
4449
  case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill);
4450
  default: return 0;
4451
}
4452
}
4453
4454
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
4455
0
  if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4456
0
    return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4457
0
  }
4458
0
  if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4459
0
    return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4460
0
  }
4461
0
  return 0;
4462
0
}
4463
4464
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
4465
0
  if ((Subtarget->hasVLX())) {
4466
0
    return fastEmitInst_r(X86::VBROADCASTSDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4467
0
  }
4468
0
  if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
4469
0
    return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4470
0
  }
4471
0
  return 0;
4472
0
}
4473
4474
0
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
4475
0
  if ((Subtarget->hasAVX512())) {
4476
0
    return fastEmitInst_r(X86::VBROADCASTSDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4477
0
  }
4478
0
  return 0;
4479
0
}
4480
4481
unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4482
switch (RetVT.SimpleTy) {
4483
  case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0, Op0IsKill);
4484
  case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill);
4485
  case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill);
4486
  default: return 0;
4487
}
4488
}
4489
4490
unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4491
  switch (VT.SimpleTy) {
4492
  case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
4493
  case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
4494
  case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
4495
  case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
4496
  case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4497
  case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4498
  case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4499
  case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4500
  default: return 0;
4501
  }
4502
}
4503
4504
// FastEmit functions for X86ISD::VBROADCASTM.
4505
4506
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
4507
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
4508
0
    return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4509
0
  }
4510
0
  return 0;
4511
0
}
4512
4513
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
4514
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
4515
0
    return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4516
0
  }
4517
0
  return 0;
4518
0
}
4519
4520
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4521
0
  if ((Subtarget->hasCDI())) {
4522
0
    return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4523
0
  }
4524
0
  return 0;
4525
0
}
4526
4527
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4528
switch (RetVT.SimpleTy) {
4529
  case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill);
4530
  case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill);
4531
  case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
4532
  default: return 0;
4533
}
4534
}
4535
4536
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
4537
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
4538
0
    return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4539
0
  }
4540
0
  return 0;
4541
0
}
4542
4543
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4544
0
  if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
4545
0
    return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4546
0
  }
4547
0
  return 0;
4548
0
}
4549
4550
0
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
4551
0
  if ((Subtarget->hasCDI())) {
4552
0
    return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4553
0
  }
4554
0
  return 0;
4555
0
}
4556
4557
unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4558
switch (RetVT.SimpleTy) {
4559
  case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill);
4560
  case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill);
4561
  case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
4562
  default: return 0;
4563
}
4564
}
4565
4566
unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4567
  switch (VT.SimpleTy) {
4568
  case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
4569
  case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
4570
  default: return 0;
4571
  }
4572
}
4573
4574
// FastEmit functions for X86ISD::VFPEXT.
4575