Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
Line
Count
Source (jump to first uncovered line)
1
//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file performs vector type splitting and scalarization for LegalizeTypes.
10
// Scalarization is the act of changing a computation in an illegal one-element
11
// vector type to be a computation in its scalar element type.  For example,
12
// implementing <1 x f32> arithmetic in a scalar f32 register.  This is needed
13
// as a base case when scalarizing vector arithmetic like <4 x f32>, which
14
// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15
// types.
16
// Splitting is the act of changing a computation in an invalid vector type to
17
// be a computation in two vectors of half the size.  For example, implementing
18
// <128 x f32> operations in terms of two <64 x f32> operations.
19
//
20
//===----------------------------------------------------------------------===//
21
22
#include "LegalizeTypes.h"
23
#include "llvm/IR/DataLayout.h"
24
#include "llvm/Support/ErrorHandling.h"
25
#include "llvm/Support/raw_ostream.h"
26
using namespace llvm;
27
28
#define DEBUG_TYPE "legalize-types"
29
30
//===----------------------------------------------------------------------===//
31
//  Result Vector Scalarization: <1 x ty> -> ty.
32
//===----------------------------------------------------------------------===//
33
34
76.3k
void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
35
76.3k
  LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
36
76.3k
             dbgs() << "\n");
37
76.3k
  SDValue R = SDValue();
38
76.3k
39
76.3k
  switch (N->getOpcode()) {
40
76.3k
  default:
41
#ifndef NDEBUG
42
    dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
43
    N->dump(&DAG);
44
    dbgs() << "\n";
45
#endif
46
    report_fatal_error("Do not know how to scalarize the result of this "
47
0
                       "operator!\n");
48
76.3k
49
76.3k
  
case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break0
;
50
76.3k
  
case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break18.1k
;
51
76.3k
  
case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break14.7k
;
52
76.3k
  
case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break5.78k
;
53
76.3k
  
case ISD::STRICT_FP_ROUND: R = ScalarizeVecRes_STRICT_FP_ROUND(N); break9
;
54
76.3k
  
case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break726
;
55
76.3k
  
case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break0
;
56
76.3k
  
case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break0
;
57
76.3k
  
case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break1.19k
;
58
76.3k
  
case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break5.97k
;
59
76.3k
  
case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break235
;
60
76.3k
  
case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break21
;
61
76.3k
  
case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break379
;
62
76.3k
  
case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break194
;
63
76.3k
  
case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break0
;
64
76.3k
  
case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break1.85k
;
65
76.3k
  
case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break333
;
66
76.3k
  
case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break0
;
67
76.3k
  case ISD::ANY_EXTEND_VECTOR_INREG:
68
8
  case ISD::SIGN_EXTEND_VECTOR_INREG:
69
8
  case ISD::ZERO_EXTEND_VECTOR_INREG:
70
8
    R = ScalarizeVecRes_VecInregOp(N);
71
8
    break;
72
19.6k
  case ISD::ABS:
73
19.6k
  case ISD::ANY_EXTEND:
74
19.6k
  case ISD::BITREVERSE:
75
19.6k
  case ISD::BSWAP:
76
19.6k
  case ISD::CTLZ:
77
19.6k
  case ISD::CTLZ_ZERO_UNDEF:
78
19.6k
  case ISD::CTPOP:
79
19.6k
  case ISD::CTTZ:
80
19.6k
  case ISD::CTTZ_ZERO_UNDEF:
81
19.6k
  case ISD::FABS:
82
19.6k
  case ISD::FCEIL:
83
19.6k
  case ISD::FCOS:
84
19.6k
  case ISD::FEXP:
85
19.6k
  case ISD::FEXP2:
86
19.6k
  case ISD::FFLOOR:
87
19.6k
  case ISD::FLOG:
88
19.6k
  case ISD::FLOG10:
89
19.6k
  case ISD::FLOG2:
90
19.6k
  case ISD::FNEARBYINT:
91
19.6k
  case ISD::FNEG:
92
19.6k
  case ISD::FP_EXTEND:
93
19.6k
  case ISD::FP_TO_SINT:
94
19.6k
  case ISD::FP_TO_UINT:
95
19.6k
  case ISD::FRINT:
96
19.6k
  case ISD::FROUND:
97
19.6k
  case ISD::FSIN:
98
19.6k
  case ISD::FSQRT:
99
19.6k
  case ISD::FTRUNC:
100
19.6k
  case ISD::SIGN_EXTEND:
101
19.6k
  case ISD::SINT_TO_FP:
102
19.6k
  case ISD::TRUNCATE:
103
19.6k
  case ISD::UINT_TO_FP:
104
19.6k
  case ISD::ZERO_EXTEND:
105
19.6k
  case ISD::FCANONICALIZE:
106
19.6k
    R = ScalarizeVecRes_UnaryOp(N);
107
19.6k
    break;
108
19.6k
109
19.6k
  case ISD::ADD:
110
6.68k
  case ISD::AND:
111
6.68k
  case ISD::FADD:
112
6.68k
  case ISD::FCOPYSIGN:
113
6.68k
  case ISD::FDIV:
114
6.68k
  case ISD::FMUL:
115
6.68k
  case ISD::FMINNUM:
116
6.68k
  case ISD::FMAXNUM:
117
6.68k
  case ISD::FMINNUM_IEEE:
118
6.68k
  case ISD::FMAXNUM_IEEE:
119
6.68k
  case ISD::FMINIMUM:
120
6.68k
  case ISD::FMAXIMUM:
121
6.68k
  case ISD::SMIN:
122
6.68k
  case ISD::SMAX:
123
6.68k
  case ISD::UMIN:
124
6.68k
  case ISD::UMAX:
125
6.68k
126
6.68k
  case ISD::SADDSAT:
127
6.68k
  case ISD::UADDSAT:
128
6.68k
  case ISD::SSUBSAT:
129
6.68k
  case ISD::USUBSAT:
130
6.68k
131
6.68k
  case ISD::FPOW:
132
6.68k
  case ISD::FREM:
133
6.68k
  case ISD::FSUB:
134
6.68k
  case ISD::MUL:
135
6.68k
  case ISD::OR:
136
6.68k
  case ISD::SDIV:
137
6.68k
  case ISD::SREM:
138
6.68k
  case ISD::SUB:
139
6.68k
  case ISD::UDIV:
140
6.68k
  case ISD::UREM:
141
6.68k
  case ISD::XOR:
142
6.68k
  case ISD::SHL:
143
6.68k
  case ISD::SRA:
144
6.68k
  case ISD::SRL:
145
6.68k
    R = ScalarizeVecRes_BinOp(N);
146
6.68k
    break;
147
6.68k
  case ISD::FMA:
148
44
    R = ScalarizeVecRes_TernaryOp(N);
149
44
    break;
150
6.68k
  case ISD::STRICT_FADD:
151
201
  case ISD::STRICT_FSUB:
152
201
  case ISD::STRICT_FMUL:
153
201
  case ISD::STRICT_FDIV:
154
201
  case ISD::STRICT_FREM:
155
201
  case ISD::STRICT_FSQRT:
156
201
  case ISD::STRICT_FMA:
157
201
  case ISD::STRICT_FPOW:
158
201
  case ISD::STRICT_FPOWI:
159
201
  case ISD::STRICT_FSIN:
160
201
  case ISD::STRICT_FCOS:
161
201
  case ISD::STRICT_FEXP:
162
201
  case ISD::STRICT_FEXP2:
163
201
  case ISD::STRICT_FLOG:
164
201
  case ISD::STRICT_FLOG10:
165
201
  case ISD::STRICT_FLOG2:
166
201
  case ISD::STRICT_FRINT:
167
201
  case ISD::STRICT_FNEARBYINT:
168
201
  case ISD::STRICT_FMAXNUM:
169
201
  case ISD::STRICT_FMINNUM:
170
201
  case ISD::STRICT_FCEIL:
171
201
  case ISD::STRICT_FFLOOR:
172
201
  case ISD::STRICT_FROUND:
173
201
  case ISD::STRICT_FTRUNC:
174
201
  case ISD::STRICT_FP_EXTEND:
175
201
    R = ScalarizeVecRes_StrictFPOp(N);
176
201
    break;
177
201
  case ISD::UADDO:
178
136
  case ISD::SADDO:
179
136
  case ISD::USUBO:
180
136
  case ISD::SSUBO:
181
136
  case ISD::UMULO:
182
136
  case ISD::SMULO:
183
136
    R = ScalarizeVecRes_OverflowOp(N, ResNo);
184
136
    break;
185
136
  case ISD::SMULFIX:
186
24
  case ISD::SMULFIXSAT:
187
24
  case ISD::UMULFIX:
188
24
    R = ScalarizeVecRes_MULFIX(N);
189
24
    break;
190
76.3k
  }
191
76.3k
192
76.3k
  // If R is null, the sub-method took care of registering the result.
193
76.3k
  if (R.getNode())
194
76.3k
    SetScalarizedVector(SDValue(N, ResNo), R);
195
76.3k
}
196
197
6.68k
SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
198
6.68k
  SDValue LHS = GetScalarizedVector(N->getOperand(0));
199
6.68k
  SDValue RHS = GetScalarizedVector(N->getOperand(1));
200
6.68k
  return DAG.getNode(N->getOpcode(), SDLoc(N),
201
6.68k
                     LHS.getValueType(), LHS, RHS, N->getFlags());
202
6.68k
}
203
204
44
SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
205
44
  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
206
44
  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
207
44
  SDValue Op2 = GetScalarizedVector(N->getOperand(2));
208
44
  return DAG.getNode(N->getOpcode(), SDLoc(N),
209
44
                     Op0.getValueType(), Op0, Op1, Op2);
210
44
}
211
212
24
SDValue DAGTypeLegalizer::ScalarizeVecRes_MULFIX(SDNode *N) {
213
24
  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
214
24
  SDValue Op1 = GetScalarizedVector(N->getOperand(1));
215
24
  SDValue Op2 = N->getOperand(2);
216
24
  return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
217
24
                     Op2);
218
24
}
219
220
201
SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
221
201
  EVT VT = N->getValueType(0).getVectorElementType();
222
201
  unsigned NumOpers = N->getNumOperands();
223
201
  SDValue Chain = N->getOperand(0);
224
201
  EVT ValueVTs[] = {VT, MVT::Other};
225
201
  SDLoc dl(N);
226
201
227
201
  SmallVector<SDValue, 4> Opers;
228
201
229
201
  // The Chain is the first operand.
230
201
  Opers.push_back(Chain);
231
201
232
201
  // Now process the remaining operands.
233
485
  for (unsigned i = 1; i < NumOpers; 
++i284
) {
234
284
    SDValue Oper = N->getOperand(i);
235
284
236
284
    if (Oper.getValueType().isVector())
237
275
      Oper = GetScalarizedVector(Oper);
238
284
239
284
    Opers.push_back(Oper);
240
284
  }
241
201
242
201
  SDValue Result = DAG.getNode(N->getOpcode(), dl, ValueVTs, Opers);
243
201
244
201
  // Legalize the chain result - switch anything that used the old chain to
245
201
  // use the new one.
246
201
  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
247
201
  return Result;
248
201
}
249
250
SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
251
136
                                                     unsigned ResNo) {
252
136
  SDLoc DL(N);
253
136
  EVT ResVT = N->getValueType(0);
254
136
  EVT OvVT = N->getValueType(1);
255
136
256
136
  SDValue ScalarLHS, ScalarRHS;
257
136
  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
258
136
    ScalarLHS = GetScalarizedVector(N->getOperand(0));
259
136
    ScalarRHS = GetScalarizedVector(N->getOperand(1));
260
136
  } else {
261
0
    SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
262
0
    DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
263
0
    DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
264
0
    ScalarLHS = ElemsLHS[0];
265
0
    ScalarRHS = ElemsRHS[0];
266
0
  }
267
136
268
136
  SDVTList ScalarVTs = DAG.getVTList(
269
136
      ResVT.getVectorElementType(), OvVT.getVectorElementType());
270
136
  SDNode *ScalarNode = DAG.getNode(
271
136
      N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
272
136
273
136
  // Replace the other vector result not being explicitly scalarized here.
274
136
  unsigned OtherNo = 1 - ResNo;
275
136
  EVT OtherVT = N->getValueType(OtherNo);
276
136
  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
277
118
    SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
278
118
  } else {
279
18
    SDValue OtherVal = DAG.getNode(
280
18
        ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
281
18
    ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
282
18
  }
283
136
284
136
  return SDValue(ScalarNode, ResNo);
285
136
}
286
287
SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
288
0
                                                       unsigned ResNo) {
289
0
  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
290
0
  return GetScalarizedVector(Op);
291
0
}
292
293
18.1k
SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
294
18.1k
  SDValue Op = N->getOperand(0);
295
18.1k
  if (Op.getValueType().isVector()
296
18.1k
      && 
Op.getValueType().getVectorNumElements() == 1297
297
18.1k
      && 
!isSimpleLegalType(Op.getValueType())128
)
298
128
    Op = GetScalarizedVector(Op);
299
18.1k
  EVT NewVT = N->getValueType(0).getVectorElementType();
300
18.1k
  return DAG.getNode(ISD::BITCAST, SDLoc(N),
301
18.1k
                     NewVT, Op);
302
18.1k
}
303
304
14.7k
SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
305
14.7k
  EVT EltVT = N->getValueType(0).getVectorElementType();
306
14.7k
  SDValue InOp = N->getOperand(0);
307
14.7k
  // The BUILD_VECTOR operands may be of wider element types and
308
14.7k
  // we may need to truncate them back to the requested return type.
309
14.7k
  if (EltVT.isInteger())
310
10.7k
    return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
311
3.97k
  return InOp;
312
3.97k
}
313
314
5.78k
SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
315
5.78k
  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
316
5.78k
                     N->getValueType(0).getVectorElementType(),
317
5.78k
                     N->getOperand(0), N->getOperand(1));
318
5.78k
}
319
320
726
SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
321
726
  EVT NewVT = N->getValueType(0).getVectorElementType();
322
726
  SDValue Op = GetScalarizedVector(N->getOperand(0));
323
726
  return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
324
726
                     NewVT, Op, N->getOperand(1));
325
726
}
326
327
9
SDValue DAGTypeLegalizer::ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N) {
328
9
  EVT NewVT = N->getValueType(0).getVectorElementType();
329
9
  SDValue Op = GetScalarizedVector(N->getOperand(1));
330
9
  SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
331
9
                            { NewVT, MVT::Other }, 
332
9
                            { N->getOperand(0), Op, N->getOperand(2) });
333
9
  // Legalize the chain result - switch anything that used the old chain to
334
9
  // use the new one.
335
9
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
336
9
  return Res;
337
9
}
338
339
0
SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
340
0
  SDValue Op = GetScalarizedVector(N->getOperand(0));
341
0
  return DAG.getNode(ISD::FPOWI, SDLoc(N),
342
0
                     Op.getValueType(), Op, N->getOperand(1));
343
0
}
344
345
1.19k
SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
346
1.19k
  // The value to insert may have a wider type than the vector element type,
347
1.19k
  // so be sure to truncate it to the element type if necessary.
348
1.19k
  SDValue Op = N->getOperand(1);
349
1.19k
  EVT EltVT = N->getValueType(0).getVectorElementType();
350
1.19k
  if (Op.getValueType() != EltVT)
351
0
    // FIXME: Can this happen for floating point types?
352
0
    Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
353
1.19k
  return Op;
354
1.19k
}
355
356
5.97k
SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
357
5.97k
  assert(N->isUnindexed() && "Indexed vector load?");
358
5.97k
359
5.97k
  SDValue Result = DAG.getLoad(
360
5.97k
      ISD::UNINDEXED, N->getExtensionType(),
361
5.97k
      N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
362
5.97k
      N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
363
5.97k
      N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
364
5.97k
      N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
365
5.97k
      N->getAAInfo());
366
5.97k
367
5.97k
  // Legalize the chain result - switch anything that used the old chain to
368
5.97k
  // use the new one.
369
5.97k
  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
370
5.97k
  return Result;
371
5.97k
}
372
373
19.6k
SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
374
19.6k
  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
375
19.6k
  EVT DestVT = N->getValueType(0).getVectorElementType();
376
19.6k
  SDValue Op = N->getOperand(0);
377
19.6k
  EVT OpVT = Op.getValueType();
378
19.6k
  SDLoc DL(N);
379
19.6k
  // The result needs scalarizing, but it's not a given that the source does.
380
19.6k
  // This is a workaround for targets where it's impossible to scalarize the
381
19.6k
  // result of a conversion, because the source type is legal.
382
19.6k
  // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
383
19.6k
  // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
384
19.6k
  // legal and was not scalarized.
385
19.6k
  // See the similar logic in ScalarizeVecRes_SETCC
386
19.6k
  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
387
19.6k
    Op = GetScalarizedVector(Op);
388
19.6k
  } else {
389
12
    EVT VT = OpVT.getVectorElementType();
390
12
    Op = DAG.getNode(
391
12
        ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
392
12
        DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
393
12
  }
394
19.6k
  return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op);
395
19.6k
}
396
397
21
SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
398
21
  EVT EltVT = N->getValueType(0).getVectorElementType();
399
21
  EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
400
21
  SDValue LHS = GetScalarizedVector(N->getOperand(0));
401
21
  return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
402
21
                     LHS, DAG.getValueType(ExtVT));
403
21
}
404
405
8
SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
406
8
  SDLoc DL(N);
407
8
  SDValue Op = N->getOperand(0);
408
8
409
8
  EVT OpVT = Op.getValueType();
410
8
  EVT OpEltVT = OpVT.getVectorElementType();
411
8
  EVT EltVT = N->getValueType(0).getVectorElementType();
412
8
413
8
  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
414
0
    Op = GetScalarizedVector(Op);
415
8
  } else {
416
8
    Op = DAG.getNode(
417
8
        ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
418
8
        DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
419
8
  }
420
8
421
8
  switch (N->getOpcode()) {
422
8
  case ISD::ANY_EXTEND_VECTOR_INREG:
423
8
    return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
424
8
  case ISD::SIGN_EXTEND_VECTOR_INREG:
425
0
    return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
426
8
  case ISD::ZERO_EXTEND_VECTOR_INREG:
427
0
    return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
428
0
  }
429
0
430
0
  llvm_unreachable("Illegal extend_vector_inreg opcode");
431
0
}
432
433
235
SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
434
235
  // If the operand is wider than the vector element type then it is implicitly
435
235
  // truncated.  Make that explicit here.
436
235
  EVT EltVT = N->getValueType(0).getVectorElementType();
437
235
  SDValue InOp = N->getOperand(0);
438
235
  if (InOp.getValueType() != EltVT)
439
0
    return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
440
235
  return InOp;
441
235
}
442
443
379
SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
444
379
  SDValue Cond = N->getOperand(0);
445
379
  EVT OpVT = Cond.getValueType();
446
379
  SDLoc DL(N);
447
379
  // The vselect result and true/value operands needs scalarizing, but it's
448
379
  // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
449
379
  // See the similar logic in ScalarizeVecRes_SETCC
450
379
  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
451
357
    Cond = GetScalarizedVector(Cond);
452
357
  } else {
453
22
    EVT VT = OpVT.getVectorElementType();
454
22
    Cond = DAG.getNode(
455
22
        ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
456
22
        DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
457
22
  }
458
379
459
379
  SDValue LHS = GetScalarizedVector(N->getOperand(1));
460
379
  TargetLowering::BooleanContent ScalarBool =
461
379
      TLI.getBooleanContents(false, false);
462
379
  TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
463
379
464
379
  // If integer and float booleans have different contents then we can't
465
379
  // reliably optimize in all cases. There is a full explanation for this in
466
379
  // DAGCombiner::visitSELECT() where the same issue affects folding
467
379
  // (select C, 0, 1) to (xor C, 1).
468
379
  if (TLI.getBooleanContents(false, false) !=
469
379
      TLI.getBooleanContents(false, true)) {
470
0
    // At least try the common case where the boolean is generated by a
471
0
    // comparison.
472
0
    if (Cond->getOpcode() == ISD::SETCC) {
473
0
      EVT OpVT = Cond->getOperand(0).getValueType();
474
0
      ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
475
0
      VecBool = TLI.getBooleanContents(OpVT);
476
0
    } else
477
0
      ScalarBool = TargetLowering::UndefinedBooleanContent;
478
0
  }
479
379
480
379
  EVT CondVT = Cond.getValueType();
481
379
  if (ScalarBool != VecBool) {
482
59
    switch (ScalarBool) {
483
59
      case TargetLowering::UndefinedBooleanContent:
484
0
        break;
485
59
      case TargetLowering::ZeroOrOneBooleanContent:
486
59
        assert(VecBool == TargetLowering::UndefinedBooleanContent ||
487
59
               VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
488
59
        // Vector read from all ones, scalar expects a single 1 so mask.
489
59
        Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
490
59
                           Cond, DAG.getConstant(1, SDLoc(N), CondVT));
491
59
        break;
492
59
      case TargetLowering::ZeroOrNegativeOneBooleanContent:
493
0
        assert(VecBool == TargetLowering::UndefinedBooleanContent ||
494
0
               VecBool == TargetLowering::ZeroOrOneBooleanContent);
495
0
        // Vector reads from a one, scalar from all ones so sign extend.
496
0
        Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
497
0
                           Cond, DAG.getValueType(MVT::i1));
498
0
        break;
499
379
    }
500
379
  }
501
379
502
379
  // Truncate the condition if needed
503
379
  auto BoolVT = getSetCCResultType(CondVT);
504
379
  if (BoolVT.bitsLT(CondVT))
505
4
    Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
506
379
507
379
  return DAG.getSelect(SDLoc(N),
508
379
                       LHS.getValueType(), Cond, LHS,
509
379
                       GetScalarizedVector(N->getOperand(2)));
510
379
}
511
512
194
SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
513
194
  SDValue LHS = GetScalarizedVector(N->getOperand(1));
514
194
  return DAG.getSelect(SDLoc(N),
515
194
                       LHS.getValueType(), N->getOperand(0), LHS,
516
194
                       GetScalarizedVector(N->getOperand(2)));
517
194
}
518
519
0
SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
520
0
  SDValue LHS = GetScalarizedVector(N->getOperand(2));
521
0
  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
522
0
                     N->getOperand(0), N->getOperand(1),
523
0
                     LHS, GetScalarizedVector(N->getOperand(3)),
524
0
                     N->getOperand(4));
525
0
}
526
527
333
SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
528
333
  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
529
333
}
530
531
0
SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
532
0
  // Figure out if the scalar is the LHS or RHS and return it.
533
0
  SDValue Arg = N->getOperand(2).getOperand(0);
534
0
  if (Arg.isUndef())
535
0
    return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
536
0
  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
537
0
  return GetScalarizedVector(N->getOperand(Op));
538
0
}
539
540
1.85k
SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
541
1.85k
  assert(N->getValueType(0).isVector() &&
542
1.85k
         N->getOperand(0).getValueType().isVector() &&
543
1.85k
         "Operand types must be vectors");
544
1.85k
  SDValue LHS = N->getOperand(0);
545
1.85k
  SDValue RHS = N->getOperand(1);
546
1.85k
  EVT OpVT = LHS.getValueType();
547
1.85k
  EVT NVT = N->getValueType(0).getVectorElementType();
548
1.85k
  SDLoc DL(N);
549
1.85k
550
1.85k
  // The result needs scalarizing, but it's not a given that the source does.
551
1.85k
  if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
552
1.84k
    LHS = GetScalarizedVector(LHS);
553
1.84k
    RHS = GetScalarizedVector(RHS);
554
1.84k
  } else {
555
8
    EVT VT = OpVT.getVectorElementType();
556
8
    LHS = DAG.getNode(
557
8
        ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
558
8
        DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
559
8
    RHS = DAG.getNode(
560
8
        ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
561
8
        DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
562
8
  }
563
1.85k
564
1.85k
  // Turn it into a scalar SETCC.
565
1.85k
  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
566
1.85k
                            N->getOperand(2));
567
1.85k
  // Vectors may have a different boolean contents to scalars.  Promote the
568
1.85k
  // value appropriately.
569
1.85k
  ISD::NodeType ExtendCode =
570
1.85k
      TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
571
1.85k
  return DAG.getNode(ExtendCode, DL, NVT, Res);
572
1.85k
}
573
574
575
//===----------------------------------------------------------------------===//
576
//  Operand Vector Scalarization <1 x ty> -> ty.
577
//===----------------------------------------------------------------------===//
578
579
23.1k
bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
580
23.1k
  LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
581
23.1k
             dbgs() << "\n");
582
23.1k
  SDValue Res = SDValue();
583
23.1k
584
23.1k
  if (!Res.getNode()) {
585
23.1k
    switch (N->getOpcode()) {
586
23.1k
    default:
587
#ifndef NDEBUG
588
      dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
589
      N->dump(&DAG);
590
      dbgs() << "\n";
591
#endif
592
      report_fatal_error("Do not know how to scalarize this operator's "
593
0
                         "operand!\n");
594
23.1k
    case ISD::BITCAST:
595
1.96k
      Res = ScalarizeVecOp_BITCAST(N);
596
1.96k
      break;
597
23.1k
    case ISD::ANY_EXTEND:
598
20
    case ISD::ZERO_EXTEND:
599
20
    case ISD::SIGN_EXTEND:
600
20
    case ISD::TRUNCATE:
601
20
    case ISD::FP_TO_SINT:
602
20
    case ISD::FP_TO_UINT:
603
20
    case ISD::SINT_TO_FP:
604
20
    case ISD::UINT_TO_FP:
605
20
      Res = ScalarizeVecOp_UnaryOp(N);
606
20
      break;
607
6.22k
    case ISD::CONCAT_VECTORS:
608
6.22k
      Res = ScalarizeVecOp_CONCAT_VECTORS(N);
609
6.22k
      break;
610
6.98k
    case ISD::EXTRACT_VECTOR_ELT:
611
6.98k
      Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
612
6.98k
      break;
613
20
    case ISD::VSELECT:
614
1
      Res = ScalarizeVecOp_VSELECT(N);
615
1
      break;
616
32
    case ISD::SETCC:
617
32
      Res = ScalarizeVecOp_VSETCC(N);
618
32
      break;
619
7.90k
    case ISD::STORE:
620
7.90k
      Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
621
7.90k
      break;
622
20
    case ISD::STRICT_FP_ROUND:
623
0
      Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
624
0
      break;
625
20
    case ISD::FP_ROUND:
626
3
      Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
627
3
      break;
628
20
    case ISD::VECREDUCE_FADD:
629
2
    case ISD::VECREDUCE_FMUL:
630
2
    case ISD::VECREDUCE_ADD:
631
2
    case ISD::VECREDUCE_MUL:
632
2
    case ISD::VECREDUCE_AND:
633
2
    case ISD::VECREDUCE_OR:
634
2
    case ISD::VECREDUCE_XOR:
635
2
    case ISD::VECREDUCE_SMAX:
636
2
    case ISD::VECREDUCE_SMIN:
637
2
    case ISD::VECREDUCE_UMAX:
638
2
    case ISD::VECREDUCE_UMIN:
639
2
    case ISD::VECREDUCE_FMAX:
640
2
    case ISD::VECREDUCE_FMIN:
641
2
      Res = ScalarizeVecOp_VECREDUCE(N);
642
2
      break;
643
23.1k
    }
644
23.1k
  }
645
23.1k
646
23.1k
  // If the result is null, the sub-method took care of registering results etc.
647
23.1k
  if (!Res.getNode()) 
return false0
;
648
23.1k
649
23.1k
  // If the result is N, the sub-method updated N in place.  Tell the legalizer
650
23.1k
  // core about this.
651
23.1k
  if (Res.getNode() == N)
652
0
    return true;
653
23.1k
654
23.1k
  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
655
23.1k
         "Invalid operand expansion");
656
23.1k
657
23.1k
  ReplaceValueWith(SDValue(N, 0), Res);
658
23.1k
  return false;
659
23.1k
}
660
661
/// If the value to convert is a vector that needs to be scalarized, it must be
662
/// <1 x ty>. Convert the element instead.
663
1.96k
SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
664
1.96k
  SDValue Elt = GetScalarizedVector(N->getOperand(0));
665
1.96k
  return DAG.getNode(ISD::BITCAST, SDLoc(N),
666
1.96k
                     N->getValueType(0), Elt);
667
1.96k
}
668
669
/// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
670
/// Do the operation on the element instead.
671
20
SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
672
20
  assert(N->getValueType(0).getVectorNumElements() == 1 &&
673
20
         "Unexpected vector type!");
674
20
  SDValue Elt = GetScalarizedVector(N->getOperand(0));
675
20
  SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
676
20
                           N->getValueType(0).getScalarType(), Elt);
677
20
  // Revectorize the result so the types line up with what the uses of this
678
20
  // expression expect.
679
20
  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
680
20
}
681
682
/// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
683
6.22k
SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
684
6.22k
  SmallVector<SDValue, 8> Ops(N->getNumOperands());
685
18.7k
  for (unsigned i = 0, e = N->getNumOperands(); i < e; 
++i12.5k
)
686
12.5k
    Ops[i] = GetScalarizedVector(N->getOperand(i));
687
6.22k
  return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
688
6.22k
}
689
690
/// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
691
/// so just return the element, ignoring the index.
692
6.98k
SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
693
6.98k
  EVT VT = N->getValueType(0);
694
6.98k
  SDValue Res = GetScalarizedVector(N->getOperand(0));
695
6.98k
  if (Res.getValueType() != VT)
696
927
    Res = VT.isFloatingPoint()
697
927
              ? 
DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)3
698
927
              : 
DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res)924
;
699
6.98k
  return Res;
700
6.98k
}
701
702
/// If the input condition is a vector that needs to be scalarized, it must be
703
/// <1 x i1>, so just convert to a normal ISD::SELECT
704
/// (still with vector output type since that was acceptable if we got here).
705
1
SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
706
1
  SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
707
1
  EVT VT = N->getValueType(0);
708
1
709
1
  return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
710
1
                     N->getOperand(2));
711
1
}
712
713
/// If the operand is a vector that needs to be scalarized then the
714
/// result must be v1i1, so just convert to a scalar SETCC and wrap
715
/// with a scalar_to_vector since the res type is legal if we got here
716
32
SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
717
32
  assert(N->getValueType(0).isVector() &&
718
32
         N->getOperand(0).getValueType().isVector() &&
719
32
         "Operand types must be vectors");
720
32
  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
721
32
722
32
  EVT VT = N->getValueType(0);
723
32
  SDValue LHS = GetScalarizedVector(N->getOperand(0));
724
32
  SDValue RHS = GetScalarizedVector(N->getOperand(1));
725
32
726
32
  EVT OpVT = N->getOperand(0).getValueType();
727
32
  EVT NVT = VT.getVectorElementType();
728
32
  SDLoc DL(N);
729
32
  // Turn it into a scalar SETCC.
730
32
  SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
731
32
      N->getOperand(2));
732
32
733
32
  // Vectors may have a different boolean contents to scalars.  Promote the
734
32
  // value appropriately.
735
32
  ISD::NodeType ExtendCode =
736
32
      TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
737
32
738
32
  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
739
32
740
32
  return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
741
32
}
742
743
/// If the value to store is a vector that needs to be scalarized, it must be
744
/// <1 x ty>. Just store the element.
745
7.90k
SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
746
7.90k
  assert(N->isUnindexed() && "Indexed store of one-element vector?");
747
7.90k
  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
748
7.90k
  SDLoc dl(N);
749
7.90k
750
7.90k
  if (N->isTruncatingStore())
751
0
    return DAG.getTruncStore(
752
0
        N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
753
0
        N->getBasePtr(), N->getPointerInfo(),
754
0
        N->getMemoryVT().getVectorElementType(), N->getAlignment(),
755
0
        N->getMemOperand()->getFlags(), N->getAAInfo());
756
7.90k
757
7.90k
  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
758
7.90k
                      N->getBasePtr(), N->getPointerInfo(),
759
7.90k
                      N->getOriginalAlignment(), N->getMemOperand()->getFlags(),
760
7.90k
                      N->getAAInfo());
761
7.90k
}
762
763
/// If the value to round is a vector that needs to be scalarized, it must be
764
/// <1 x ty>. Convert the element instead.
765
3
SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
766
3
  SDValue Elt = GetScalarizedVector(N->getOperand(0));
767
3
  SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
768
3
                            N->getValueType(0).getVectorElementType(), Elt,
769
3
                            N->getOperand(1));
770
3
  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
771
3
}
772
773
SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, 
774
0
                                                         unsigned OpNo) {
775
0
  assert(OpNo == 1 && "Wrong operand for scalarization!");
776
0
  SDValue Elt = GetScalarizedVector(N->getOperand(1));
777
0
  SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
778
0
                            { N->getValueType(0).getVectorElementType(), 
779
0
                              MVT::Other },
780
0
                            { N->getOperand(0), Elt, N->getOperand(2) });
781
0
  // Legalize the chain result - switch anything that used the old chain to
782
0
  // use the new one.
783
0
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
784
0
  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
785
0
} 
786
787
2
SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
788
2
  SDValue Res = GetScalarizedVector(N->getOperand(0));
789
2
  // Result type may be wider than element type.
790
2
  if (Res.getValueType() != N->getValueType(0))
791
0
    Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
792
2
  return Res;
793
2
}
794
795
//===----------------------------------------------------------------------===//
796
//  Result Vector Splitting
797
//===----------------------------------------------------------------------===//
798
799
/// This method is called when the specified result of the specified node is
800
/// found to need vector splitting. At this point, the node may also have
801
/// invalid operands or may have other results that need legalization, we just
802
/// know that (at least) one result needs vector splitting.
803
173k
void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
804
173k
  LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
805
173k
  SDValue Lo, Hi;
806
173k
807
173k
  // See if the target wants to custom expand this node.
808
173k
  if (CustomLowerNode(N, N->getValueType(ResNo), true))
809
497
    return;
810
173k
811
173k
  switch (N->getOpcode()) {
812
173k
  default:
813
#ifndef NDEBUG
814
    dbgs() << "SplitVectorResult #" << ResNo << ": ";
815
    N->dump(&DAG);
816
    dbgs() << "\n";
817
#endif
818
    report_fatal_error("Do not know how to split the result of this "
819
0
                       "operator!\n");
820
173k
821
173k
  
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break0
;
822
173k
  case ISD::VSELECT:
823
1.14k
  case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
824
1.14k
  
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break0
;
825
2.77k
  case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
826
22.6k
  case ISD::BITCAST:           SplitVecRes_BITCAST(N, Lo, Hi); break;
827
23.6k
  case ISD::BUILD_VECTOR:      SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
828
30.3k
  case ISD::CONCAT_VECTORS:    SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
829
3.20k
  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
830
2.18k
  case ISD::INSERT_SUBVECTOR:  SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
831
1.14k
  
case ISD::FP_ROUND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break0
;
832
1.14k
  
case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break8
;
833
1.14k
  
case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break11
;
834
9.97k
  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
835
1.14k
  
case ISD::SCALAR_TO_VECTOR: SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break327
;
836
1.14k
  
case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break150
;
837
5.03k
  case ISD::LOAD:
838
5.03k
    SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
839
5.03k
    break;
840
1.14k
  case ISD::MLOAD:
841
18
    SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
842
18
    break;
843
1.14k
  case ISD::MGATHER:
844
16
    SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
845
16
    break;
846
6.58k
  case ISD::SETCC:
847
6.58k
    SplitVecRes_SETCC(N, Lo, Hi);
848
6.58k
    break;
849
5.36k
  case ISD::VECTOR_SHUFFLE:
850
5.36k
    SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
851
5.36k
    break;
852
1.14k
  case ISD::VAARG:
853
5
    SplitVecRes_VAARG(N, Lo, Hi);
854
5
    break;
855
1.14k
856
1.14k
  case ISD::ANY_EXTEND_VECTOR_INREG:
857
460
  case ISD::SIGN_EXTEND_VECTOR_INREG:
858
460
  case ISD::ZERO_EXTEND_VECTOR_INREG:
859
460
    SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
860
460
    break;
861
460
862
18.2k
  case ISD::ABS:
863
18.2k
  case ISD::BITREVERSE:
864
18.2k
  case ISD::BSWAP:
865
18.2k
  case ISD::CTLZ:
866
18.2k
  case ISD::CTTZ:
867
18.2k
  case ISD::CTLZ_ZERO_UNDEF:
868
18.2k
  case ISD::CTTZ_ZERO_UNDEF:
869
18.2k
  case ISD::CTPOP:
870
18.2k
  case ISD::FABS:
871
18.2k
  case ISD::FCEIL:
872
18.2k
  case ISD::FCOS:
873
18.2k
  case ISD::FEXP:
874
18.2k
  case ISD::FEXP2:
875
18.2k
  case ISD::FFLOOR:
876
18.2k
  case ISD::FLOG:
877
18.2k
  case ISD::FLOG10:
878
18.2k
  case ISD::FLOG2:
879
18.2k
  case ISD::FNEARBYINT:
880
18.2k
  case ISD::FNEG:
881
18.2k
  case ISD::FP_EXTEND:
882
18.2k
  case ISD::STRICT_FP_EXTEND:
883
18.2k
  case ISD::FP_ROUND:
884
18.2k
  case ISD::STRICT_FP_ROUND:
885
18.2k
  case ISD::FP_TO_SINT:
886
18.2k
  case ISD::FP_TO_UINT:
887
18.2k
  case ISD::FRINT:
888
18.2k
  case ISD::FROUND:
889
18.2k
  case ISD::FSIN:
890
18.2k
  case ISD::FSQRT:
891
18.2k
  case ISD::FTRUNC:
892
18.2k
  case ISD::SINT_TO_FP:
893
18.2k
  case ISD::TRUNCATE:
894
18.2k
  case ISD::UINT_TO_FP:
895
18.2k
  case ISD::FCANONICALIZE:
896
18.2k
    SplitVecRes_UnaryOp(N, Lo, Hi);
897
18.2k
    break;
898
18.2k
899
18.2k
  case ISD::ANY_EXTEND:
900
10.9k
  case ISD::SIGN_EXTEND:
901
10.9k
  case ISD::ZERO_EXTEND:
902
10.9k
    SplitVecRes_ExtendOp(N, Lo, Hi);
903
10.9k
    break;
904
10.9k
905
29.3k
  case ISD::ADD:
906
29.3k
  case ISD::SUB:
907
29.3k
  case ISD::MUL:
908
29.3k
  case ISD::MULHS:
909
29.3k
  case ISD::MULHU:
910
29.3k
  case ISD::FADD:
911
29.3k
  case ISD::FSUB:
912
29.3k
  case ISD::FMUL:
913
29.3k
  case ISD::FMINNUM:
914
29.3k
  case ISD::FMAXNUM:
915
29.3k
  case ISD::FMINIMUM:
916
29.3k
  case ISD::FMAXIMUM:
917
29.3k
  case ISD::SDIV:
918
29.3k
  case ISD::UDIV:
919
29.3k
  case ISD::FDIV:
920
29.3k
  case ISD::FPOW:
921
29.3k
  case ISD::AND:
922
29.3k
  case ISD::OR:
923
29.3k
  case ISD::XOR:
924
29.3k
  case ISD::SHL:
925
29.3k
  case ISD::SRA:
926
29.3k
  case ISD::SRL:
927
29.3k
  case ISD::UREM:
928
29.3k
  case ISD::SREM:
929
29.3k
  case ISD::FREM:
930
29.3k
  case ISD::SMIN:
931
29.3k
  case ISD::SMAX:
932
29.3k
  case ISD::UMIN:
933
29.3k
  case ISD::UMAX:
934
29.3k
  case ISD::SADDSAT:
935
29.3k
  case ISD::UADDSAT:
936
29.3k
  case ISD::SSUBSAT:
937
29.3k
  case ISD::USUBSAT:
938
29.3k
    SplitVecRes_BinOp(N, Lo, Hi);
939
29.3k
    break;
940
29.3k
  case ISD::FMA:
941
201
    SplitVecRes_TernaryOp(N, Lo, Hi);
942
201
    break;
943
29.3k
  case ISD::STRICT_FADD:
944
156
  case ISD::STRICT_FSUB:
945
156
  case ISD::STRICT_FMUL:
946
156
  case ISD::STRICT_FDIV:
947
156
  case ISD::STRICT_FREM:
948
156
  case ISD::STRICT_FSQRT:
949
156
  case ISD::STRICT_FMA:
950
156
  case ISD::STRICT_FPOW:
951
156
  case ISD::STRICT_FPOWI:
952
156
  case ISD::STRICT_FSIN:
953
156
  case ISD::STRICT_FCOS:
954
156
  case ISD::STRICT_FEXP:
955
156
  case ISD::STRICT_FEXP2:
956
156
  case ISD::STRICT_FLOG:
957
156
  case ISD::STRICT_FLOG10:
958
156
  case ISD::STRICT_FLOG2:
959
156
  case ISD::STRICT_FRINT:
960
156
  case ISD::STRICT_FNEARBYINT:
961
156
  case ISD::STRICT_FMAXNUM:
962
156
  case ISD::STRICT_FMINNUM:
963
156
  case ISD::STRICT_FCEIL:
964
156
  case ISD::STRICT_FFLOOR:
965
156
  case ISD::STRICT_FROUND:
966
156
  case ISD::STRICT_FTRUNC:
967
156
    SplitVecRes_StrictFPOp(N, Lo, Hi);
968
156
    break;
969
156
  case ISD::UADDO:
970
156
  case ISD::SADDO:
971
156
  case ISD::USUBO:
972
156
  case ISD::SSUBO:
973
156
  case ISD::UMULO:
974
156
  case ISD::SMULO:
975
156
    SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
976
156
    break;
977
156
  case ISD::SMULFIX:
978
18
  case ISD::SMULFIXSAT:
979
18
  case ISD::UMULFIX:
980
18
    SplitVecRes_MULFIX(N, Lo, Hi);
981
18
    break;
982
173k
  }
983
173k
984
173k
  // If Lo/Hi is null, the sub-method took care of registering results etc.
985
173k
  if (Lo.getNode())
986
173k
    SetSplitVector(SDValue(N, ResNo), Lo, Hi);
987
173k
}
988
989
void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
990
29.3k
                                         SDValue &Hi) {
991
29.3k
  SDValue LHSLo, LHSHi;
992
29.3k
  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
993
29.3k
  SDValue RHSLo, RHSHi;
994
29.3k
  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
995
29.3k
  SDLoc dl(N);
996
29.3k
997
29.3k
  const SDNodeFlags Flags = N->getFlags();
998
29.3k
  unsigned Opcode = N->getOpcode();
999
29.3k
  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1000
29.3k
  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1001
29.3k
}
1002
1003
void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1004
201
                                             SDValue &Hi) {
1005
201
  SDValue Op0Lo, Op0Hi;
1006
201
  GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1007
201
  SDValue Op1Lo, Op1Hi;
1008
201
  GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1009
201
  SDValue Op2Lo, Op2Hi;
1010
201
  GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1011
201
  SDLoc dl(N);
1012
201
1013
201
  Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(),
1014
201
                   Op0Lo, Op1Lo, Op2Lo);
1015
201
  Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(),
1016
201
                   Op0Hi, Op1Hi, Op2Hi);
1017
201
}
1018
1019
18
void DAGTypeLegalizer::SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1020
18
  SDValue LHSLo, LHSHi;
1021
18
  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1022
18
  SDValue RHSLo, RHSHi;
1023
18
  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1024
18
  SDLoc dl(N);
1025
18
  SDValue Op2 = N->getOperand(2);
1026
18
1027
18
  unsigned Opcode = N->getOpcode();
1028
18
  Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2);
1029
18
  Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2);
1030
18
}
1031
1032
void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1033
22.6k
                                           SDValue &Hi) {
1034
22.6k
  // We know the result is a vector.  The input may be either a vector or a
1035
22.6k
  // scalar value.
1036
22.6k
  EVT LoVT, HiVT;
1037
22.6k
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1038
22.6k
  SDLoc dl(N);
1039
22.6k
1040
22.6k
  SDValue InOp = N->getOperand(0);
1041
22.6k
  EVT InVT = InOp.getValueType();
1042
22.6k
1043
22.6k
  // Handle some special cases efficiently.
1044
22.6k
  switch (getTypeAction(InVT)) {
1045
22.6k
  case TargetLowering::TypeLegal:
1046
17.6k
  case TargetLowering::TypePromoteInteger:
1047
17.6k
  case TargetLowering::TypePromoteFloat:
1048
17.6k
  case TargetLowering::TypeSoftenFloat:
1049
17.6k
  case TargetLowering::TypeScalarizeVector:
1050
17.6k
  case TargetLowering::TypeWidenVector:
1051
17.6k
    break;
1052
17.6k
  case TargetLowering::TypeExpandInteger:
1053
2.19k
  case TargetLowering::TypeExpandFloat:
1054
2.19k
    // A scalar to vector conversion, where the scalar needs expansion.
1055
2.19k
    // If the vector is being split in two then we can just convert the
1056
2.19k
    // expanded pieces.
1057
2.19k
    if (LoVT == HiVT) {
1058
2.19k
      GetExpandedOp(InOp, Lo, Hi);
1059
2.19k
      if (DAG.getDataLayout().isBigEndian())
1060
2
        std::swap(Lo, Hi);
1061
2.19k
      Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1062
2.19k
      Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1063
2.19k
      return;
1064
2.19k
    }
1065
0
    break;
1066
2.80k
  case TargetLowering::TypeSplitVector:
1067
2.80k
    // If the input is a vector that needs to be split, convert each split
1068
2.80k
    // piece of the input now.
1069
2.80k
    GetSplitVector(InOp, Lo, Hi);
1070
2.80k
    Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1071
2.80k
    Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1072
2.80k
    return;
1073
17.6k
  }
1074
17.6k
1075
17.6k
  // In the general case, convert the input to an integer and split it by hand.
1076
17.6k
  EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1077
17.6k
  EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1078
17.6k
  if (DAG.getDataLayout().isBigEndian())
1079
260
    std::swap(LoIntVT, HiIntVT);
1080
17.6k
1081
17.6k
  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1082
17.6k
1083
17.6k
  if (DAG.getDataLayout().isBigEndian())
1084
260
    std::swap(Lo, Hi);
1085
17.6k
  Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1086
17.6k
  Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1087
17.6k
}
1088
1089
void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1090
23.6k
                                                SDValue &Hi) {
1091
23.6k
  EVT LoVT, HiVT;
1092
23.6k
  SDLoc dl(N);
1093
23.6k
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1094
23.6k
  unsigned LoNumElts = LoVT.getVectorNumElements();
1095
23.6k
  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1096
23.6k
  Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1097
23.6k
1098
23.6k
  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1099
23.6k
  Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1100
23.6k
}
1101
1102
void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1103
30.3k
                                                  SDValue &Hi) {
1104
30.3k
  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1105
30.3k
  SDLoc dl(N);
1106
30.3k
  unsigned NumSubvectors = N->getNumOperands() / 2;
1107
30.3k
  if (NumSubvectors == 1) {
1108
23.4k
    Lo = N->getOperand(0);
1109
23.4k
    Hi = N->getOperand(1);
1110
23.4k
    return;
1111
23.4k
  }
1112
6.92k
1113
6.92k
  EVT LoVT, HiVT;
1114
6.92k
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1115
6.92k
1116
6.92k
  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1117
6.92k
  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1118
6.92k
1119
6.92k
  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1120
6.92k
  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1121
6.92k
}
1122
1123
void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1124
3.20k
                                                     SDValue &Hi) {
1125
3.20k
  SDValue Vec = N->getOperand(0);
1126
3.20k
  SDValue Idx = N->getOperand(1);
1127
3.20k
  SDLoc dl(N);
1128
3.20k
1129
3.20k
  EVT LoVT, HiVT;
1130
3.20k
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1131
3.20k
1132
3.20k
  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1133
3.20k
  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1134
3.20k
  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1135
3.20k
                   DAG.getConstant(IdxVal + LoVT.getVectorNumElements(), dl,
1136
3.20k
                                   TLI.getVectorIdxTy(DAG.getDataLayout())));
1137
3.20k
}
1138
1139
void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1140
2.18k
                                                    SDValue &Hi) {
1141
2.18k
  SDValue Vec = N->getOperand(0);
1142
2.18k
  SDValue SubVec = N->getOperand(1);
1143
2.18k
  SDValue Idx = N->getOperand(2);
1144
2.18k
  SDLoc dl(N);
1145
2.18k
  GetSplitVector(Vec, Lo, Hi);
1146
2.18k
1147
2.18k
  EVT VecVT = Vec.getValueType();
1148
2.18k
  unsigned VecElems = VecVT.getVectorNumElements();
1149
2.18k
  unsigned SubElems = SubVec.getValueType().getVectorNumElements();
1150
2.18k
1151
2.18k
  // If we know the index is 0, and we know the subvector doesn't cross the
1152
2.18k
  // boundary between the halves, we can avoid spilling the vector, and insert
1153
2.18k
  // into the lower half of the split vector directly.
1154
2.18k
  // TODO: The IdxVal == 0 constraint is artificial, we could do this whenever
1155
2.18k
  // the index is constant and there is no boundary crossing. But those cases
1156
2.18k
  // don't seem to get hit in practice.
1157
2.18k
  if (ConstantSDNode *ConstIdx = dyn_cast<ConstantSDNode>(Idx)) {
1158
2.18k
    unsigned IdxVal = ConstIdx->getZExtValue();
1159
2.18k
    if ((IdxVal == 0) && (IdxVal + SubElems <= VecElems / 2)) {
1160
2.18k
      EVT LoVT, HiVT;
1161
2.18k
      std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1162
2.18k
      Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1163
2.18k
      return;
1164
2.18k
    }
1165
0
  }
1166
0
1167
0
  // Spill the vector to the stack.
1168
0
  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1169
0
  SDValue Store =
1170
0
      DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, MachinePointerInfo());
1171
0
1172
0
  // Store the new subvector into the specified index.
1173
0
  SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1174
0
  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1175
0
  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1176
0
  Store = DAG.getStore(Store, dl, SubVec, SubVecPtr, MachinePointerInfo());
1177
0
1178
0
  // Load the Lo part from the stack slot.
1179
0
  Lo =
1180
0
      DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, MachinePointerInfo());
1181
0
1182
0
  // Increment the pointer to the other part.
1183
0
  unsigned IncrementSize = Lo.getValueSizeInBits() / 8;
1184
0
  StackPtr =
1185
0
      DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1186
0
                  DAG.getConstant(IncrementSize, dl, StackPtr.getValueType()));
1187
0
1188
0
  // Load the Hi part from the stack slot.
1189
0
  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MachinePointerInfo(),
1190
0
                   MinAlign(Alignment, IncrementSize));
1191
0
}
1192
1193
void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1194
8
                                         SDValue &Hi) {
1195
8
  SDLoc dl(N);
1196
8
  GetSplitVector(N->getOperand(0), Lo, Hi);
1197
8
  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1198
8
  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1199
8
}
1200
1201
void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1202
11
                                             SDValue &Hi) {
1203
11
  SDValue LHSLo, LHSHi;
1204
11
  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1205
11
  SDLoc DL(N);
1206
11
1207
11
  SDValue RHSLo, RHSHi;
1208
11
  SDValue RHS = N->getOperand(1);
1209
11
  EVT RHSVT = RHS.getValueType();
1210
11
  if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1211
9
    GetSplitVector(RHS, RHSLo, RHSHi);
1212
2
  else
1213
2
    std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1214
11
1215
11
1216
11
  Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1217
11
  Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1218
11
}
1219
1220
void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1221
150
                                           SDValue &Hi) {
1222
150
  SDValue LHSLo, LHSHi;
1223
150
  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1224
150
  SDLoc dl(N);
1225
150
1226
150
  EVT LoVT, HiVT;
1227
150
  std::tie(LoVT, HiVT) =
1228
150
    DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1229
150
1230
150
  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1231
150
                   DAG.getValueType(LoVT));
1232
150
  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1233
150
                   DAG.getValueType(HiVT));
1234
150
}
1235
1236
void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1237
466
                                                 SDValue &Hi) {
1238
466
  unsigned Opcode = N->getOpcode();
1239
466
  SDValue N0 = N->getOperand(0);
1240
466
1241
466
  SDLoc dl(N);
1242
466
  SDValue InLo, InHi;
1243
466
1244
466
  if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1245
466
    GetSplitVector(N0, InLo, InHi);
1246
0
  else
1247
0
    std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1248
466
1249
466
  EVT InLoVT = InLo.getValueType();
1250
466
  unsigned InNumElements = InLoVT.getVectorNumElements();
1251
466
1252
466
  EVT OutLoVT, OutHiVT;
1253
466
  std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1254
466
  unsigned OutNumElements = OutLoVT.getVectorNumElements();
1255
466
  assert((2 * OutNumElements) <= InNumElements &&
1256
466
         "Illegal extend vector in reg split");
1257
466
1258
466
  // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1259
466
  // input vector (i.e. we only use InLo):
1260
466
  // OutLo will extend the first OutNumElements from InLo.
1261
466
  // OutHi will extend the next OutNumElements from InLo.
1262
466
1263
466
  // Shuffle the elements from InLo for OutHi into the bottom elements to
1264
466
  // create a 'fake' InHi.
1265
466
  SmallVector<int, 8> SplitHi(InNumElements, -1);
1266
3.03k
  for (unsigned i = 0; i != OutNumElements; 
++i2.56k
)
1267
2.56k
    SplitHi[i] = i + OutNumElements;
1268
466
  InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1269
466
1270
466
  Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1271
466
  Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1272
466
}
1273
1274
void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1275
156
                                              SDValue &Hi) {
1276
156
  unsigned NumOps = N->getNumOperands();
1277
156
  SDValue Chain = N->getOperand(0);
1278
156
  EVT LoVT, HiVT;
1279
156
  SDLoc dl(N);
1280
156
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1281
156
1282
156
  SmallVector<SDValue, 4> OpsLo;
1283
156
  SmallVector<SDValue, 4> OpsHi;
1284
156
1285
156
  // The Chain is the first operand.
1286
156
  OpsLo.push_back(Chain);
1287
156
  OpsHi.push_back(Chain);
1288
156
1289
156
  // Now process the remaining operands.
1290
384
  for (unsigned i = 1; i < NumOps; 
++i228
) {
1291
228
    SDValue Op = N->getOperand(i);
1292
228
    SDValue OpLo = Op;
1293
228
    SDValue OpHi = Op;
1294
228
1295
228
    EVT InVT = Op.getValueType();
1296
228
    if (InVT.isVector()) {
1297
220
      // If the input also splits, handle it directly for a
1298
220
      // compile time speedup. Otherwise split it by hand.
1299
220
      if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1300
220
        GetSplitVector(Op, OpLo, OpHi);
1301
0
      else
1302
0
        std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1303
220
    }
1304
228
1305
228
    OpsLo.push_back(OpLo);
1306
228
    OpsHi.push_back(OpHi);
1307
228
  }
1308
156
1309
156
  EVT LoValueVTs[] = {LoVT, MVT::Other};
1310
156
  EVT HiValueVTs[] = {HiVT, MVT::Other};
1311
156
  Lo = DAG.getNode(N->getOpcode(), dl, LoValueVTs, OpsLo);
1312
156
  Hi = DAG.getNode(N->getOpcode(), dl, HiValueVTs, OpsHi);
1313
156
1314
156
  // Build a factor node to remember that this Op is independent of the
1315
156
  // other one.
1316
156
  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1317
156
                      Lo.getValue(1), Hi.getValue(1));
1318
156
1319
156
  // Legalize the chain result - switch anything that used the old chain to
1320
156
  // use the new one.
1321
156
  ReplaceValueWith(SDValue(N, 1), Chain);
1322
156
}
1323
1324
46
SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1325
46
  SDValue Chain = N->getOperand(0);
1326
46
  EVT VT = N->getValueType(0);
1327
46
  unsigned NE = VT.getVectorNumElements();
1328
46
  EVT EltVT = VT.getVectorElementType();
1329
46
  SDLoc dl(N);
1330
46
1331
46
  SmallVector<SDValue, 8> Scalars;
1332
46
  SmallVector<SDValue, 4> Operands(N->getNumOperands());
1333
46
1334
46
  // If ResNE is 0, fully unroll the vector op.
1335
46
  if (ResNE == 0)
1336
0
    ResNE = NE;
1337
46
  else if (NE > ResNE)
1338
0
    NE = ResNE;
1339
46
1340
46
  //The results of each unrolled operation, including the chain.
1341
46
  EVT ChainVTs[] = {EltVT, MVT::Other};
1342
46
  SmallVector<SDValue, 8> Chains;
1343
46
1344
46
  unsigned i;
1345
184
  for (i = 0; i != NE; 
++i138
) {
1346
138
    Operands[0] = Chain;
1347
330
    for (unsigned j = 1, e = N->getNumOperands(); j != e; 
++j192
) {
1348
192
      SDValue Operand = N->getOperand(j);
1349
192
      EVT OperandVT = Operand.getValueType();
1350
192
      if (OperandVT.isVector()) {
1351
186
        EVT OperandEltVT = OperandVT.getVectorElementType();
1352
186
        Operands[j] =
1353
186
            DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
1354
186
                    DAG.getConstant(i, dl, TLI.getVectorIdxTy(
1355
186
                          DAG.getDataLayout())));
1356
186
      } else {
1357
6
        Operands[j] = Operand;
1358
6
      }
1359
192
    }
1360
138
    SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1361
138
    Scalar.getNode()->setFlags(N->getFlags());
1362
138
1363
138
    //Add in the scalar as well as its chain value to the
1364
138
    //result vectors.
1365
138
    Scalars.push_back(Scalar);
1366
138
    Chains.push_back(Scalar.getValue(1));
1367
138
  }
1368
46
1369
92
  for (; i < ResNE; 
++i46
)
1370
46
    Scalars.push_back(DAG.getUNDEF(EltVT));
1371
46
1372
46
  // Build a new factor node to connect the chain back together.
1373
46
  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1374
46
  ReplaceValueWith(SDValue(N, 1), Chain);
1375
46
1376
46
  // Create a new BUILD_VECTOR node
1377
46
  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1378
46
  return DAG.getBuildVector(VecVT, dl, Scalars);
1379
46
}
1380
1381
void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1382
156
                                              SDValue &Lo, SDValue &Hi) {
1383
156
  SDLoc dl(N);
1384
156
  EVT ResVT = N->getValueType(0);
1385
156
  EVT OvVT = N->getValueType(1);
1386
156
  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1387
156
  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1388
156
  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1389
156
1390
156
  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1391
156
  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1392
144
    GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1393
144
    GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1394
144
  } else {
1395
12
    std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1396
12
    std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1397
12
  }
1398
156
1399
156
  unsigned Opcode = N->getOpcode();
1400
156
  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1401
156
  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1402
156
  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1403
156
  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1404
156
1405
156
  Lo = SDValue(LoNode, ResNo);
1406
156
  Hi = SDValue(HiNode, ResNo);
1407
156
1408
156
  // Replace the other vector result not being explicitly split here.
1409
156
  unsigned OtherNo = 1 - ResNo;
1410
156
  EVT OtherVT = N->getValueType(OtherNo);
1411
156
  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1412
0
    SetSplitVector(SDValue(N, OtherNo),
1413
0
                   SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1414
156
  } else {
1415
156
    SDValue OtherVal = DAG.getNode(
1416
156
        ISD::CONCAT_VECTORS, dl, OtherVT,
1417
156
        SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1418
156
    ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1419
156
  }
1420
156
}
1421
1422
void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1423
9.97k
                                                     SDValue &Hi) {
1424
9.97k
  SDValue Vec = N->getOperand(0);
1425
9.97k
  SDValue Elt = N->getOperand(1);
1426
9.97k
  SDValue Idx = N->getOperand(2);
1427
9.97k
  SDLoc dl(N);
1428
9.97k
  GetSplitVector(Vec, Lo, Hi);
1429
9.97k
1430
9.97k
  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1431
9.91k
    unsigned IdxVal = CIdx->getZExtValue();
1432
9.91k
    unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
1433
9.91k
    if (IdxVal < LoNumElts)
1434
5.06k
      Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1435
5.06k
                       Lo.getValueType(), Lo, Elt, Idx);
1436
4.84k
    else
1437
4.84k
      Hi =
1438
4.84k
          DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1439
4.84k
                      DAG.getConstant(IdxVal - LoNumElts, dl,
1440
4.84k
                                      TLI.getVectorIdxTy(DAG.getDataLayout())));
1441
9.91k
    return;
1442
9.91k
  }
1443
61
1444
61
  // See if the target wants to custom expand this node.
1445
61
  if (CustomLowerNode(N, N->getValueType(0), true))
1446
0
    return;
1447
61
1448
61
  // Make the vector elements byte-addressable if they aren't already.
1449
61
  EVT VecVT = Vec.getValueType();
1450
61
  EVT EltVT = VecVT.getVectorElementType();
1451
61
  if (VecVT.getScalarSizeInBits() < 8) {
1452
9
    EltVT = MVT::i8;
1453
9
    VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1454
9
                             VecVT.getVectorNumElements());
1455
9
    Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1456
9
    // Extend the element type to match if needed.
1457
9
    if (EltVT.bitsGT(Elt.getValueType()))
1458
9
      Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1459
9
  }
1460
61
1461
61
  // Spill the vector to the stack.
1462
61
  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1463
61
  auto &MF = DAG.getMachineFunction();
1464
61
  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1465
61
  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1466
61
  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1467
61
1468
61
  // Store the new element.  This may be larger than the vector element type,
1469
61
  // so use a truncating store.
1470
61
  SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1471
61
  Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
1472
61
  unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(VecType);
1473
61
  Store = DAG.getTruncStore(Store, dl, Elt, EltPtr,
1474
61
                            MachinePointerInfo::getUnknownStack(MF), EltVT);
1475
61
1476
61
  EVT LoVT, HiVT;
1477
61
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1478
61
1479
61
  // Load the Lo part from the stack slot.
1480
61
  Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo);
1481
61
1482
61
  // Increment the pointer to the other part.
1483
61
  unsigned IncrementSize = LoVT.getSizeInBits() / 8;
1484
61
  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
1485
61
                         DAG.getConstant(IncrementSize, dl,
1486
61
                                         StackPtr.getValueType()));
1487
61
1488
61
  // Load the Hi part from the stack slot.
1489
61
  Hi = DAG.getLoad(HiVT, dl, Store, StackPtr,
1490
61
                   PtrInfo.getWithOffset(IncrementSize),
1491
61
                   MinAlign(Alignment, IncrementSize));
1492
61
1493
61
  // If we adjusted the original type, we need to truncate the results.
1494
61
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1495
61
  if (LoVT != Lo.getValueType())
1496
9
    Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1497
61
  if (HiVT != Hi.getValueType())
1498
9
    Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1499
61
}
1500
1501
void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
1502
327
                                                    SDValue &Hi) {
1503
327
  EVT LoVT, HiVT;
1504
327
  SDLoc dl(N);
1505
327
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1506
327
  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
1507
327
  Hi = DAG.getUNDEF(HiVT);
1508
327
}
1509
1510
void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1511
5.03k
                                        SDValue &Hi) {
1512
5.03k
  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1513
5.03k
  EVT LoVT, HiVT;
1514
5.03k
  SDLoc dl(LD);
1515
5.03k
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1516
5.03k
1517
5.03k
  ISD::LoadExtType ExtType = LD->getExtensionType();
1518
5.03k
  SDValue Ch = LD->getChain();
1519
5.03k
  SDValue Ptr = LD->getBasePtr();
1520
5.03k
  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1521
5.03k
  EVT MemoryVT = LD->getMemoryVT();
1522
5.03k
  unsigned Alignment = LD->getOriginalAlignment();
1523
5.03k
  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1524
5.03k
  AAMDNodes AAInfo = LD->getAAInfo();
1525
5.03k
1526
5.03k
  EVT LoMemVT, HiMemVT;
1527
5.03k
  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1528
5.03k
1529
5.03k
  Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1530
5.03k
                   LD->getPointerInfo(), LoMemVT, Alignment, MMOFlags, AAInfo);
1531
5.03k
1532
5.03k
  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1533
5.03k
  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
1534
5.03k
  Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset,
1535
5.03k
                   LD->getPointerInfo().getWithOffset(IncrementSize), HiMemVT,
1536
5.03k
                   Alignment, MMOFlags, AAInfo);
1537
5.03k
1538
5.03k
  // Build a factor node to remember that this load is independent of the
1539
5.03k
  // other one.
1540
5.03k
  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1541
5.03k
                   Hi.getValue(1));
1542
5.03k
1543
5.03k
  // Legalize the chain result - switch anything that used the old chain to
1544
5.03k
  // use the new one.
1545
5.03k
  ReplaceValueWith(SDValue(LD, 1), Ch);
1546
5.03k
}
1547
1548
void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1549
18
                                         SDValue &Lo, SDValue &Hi) {
1550
18
  EVT LoVT, HiVT;
1551
18
  SDLoc dl(MLD);
1552
18
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1553
18
1554
18
  SDValue Ch = MLD->getChain();
1555
18
  SDValue Ptr = MLD->getBasePtr();
1556
18
  SDValue Mask = MLD->getMask();
1557
18
  SDValue PassThru = MLD->getPassThru();
1558
18
  unsigned Alignment = MLD->getOriginalAlignment();
1559
18
  ISD::LoadExtType ExtType = MLD->getExtensionType();
1560
18
1561
18
  // Split Mask operand
1562
18
  SDValue MaskLo, MaskHi;
1563
18
  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1564
1
    GetSplitVector(Mask, MaskLo, MaskHi);
1565
17
  else
1566
17
    std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1567
18
1568
18
  EVT MemoryVT = MLD->getMemoryVT();
1569
18
  EVT LoMemVT, HiMemVT;
1570
18
  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1571
18
1572
18
  SDValue PassThruLo, PassThruHi;
1573
18
  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1574
18
    GetSplitVector(PassThru, PassThruLo, PassThruHi);
1575
0
  else
1576
0
    std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1577
18
1578
18
  MachineMemOperand *MMO = DAG.getMachineFunction().
1579
18
    getMachineMemOperand(MLD->getPointerInfo(),
1580
18
                         MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
1581
18
                         Alignment, MLD->getAAInfo(), MLD->getRanges());
1582
18
1583
18
  Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, MaskLo, PassThruLo, LoMemVT, MMO,
1584
18
                         ExtType, MLD->isExpandingLoad());
1585
18
1586
18
  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1587
18
                                   MLD->isExpandingLoad());
1588
18
  unsigned HiOffset = LoMemVT.getStoreSize();
1589
18
1590
18
  MMO = DAG.getMachineFunction().getMachineMemOperand(
1591
18
      MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad,
1592
18
      HiMemVT.getStoreSize(), Alignment, MLD->getAAInfo(),
1593
18
      MLD->getRanges());
1594
18
1595
18
  Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO,
1596
18
                         ExtType, MLD->isExpandingLoad());
1597
18
1598
18
  // Build a factor node to remember that this load is independent of the
1599
18
  // other one.
1600
18
  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1601
18
                   Hi.getValue(1));
1602
18
1603
18
  // Legalize the chain result - switch anything that used the old chain to
1604
18
  // use the new one.
1605
18
  ReplaceValueWith(SDValue(MLD, 1), Ch);
1606
18
1607
18
}
1608
1609
void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1610
16
                                         SDValue &Lo, SDValue &Hi) {
1611
16
  EVT LoVT, HiVT;
1612
16
  SDLoc dl(MGT);
1613
16
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1614
16
1615
16
  SDValue Ch = MGT->getChain();
1616
16
  SDValue Ptr = MGT->getBasePtr();
1617
16
  SDValue Mask = MGT->getMask();
1618
16
  SDValue PassThru = MGT->getPassThru();
1619
16
  SDValue Index = MGT->getIndex();
1620
16
  SDValue Scale = MGT->getScale();
1621
16
  unsigned Alignment = MGT->getOriginalAlignment();
1622
16
1623
16
  // Split Mask operand
1624
16
  SDValue MaskLo, MaskHi;
1625
16
  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1626
0
    GetSplitVector(Mask, MaskLo, MaskHi);
1627
16
  else
1628
16
    std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1629
16
1630
16
  EVT MemoryVT = MGT->getMemoryVT();
1631
16
  EVT LoMemVT, HiMemVT;
1632
16
  // Split MemoryVT
1633
16
  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1634
16
1635
16
  SDValue PassThruLo, PassThruHi;
1636
16
  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1637
16
    GetSplitVector(PassThru, PassThruLo, PassThruHi);
1638
0
  else
1639
0
    std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1640
16
1641
16
  SDValue IndexHi, IndexLo;
1642
16
  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1643
12
    GetSplitVector(Index, IndexLo, IndexHi);
1644
4
  else
1645
4
    std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1646
16
1647
16
  MachineMemOperand *MMO = DAG.getMachineFunction().
1648
16
    getMachineMemOperand(MGT->getPointerInfo(),
1649
16
                         MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
1650
16
                         Alignment, MGT->getAAInfo(), MGT->getRanges());
1651
16
1652
16
  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1653
16
  Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl, OpsLo,
1654
16
                           MMO);
1655
16
1656
16
  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1657
16
  Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl, OpsHi,
1658
16
                           MMO);
1659
16
1660
16
  // Build a factor node to remember that this load is independent of the
1661
16
  // other one.
1662
16
  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1663
16
                   Hi.getValue(1));
1664
16
1665
16
  // Legalize the chain result - switch anything that used the old chain to
1666
16
  // use the new one.
1667
16
  ReplaceValueWith(SDValue(MGT, 1), Ch);
1668
16
}
1669
1670
1671
6.58k
void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1672
6.58k
  assert(N->getValueType(0).isVector() &&
1673
6.58k
         N->getOperand(0).getValueType().isVector() &&
1674
6.58k
         "Operand types must be vectors");
1675
6.58k
1676
6.58k
  EVT LoVT, HiVT;
1677
6.58k
  SDLoc DL(N);
1678
6.58k
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1679
6.58k
1680
6.58k
  // If the input also splits, handle it directly. Otherwise split it by hand.
1681
6.58k
  SDValue LL, LH, RL, RH;
1682
6.58k
  if (getTypeAction(N->getOperand(0).getValueType()) ==
1683
6.58k
      TargetLowering::TypeSplitVector)
1684
6.37k
    GetSplitVector(N->getOperand(0), LL, LH);
1685
208
  else
1686
208
    std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1687
6.58k
1688
6.58k
  if (getTypeAction(N->getOperand(1).getValueType()) ==
1689
6.58k
      TargetLowering::TypeSplitVector)
1690
6.37k
    GetSplitVector(N->getOperand(1), RL, RH);
1691
208
  else
1692
208
    std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1693
6.58k
1694
6.58k
  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1695
6.58k
  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1696
6.58k
}
1697
1698
void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1699
28.7k
                                           SDValue &Hi) {
1700
28.7k
  // Get the dest types - they may not match the input types, e.g. int_to_fp.
1701
28.7k
  EVT LoVT, HiVT;
1702
28.7k
  SDLoc dl(N);
1703
28.7k
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1704
28.7k
1705
28.7k
  // If the input also splits, handle it directly for a compile time speedup.
1706
28.7k
  // Otherwise split it by hand.
1707
28.7k
  unsigned OpNo = N->isStrictFPOpcode() ? 
112
:
028.7k
;
1708
28.7k
  EVT InVT = N->getOperand(OpNo).getValueType();
1709
28.7k
  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1710
10.7k
    GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1711
17.9k
  else
1712
17.9k
    std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1713
28.7k
1714
28.7k
  if (N->getOpcode() == ISD::FP_ROUND) {
1715
592
    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1));
1716
592
    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1));
1717
28.1k
  } else if (N->getOpcode() == ISD::STRICT_FP_ROUND) {
1718
4
    Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other }, 
1719
4
                     { N->getOperand(0), Lo, N->getOperand(2) });
1720
4
    Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other }, 
1721
4
                     { N->getOperand(0), Hi, N->getOperand(2) });
1722
4
    SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 
1723
4
                                   Lo.getValue(1), Hi.getValue(1));
1724
4
    ReplaceValueWith(SDValue(N, 1), NewChain);
1725
28.1k
  } else if (N->isStrictFPOpcode()) {
1726
8
    Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other }, 
1727
8
                     { N->getOperand(0), Lo });
1728
8
    Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other }, 
1729
8
                     { N->getOperand(0), Hi });
1730
8
    // Legalize the chain result - switch anything that used the old chain to
1731
8
    // use the new one.
1732
8
    SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 
1733
8
                                   Lo.getValue(1), Hi.getValue(1));
1734
8
    ReplaceValueWith(SDValue(N, 1), NewChain);
1735
28.1k
  } else {
1736
28.1k
    Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1737
28.1k
    Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1738
28.1k
  }
1739
28.7k
}
1740
1741
void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1742
10.9k
                                            SDValue &Hi) {
1743
10.9k
  SDLoc dl(N);
1744
10.9k
  EVT SrcVT = N->getOperand(0).getValueType();
1745
10.9k
  EVT DestVT = N->getValueType(0);
1746
10.9k
  EVT LoVT, HiVT;
1747
10.9k
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1748
10.9k
1749
10.9k
  // We can do better than a generic split operation if the extend is doing
1750
10.9k
  // more than just doubling the width of the elements and the following are
1751
10.9k
  // true:
1752
10.9k
  //   - The number of vector elements is even,
1753
10.9k
  //   - the source type is legal,
1754
10.9k
  //   - the type of a split source is illegal,
1755
10.9k
  //   - the type of an extended (by doubling element size) source is legal, and
1756
10.9k
  //   - the type of that extended source when split is legal.
1757
10.9k
  //
1758
10.9k
  // This won't necessarily completely legalize the operation, but it will
1759
10.9k
  // more effectively move in the right direction and prevent falling down
1760
10.9k
  // to scalarization in many cases due to the input vector being split too
1761
10.9k
  // far.
1762
10.9k
  unsigned NumElements = SrcVT.getVectorNumElements();
1763
10.9k
  if ((NumElements & 1) == 0 &&
1764
10.9k
      SrcVT.getSizeInBits() * 2 < DestVT.getSizeInBits()) {
1765
6.49k
    LLVMContext &Ctx = *DAG.getContext();
1766
6.49k
    EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1767
6.49k
    EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1768
6.49k
1769
6.49k
    EVT SplitLoVT, SplitHiVT;
1770
6.49k
    std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1771
6.49k
    if (TLI.isTypeLegal(SrcVT) && 
!TLI.isTypeLegal(SplitSrcVT)972
&&
1772
6.49k
        
TLI.isTypeLegal(NewSrcVT)515
&&
TLI.isTypeLegal(SplitLoVT)442
) {
1773
442
      LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1774
442
                 N->dump(&DAG); dbgs() << "\n");
1775
442
      // Extend the source vector by one step.
1776
442
      SDValue NewSrc =
1777
442
          DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1778
442
      // Get the low and high halves of the new, extended one step, vector.
1779
442
      std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1780
442
      // Extend those vector halves the rest of the way.
1781
442
      Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1782
442
      Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1783
442
      return;
1784
442
    }
1785
10.4k
  }
1786
10.4k
  // Fall back to the generic unary operator splitting otherwise.
1787
10.4k
  SplitVecRes_UnaryOp(N, Lo, Hi);
1788
10.4k
}
1789
1790
void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1791
5.36k
                                                  SDValue &Lo, SDValue &Hi) {
1792
5.36k
  // The low and high parts of the original input give four input vectors.
1793
5.36k
  SDValue Inputs[4];
1794
5.36k
  SDLoc dl(N);
1795
5.36k
  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1796
5.36k
  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1797
5.36k
  EVT NewVT = Inputs[0].getValueType();
1798
5.36k
  unsigned NewElts = NewVT.getVectorNumElements();
1799
5.36k
1800
5.36k
  // If Lo or Hi uses elements from at most two of the four input vectors, then
1801
5.36k
  // express it as a vector shuffle of those two inputs.  Otherwise extract the
1802
5.36k
  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1803
5.36k
  SmallVector<int, 16> Ops;
1804
16.1k
  for (unsigned High = 0; High < 2; 
++High10.7k
) {
1805
10.7k
    SDValue &Output = High ? 
Hi5.36k
:
Lo5.36k
;
1806
10.7k
1807
10.7k
    // Build a shuffle mask for the output, discovering on the fly which
1808
10.7k
    // input vectors to use as shuffle operands (recorded in InputUsed).
1809
10.7k
    // If building a suitable shuffle vector proves too hard, then bail
1810
10.7k
    // out with useBuildVector set.
1811
10.7k
    unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
1812
10.7k
    unsigned FirstMaskIdx = High * NewElts;
1813
10.7k
    bool useBuildVector = false;
1814
195k
    for (unsigned MaskOffset = 0; MaskOffset < NewElts; 
++MaskOffset184k
) {
1815
184k
      // The mask element.  This indexes into the input.
1816
184k
      int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1817
184k
1818
184k
      // The input vector this mask element indexes into.
1819
184k
      unsigned Input = (unsigned)Idx / NewElts;
1820
184k
1821
184k
      if (Input >= array_lengthof(Inputs)) {
1822
140k
        // The mask element does not index into any input vector.
1823
140k
        Ops.push_back(-1);
1824
140k
        continue;
1825
140k
      }
1826
44.3k
1827
44.3k
      // Turn the index into an offset from the start of the input vector.
1828
44.3k
      Idx -= Input * NewElts;
1829
44.3k
1830
44.3k
      // Find or create a shuffle vector operand to hold this input.
1831
44.3k
      unsigned OpNo;
1832
50.8k
      for (OpNo = 0; OpNo < array_lengthof(InputUsed); 
++OpNo6.51k
) {
1833
50.7k
        if (InputUsed[OpNo] == Input) {
1834
36.6k
          // This input vector is already an operand.
1835
36.6k
          break;
1836
36.6k
        } else 
if (14.1k
InputUsed[OpNo] == -1U14.1k
) {
1837
7.60k
          // Create a new operand for this input vector.
1838
7.60k
          InputUsed[OpNo] = Input;
1839
7.60k
          break;
1840
7.60k
        }
1841
50.7k
      }
1842
44.3k
1843
44.3k
      if (OpNo >= array_lengthof(InputUsed)) {
1844
106
        // More than two input vectors used!  Give up on trying to create a
1845
106
        // shuffle vector.  Insert all elements into a BUILD_VECTOR instead.
1846
106
        useBuildVector = true;
1847
106
        break;
1848
106
      }
1849
44.2k
1850
44.2k
      // Add the mask index for the new shuffle vector.
1851
44.2k
      Ops.push_back(Idx + OpNo * NewElts);
1852
44.2k
    }
1853
10.7k
1854
10.7k
    if (useBuildVector) {
1855
106
      EVT EltVT = NewVT.getVectorElementType();
1856
106
      SmallVector<SDValue, 16> SVOps;
1857
106
1858
106
      // Extract the input elements by hand.
1859
1.22k
      for (unsigned MaskOffset = 0; MaskOffset < NewElts; 
++MaskOffset1.12k
) {
1860
1.12k
        // The mask element.  This indexes into the input.
1861
1.12k
        int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
1862
1.12k
1863
1.12k
        // The input vector this mask element indexes into.
1864
1.12k
        unsigned Input = (unsigned)Idx / NewElts;
1865
1.12k
1866
1.12k
        if (Input >= array_lengthof(Inputs)) {
1867
122
          // The mask element is "undef" or indexes off the end of the input.
1868
122
          SVOps.push_back(DAG.getUNDEF(EltVT));
1869
122
          continue;
1870
122
        }
1871
998
1872
998
        // Turn the index into an offset from the start of the input vector.
1873
998
        Idx -= Input * NewElts;
1874
998
1875
998
        // Extract the vector element by hand.
1876
998
        SVOps.push_back(DAG.getNode(
1877
998
            ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Inputs[Input],
1878
998
            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
1879
998
      }
1880
106
1881
106
      // Construct the Lo/Hi output using a BUILD_VECTOR.
1882
106
      Output = DAG.getBuildVector(NewVT, dl, SVOps);
1883
10.6k
    } else if (InputUsed[0] == -1U) {
1884
4.26k
      // No input vectors were used!  The result is undefined.
1885
4.26k
      Output = DAG.getUNDEF(NewVT);
1886
6.35k
    } else {
1887
6.35k
      SDValue Op0 = Inputs[InputUsed[0]];
1888
6.35k
      // If only one input was used, use an undefined vector for the other.
1889
6.35k
      SDValue Op1 = InputUsed[1] == -1U ?
1890
5.32k
        DAG.getUNDEF(NewVT) : 
Inputs[InputUsed[1]]1.03k
;
1891
6.35k
      // At least one input vector was used.  Create a new shuffle vector.
1892
6.35k
      Output =  DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
1893
6.35k
    }
1894
10.7k
1895
10.7k
    Ops.clear();
1896
10.7k
  }
1897
5.36k
}
1898
1899
5
void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
1900
5
  EVT OVT = N->getValueType(0);
1901
5
  EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
1902
5
  SDValue Chain = N->getOperand(0);
1903
5
  SDValue Ptr = N->getOperand(1);
1904
5
  SDValue SV = N->getOperand(2);
1905
5
  SDLoc dl(N);
1906
5
1907
5
  const unsigned Alignment = DAG.getDataLayout().getABITypeAlignment(
1908
5
      NVT.getTypeForEVT(*DAG.getContext()));
1909
5
1910
5
  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment);
1911
5
  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment);
1912
5
  Chain = Hi.getValue(1);
1913
5
1914
5
  // Modified the chain - switch anything that used the old chain to use
1915
5
  // the new one.
1916
5
  ReplaceValueWith(SDValue(N, 1), Chain);
1917
5
}
1918
1919
1920
//===----------------------------------------------------------------------===//
1921
//  Operand Vector Splitting
1922
//===----------------------------------------------------------------------===//
1923
1924
/// This method is called when the specified operand of the specified node is
1925
/// found to need vector splitting. At this point, all of the result types of
1926
/// the node are known to be legal, but other operands of the node may need
1927
/// legalization as well as the specified one.
1928
176k
bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
1929
176k
  LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
1930
176k
  SDValue Res = SDValue();
1931
176k
1932
176k
  // See if the target wants to custom split this node.
1933
176k
  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1934
3.13k
    return false;
1935
173k
1936
173k
  if (!Res.getNode()) {
1937
173k
    switch (N->getOpcode()) {
1938
173k
    default:
1939
#ifndef NDEBUG
1940
      dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
1941
      N->dump(&DAG);
1942
      dbgs() << "\n";
1943
#endif
1944
      report_fatal_error("Do not know how to split this operator's "
1945
0
                         "operand!\n");
1946
173k
1947
173k
    
case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break146
;
1948
173k
    
case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break5.68k
;
1949
173k
    
case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break84.1k
;
1950
173k
    
case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break50.6k
;
1951
173k
    
case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break2
;
1952
173k
    case ISD::TRUNCATE:
1953
9.44k
      Res = SplitVecOp_TruncateHelper(N);
1954
9.44k
      break;
1955
173k
    case ISD::STRICT_FP_ROUND:
1956
290
    case ISD::FP_ROUND:          Res = SplitVecOp_FP_ROUND(N); break;
1957
290
    
case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break8
;
1958
12.4k
    case ISD::STORE:
1959
12.4k
      Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
1960
12.4k
      break;
1961
290
    case ISD::MSTORE:
1962
7
      Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
1963
7
      break;
1964
290
    case ISD::MSCATTER:
1965
22
      Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
1966
22
      break;
1967
290
    case ISD::MGATHER:
1968
33
      Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
1969
33
      break;
1970
290
    case ISD::VSELECT:
1971
147
      Res = SplitVecOp_VSELECT(N, OpNo);
1972
147
      break;
1973
403
    case ISD::SINT_TO_FP:
1974
403
    case ISD::UINT_TO_FP:
1975
403
      if (N->getValueType(0).bitsLT(N->getOperand(0).getValueType()))
1976
245
        Res = SplitVecOp_TruncateHelper(N);
1977
158
      else
1978
158
        Res = SplitVecOp_UnaryOp(N);
1979
403
      break;
1980
9.58k
    case ISD::FP_TO_SINT:
1981
9.58k
    case ISD::FP_TO_UINT:
1982
9.58k
    case ISD::CTTZ:
1983
9.58k
    case ISD::CTLZ:
1984
9.58k
    case ISD::CTPOP:
1985
9.58k
    case ISD::STRICT_FP_EXTEND:
1986
9.58k
    case ISD::FP_EXTEND:
1987
9.58k
    case ISD::SIGN_EXTEND:
1988
9.58k
    case ISD::ZERO_EXTEND:
1989
9.58k
    case ISD::ANY_EXTEND:
1990
9.58k
    case ISD::FTRUNC:
1991
9.58k
    case ISD::FCANONICALIZE:
1992
9.58k
      Res = SplitVecOp_UnaryOp(N);
1993
9.58k
      break;
1994
9.58k
1995
9.58k
    case ISD::ANY_EXTEND_VECTOR_INREG:
1996
6
    case ISD::SIGN_EXTEND_VECTOR_INREG:
1997
6
    case ISD::ZERO_EXTEND_VECTOR_INREG:
1998
6
      Res = SplitVecOp_ExtVecInRegOp(N);
1999
6
      break;
2000
6
2001
103
    case ISD::VECREDUCE_FADD:
2002
103
    case ISD::VECREDUCE_FMUL:
2003
103
    case ISD::VECREDUCE_ADD:
2004
103
    case ISD::VECREDUCE_MUL:
2005
103
    case ISD::VECREDUCE_AND:
2006
103
    case ISD::VECREDUCE_OR:
2007
103
    case ISD::VECREDUCE_XOR:
2008
103
    case ISD::VECREDUCE_SMAX:
2009
103
    case ISD::VECREDUCE_SMIN:
2010
103
    case ISD::VECREDUCE_UMAX:
2011
103
    case ISD::VECREDUCE_UMIN:
2012
103
    case ISD::VECREDUCE_FMAX:
2013
103
    case ISD::VECREDUCE_FMIN:
2014
103
      Res = SplitVecOp_VECREDUCE(N, OpNo);
2015
103
      break;
2016
173k
    }
2017
173k
  }
2018
173k
2019
173k
  // If the result is null, the sub-method took care of registering results etc.
2020
173k
  if (!Res.getNode()) 
return false33
;
2021
173k
2022
173k
  // If the result is N, the sub-method updated N in place.  Tell the legalizer
2023
173k
  // core about this.
2024
173k
  if (Res.getNode() == N)
2025
49.6k
    return true;
2026
123k
2027
123k
  if (N->isStrictFPOpcode())
2028
123k
    assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2029
123k
           "Invalid operand expansion");
2030
123k
  else
2031
123k
    assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2032
123k
         "Invalid operand expansion");
2033
123k
2034
123k
  ReplaceValueWith(SDValue(N, 0), Res);
2035
123k
  return false;
2036
123k
}
2037
2038
151
SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2039
151
  // The only possibility for an illegal operand is the mask, since result type
2040
151
  // legalization would have handled this node already otherwise.
2041
151
  assert(OpNo == 0 && "Illegal operand must be mask");
2042
151
2043
151
  SDValue Mask = N->getOperand(0);
2044
151
  SDValue Src0 = N->getOperand(1);
2045
151
  SDValue Src1 = N->getOperand(2);
2046
151
  EVT Src0VT = Src0.getValueType();
2047
151
  SDLoc DL(N);
2048
151
  assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2049
151
2050
151
  SDValue Lo, Hi;
2051
151
  GetSplitVector(N->getOperand(0), Lo, Hi);
2052
151
  assert(Lo.getValueType() == Hi.getValueType() &&
2053
151
         "Lo and Hi have differing types");
2054
151
2055
151
  EVT LoOpVT, HiOpVT;
2056
151
  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2057
151
  assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2058
151
2059
151
  SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2060
151
  std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2061
151
  std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2062
151
  std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2063
151
2064
151
  SDValue LoSelect =
2065
151
    DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2066
151
  SDValue HiSelect =
2067
151
    DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2068
151
2069
151
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2070
151
}
2071
2072
103
SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2073
103
  EVT ResVT = N->getValueType(0);
2074
103
  SDValue Lo, Hi;
2075
103
  SDLoc dl(N);
2076
103
2077
103
  SDValue VecOp = N->getOperand(OpNo);
2078
103
  EVT VecVT = VecOp.getValueType();
2079
103
  assert(VecVT.isVector() && "Can only split reduce vector operand");
2080
103
  GetSplitVector(VecOp, Lo, Hi);
2081
103
  EVT LoOpVT, HiOpVT;
2082
103
  std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2083
103
2084
103
  bool NoNaN = N->getFlags().hasNoNaNs();
2085
103
  unsigned CombineOpc = 0;
2086
103
  switch (N->getOpcode()) {
2087
103
  
case ISD::VECREDUCE_FADD: CombineOpc = ISD::FADD; break9
;
2088
103
  
case ISD::VECREDUCE_FMUL: CombineOpc = ISD::FMUL; break0
;
2089
103
  
case ISD::VECREDUCE_ADD: CombineOpc = ISD::ADD; break73
;
2090
103
  
case ISD::VECREDUCE_MUL: CombineOpc = ISD::MUL; break0
;
2091
103
  
case ISD::VECREDUCE_AND: CombineOpc = ISD::AND; break3
;
2092
103
  
case ISD::VECREDUCE_OR: CombineOpc = ISD::OR; break1
;
2093
103
  
case ISD::VECREDUCE_XOR: CombineOpc = ISD::XOR; break0
;
2094
103
  
case ISD::VECREDUCE_SMAX: CombineOpc = ISD::SMAX; break3
;
2095
103
  
case ISD::VECREDUCE_SMIN: CombineOpc = ISD::SMIN; break3
;
2096
103
  
case ISD::VECREDUCE_UMAX: CombineOpc = ISD::UMAX; break5
;
2097
103
  
case ISD::VECREDUCE_UMIN: CombineOpc = ISD::UMIN; break3
;
2098
103
  case ISD::VECREDUCE_FMAX:
2099
3
    CombineOpc = NoNaN ? ISD::FMAXNUM : 
ISD::FMAXIMUM0
;
2100
3
    break;
2101
103
  case ISD::VECREDUCE_FMIN:
2102
0
    CombineOpc = NoNaN ? ISD::FMINNUM : ISD::FMINIMUM;
2103
0
    break;
2104
103
  default:
2105
0
    llvm_unreachable("Unexpected reduce ISD node");
2106
103
  }
2107
103
2108
103
  // Use the appropriate scalar instruction on the split subvectors before
2109
103
  // reducing the now partially reduced smaller vector.
2110
103
  SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2111
103
  return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2112
103
}
2113
2114
17.2k
SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2115
17.2k
  // The result has a legal vector type, but the input needs splitting.
2116
17.2k
  EVT ResVT = N->getValueType(0);
2117
17.2k
  SDValue Lo, Hi;
2118
17.2k
  SDLoc dl(N);
2119
17.2k
  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 
10
: 0), Lo, Hi);
2120
17.2k
  EVT InVT = Lo.getValueType();
2121
17.2k
2122
17.2k
  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2123
17.2k
                               InVT.getVectorNumElements());
2124
17.2k
2125
17.2k
  if (N->isStrictFPOpcode()) {
2126
0
    Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 
2127
0
                     { N->getOperand(0), Lo });
2128
0
    Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, 
2129
0
                     { N->getOperand(0), Hi });
2130
0
2131
0
    // Build a factor node to remember that this operation is independent
2132
0
    // of the other one.
2133
0
    SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2134
0
                             Hi.getValue(1));
2135
0
  
2136
0
    // Legalize the chain result - switch anything that used the old chain to
2137
0
    // use the new one.
2138
0
    ReplaceValueWith(SDValue(N, 1), Ch);
2139
17.2k
  } else {
2140
17.2k
    Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2141
17.2k
    Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2142
17.2k
  }
2143
17.2k
2144
17.2k
  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2145
17.2k
}
2146
2147
5.68k
SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2148
5.68k
  // For example, i64 = BITCAST v4i16 on alpha.  Typically the vector will
2149
5.68k
  // end up being split all the way down to individual components.  Convert the
2150
5.68k
  // split pieces into integers and reassemble.
2151
5.68k
  SDValue Lo, Hi;
2152
5.68k
  GetSplitVector(N->getOperand(0), Lo, Hi);
2153
5.68k
  Lo = BitConvertToInteger(Lo);
2154
5.68k
  Hi = BitConvertToInteger(Hi);
2155
5.68k
2156
5.68k
  if (DAG.getDataLayout().isBigEndian())
2157
72
    std::swap(Lo, Hi);
2158
5.68k
2159
5.68k
  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2160
5.68k
                     JoinIntegers(Lo, Hi));
2161
5.68k
}
2162
2163
84.1k
SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2164
84.1k
  // We know that the extracted result type is legal.
2165
84.1k
  EVT SubVT = N->getValueType(0);
2166
84.1k
  SDValue Idx = N->getOperand(1);
2167
84.1k
  SDLoc dl(N);
2168
84.1k
  SDValue Lo, Hi;
2169
84.1k
  GetSplitVector(N->getOperand(0), Lo, Hi);
2170
84.1k
2171
84.1k
  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2172
84.1k
  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2173
84.1k
2174
84.1k
  if (IdxVal < LoElts) {
2175
44.8k
    assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
2176
44.8k
           "Extracted subvector crosses vector split!");
2177
44.8k
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2178
44.8k
  } else {
2179
39.2k
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2180
39.2k
                       DAG.getConstant(IdxVal - LoElts, dl,
2181
39.2k
                                       Idx.getValueType()));
2182
39.2k
  }
2183
84.1k
}
2184
2185
50.6k
SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2186
50.6k
  SDValue Vec = N->getOperand(0);
2187
50.6k
  SDValue Idx = N->getOperand(1);
2188
50.6k
  EVT VecVT = Vec.getValueType();
2189
50.6k
2190
50.6k
  if (isa<ConstantSDNode>(Idx)) {
2191
50.4k
    uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2192
50.4k
2193
50.4k
    SDValue Lo, Hi;
2194
50.4k
    GetSplitVector(Vec, Lo, Hi);
2195
50.4k
2196
50.4k
    uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2197
50.4k
2198
50.4k
    if (IdxVal < LoElts)
2199
27.8k
      return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2200
22.6k
    return SDValue(DAG.UpdateNodeOperands(N, Hi,
2201
22.6k
                                  DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2202
22.6k
                                                  Idx.getValueType())), 0);
2203
22.6k
  }
2204
272
2205
272
  // See if the target wants to custom expand this node.
2206
272
  if (CustomLowerNode(N, N->getValueType(0), true))
2207
0
    return SDValue();
2208
272
2209
272
  // Make the vector elements byte-addressable if they aren't already.
2210
272
  SDLoc dl(N);
2211
272
  EVT EltVT = VecVT.getVectorElementType();
2212
272
  if (VecVT.getScalarSizeInBits() < 8) {
2213
3
    EltVT = MVT::i8;
2214
3
    VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2215
3
                             VecVT.getVectorNumElements());
2216
3
    Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2217
3
  }
2218
272
2219
272
  // Store the vector to the stack.
2220
272
  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
2221
272
  auto &MF = DAG.getMachineFunction();
2222
272
  auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2223
272
  auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2224
272
  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
2225
272
2226
272
  // Load back the required element.
2227
272
  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2228
272
2229
272
  // FIXME: This is to handle i1 vectors with elements promoted to i8.
2230
272
  // i1 vector handling needs general improvement.
2231
272
  if (N->getValueType(0).bitsLT(EltVT)) {
2232
1
    SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2233
1
      MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
2234
1
    return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2235
1
  }
2236
271
2237
271
  return DAG.getExtLoad(
2238
271
      ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2239
271
      MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
2240
271
}
2241
2242
6
SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2243
6
  SDValue Lo, Hi;
2244
6
2245
6
  // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2246
6
  // splitting the result has the same effect as splitting the input operand.
2247
6
  SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2248
6
2249
6
  return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2250
6
}
2251
2252
SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2253
33
                                             unsigned OpNo) {
2254
33
  EVT LoVT, HiVT;
2255
33
  SDLoc dl(MGT);
2256
33
  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2257
33
2258
33
  SDValue Ch = MGT->getChain();
2259
33
  SDValue Ptr = MGT->getBasePtr();
2260
33
  SDValue Index = MGT->getIndex();
2261
33
  SDValue Scale = MGT->getScale();
2262
33
  SDValue Mask = MGT->getMask();
2263
33
  SDValue PassThru = MGT->getPassThru();
2264
33
  unsigned Alignment = MGT->getOriginalAlignment();
2265
33
2266
33
  SDValue MaskLo, MaskHi;
2267
33
  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2268
0
    // Split Mask operand
2269
0
    GetSplitVector(Mask, MaskLo, MaskHi);
2270
33
  else
2271
33
    std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2272
33
2273
33
  EVT MemoryVT = MGT->getMemoryVT();
2274
33
  EVT LoMemVT, HiMemVT;
2275
33
  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2276
33
2277
33
  SDValue PassThruLo, PassThruHi;
2278
33
  if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2279
0
    GetSplitVector(PassThru, PassThruLo, PassThruHi);
2280
33
  else
2281
33
    std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2282
33
2283
33
  SDValue IndexHi, IndexLo;
2284
33
  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2285
33
    GetSplitVector(Index, IndexLo, IndexHi);
2286
0
  else
2287
0
    std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2288
33
2289
33
  MachineMemOperand *MMO = DAG.getMachineFunction().
2290
33
    getMachineMemOperand(MGT->getPointerInfo(),
2291
33
                         MachineMemOperand::MOLoad,  LoMemVT.getStoreSize(),
2292
33
                         Alignment, MGT->getAAInfo(), MGT->getRanges());
2293
33
2294
33
  SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2295
33
  SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoVT, dl,
2296
33
                                   OpsLo, MMO);
2297
33
2298
33
  MMO = DAG.getMachineFunction().
2299
33
    getMachineMemOperand(MGT->getPointerInfo(),
2300
33
                         MachineMemOperand::MOLoad,  HiMemVT.getStoreSize(),
2301
33
                         Alignment, MGT->getAAInfo(),
2302
33
                         MGT->getRanges());
2303
33
2304
33
  SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2305
33
  SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiVT, dl,
2306
33
                                   OpsHi, MMO);
2307
33
2308
33
  // Build a factor node to remember that this load is independent of the
2309
33
  // other one.
2310
33
  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2311
33
                   Hi.getValue(1));
2312
33
2313
33
  // Legalize the chain result - switch anything that used the old chain to
2314
33
  // use the new one.
2315
33
  ReplaceValueWith(SDValue(MGT, 1), Ch);
2316
33
2317
33
  SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2318
33
                            Hi);
2319
33
  ReplaceValueWith(SDValue(MGT, 0), Res);
2320
33
  return SDValue();
2321
33
}
2322
2323
SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2324
7
                                            unsigned OpNo) {
2325
7
  SDValue Ch  = N->getChain();
2326
7
  SDValue Ptr = N->getBasePtr();
2327
7
  SDValue Mask = N->getMask();
2328
7
  SDValue Data = N->getValue();
2329
7
  EVT MemoryVT = N->getMemoryVT();
2330
7
  unsigned Alignment = N->getOriginalAlignment();
2331
7
  SDLoc DL(N);
2332
7
2333
7
  EVT LoMemVT, HiMemVT;
2334
7
  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2335
7
2336
7
  SDValue DataLo, DataHi;
2337
7
  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2338
7
    // Split Data operand
2339
7
    GetSplitVector(Data, DataLo, DataHi);
2340
0
  else
2341
0
    std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2342
7
2343
7
  SDValue MaskLo, MaskHi;
2344
7
  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2345
0
    // Split Mask operand
2346
0
    GetSplitVector(Mask, MaskLo, MaskHi);
2347
7
  else
2348
7
    std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2349
7
2350
7
  SDValue Lo, Hi;
2351
7
  MachineMemOperand *MMO = DAG.getMachineFunction().
2352
7
    getMachineMemOperand(N->getPointerInfo(),
2353
7
                         MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
2354
7
                         Alignment, N->getAAInfo(), N->getRanges());
2355
7
2356
7
  Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, MaskLo, LoMemVT, MMO,
2357
7
                          N->isTruncatingStore(),
2358
7
                          N->isCompressingStore());
2359
7
2360
7
  Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2361
7
                                   N->isCompressingStore());
2362
7
  unsigned HiOffset = LoMemVT.getStoreSize();
2363
7
2364
7
  MMO = DAG.getMachineFunction().getMachineMemOperand(
2365
7
      N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore,
2366
7
      HiMemVT.getStoreSize(), Alignment, N->getAAInfo(),
2367
7
      N->getRanges());
2368
7
2369
7
  Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO,
2370
7
                          N->isTruncatingStore(), N->isCompressingStore());
2371
7
2372
7
  // Build a factor node to remember that this store is independent of the
2373
7
  // other one.
2374
7
  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2375
7
}
2376
2377
SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2378
22
                                              unsigned OpNo) {
2379
22
  SDValue Ch  = N->getChain();
2380
22
  SDValue Ptr = N->getBasePtr();
2381
22
  SDValue Mask = N->getMask();
2382
22
  SDValue Index = N->getIndex();
2383
22
  SDValue Scale = N->getScale();
2384
22
  SDValue Data = N->getValue();
2385
22
  EVT MemoryVT = N->getMemoryVT();
2386
22
  unsigned Alignment = N->getOriginalAlignment();
2387
22
  SDLoc DL(N);
2388
22
2389
22
  // Split all operands
2390
22
  EVT LoMemVT, HiMemVT;
2391
22
  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2392
22
2393
22
  SDValue DataLo, DataHi;
2394
22
  if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2395
13
    // Split Data operand
2396
13
    GetSplitVector(Data, DataLo, DataHi);
2397
9
  else
2398
9
    std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2399
22
2400
22
  SDValue MaskLo, MaskHi;
2401
22
  if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2402
0
    // Split Mask operand
2403
0
    GetSplitVector(Mask, MaskLo, MaskHi);
2404
22
  else
2405
22
    std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2406
22
2407
22
  SDValue IndexHi, IndexLo;
2408
22
  if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2409
18
    GetSplitVector(Index, IndexLo, IndexHi);
2410
4
  else
2411
4
    std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2412
22
2413
22
  SDValue Lo;
2414
22
  MachineMemOperand *MMO = DAG.getMachineFunction().
2415
22
    getMachineMemOperand(N->getPointerInfo(),
2416
22
                         MachineMemOperand::MOStore, LoMemVT.getStoreSize(),
2417
22
                         Alignment, N->getAAInfo(), N->getRanges());
2418
22
2419
22
  SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2420
22
  Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataLo.getValueType(),
2421
22
                            DL, OpsLo, MMO);
2422
22
2423
22
  MMO = DAG.getMachineFunction().
2424
22
    getMachineMemOperand(N->getPointerInfo(),
2425
22
                         MachineMemOperand::MOStore,  HiMemVT.getStoreSize(),
2426
22
                         Alignment, N->getAAInfo(), N->getRanges());
2427
22
2428
22
  // The order of the Scatter operation after split is well defined. The "Hi"
2429
22
  // part comes after the "Lo". So these two operations should be chained one
2430
22
  // after another.
2431
22
  SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2432
22
  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), DataHi.getValueType(),
2433
22
                              DL, OpsHi, MMO);
2434
22
}
2435
2436
12.4k
SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2437
12.4k
  assert(N->isUnindexed() && "Indexed store of vector?");
2438
12.4k
  assert(OpNo == 1 && "Can only split the stored value");
2439
12.4k
  SDLoc DL(N);
2440
12.4k
2441
12.4k
  bool isTruncating = N->isTruncatingStore();
2442
12.4k
  SDValue Ch  = N->getChain();
2443
12.4k
  SDValue Ptr = N->getBasePtr();
2444
12.4k
  EVT MemoryVT = N->getMemoryVT();
2445
12.4k
  unsigned Alignment = N->getOriginalAlignment();
2446
12.4k
  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2447
12.4k
  AAMDNodes AAInfo = N->getAAInfo();
2448
12.4k
  SDValue Lo, Hi;
2449
12.4k
  GetSplitVector(N->getOperand(1), Lo, Hi);
2450
12.4k
2451
12.4k
  EVT LoMemVT, HiMemVT;
2452
12.4k
  std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2453
12.4k
2454
12.4k
  // Scalarize if the split halves are not byte-sized.
2455
12.4k
  if (!LoMemVT.isByteSized() || 
!HiMemVT.isByteSized()12.2k
)
2456
145
    return TLI.scalarizeVectorStore(N, DAG);
2457
12.2k
2458
12.2k
  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
2459
12.2k
2460
12.2k
  if (isTruncating)
2461
0
    Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2462
0
                           Alignment, MMOFlags, AAInfo);
2463
12.2k
  else
2464
12.2k
    Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2465
12.2k
                      AAInfo);
2466
12.2k
2467
12.2k
  // Increment the pointer to the other half.
2468
12.2k
  Ptr = DAG.getObjectPtrOffset(DL, Ptr, IncrementSize);
2469
12.2k
2470
12.2k
  if (isTruncating)
2471
0
    Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr,
2472
0
                           N->getPointerInfo().getWithOffset(IncrementSize),
2473
0
                           HiMemVT, Alignment, MMOFlags, AAInfo);
2474
12.2k
  else
2475
12.2k
    Hi = DAG.getStore(Ch, DL, Hi, Ptr,
2476
12.2k
                      N->getPointerInfo().getWithOffset(IncrementSize),
2477
12.2k
                      Alignment, MMOFlags, AAInfo);
2478
12.2k
2479
12.2k
  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2480
12.2k
}
2481
2482
2
SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2483
2
  SDLoc DL(N);
2484
2
2485
2
  // The input operands all must have the same type, and we know the result
2486
2
  // type is valid.  Convert this to a buildvector which extracts all the
2487
2
  // input elements.
2488
2
  // TODO: If the input elements are power-two vectors, we could convert this to
2489
2
  // a new CONCAT_VECTORS node with elements that are half-wide.
2490
2
  SmallVector<SDValue, 32> Elts;
2491
2
  EVT EltVT = N->getValueType(0).getVectorElementType();
2492
8
  for (const SDValue &Op : N->op_values()) {
2493
8
    for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2494
56
         i != e; 
++i48
) {
2495
48
      Elts.push_back(DAG.getNode(
2496
48
          ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2497
48
          DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2498
48
    }
2499
8
  }
2500
2
2501
2
  return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2502
2
}
2503
2504
9.69k
SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2505
9.69k
  // The result type is legal, but the input type is illegal.  If splitting
2506
9.69k
  // ends up with the result type of each half still being legal, just
2507
9.69k
  // do that.  If, however, that would result in an illegal result type,
2508
9.69k
  // we can try to get more clever with power-two vectors. Specifically,
2509
9.69k
  // split the input type, but also widen the result element size, then
2510
9.69k
  // concatenate the halves and truncate again.  For example, consider a target
2511
9.69k
  // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2512
9.69k
  // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2513
9.69k
  //   %inlo = v4i32 extract_subvector %in, 0
2514
9.69k
  //   %inhi = v4i32 extract_subvector %in, 4
2515
9.69k
  //   %lo16 = v4i16 trunc v4i32 %inlo
2516
9.69k
  //   %hi16 = v4i16 trunc v4i32 %inhi
2517
9.69k
  //   %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2518
9.69k
  //   %res = v8i8 trunc v8i16 %in16
2519
9.69k
  //
2520
9.69k
  // Without this transform, the original truncate would end up being
2521
9.69k
  // scalarized, which is pretty much always a last resort.
2522
9.69k
  SDValue InVec = N->getOperand(0);
2523
9.69k
  EVT InVT = InVec->getValueType(0);
2524
9.69k
  EVT OutVT = N->getValueType(0);
2525
9.69k
  unsigned NumElements = OutVT.getVectorNumElements();
2526
9.69k
  bool IsFloat = OutVT.isFloatingPoint();
2527
9.69k
2528
9.69k
  // Widening should have already made sure this is a power-two vector
2529
9.69k
  // if we're trying to split it at all. assert() that's true, just in case.
2530
9.69k
  assert(!(NumElements & 1) && "Splitting vector, but not in half!");
2531
9.69k
2532
9.69k
  unsigned InElementSize = InVT.getScalarSizeInBits();
2533
9.69k
  unsigned OutElementSize = OutVT.getScalarSizeInBits();
2534
9.69k
2535
9.69k
  // Determine the split output VT. If its legal we can just split dirctly.
2536
9.69k
  EVT LoOutVT, HiOutVT;
2537
9.69k
  std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2538
9.69k
  assert(LoOutVT == HiOutVT && "Unequal split?");
2539
9.69k
2540
9.69k
  // If the input elements are only 1/2 the width of the result elements,
2541
9.69k
  // just use the normal splitting. Our trick only work if there's room
2542
9.69k
  // to split more than once.
2543
9.69k
  if (isTypeLegal(LoOutVT) ||
2544
9.69k
      
InElementSize <= OutElementSize * 25.32k
)
2545
7.47k
    return SplitVecOp_UnaryOp(N);
2546
2.21k
  SDLoc DL(N);
2547
2.21k
2548
2.21k
  // Don't touch if this will be scalarized.
2549
2.21k
  EVT FinalVT = InVT;
2550
5.33k
  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2551
3.12k
    FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2552
2.21k
2553
2.21k
  if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2554
12
    return SplitVecOp_UnaryOp(N);
2555
2.20k
2556
2.20k
  // Get the split input vector.
2557
2.20k
  SDValue InLoVec, InHiVec;
2558
2.20k
  GetSplitVector(InVec, InLoVec, InHiVec);
2559
2.20k
2560
2.20k
  // Truncate them to 1/2 the element size.
2561
2.20k
  EVT HalfElementVT = IsFloat ?
2562
21
    EVT::getFloatingPointVT(InElementSize/2) :
2563
2.20k
    
EVT::getIntegerVT(*DAG.getContext(), InElementSize/2)2.18k
;
2564
2.20k
  EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2565
2.20k
                                NumElements/2);
2566
2.20k
  SDValue HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2567
2.20k
  SDValue HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2568
2.20k
  // Concatenate them to get the full intermediate truncation result.
2569
2.20k
  EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2570
2.20k
  SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2571
2.20k
                                 HalfHi);
2572
2.20k
  // Now finish up by truncating all the way down to the original result
2573
2.20k
  // type. This should normally be something that ends up being legal directly,
2574
2.20k
  // but in theory if a target has very wide vectors and an annoyingly
2575
2.20k
  // restricted set of legal types, this split can chain to build things up.
2576
2.20k
  return IsFloat
2577
2.20k
             ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2578
21
                           DAG.getTargetConstant(
2579
21
                               0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2580
2.20k
             : 
DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec)2.18k
;
2581
2.20k
}
2582
2583
148
SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2584
148
  assert(N->getValueType(0).isVector() &&
2585
148
         N->getOperand(0).getValueType().isVector() &&
2586
148
         "Operand types must be vectors");
2587
148
  // The result has a legal vector type, but the input needs splitting.
2588
148
  SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2589
148
  SDLoc DL(N);
2590
148
  GetSplitVector(N->getOperand(0), Lo0, Hi0);
2591
148
  GetSplitVector(N->getOperand(1), Lo1, Hi1);
2592
148
  unsigned PartElements = Lo0.getValueType().getVectorNumElements();
2593
148
  EVT PartResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, PartElements);
2594
148
  EVT WideResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, 2*PartElements);
2595
148
2596
148
  LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2597
148
  HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2598
148
  SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2599
148
  return PromoteTargetBoolean(Con, N->getValueType(0));
2600
148
}
2601
2602
2603
290
SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2604
290
  // The result has a legal vector type, but the input needs splitting.
2605
290
  EVT ResVT = N->getValueType(0);
2606
290
  SDValue Lo, Hi;
2607
290
  SDLoc DL(N);
2608
290
  GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 
14
:
0286
), Lo, Hi);
2609
290
  EVT InVT = Lo.getValueType();
2610
290
2611
290
  EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2612
290
                               InVT.getVectorNumElements());
2613
290
2614
290
  if (N->isStrictFPOpcode()) {
2615
4
    Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 
2616
4
                     { N->getOperand(0), Lo, N->getOperand(2) });
2617
4
    Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, 
2618
4
                     { N->getOperand(0), Hi, N->getOperand(2) });
2619
4
    // Legalize the chain result - switch anything that used the old chain to
2620
4
    // use the new one.
2621
4
    SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, 
2622
4
                                   Lo.getValue(1), Hi.getValue(1));
2623
4
    ReplaceValueWith(SDValue(N, 1), NewChain);
2624
286
  } else {
2625
286
    Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2626
286
    Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2627
286
  }
2628
290
2629
290
  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2630
290
}
2631
2632
8
SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2633
8
  // The result (and the first input) has a legal vector type, but the second
2634
8
  // input needs splitting.
2635
8
  return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2636
8
}
2637
2638
2639
//===----------------------------------------------------------------------===//
2640
//  Result Vector Widening
2641
//===----------------------------------------------------------------------===//
2642
2643
24.4k
void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2644
24.4k
  LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2645
24.4k
             dbgs() << "\n");
2646
24.4k
2647
24.4k
  // See if the target wants to custom widen this node.
2648
24.4k
  if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2649
1.49k
    return;
2650
22.9k
2651
22.9k
  SDValue Res = SDValue();
2652
22.9k
  switch (N->getOpcode()) {
2653
22.9k
  default:
2654
#ifndef NDEBUG
2655
    dbgs() << "WidenVectorResult #" << ResNo << ": ";
2656
    N->dump(&DAG);
2657
    dbgs() << "\n";
2658
#endif
2659
0
    llvm_unreachable("Do not know how to widen the result of this operator!");
2660
22.9k
2661
22.9k
  
case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break0
;
2662
22.9k
  
case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break2.11k
;
2663
22.9k
  
case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break1.94k
;
2664
22.9k
  
case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break68
;
2665
22.9k
  
case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break6.60k
;
2666
22.9k
  
case ISD::FP_ROUND_INREG: Res = WidenVecRes_InregOp(N); break0
;
2667
22.9k
  
case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break24
;
2668
22.9k
  
case ISD::LOAD: Res = WidenVecRes_LOAD(N); break2.75k
;
2669
22.9k
  
case ISD::SCALAR_TO_VECTOR: Res = WidenVecRes_SCALAR_TO_VECTOR(N); break6
;
2670
22.9k
  
case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break13
;
2671
22.9k
  case ISD::VSELECT:
2672
123
  case ISD::SELECT:            Res = WidenVecRes_SELECT(N); break;
2673
123
  
case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break0
;
2674
152
  case ISD::SETCC:             Res = WidenVecRes_SETCC(N); break;
2675
2.91k
  case ISD::UNDEF:             Res = WidenVecRes_UNDEF(N); break;
2676
884
  case ISD::VECTOR_SHUFFLE:
2677
884
    Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2678
884
    break;
2679
123
  case ISD::MLOAD:
2680
13
    Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
2681
13
    break;
2682
123
  case ISD::MGATHER:
2683
22
    Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
2684
22
    break;
2685
123
2686
1.15k
  case ISD::ADD:
2687
1.15k
  case ISD::AND:
2688
1.15k
  case ISD::MUL:
2689
1.15k
  case ISD::MULHS:
2690
1.15k
  case ISD::MULHU:
2691
1.15k
  case ISD::OR:
2692
1.15k
  case ISD::SUB:
2693
1.15k
  case ISD::XOR:
2694
1.15k
  case ISD::FMINNUM:
2695
1.15k
  case ISD::FMAXNUM:
2696
1.15k
  case ISD::FMINIMUM:
2697
1.15k
  case ISD::FMAXIMUM:
2698
1.15k
  case ISD::SMIN:
2699
1.15k
  case ISD::SMAX:
2700
1.15k
  case ISD::UMIN:
2701
1.15k
  case ISD::UMAX:
2702
1.15k
  case ISD::UADDSAT:
2703
1.15k
  case ISD::SADDSAT:
2704
1.15k
  case ISD::USUBSAT:
2705
1.15k
  case ISD::SSUBSAT:
2706
1.15k
    Res = WidenVecRes_Binary(N);
2707
1.15k
    break;
2708
1.15k
2709
1.15k
  case ISD::FADD:
2710
183
  case ISD::FMUL:
2711
183
  case ISD::FPOW:
2712
183
  case ISD::FSUB:
2713
183
  case ISD::FDIV:
2714
183
  case ISD::FREM:
2715
183
  case ISD::SDIV:
2716
183
  case ISD::UDIV:
2717
183
  case ISD::SREM:
2718
183
  case ISD::UREM:
2719
183
    Res = WidenVecRes_BinaryCanTrap(N);
2720
183
    break;
2721
183
2722
347
  case ISD::STRICT_FADD:
2723
347
  case ISD::STRICT_FSUB:
2724
347
  case ISD::STRICT_FMUL:
2725
347
  case ISD::STRICT_FDIV:
2726
347
  case ISD::STRICT_FREM:
2727
347
  case ISD::STRICT_FSQRT:
2728
347
  case ISD::STRICT_FMA:
2729
347
  case ISD::STRICT_FPOW:
2730
347
  case ISD::STRICT_FPOWI:
2731
347
  case ISD::STRICT_FSIN:
2732
347
  case ISD::STRICT_FCOS:
2733
347
  case ISD::STRICT_FEXP:
2734
347
  case ISD::STRICT_FEXP2:
2735
347
  case ISD::STRICT_FLOG:
2736
347
  case ISD::STRICT_FLOG10:
2737
347
  case ISD::STRICT_FLOG2:
2738
347
  case ISD::STRICT_FRINT:
2739
347
  case ISD::STRICT_FNEARBYINT:
2740
347
  case ISD::STRICT_FMAXNUM:
2741
347
  case ISD::STRICT_FMINNUM:
2742
347
  case ISD::STRICT_FCEIL:
2743
347
  case ISD::STRICT_FFLOOR:
2744
347
  case ISD::STRICT_FROUND:
2745
347
  case ISD::STRICT_FTRUNC:
2746
347
    Res = WidenVecRes_StrictFP(N);
2747
347
    break;
2748
347
2749
347
  case ISD::UADDO:
2750
78
  case ISD::SADDO:
2751
78
  case ISD::USUBO:
2752
78
  case ISD::SSUBO:
2753
78
  case ISD::UMULO:
2754
78
  case ISD::SMULO:
2755
78
    Res = WidenVecRes_OverflowOp(N, ResNo);
2756
78
    break;
2757
78
2758
78
  case ISD::FCOPYSIGN:
2759
13
    Res = WidenVecRes_FCOPYSIGN(N);
2760
13
    break;
2761
78
2762
78
  case ISD::FPOWI:
2763
1
    Res = WidenVecRes_POWI(N);
2764
1
    break;
2765
78
2766
935
  case ISD::SHL:
2767
935
  case ISD::SRA:
2768
935
  case ISD::SRL:
2769
935
    Res = WidenVecRes_Shift(N);
2770
935
    break;
2771
935
2772
935
  case ISD::ANY_EXTEND_VECTOR_INREG:
2773
16
  case ISD::SIGN_EXTEND_VECTOR_INREG:
2774
16
  case ISD::ZERO_EXTEND_VECTOR_INREG:
2775
16
    Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
2776
16
    break;
2777
16
2778
2.46k
  case ISD::ANY_EXTEND:
2779
2.46k
  case ISD::FP_EXTEND:
2780
2.46k
  case ISD::FP_ROUND:
2781
2.46k
  case ISD::FP_TO_SINT:
2782
2.46k
  case ISD::FP_TO_UINT:
2783
2.46k
  case ISD::SIGN_EXTEND:
2784
2.46k
  case ISD::SINT_TO_FP:
2785
2.46k
  case ISD::TRUNCATE:
2786
2.46k
  case ISD::UINT_TO_FP:
2787
2.46k
  case ISD::ZERO_EXTEND:
2788
2.46k
    Res = WidenVecRes_Convert(N);
2789
2.46k
    break;
2790
2.46k
2791
2.46k
  case ISD::STRICT_FP_EXTEND:
2792
31
  case ISD::STRICT_FP_ROUND:
2793
31
    Res = WidenVecRes_Convert_StrictFP(N);
2794
31
    break;
2795
31
2796
51
  case ISD::FABS:
2797
51
  case ISD::FCEIL:
2798
51
  case ISD::FCOS:
2799
51
  case ISD::FEXP:
2800
51
  case ISD::FEXP2:
2801
51
  case ISD::FFLOOR:
2802
51
  case ISD::FLOG:
2803
51
  case ISD::FLOG10:
2804
51
  case ISD::FLOG2:
2805
51
  case ISD::FNEARBYINT:
2806
51
  case ISD::FRINT:
2807
51
  case ISD::FROUND:
2808
51
  case ISD::FSIN:
2809
51
  case ISD::FSQRT:
2810
51
  case ISD::FTRUNC: {
2811
51
    // We're going to widen this vector op to a legal type by padding with undef
2812
51
    // elements. If the wide vector op is eventually going to be expanded to
2813
51
    // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
2814
51
    // libcalls on the undef elements.
2815
51
    EVT VT = N->getValueType(0);
2816
51
    EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2817
51
    if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
2818
51
        
TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())29
) {
2819
24
      Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
2820
24
      break;
2821
24
    }
2822
27
  }
2823
27
  // If the target has custom/legal support for the scalar FP intrinsic ops
2824
27
  // (they are probably not destined to become libcalls), then widen those like
2825
27
  // any other unary ops.
2826
27
  LLVM_FALLTHROUGH;
2827
27
2828
39
  case ISD::ABS:
2829
39
  case ISD::BITREVERSE:
2830
39
  case ISD::BSWAP:
2831
39
  case ISD::CTLZ:
2832
39
  case ISD::CTLZ_ZERO_UNDEF:
2833
39
  case ISD::CTPOP:
2834
39
  case ISD::CTTZ:
2835
39
  case ISD::CTTZ_ZERO_UNDEF:
2836
39
  case ISD::FNEG:
2837
39
  case ISD::FCANONICALIZE:
2838
39
    Res = WidenVecRes_Unary(N);
2839
39
    break;
2840
39
  case ISD::FMA:
2841
2
    Res = WidenVecRes_Ternary(N);
2842
2
    break;
2843
22.9k
  }
2844
22.9k
2845
22.9k
  // If Res is null, the sub-method took care of registering the result.
2846
22.9k
  if (Res.getNode())
2847
22.9k
    SetWidenedVector(SDValue(N, ResNo), Res);
2848
22.9k
}
2849
2850
2
SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
2851
2
  // Ternary op widening.
2852
2
  SDLoc dl(N);
2853
2
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2854
2
  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2855
2
  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2856
2
  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
2857
2
  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
2858
2
}
2859
2860
1.15k
SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
2861
1.15k
  // Binary op widening.
2862
1.15k
  SDLoc dl(N);
2863
1.15k
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2864
1.15k
  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2865
1.15k
  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2866
1.15k
  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
2867
1.15k
}
2868
2869
// Given a vector of operations that have been broken up to widen, see
2870
// if we can collect them together into the next widest legal VT. This
2871
// implementation is trap-safe.
2872
static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
2873
                                 SmallVectorImpl<SDValue> &ConcatOps,
2874
                                 unsigned ConcatEnd, EVT VT, EVT MaxVT,
2875
352
                                 EVT WidenVT) {
2876
352
  // Check to see if we have a single operation with the widen type.
2877
352
  if (ConcatEnd == 1) {
2878
81
    VT = ConcatOps[0].getValueType();
2879
81
    if (VT == WidenVT)
2880
0
      return ConcatOps[0];
2881
352
  }
2882
352
2883
352
  SDLoc dl(ConcatOps[0]);
2884
352
  EVT WidenEltVT = WidenVT.getVectorElementType();
2885
352
2886
352
  // while (Some element of ConcatOps is not of type MaxVT) {
2887
352
  //   From the end of ConcatOps, collect elements of the same type and put
2888
352
  //   them into an op of the next larger supported type
2889
352
  // }
2890
736
  while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
2891
384
    int Idx = ConcatEnd - 1;
2892
384
    VT = ConcatOps[Idx--].getValueType();
2893
704
    while (Idx >= 0 && 
ConcatOps[Idx].getValueType() == VT459
)
2894
320
      Idx--;
2895
384
2896
384
    int NextSize = VT.isVector() ? 
VT.getVectorNumElements()32
:
1352
;
2897
384
    EVT NextVT;
2898
631
    do {
2899
631
      NextSize *= 2;
2900
631
      NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
2901
631
    } while (!TLI.isTypeLegal(NextVT));
2902
384
2903
384
    if (!VT.isVector()) {
2904
352
      // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
2905
352
      SDValue VecOp = DAG.getUNDEF(NextVT);
2906
352
      unsigned NumToInsert = ConcatEnd - Idx - 1;
2907
1.00k
      for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; 
i++, OpIdx++648
) {
2908
648
        VecOp = DAG.getNode(
2909
648
            ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp, ConcatOps[OpIdx],
2910
648
            DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2911
648
      }
2912
352
      ConcatOps[Idx+1] = VecOp;
2913
352
      ConcatEnd = Idx + 2;
2914
352
    } else {
2915
32
      // Vector type, create a CONCAT_VECTORS of type NextVT
2916
32
      SDValue undefVec = DAG.getUNDEF(VT);
2917
32
      unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
2918
32
      SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
2919
32
      unsigned RealVals = ConcatEnd - Idx - 1;
2920
32
      unsigned SubConcatEnd = 0;
2921
32
      unsigned SubConcatIdx = Idx + 1;
2922
88
      while (SubConcatEnd < RealVals)
2923
56
        SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
2924
40
      while (SubConcatEnd < OpsToConcat)
2925
8
        SubConcatOps[SubConcatEnd++] = undefVec;
2926
32
      ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
2927
32
                                            NextVT, SubConcatOps);
2928
32
      ConcatEnd = SubConcatIdx + 1;
2929
32
    }
2930
384
  }
2931
352
2932
352
  // Check to see if we have a single operation with the widen type.
2933
352
  if (ConcatEnd == 1) {
2934
245
    VT = ConcatOps[0].getValueType();
2935
245
    if (VT == WidenVT)
2936
245
      return ConcatOps[0];
2937
107
  }
2938
107
2939
107
  // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
2940
107
  unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
2941
107
  if (NumOps != ConcatEnd ) {
2942
5
    SDValue UndefVal = DAG.getUNDEF(MaxVT);
2943
10
    for (unsigned j = ConcatEnd; j < NumOps; 
++j5
)
2944
5
      ConcatOps[j] = UndefVal;
2945
5
  }
2946
107
  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
2947
107
                     makeArrayRef(ConcatOps.data(), NumOps));
2948
107
}
2949
2950
195
SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
2951
195
  // Binary op widening for operations that can trap.
2952
195
  unsigned Opcode = N->getOpcode();
2953
195
  SDLoc dl(N);
2954
195
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2955
195
  EVT WidenEltVT = WidenVT.getVectorElementType();
2956
195
  EVT VT = WidenVT;
2957
195
  unsigned NumElts =  VT.getVectorNumElements();
2958
195
  const SDNodeFlags Flags = N->getFlags();
2959
242
  while (!TLI.isTypeLegal(VT) && 
NumElts != 151
) {
2960
47
    NumElts = NumElts / 2;
2961
47
    VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
2962
47
  }
2963
195
2964
195
  if (NumElts != 1 && 
!TLI.canOpTrap(N->getOpcode(), VT)191
) {
2965
140
    // Operation doesn't trap so just widen as normal.
2966
140
    SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2967
140
    SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2968
140
    return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
2969
140
  }
2970
55
2971
55
  // No legal vector version so unroll the vector operation and then widen.
2972
55
  if (NumElts == 1)
2973
4
    return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
2974
51
2975
51
  // Since the operation can trap, apply operation on the original vector.
2976
51
  EVT MaxVT = VT;
2977
51
  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
2978
51
  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
2979
51
  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
2980
51
2981
51
  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
2982
51
  unsigned ConcatEnd = 0;  // Current ConcatOps index.
2983
51
  int Idx = 0;        // Current Idx into input vectors.
2984
51
2985
51
  // NumElts := greatest legal vector size (at most WidenVT)
2986
51
  // while (orig. vector has unhandled elements) {
2987
51
  //   take munches of size NumElts from the beginning and add to ConcatOps
2988
51
  //   NumElts := next smaller supported vector size or 1
2989
51
  // }
2990
110
  while (CurNumElts != 0) {
2991
79
    while (CurNumElts >= NumElts) {
2992
20
      SDValue EOp1 = DAG.getNode(
2993
20
          ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
2994
20
          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2995
20
      SDValue EOp2 = DAG.getNode(
2996
20
          ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
2997
20
          DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
2998
20
      ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
2999
20
      Idx += NumElts;
3000
20
      CurNumElts -= NumElts;
3001
20
    }
3002
121
    do {
3003
121
      NumElts = NumElts / 2;
3004
121
      VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3005
121
    } while (!TLI.isTypeLegal(VT) && 
NumElts != 1113
);
3006
59
3007
59
    if (NumElts == 1) {
3008
166
      for (unsigned i = 0; i != CurNumElts; 
++i, ++Idx115
) {
3009
115
        SDValue EOp1 = DAG.getNode(
3010
115
            ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp1,
3011
115
            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3012
115
        SDValue EOp2 = DAG.getNode(
3013
115
            ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, InOp2,
3014
115
            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3015
115
        ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3016
115
                                             EOp1, EOp2, Flags);
3017
115
      }
3018
51
      CurNumElts = 0;
3019
51
    }
3020
59
  }
3021
51
3022
51
  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3023
51
}
3024
3025
347
SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3026
347
  // StrictFP op widening for operations that can trap.
3027
347
  unsigned NumOpers = N->getNumOperands();
3028
347
  unsigned Opcode = N->getOpcode();
3029
347
  SDLoc dl(N);
3030
347
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3031
347
  EVT WidenEltVT = WidenVT.getVectorElementType();
3032
347
  EVT VT = WidenVT;
3033
347
  unsigned NumElts = VT.getVectorNumElements();
3034
531
  while (!TLI.isTypeLegal(VT) && 
NumElts != 1230
) {
3035
184
    NumElts = NumElts / 2;
3036
184
    VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3037
184
  }
3038
347
3039
347
  // No legal vector version so unroll the vector operation and then widen.
3040
347
  if (NumElts == 1)
3041
46
    return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3042
301
3043
301
  // Since the operation can trap, apply operation on the original vector.
3044
301
  EVT MaxVT = VT;
3045
301
  SmallVector<SDValue, 4> InOps;
3046
301
  unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3047
301
3048
301
  SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3049
301
  SmallVector<SDValue, 16> Chains;
3050
301
  unsigned ConcatEnd = 0;  // Current ConcatOps index.
3051
301
  int Idx = 0;        // Current Idx into input vectors.
3052
301
3053
301
  // The Chain is the first operand.
3054
301
  InOps.push_back(N->getOperand(0));
3055
301
3056
301
  // Now process the remaining operands.
3057
723
  for (unsigned i = 1; i < NumOpers; 
++i422
) {
3058
422
    SDValue Oper = N->getOperand(i);
3059
422
3060
422
    if (Oper.getValueType().isVector()) {
3061
409
      assert(Oper.getValueType() == N->getValueType(0) && 
3062
409
             "Invalid operand type to widen!");
3063
409
      Oper = GetWidenedVector(Oper);
3064
409
    }
3065
422
3066
422
    InOps.push_back(Oper);
3067
422
  }
3068
301
3069
301
  // NumElts := greatest legal vector size (at most WidenVT)
3070
301
  // while (orig. vector has unhandled elements) {
3071
301
  //   take munches of size NumElts from the beginning and add to ConcatOps
3072
301
  //   NumElts := next smaller supported vector size or 1
3073
301
  // }
3074
626
  while (CurNumElts != 0) {
3075
441
    while (CurNumElts >= NumElts) {
3076
116
      SmallVector<SDValue, 4> EOps;
3077
116
      
3078
395
      for (unsigned i = 0; i < NumOpers; 
++i279
) {
3079
279
        SDValue Op = InOps[i];
3080
279
        
3081
279
        if (Op.getValueType().isVector()) 
3082
158
          Op = DAG.getNode(
3083
158
            ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3084
158
            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3085
279
3086
279
        EOps.push_back(Op);
3087
279
      }
3088
116
3089
116
      EVT OperVT[] = {VT, MVT::Other};
3090
116
      SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3091
116
      ConcatOps[ConcatEnd++] = Oper;
3092
116
      Chains.push_back(Oper.getValue(1));
3093
116
      Idx += NumElts;
3094
116
      CurNumElts -= NumElts;
3095
116
    }
3096
510
    do {
3097
510
      NumElts = NumElts / 2;
3098
510
      VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3099
510
    } while (!TLI.isTypeLegal(VT) && 
NumElts != 1486
);
3100
325
3101
325
    if (NumElts == 1) {
3102
834
      for (unsigned i = 0; i != CurNumElts; 
++i, ++Idx533
) {
3103
533
        SmallVector<SDValue, 4> EOps;
3104
533
3105
1.81k
        for (unsigned i = 0; i < NumOpers; 
++i1.28k
) {
3106
1.28k
          SDValue Op = InOps[i];
3107
1.28k
3108
1.28k
          if (Op.getValueType().isVector())
3109
725
            Op = DAG.getNode(
3110
725
              ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3111
725
              DAG.getConstant(Idx, dl, 
3112
725
                              TLI.getVectorIdxTy(DAG.getDataLayout())));
3113
1.28k
3114
1.28k
          EOps.push_back(Op);
3115
1.28k
        }
3116
533
3117
533
        EVT WidenVT[] = {WidenEltVT, MVT::Other}; 
3118
533
        SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3119
533
        ConcatOps[ConcatEnd++] = Oper;
3120
533
        Chains.push_back(Oper.getValue(1));
3121
533
      }
3122
301
      CurNumElts = 0;
3123
301
    }
3124
325
  }
3125
301
3126
301
  // Build a factor node to remember all the Ops that have been created.
3127
301
  SDValue NewChain;
3128
301
  if (Chains.size() == 1)
3129
69
    NewChain = Chains[0];
3130
232
  else
3131
232
    NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3132
301
  ReplaceValueWith(SDValue(N, 1), NewChain);
3133
301
3134
301
  return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3135
301
}
3136
3137
78
SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3138
78
  SDLoc DL(N);
3139
78
  EVT ResVT = N->getValueType(0);
3140
78
  EVT OvVT = N->getValueType(1);
3141
78
  EVT WideResVT, WideOvVT;
3142
78
  SDValue WideLHS, WideRHS;
3143
78
3144
78
  // TODO: This might result in a widen/split loop.
3145
78
  if (ResNo == 0) {
3146
78
    WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3147
78
    WideOvVT = EVT::getVectorVT(
3148
78
        *DAG.getContext(), OvVT.getVectorElementType(),
3149
78
        WideResVT.getVectorNumElements());
3150
78
3151
78
    WideLHS = GetWidenedVector(N->getOperand(0));
3152
78
    WideRHS = GetWidenedVector(N->getOperand(1));
3153
78
  } else {
3154
0
    WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3155
0
    WideResVT = EVT::getVectorVT(
3156
0
        *DAG.getContext(), ResVT.getVectorElementType(),
3157
0
        WideOvVT.getVectorNumElements());
3158
0
3159
0
    SDValue Zero = DAG.getConstant(
3160
0
        0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3161
0
    WideLHS = DAG.getNode(
3162
0
        ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3163
0
        N->getOperand(0), Zero);
3164
0
    WideRHS = DAG.getNode(
3165
0
        ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3166
0
        N->getOperand(1), Zero);
3167
0
  }
3168
78
3169
78
  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3170
78
  SDNode *WideNode = DAG.getNode(
3171
78
      N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3172
78
3173
78
  // Replace the other vector result not being explicitly widened here.
3174
78
  unsigned OtherNo = 1 - ResNo;
3175
78
  EVT OtherVT = N->getValueType(OtherNo);
3176
78
  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3177
76
    SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3178
76
  } else {
3179
2
    SDValue Zero = DAG.getConstant(
3180
2
        0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
3181
2
    SDValue OtherVal = DAG.getNode(
3182
2
        ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3183
2
    ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3184
2
  }
3185
78
3186
78
  return SDValue(WideNode, ResNo);
3187
78
}
3188
3189
2.46k
SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3190
2.46k
  SDValue InOp = N->getOperand(0);
3191
2.46k
  SDLoc DL(N);
3192
2.46k
3193
2.46k
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3194
2.46k
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3195
2.46k
3196
2.46k
  EVT InVT = InOp.getValueType();
3197
2.46k
  EVT InEltVT = InVT.getVectorElementType();
3198
2.46k
  EVT InWidenVT = EVT::getVectorVT(*DAG.getContext(), InEltVT, WidenNumElts);
3199
2.46k
3200
2.46k
  unsigned Opcode = N->getOpcode();
3201
2.46k
  unsigned InVTNumElts = InVT.getVectorNumElements();
3202
2.46k
  const SDNodeFlags Flags = N->getFlags();
3203
2.46k
  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3204
1.08k
    InOp = GetWidenedVector(N->getOperand(0));
3205
1.08k
    InVT = InOp.getValueType();
3206
1.08k
    InVTNumElts = InVT.getVectorNumElements();
3207
1.08k
    if (InVTNumElts == WidenNumElts) {
3208
720
      if (N->getNumOperands() == 1)
3209
719
        return DAG.getNode(Opcode, DL, WidenVT, InOp);
3210
1
      return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3211
1
    }
3212
365
    if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3213
365
      // If both input and result vector types are of same width, extend
3214
365
      // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3215
365
      // accepts fewer elements in the result than in the input.
3216
365
      if (Opcode == ISD::ANY_EXTEND)
3217
68
        return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3218
297
      if (Opcode == ISD::SIGN_EXTEND)
3219
144
        return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3220
153
      if (Opcode == ISD::ZERO_EXTEND)
3221
121
        return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3222
1.40k
    }
3223
365
  }
3224
1.40k
3225
1.40k
  if (TLI.isTypeLegal(InWidenVT)) {
3226
137
    // Because the result and the input are different vector types, widening
3227
137
    // the result could create a legal type but widening the input might make
3228
137
    // it an illegal type that might lead to repeatedly splitting the input
3229
137
    // and then widening it. To avoid this, we widen the input only if
3230
137
    // it results in a legal type.
3231
137
    if (WidenNumElts % InVTNumElts == 0) {
3232
121
      // Widen the input and call convert on the widened input vector.
3233
121
      unsigned NumConcat = WidenNumElts/InVTNumElts;
3234
121
      SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3235
121
      Ops[0] = InOp;
3236
121
      SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3237
121
      if (N->getNumOperands() == 1)
3238
121
        return DAG.getNode(Opcode, DL, WidenVT, InVec);
3239
0
      return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3240
0
    }
3241
16
3242
16
    if (InVTNumElts % WidenNumElts == 0) {
3243
0
      SDValue InVal = DAG.getNode(
3244
0
          ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3245
0
          DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3246
0
      // Extract the input and convert the shorten input vector.
3247
0
      if (N->getNumOperands() == 1)
3248
0
        return DAG.getNode(Opcode, DL, WidenVT, InVal);
3249
0
      return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3250
0
    }
3251
16
  }
3252
1.28k
3253
1.28k
  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3254
1.28k
  EVT EltVT = WidenVT.getVectorElementType();
3255
1.28k
  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3256
1.28k
  // Use the original element count so we don't do more scalar opts than
3257
1.28k
  // necessary.
3258
1.28k
  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3259
4.95k
  for (unsigned i=0; i < MinElts; 
++i3.67k
) {
3260
3.67k
    SDValue Val = DAG.getNode(
3261
3.67k
        ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3262
3.67k
        DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3263
3.67k
    if (N->getNumOperands() == 1)
3264
3.59k
      Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3265
73
    else
3266
73
      Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3267
3.67k
  }
3268
1.28k
3269
1.28k
  return DAG.getBuildVector(WidenVT, DL, Ops);
3270
1.28k
}
3271
3272
31
SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3273
31
  SDValue InOp = N->getOperand(1);
3274
31
  SDLoc DL(N);
3275
31
  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3276
31
3277
31
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3278
31
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3279
31
  SmallVector<EVT, 2> WidenVTs = { WidenVT, MVT::Other };
3280
31
3281
31
  EVT InVT = InOp.getValueType();
3282
31
  EVT InEltVT = InVT.getVectorElementType();
3283
31
3284
31
  unsigned Opcode = N->getOpcode();
3285
31
3286
31
  // FIXME: Optimizations need to be implemented here.
3287
31
3288
31
  // Otherwise unroll into some nasty scalar code and rebuild the vector.
3289
31
  EVT EltVT = WidenVT.getVectorElementType();
3290
31
  SmallVector<EVT, 2> EltVTs = { EltVT, MVT::Other };
3291
31
  SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3292
31
  SmallVector<SDValue, 32> OpChains;
3293
31
  // Use the original element count so we don't do more scalar opts than
3294
31
  // necessary.
3295
31
  unsigned MinElts = N->getValueType(0).getVectorNumElements();
3296
99
  for (unsigned i=0; i < MinElts; 
++i68
) {
3297
68
    NewOps[1] = DAG.getNode(
3298
68
        ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3299
68
        DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3300
68
    Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3301
68
    OpChains.push_back(Ops[i].getValue(1));
3302
68
  }
3303
31
  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3304
31
  ReplaceValueWith(SDValue(N, 1), NewChain);
3305
31
3306
31
  return DAG.getBuildVector(WidenVT, DL, Ops);
3307
31
}
3308
3309
16
SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3310
16
  unsigned Opcode = N->getOpcode();
3311
16
  SDValue InOp = N->getOperand(0);
3312
16
  SDLoc DL(N);
3313
16
3314
16
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3315
16
  EVT WidenSVT = WidenVT.getVectorElementType();
3316
16
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3317
16
3318
16
  EVT InVT = InOp.getValueType();
3319
16
  EVT InSVT = InVT.getVectorElementType();
3320
16
  unsigned InVTNumElts = InVT.getVectorNumElements();
3321
16
3322
16
  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3323
16
    InOp = GetWidenedVector(InOp);
3324
16
    InVT = InOp.getValueType();
3325
16
    if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3326
16
      switch (Opcode) {
3327
16
      case ISD::ANY_EXTEND_VECTOR_INREG:
3328
16
      case ISD::SIGN_EXTEND_VECTOR_INREG:
3329
16
      case ISD::ZERO_EXTEND_VECTOR_INREG:
3330
16
        return DAG.getNode(Opcode, DL, WidenVT, InOp);
3331
0
      }
3332
0
    }
3333
16
  }
3334
0
3335
0
  // Unroll, extend the scalars and rebuild the vector.
3336
0
  SmallVector<SDValue, 16> Ops;
3337
0
  for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3338
0
    SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3339
0
      DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3340
0
    switch (Opcode) {
3341
0
    case ISD::ANY_EXTEND_VECTOR_INREG:
3342
0
      Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3343
0
      break;
3344
0
    case ISD::SIGN_EXTEND_VECTOR_INREG:
3345
0
      Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3346
0
      break;
3347
0
    case ISD::ZERO_EXTEND_VECTOR_INREG:
3348
0
      Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3349
0
      break;
3350
0
    default:
3351
0
      llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3352
0
    }
3353
0
    Ops.push_back(Val);
3354
0
  }
3355
0
3356
0
  while (Ops.size() != WidenNumElts)
3357
0
    Ops.push_back(DAG.getUNDEF(WidenSVT));
3358
0
3359
0
  return DAG.getBuildVector(WidenVT, DL, Ops);
3360
0
}
3361
3362
13
SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3363
13
  // If this is an FCOPYSIGN with same input types, we can treat it as a
3364
13
  // normal (can trap) binary op.
3365
13
  if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3366
12
    return WidenVecRes_BinaryCanTrap(N);
3367
1
3368
1
  // If the types are different, fall back to unrolling.
3369
1
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3370
1
  return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3371
1
}
3372
3373
1
SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3374
1
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3375
1
  SDValue InOp = GetWidenedVector(N->getOperand(0));
3376
1
  SDValue ShOp = N->getOperand(1);
3377
1
  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3378
1
}
3379
3380
935
SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
3381
935
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3382
935
  SDValue InOp = GetWidenedVector(N->getOperand(0));
3383
935
  SDValue ShOp = N->getOperand(1);
3384
935
3385
935
  EVT ShVT = ShOp.getValueType();
3386
935
  if (getTypeAction(ShVT) == TargetLowering::TypeWidenVector) {
3387
935
    ShOp = GetWidenedVector(ShOp);
3388
935
    ShVT = ShOp.getValueType();
3389
935
  }
3390
935
  EVT ShWidenVT = EVT::getVectorVT(*DAG.getContext(),
3391
935
                                   ShVT.getVectorElementType(),
3392
935
                                   WidenVT.getVectorNumElements());
3393
935
  if (ShVT != ShWidenVT)
3394
0
    ShOp = ModifyToType(ShOp, ShWidenVT);
3395
935
3396
935
  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3397
935
}
3398
3399
39
SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3400
39
  // Unary op widening.
3401
39
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3402
39
  SDValue InOp = GetWidenedVector(N->getOperand(0));
3403
39
  return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3404
39
}
3405
3406
13
SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3407
13
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3408
13
  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3409
13
                               cast<VTSDNode>(N->getOperand(1))->getVT()
3410
13
                                 .getVectorElementType(),
3411
13
                               WidenVT.getVectorNumElements());
3412
13
  SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3413
13
  return DAG.getNode(N->getOpcode(), SDLoc(N),
3414
13
                     WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3415
13
}
3416
3417
0
SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3418
0
  SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3419
0
  return GetWidenedVector(WidenVec);
3420
0
}
3421
3422
2.11k
SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3423
2.11k
  SDValue InOp = N->getOperand(0);
3424
2.11k
  EVT InVT = InOp.getValueType();
3425
2.11k
  EVT VT = N->getValueType(0);
3426
2.11k
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3427
2.11k
  SDLoc dl(N);
3428
2.11k
3429
2.11k
  switch (getTypeAction(InVT)) {
3430
2.11k
  case TargetLowering::TypeLegal:
3431
179
    break;
3432
2.11k
  case TargetLowering::TypePromoteInteger:
3433
52
    // If the incoming type is a vector that is being promoted, then
3434
52
    // we know that the elements are arranged differently and that we
3435
52
    // must perform the conversion using a stack slot.
3436
52
    if (InVT.isVector())
3437
18
      break;
3438
34
3439
34
    // If the InOp is promoted to the same size, convert it.  Otherwise,
3440
34
    // fall out of the switch and widen the promoted input.
3441
34
    InOp = GetPromotedInteger(InOp);
3442
34
    InVT = InOp.getValueType();
3443
34
    if (WidenVT.bitsEq(InVT))
3444
14
      return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3445
20
    break;
3446
20
  case TargetLowering::TypeSoftenFloat:
3447
2
  case TargetLowering::TypePromoteFloat:
3448
2
  case TargetLowering::TypeExpandInteger:
3449
2
  case TargetLowering::TypeExpandFloat:
3450
2
  case TargetLowering::TypeScalarizeVector:
3451
2
  case TargetLowering::TypeSplitVector:
3452
2
    break;
3453
1.87k
  case TargetLowering::TypeWidenVector:
3454
1.87k
    // If the InOp is widened to the same size, convert it.  Otherwise, fall
3455
1.87k
    // out of the switch and widen the widened input.
3456
1.87k
    InOp = GetWidenedVector(InOp);
3457
1.87k
    InVT = InOp.getValueType();
3458
1.87k
    if (WidenVT.bitsEq(InVT))
3459
1.87k
      // The input widens to the same size. Convert to the widen value.
3460
1.87k
      return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3461
0
    break;
3462
219
  }
3463
219
3464
219
  unsigned WidenSize = WidenVT.getSizeInBits();
3465
219
  unsigned InSize = InVT.getSizeInBits();
3466
219
  // x86mmx is not an acceptable vector element type, so don't try.
3467
219
  if (WidenSize % InSize == 0 && 
InVT != MVT::x86mmx209
) {
3468
192
    // Determine new input vector type.  The new input vector type will use
3469
192
    // the same element type (if its a vector) or use the input type as a
3470
192
    // vector.  It is the same size as the type to widen to.
3471
192
    EVT NewInVT;
3472
192
    unsigned NewNumElts = WidenSize / InSize;
3473
192
    if (InVT.isVector()) {
3474
18
      EVT InEltVT = InVT.getVectorElementType();
3475
18
      NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3476
18
                                 WidenSize / InEltVT.getSizeInBits());
3477
174
    } else {
3478
174
      NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3479
174
    }
3480
192
3481
192
    if (TLI.isTypeLegal(NewInVT)) {
3482
190
      SDValue NewVec;
3483
190
      if (InVT.isVector()) {
3484
18
        // Because the result and the input are different vector types, widening
3485
18
        // the result could create a legal type but widening the input might make
3486
18
        // it an illegal type that might lead to repeatedly splitting the input
3487
18
        // and then widening it. To avoid this, we widen the input only if
3488
18
        // it results in a legal type.
3489
18
        SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3490
18
        Ops[0] = InOp;
3491
18
3492
18
        NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3493
172
      } else {
3494
172
        NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3495
172
      }
3496
190
      return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3497
190
    }
3498
29
  }
3499
29
3500
29
  return CreateStackStoreLoad(InOp, WidenVT);
3501
29
}
3502
3503
1.94k
SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3504
1.94k
  SDLoc dl(N);
3505
1.94k
  // Build a vector with undefined for the new nodes.
3506
1.94k
  EVT VT = N->getValueType(0);
3507
1.94k
3508
1.94k
  // Integer BUILD_VECTOR operands may be larger than the node's vector element
3509
1.94k
  // type. The UNDEFs need to have the same type as the existing operands.
3510
1.94k
  EVT EltVT = N->getOperand(0).getValueType();
3511
1.94k
  unsigned NumElts = VT.getVectorNumElements();
3512
1.94k
3513
1.94k
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3514
1.94k
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3515
1.94k
3516
1.94k
  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3517
1.94k
  assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3518
1.94k
  NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3519
1.94k
3520
1.94k
  return DAG.getBuildVector(WidenVT, dl, NewOps);
3521
1.94k
}
3522
3523
68
SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3524
68
  EVT InVT = N->getOperand(0).getValueType();
3525
68
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3526
68
  SDLoc dl(N);
3527
68
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3528
68
  unsigned NumInElts = InVT.getVectorNumElements();
3529
68
  unsigned NumOperands = N->getNumOperands();
3530
68
3531
68
  bool InputWidened = false; // Indicates we need to widen the input.
3532
68
  if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3533
33
    if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
3534
32
      // Add undef vectors to widen to correct length.
3535
32
      unsigned NumConcat = WidenVT.getVectorNumElements() /
3536
32
                           InVT.getVectorNumElements();
3537
32
      SDValue UndefVal = DAG.getUNDEF(InVT);
3538
32
      SmallVector<SDValue, 16> Ops(NumConcat);
3539
116
      for (unsigned i=0; i < NumOperands; 
++i84
)
3540
84
        Ops[i] = N->getOperand(i);
3541
92
      for (unsigned i = NumOperands; i != NumConcat; 
++i60
)
3542
60
        Ops[i] = UndefVal;
3543
32
      return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3544
32
    }
3545
35
  } else {
3546
35
    InputWidened = true;
3547
35
    if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3548
4
      // The inputs and the result are widen to the same value.
3549
4
      unsigned i;
3550
4
      for (i=1; i < NumOperands; 
++i0
)
3551
4
        if (!N->getOperand(i).isUndef())
3552
4
          break;
3553
4
3554
4
      if (i == NumOperands)
3555
0
        // Everything but the first operand is an UNDEF so just return the
3556
0
        // widened first operand.
3557
0
        return GetWidenedVector(N->getOperand(0));
3558
4
3559
4
      if (NumOperands == 2) {
3560
4
        // Replace concat of two operands with a shuffle.
3561
4
        SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3562
16
        for (unsigned i = 0; i < NumInElts; 
++i12
) {
3563
12
          MaskOps[i] = i;
3564
12
          MaskOps[i + NumInElts] = i + WidenNumElts;
3565
12
        }
3566
4
        return DAG.getVectorShuffle(WidenVT, dl,
3567
4
                                    GetWidenedVector(N->getOperand(0)),
3568
4
                                    GetWidenedVector(N->getOperand(1)),
3569
4
                                    MaskOps);
3570
4
      }
3571
32
    }
3572
35
  }
3573
32
3574
32
  // Fall back to use extracts and build vector.
3575
32
  EVT EltVT = WidenVT.getVectorElementType();
3576
32
  SmallVector<SDValue, 16> Ops(WidenNumElts);
3577
32
  unsigned Idx = 0;
3578
112
  for (unsigned i=0; i < NumOperands; 
++i80
) {
3579
80
    SDValue InOp = N->getOperand(i);
3580
80
    if (InputWidened)
3581
78
      InOp = GetWidenedVector(InOp);
3582
730
    for (unsigned j=0; j < NumInElts; 
++j650
)
3583
650
      Ops[Idx++] = DAG.getNode(
3584
650
          ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3585
650
          DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3586
80
  }
3587
32
  SDValue UndefVal = DAG.getUNDEF(EltVT);
3588
278
  for (; Idx < WidenNumElts; 
++Idx246
)
3589
246
    Ops[Idx] = UndefVal;
3590
32
  return DAG.getBuildVector(WidenVT, dl, Ops);
3591
32
}
3592
3593
6.60k
SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3594
6.60k
  EVT      VT = N->getValueType(0);
3595
6.60k
  EVT      WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3596
6.60k
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3597
6.60k
  SDValue  InOp = N->getOperand(0);
3598
6.60k
  SDValue  Idx  = N->getOperand(1);
3599
6.60k
  SDLoc dl(N);
3600
6.60k
3601
6.60k
  if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3602
673
    InOp = GetWidenedVector(InOp);
3603
6.60k
3604
6.60k
  EVT InVT = InOp.getValueType();
3605
6.60k
3606
6.60k
  // Check if we can just return the input vector after widening.
3607
6.60k
  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3608
6.60k
  if (IdxVal == 0 && 
InVT == WidenVT5.47k
)
3609
5.46k
    return InOp;
3610
1.14k
3611
1.14k
  // Check if we can extract from the vector.
3612
1.14k
  unsigned InNumElts = InVT.getVectorNumElements();
3613
1.14k
  if (IdxVal % WidenNumElts == 0 && 
IdxVal + WidenNumElts < InNumElts7
)
3614
6
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3615
1.13k
3616
1.13k
  // We could try widening the input to the right length but for now, extract
3617
1.13k
  // the original elements, fill the rest with undefs and build a vector.
3618
1.13k
  SmallVector<SDValue, 16> Ops(WidenNumElts);
3619
1.13k
  EVT EltVT = VT.getVectorElementType();
3620
1.13k
  unsigned NumElts = VT.getVectorNumElements();
3621
1.13k
  unsigned i;
3622
5.40k
  for (i=0; i < NumElts; 
++i4.26k
)
3623
4.26k
    Ops[i] =
3624
4.26k
        DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3625
4.26k
                    DAG.getConstant(IdxVal + i, dl,
3626
4.26k
                                    TLI.getVectorIdxTy(DAG.getDataLayout())));
3627
1.13k
3628
1.13k
  SDValue UndefVal = DAG.getUNDEF(EltVT);
3629
8.04k
  for (; i < WidenNumElts; 
++i6.91k
)
3630
6.91k
    Ops[i] = UndefVal;
3631
1.13k
  return DAG.getBuildVector(WidenVT, dl, Ops);
3632
1.13k
}
3633
3634
24
SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
3635
24
  SDValue InOp = GetWidenedVector(N->getOperand(0));
3636
24
  return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
3637
24
                     InOp.getValueType(), InOp,
3638
24
                     N->getOperand(1), N->getOperand(2));
3639
24
}
3640
3641
2.75k
SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
3642
2.75k
  LoadSDNode *LD = cast<LoadSDNode>(N);
3643
2.75k
  ISD::LoadExtType ExtType = LD->getExtensionType();
3644
2.75k
3645
2.75k
  SDValue Result;
3646
2.75k
  SmallVector<SDValue, 16> LdChain;  // Chain for the series of load
3647
2.75k
  if (ExtType != ISD::NON_EXTLOAD)
3648
0
    Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
3649
2.75k
  else
3650
2.75k
    Result = GenWidenVectorLoads(LdChain, LD);
3651
2.75k
3652
2.75k
  // If we generate a single load, we can use that for the chain.  Otherwise,
3653
2.75k
  // build a factor node to remember the multiple loads are independent and
3654
2.75k
  // chain to that.
3655
2.75k
  SDValue NewChain;
3656
2.75k
  if (LdChain.size() == 1)
3657
635
    NewChain = LdChain[0];
3658
2.11k
  else
3659
2.11k
    NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
3660
2.75k
3661
2.75k
  // Modified the chain - switch anything that used the old chain to use
3662
2.75k
  // the new one.
3663
2.75k
  ReplaceValueWith(SDValue(N, 1), NewChain);
3664
2.75k
3665
2.75k
  return Result;
3666
2.75k
}
3667
3668
13
SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
3669
13
3670
13
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
3671
13
  SDValue Mask = N->getMask();
3672
13
  EVT MaskVT = Mask.getValueType();
3673
13
  SDValue PassThru = GetWidenedVector(N->getPassThru());
3674
13
  ISD::LoadExtType ExtType = N->getExtensionType();
3675
13
  SDLoc dl(N);
3676
13
3677
13
  // The mask should be widened as well
3678
13
  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3679
13
                                    MaskVT.getVectorElementType(),
3680
13
                                    WidenVT.getVectorNumElements());
3681
13
  Mask = ModifyToType(Mask, WideMaskVT, true);
3682
13
3683
13
  SDValue Res = DAG.getMaskedLoad(WidenVT, dl, N->getChain(), N->getBasePtr(),
3684
13
                                  Mask, PassThru, N->getMemoryVT(),
3685
13
                                  N->getMemOperand(), ExtType,
3686
13
                                  N->isExpandingLoad());
3687
13
  // Legalize the chain result - switch anything that used the old chain to
3688
13
  // use the new one.
3689
13
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3690
13
  return Res;
3691
13
}
3692
3693
22
SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
3694
22
3695
22
  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3696
22
  SDValue Mask = N->getMask();
3697
22
  EVT MaskVT = Mask.getValueType();
3698
22
  SDValue PassThru = GetWidenedVector(N->getPassThru());
3699
22
  SDValue Scale = N->getScale();
3700
22
  unsigned NumElts = WideVT.getVectorNumElements();
3701
22
  SDLoc dl(N);
3702
22
3703
22
  // The mask should be widened as well
3704
22
  EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
3705
22
                                    MaskVT.getVectorElementType(),
3706
22
                                    WideVT.getVectorNumElements());
3707
22
  Mask = ModifyToType(Mask, WideMaskVT, true);
3708
22
3709
22
  // Widen the Index operand
3710
22
  SDValue Index = N->getIndex();
3711
22
  EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
3712
22
                                     Index.getValueType().getScalarType(),
3713
22
                                     NumElts);
3714
22
  Index = ModifyToType(Index, WideIndexVT);
3715
22
  SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
3716
22
                    Scale };
3717
22
  SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
3718
22
                                    N->getMemoryVT(), dl, Ops,
3719
22
                                    N->getMemOperand());
3720
22
3721
22
  // Legalize the chain result - switch anything that used the old chain to
3722
22
  // use the new one.
3723
22
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3724
22
  return Res;
3725
22
}
3726
3727
6
SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
3728
6
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3729
6
  return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N),
3730
6
                     WidenVT, N->getOperand(0));
3731
6
}
3732
3733
// Return true if this is a node that could have two SETCCs as operands.
3734
704
static inline bool isLogicalMaskOp(unsigned Opcode) {
3735
704
  switch (Opcode) {
3736
704
  case ISD::AND:
3737
171
  case ISD::OR:
3738
171
  case ISD::XOR:
3739
171
    return true;
3740
533
  }
3741
533
  return false;
3742
533
}
3743
3744
// This is used just for the assert in convertMask(). Check that this either
3745
// a SETCC or a previously handled SETCC by convertMask().
3746
#ifndef NDEBUG
3747
static inline bool isSETCCorConvertedSETCC(SDValue N) {
3748
  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
3749
    N = N.getOperand(0);
3750
  else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
3751
    for (unsigned i = 1; i < N->getNumOperands(); ++i)
3752
      if (!N->getOperand(i)->isUndef())
3753
        return false;
3754
    N = N.getOperand(0);
3755
  }
3756
3757
  if (N.getOpcode() == ISD::TRUNCATE)
3758
    N = N.getOperand(0);
3759
  else if (N.getOpcode() == ISD::SIGN_EXTEND)
3760
    N = N.getOperand(0);
3761
3762
  if (isLogicalMaskOp(N.getOpcode()))
3763
    return isSETCCorConvertedSETCC(N.getOperand(0)) &&
3764
           isSETCCorConvertedSETCC(N.getOperand(1));
3765
3766
  return (N.getOpcode() == ISD::SETCC ||
3767
          ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
3768
}
3769
#endif
3770
3771
// Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
3772
// to ToMaskVT if needed with vector extension or truncation.
3773
SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
3774
2.51k
                                      EVT ToMaskVT) {
3775
2.51k
  // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
3776
2.51k
  // FIXME: This code seems to be too restrictive, we might consider
3777
2.51k
  // generalizing it or dropping it.
3778
2.51k
  assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
3779
2.51k
3780
2.51k
  // Make a new Mask node, with a legal result VT.
3781
2.51k
  SmallVector<SDValue, 4> Ops;
3782
9.98k
  for (unsigned i = 0, e = InMask->getNumOperands(); i < e; 
++i7.46k
)
3783
7.46k
    Ops.push_back(InMask->getOperand(i));
3784
2.51k
  SDValue Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
3785
2.51k
3786
2.51k
  // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
3787
2.51k
  // extend or truncate is needed.
3788
2.51k
  LLVMContext &Ctx = *DAG.getContext();
3789
2.51k
  unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
3790
2.51k
  unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
3791
2.51k
  if (MaskScalarBits < ToMaskScalBits) {
3792
204
    EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3793
204
                                 MaskVT.getVectorNumElements());
3794
204
    Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
3795
2.31k
  } else if (MaskScalarBits > ToMaskScalBits) {
3796
113
    EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
3797
113
                                   MaskVT.getVectorNumElements());
3798
113
    Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
3799
113
  }
3800
2.51k
3801
2.51k
  assert(Mask->getValueType(0).getScalarSizeInBits() ==
3802
2.51k
             ToMaskVT.getScalarSizeInBits() &&
3803
2.51k
         "Mask should have the right element size by now.");
3804
2.51k
3805
2.51k
  // Adjust Mask to the right number of elements.
3806
2.51k
  unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
3807
2.51k
  if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
3808
0
    MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
3809
0
    SDValue ZeroIdx = DAG.getConstant(0, SDLoc(Mask), IdxTy);
3810
0
    Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
3811
0
                       ZeroIdx);
3812
2.51k
  } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
3813
67
    unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
3814
67
    EVT SubVT = Mask->getValueType(0);
3815
67
    SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
3816
67
    SubOps[0] = Mask;
3817
67
    Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
3818
67
  }
3819
2.51k
3820
2.51k
  assert((Mask->getValueType(0) == ToMaskVT) &&
3821
2.51k
         "A mask of ToMaskVT should have been produced by now.");
3822
2.51k
3823
2.51k
  return Mask;
3824
2.51k
}
3825
3826
// This method tries to handle VSELECT and its mask by legalizing operands
3827
// (which may require widening) and if needed adjusting the mask vector type
3828
// to match that of the VSELECT. Without it, many cases end up with
3829
// scalarization of the SETCC, with many unnecessary instructions.
3830
3.06k
SDValue DAGTypeLegalizer::WidenVSELECTAndMask(SDNode *N) {
3831
3.06k
  LLVMContext &Ctx = *DAG.getContext();
3832
3.06k
  SDValue Cond = N->getOperand(0);
3833
3.06k
3834
3.06k
  if (N->getOpcode() != ISD::VSELECT)
3835
0
    return SDValue();
3836
3.06k
3837
3.06k
  if (Cond->getOpcode() != ISD::SETCC && 
!isLogicalMaskOp(Cond->getOpcode())622
)
3838
533
    return SDValue();
3839
2.53k
3840
2.53k
  // If this is a splitted VSELECT that was previously already handled, do
3841
2.53k
  // nothing.
3842
2.53k
  EVT CondVT = Cond->getValueType(0);
3843
2.53k
  if (CondVT.getScalarSizeInBits() != 1)
3844
38
    return SDValue();
3845
2.49k
3846
2.49k
  EVT VSelVT = N->getValueType(0);
3847
2.49k
  // Only handle vector types which are a power of 2.
3848
2.49k
  if (!isPowerOf2_64(VSelVT.getSizeInBits()))
3849
39
    return SDValue();
3850
2.45k
3851
2.45k
  // Don't touch if this will be scalarized.
3852
2.45k
  EVT FinalVT = VSelVT;
3853
3.12k
  while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
3854
670
    FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
3855
2.45k
3856
2.45k
  if (FinalVT.getVectorNumElements() == 1)
3857
75
    return SDValue();
3858
2.38k
3859
2.38k
  // If there is support for an i1 vector mask, don't touch.
3860
2.38k
  if (Cond.getOpcode() == ISD::SETCC) {
3861
2.29k
    EVT SetCCOpVT = Cond->getOperand(0).getValueType();
3862
3.08k
    while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
3863
787
      SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
3864
2.29k
    EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
3865
2.29k
    if (SetCCResVT.getScalarSizeInBits() == 1)
3866
8
      return SDValue();
3867
86
  } else if (CondVT.getScalarType() == MVT::i1) {
3868
86
    // If there is support for an i1 vector mask (or only scalar i1 conditions),
3869
86
    // don't touch.
3870
176
    while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
3871
90
      CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
3872
86
3873
86
    if (CondVT.getScalarType() == MVT::i1)
3874
4
      return SDValue();
3875
2.36k
  }
3876
2.36k
3877
2.36k
  // Get the VT and operands for VSELECT, and widen if needed.
3878
2.36k
  SDValue VSelOp1 = N->getOperand(1);
3879
2.36k
  SDValue VSelOp2 = N->getOperand(2);
3880
2.36k
  if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) {
3881
67
    VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
3882
67
    VSelOp1 = GetWidenedVector(VSelOp1);
3883
67
    VSelOp2 = GetWidenedVector(VSelOp2);
3884
67
  }
3885
2.36k
3886
2.36k
  // The mask of the VSELECT should have integer elements.
3887
2.36k
  EVT ToMaskVT = VSelVT;
3888
2.36k
  if (!ToMaskVT.getScalarType().isInteger())
3889
613
    ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
3890
2.36k
3891
2.36k
  SDValue Mask;
3892
2.36k
  if (Cond->getOpcode() == ISD::SETCC) {
3893
2.28k
    EVT MaskVT = getSetCCResultType(Cond.getOperand(0).getValueType());
3894
2.28k
    Mask = convertMask(Cond, MaskVT, ToMaskVT);
3895
2.28k
  } else 
if (82
isLogicalMaskOp(Cond->getOpcode())82
&&
3896
82
             Cond->getOperand(0).getOpcode() == ISD::SETCC &&
3897
82
             
Cond->getOperand(1).getOpcode() == ISD::SETCC78
) {
3898
76
    // Cond is (AND/OR/XOR (SETCC, SETCC))
3899
76
    SDValue SETCC0 = Cond->getOperand(0);
3900
76
    SDValue SETCC1 = Cond->getOperand(1);
3901
76
    EVT VT0 = getSetCCResultType(SETCC0.getOperand(0).getValueType());
3902
76
    EVT VT1 = getSetCCResultType(SETCC1.getOperand(0).getValueType());
3903
76
    unsigned ScalarBits0 = VT0.getScalarSizeInBits();
3904
76
    unsigned ScalarBits1 = VT1.getScalarSizeInBits();
3905
76
    unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
3906
76
    EVT MaskVT;
3907
76
    // If the two SETCCs have different VTs, either extend/truncate one of
3908
76
    // them to the other "towards" ToMaskVT, or truncate one and extend the
3909
76
    // other to ToMaskVT.
3910
76
    if (ScalarBits0 != ScalarBits1) {
3911
40
      EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? 
VT020
:
VT120
);
3912
40
      EVT WideVT = ((NarrowVT == VT0) ? 
VT120
:
VT020
);
3913
40
      if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
3914
18
        MaskVT = WideVT;
3915
22
      else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
3916
22
        MaskVT = NarrowVT;
3917
0
      else
3918
0
        MaskVT = ToMaskVT;
3919
40
    } else
3920
36
      // If the two SETCCs have the same VT, don't change it.
3921
36
      MaskVT = VT0;
3922
76
3923
76
    // Make new SETCCs and logical nodes.
3924
76
    SETCC0 = convertMask(SETCC0, VT0, MaskVT);
3925
76
    SETCC1 = convertMask(SETCC1, VT1, MaskVT);
3926
76
    Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
3927
76
3928
76
    // Convert the logical op for VSELECT if needed.
3929
76
    Mask = convertMask(Cond, MaskVT, ToMaskVT);
3930
76
  } else
3931
6
    return SDValue();
3932
2.36k
3933
2.36k
  return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2);
3934
2.36k
}
3935
3936
123
SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
3937
123
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3938
123
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3939
123
3940
123
  SDValue Cond1 = N->getOperand(0);
3941
123
  EVT CondVT = Cond1.getValueType();
3942
123
  if (CondVT.isVector()) {
3943
113
    if (SDValue Res = WidenVSELECTAndMask(N))
3944
67
      return Res;
3945
46
3946
46
    EVT CondEltVT = CondVT.getVectorElementType();
3947
46
    EVT CondWidenVT =  EVT::getVectorVT(*DAG.getContext(),
3948
46
                                        CondEltVT, WidenNumElts);
3949
46
    if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
3950
40
      Cond1 = GetWidenedVector(Cond1);
3951
46
3952
46
    // If we have to split the condition there is no point in widening the
3953
46
    // select. This would result in an cycle of widening the select ->
3954
46
    // widening the condition operand -> splitting the condition operand ->
3955
46
    // splitting the select -> widening the select. Instead split this select
3956
46
    // further and widen the resulting type.
3957
46
    if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
3958
4
      SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
3959
4
      SDValue Res = ModifyToType(SplitSelect, WidenVT);
3960
4
      return Res;
3961
4
    }
3962
42
3963
42
    if (Cond1.getValueType() != CondWidenVT)
3964
2
      Cond1 = ModifyToType(Cond1, CondWidenVT);
3965
42
  }
3966
123
3967
123
  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
3968
52
  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
3969
52
  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
3970
52
  return DAG.getNode(N->getOpcode(), SDLoc(N),
3971
52
                     WidenVT, Cond1, InOp1, InOp2);
3972
123
}
3973
3974
0
SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
3975
0
  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
3976
0
  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
3977
0
  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
3978
0
                     InOp1.getValueType(), N->getOperand(0),
3979
0
                     N->getOperand(1), InOp1, InOp2, N->getOperand(4));
3980
0
}
3981
3982
2.91k
SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
3983
2.91k
 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3984
2.91k
 return DAG.getUNDEF(WidenVT);
3985
2.91k
}
3986
3987
884
SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
3988
884
  EVT VT = N->getValueType(0);
3989
884
  SDLoc dl(N);
3990
884
3991
884
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3992
884
  unsigned NumElts = VT.getVectorNumElements();
3993
884
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
3994
884
3995
884
  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3996
884
  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3997
884
3998
884
  // Adjust mask based on new input vector length.
3999
884
  SmallVector<int, 16> NewMask;
4000
4.66k
  for (unsigned i = 0; i != NumElts; 
++i3.78k
) {
4001
3.78k
    int Idx = N->getMaskElt(i);
4002
3.78k
    if (Idx < (int)NumElts)
4003
3.63k
      NewMask.push_back(Idx);
4004
142
    else
4005
142
      NewMask.push_back(Idx - NumElts + WidenNumElts);
4006
3.78k
  }
4007
7.41k
  for (unsigned i = NumElts; i != WidenNumElts; 
++i6.53k
)
4008
6.53k
    NewMask.push_back(-1);
4009
884
  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
4010
884
}
4011
4012
152
SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4013
152
  assert(N->getValueType(0).isVector() &&
4014
152
         N->getOperand(0).getValueType().isVector() &&
4015
152
         "Operands must be vectors");
4016
152
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4017
152
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4018
152
4019
152
  SDValue InOp1 = N->getOperand(0);
4020
152
  EVT InVT = InOp1.getValueType();
4021
152
  assert(InVT.isVector() && "can not widen non-vector type");
4022
152
  EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
4023
152
                                   InVT.getVectorElementType(), WidenNumElts);
4024
152
4025
152
  // The input and output types often differ here, and it could be that while
4026
152
  // we'd prefer to widen the result type, the input operands have been split.
4027
152
  // In this case, we also need to split the result of this node as well.
4028
152
  if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
4029
2
    SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4030
2
    SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4031
2
    return Res;
4032
2
  }
4033
150
4034
150
  // If the inputs also widen, handle them directly. Otherwise widen by hand.
4035
150
  SDValue InOp2 = N->getOperand(1);
4036
150
  if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4037
141
    InOp1 = GetWidenedVector(InOp1);
4038
141
    InOp2 = GetWidenedVector(InOp2);
4039
141
  } else {
4040
9
    InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
4041
9
    InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
4042
9
  }
4043
150
4044
150
  // Assume that the input and output will be widen appropriately.  If not,
4045
150
  // we will have to unroll it at some point.
4046
150
  assert(InOp1.getValueType() == WidenInVT &&
4047
150
         InOp2.getValueType() == WidenInVT &&
4048
150
         "Input not widened to expected type!");
4049
150
  (void)WidenInVT;
4050
150
  return DAG.getNode(ISD::SETCC, SDLoc(N),
4051
150
                     WidenVT, InOp1, InOp2, N->getOperand(2));
4052
150
}
4053
4054
4055
//===----------------------------------------------------------------------===//
4056
// Widen Vector Operand
4057
//===----------------------------------------------------------------------===//
4058
21.5k
bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
4059
21.5k
  LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
4060
21.5k
             dbgs() << "\n");
4061
21.5k
  SDValue Res = SDValue();
4062
21.5k
4063
21.5k
  // See if the target wants to custom widen this node.
4064
21.5k
  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4065
1.26k
    return false;
4066
20.2k
4067
20.2k
  switch (N->getOpcode()) {
4068
20.2k
  default:
4069
#ifndef NDEBUG
4070
    dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
4071
    N->dump(&DAG);
4072
    dbgs() << "\n";
4073
#endif
4074
0
    llvm_unreachable("Do not know how to widen this operator's operand!");
4075
20.2k
4076
20.2k
  
case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break137
;
4077
20.2k
  
case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break4.26k
;
4078
20.2k
  
case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break2.23k
;
4079
20.2k
  
case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break10.7k
;
4080
20.2k
  
case ISD::STORE: Res = WidenVecOp_STORE(N); break983
;
4081
20.2k
  
case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break13
;
4082
20.2k
  
case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break3
;
4083
20.2k
  
case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break5
;
4084
20.2k
  
case ISD::SETCC: Res = WidenVecOp_SETCC(N); break17
;
4085
20.2k
  
case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break8
;
4086
20.2k
  
case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break3
;
4087
20.2k
4088
20.2k
  case ISD::ANY_EXTEND:
4089
1.43k
  case ISD::SIGN_EXTEND:
4090
1.43k
  case ISD::ZERO_EXTEND:
4091
1.43k
    Res = WidenVecOp_EXTEND(N);
4092
1.43k
    break;
4093
1.43k
4094
1.43k
  case ISD::FP_EXTEND:
4095
421
  case ISD::STRICT_FP_EXTEND:
4096
421
  case ISD::FP_TO_SINT:
4097
421
  case ISD::FP_TO_UINT:
4098
421
  case ISD::SINT_TO_FP:
4099
421
  case ISD::UINT_TO_FP:
4100
421
  case ISD::TRUNCATE:
4101
421
    Res = WidenVecOp_Convert(N);
4102
421
    break;
4103
421
4104
421
  case ISD::VECREDUCE_FADD:
4105
11
  case ISD::VECREDUCE_FMUL:
4106
11
  case ISD::VECREDUCE_ADD:
4107
11
  case ISD::VECREDUCE_MUL:
4108
11
  case ISD::VECREDUCE_AND:
4109
11
  case ISD::VECREDUCE_OR:
4110
11
  case ISD::VECREDUCE_XOR:
4111
11
  case ISD::VECREDUCE_SMAX:
4112
11
  case ISD::VECREDUCE_SMIN:
4113
11
  case ISD::VECREDUCE_UMAX:
4114
11
  case ISD::VECREDUCE_UMIN:
4115
11
  case ISD::VECREDUCE_FMAX:
4116
11
  case ISD::VECREDUCE_FMIN:
4117
11
    Res = WidenVecOp_VECREDUCE(N);
4118
11
    break;
4119
20.2k
  }
4120
20.2k
4121
20.2k
  // If Res is null, the sub-method took care of registering the result.
4122
20.2k
  if (!Res.getNode()) 
return false3
;
4123
20.2k
4124
20.2k
  // If the result is N, the sub-method updated N in place.  Tell the legalizer
4125
20.2k
  // core about this.
4126
20.2k
  if (Res.getNode() == N)
4127
0
    return true;
4128
20.2k
4129
20.2k
4130
20.2k
  if (N->isStrictFPOpcode())
4131
20.2k
    assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4132
20.2k
           "Invalid operand expansion");
4133
20.2k
  else
4134
20.2k
    assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4135
20.2k
           "Invalid operand expansion");
4136
20.2k
4137
20.2k
  ReplaceValueWith(SDValue(N, 0), Res);
4138
20.2k
  return false;
4139
20.2k
}
4140
4141
1.43k
SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4142
1.43k
  SDLoc DL(N);
4143
1.43k
  EVT VT = N->getValueType(0);
4144
1.43k
4145
1.43k
  SDValue InOp = N->getOperand(0);
4146
1.43k
  assert(getTypeAction(InOp.getValueType()) ==
4147
1.43k
             TargetLowering::TypeWidenVector &&
4148
1.43k
         "Unexpected type action");
4149
1.43k
  InOp = GetWidenedVector(InOp);
4150
1.43k
  assert(VT.getVectorNumElements() <
4151
1.43k
             InOp.getValueType().getVectorNumElements() &&
4152
1.43k
         "Input wasn't widened!");
4153
1.43k
4154
1.43k
  // We may need to further widen the operand until it has the same total
4155
1.43k
  // vector size as the result.
4156
1.43k
  EVT InVT = InOp.getValueType();
4157
1.43k
  if (InVT.getSizeInBits() != VT.getSizeInBits()) {
4158
152
    EVT InEltVT = InVT.getVectorElementType();
4159
8.67k
    for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; 
++i8.52k
) {
4160
8.61k
      EVT FixedVT = (MVT::SimpleValueType)i;
4161
8.61k
      EVT FixedEltVT = FixedVT.getVectorElementType();
4162
8.61k
      if (TLI.isTypeLegal(FixedVT) &&
4163
8.61k
          
FixedVT.getSizeInBits() == VT.getSizeInBits()1.51k
&&
4164
8.61k
          
FixedEltVT == InEltVT226
) {
4165
87
        assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
4166
87
               "Not enough elements in the fixed type for the operand!");
4167
87
        assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
4168
87
               "We can't have the same type as we started with!");
4169
87
        if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
4170
87
          InOp = DAG.getNode(
4171
87
              ISD::INSERT_SUBVECTOR, DL, FixedVT, DAG.getUNDEF(FixedVT), InOp,
4172
87
              DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4173
0
        else
4174
0
          InOp = DAG.getNode(
4175
0
              ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
4176
0
              DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4177
87
        break;
4178
87
      }
4179
8.61k
    }
4180
152
    InVT = InOp.getValueType();
4181
152
    if (InVT.getSizeInBits() != VT.getSizeInBits())
4182
65
      // We couldn't find a legal vector type that was a widening of the input
4183
65
      // and could be extended in-register to the result type, so we have to
4184
65
      // scalarize.
4185
65
      return WidenVecOp_Convert(N);
4186
1.37k
  }
4187
1.37k
4188
1.37k
  // Use special DAG nodes to represent the operation of extending the
4189
1.37k
  // low lanes.
4190
1.37k
  switch (N->getOpcode()) {
4191
1.37k
  default:
4192
0
    llvm_unreachable("Extend legalization on extend operation!");
4193
1.37k
  case ISD::ANY_EXTEND:
4194
68
    return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
4195
1.37k
  case ISD::SIGN_EXTEND:
4196
676
    return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
4197
1.37k
  case ISD::ZERO_EXTEND:
4198
628
    return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
4199
1.37k
  }
4200
1.37k
}
4201
4202
3
SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4203
3
  // The result (and first input) is legal, but the second input is illegal.
4204
3
  // We can't do much to fix that, so just unroll and let the extracts off of
4205
3
  // the second input be widened as needed later.
4206
3
  return DAG.UnrollVectorOp(N);
4207
3
}
4208
4209
486
SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4210
486
  // Since the result is legal and the input is illegal.
4211
486
  EVT VT = N->getValueType(0);
4212
486
  EVT EltVT = VT.getVectorElementType();
4213
486
  SDLoc dl(N);
4214
486
  unsigned NumElts = VT.getVectorNumElements();
4215
486
  SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 
113
:
0473
);
4216
486
  assert(getTypeAction(InOp.getValueType()) ==
4217
486
             TargetLowering::TypeWidenVector &&
4218
486
         "Unexpected type action");
4219
486
  InOp = GetWidenedVector(InOp);
4220
486
  EVT InVT = InOp.getValueType();
4221
486
  unsigned Opcode = N->getOpcode();
4222
486
4223
486
  // See if a widened result type would be legal, if so widen the node.
4224
486
  // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
4225
486
  EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4226
486
                                InVT.getVectorNumElements());
4227
486
  if (TLI.isTypeLegal(WideVT) && 
!N->isStrictFPOpcode()177
) {
4228
176
    SDValue Res;
4229
176
    if (N->isStrictFPOpcode()) {
4230
0
      Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, 
4231
0
                        { N->getOperand(0), InOp });
4232
0
      // Legalize the chain result - switch anything that used the old chain to
4233
0
      // use the new one.
4234
0
      ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4235
0
    } else
4236
176
      Res = DAG.getNode(Opcode, dl, WideVT, InOp);
4237
176
    return DAG.getNode(
4238
176
        ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
4239
176
        DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4240
176
  }
4241
310
4242
310
  EVT InEltVT = InVT.getVectorElementType();
4243
310
4244
310
  // Unroll the convert into some scalar code and create a nasty build vector.
4245
310
  SmallVector<SDValue, 16> Ops(NumElts);
4246
310
  if (N->isStrictFPOpcode()) {
4247
13
    SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4248
13
    SmallVector<SDValue, 32> OpChains;
4249
39
    for (unsigned i=0; i < NumElts; 
++i26
) {
4250
26
      NewOps[1] = DAG.getNode(
4251
26
          ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4252
26
          DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4253
26
      Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
4254
26
      OpChains.push_back(Ops[i].getValue(1));
4255
26
    }
4256
13
    SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4257
13
    ReplaceValueWith(SDValue(N, 1), NewChain);
4258
297
  } else {
4259
891
    for (unsigned i = 0; i < NumElts; 
++i594
)
4260
594
      Ops[i] = DAG.getNode(
4261
594
          Opcode, dl, EltVT,
4262
594
          DAG.getNode(
4263
594
              ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4264
594
              DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
4265
297
  }
4266
310
4267
310
  return DAG.getBuildVector(VT, dl, Ops);
4268
310
}
4269
4270
137
SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4271
137
  EVT VT = N->getValueType(0);
4272
137
  SDValue InOp = GetWidenedVector(N->getOperand(0));
4273
137
  EVT InWidenVT = InOp.getValueType();
4274
137
  SDLoc dl(N);
4275
137
4276
137
  // Check if we can convert between two legal vector types and extract.
4277
137
  unsigned InWidenSize = InWidenVT.getSizeInBits();
4278
137
  unsigned Size = VT.getSizeInBits();
4279
137
  // x86mmx is not an acceptable vector element type, so don't try.
4280
137
  if (InWidenSize % Size == 0 && 
!VT.isVector()136
&&
VT != MVT::x86mmx136
) {
4281
136
    unsigned NewNumElts = InWidenSize / Size;
4282
136
    EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4283
136
    if (TLI.isTypeLegal(NewVT)) {
4284
136
      SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4285
136
      return DAG.getNode(
4286
136
          ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
4287
136
          DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4288
136
    }
4289
1
  }
4290
1
4291
1
  // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
4292
1
  // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
4293
1
  // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
4294
1
  // having to copy via memory.
4295
1
  if (VT.isVector()) {
4296
1
    EVT EltVT = VT.getVectorElementType();
4297
1
    unsigned EltSize = EltVT.getSizeInBits();
4298
1
    if (InWidenSize % EltSize == 0) {
4299
1
      unsigned NewNumElts = InWidenSize / EltSize;
4300
1
      EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
4301
1
      if (TLI.isTypeLegal(NewVT)) {
4302
1
        SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4303
1
        return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
4304
1
            DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4305
1
      }
4306
0
    }
4307
1
  }
4308
0
4309
0
  return CreateStackStoreLoad(InOp, VT);
4310
0
}
4311
4312
4.26k
SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4313
4.26k
  EVT VT = N->getValueType(0);
4314
4.26k
  EVT EltVT = VT.getVectorElementType();
4315
4.26k
  EVT InVT = N->getOperand(0).getValueType();
4316
4.26k
  SDLoc dl(N);
4317
4.26k
4318
4.26k
  // If the widen width for this operand is the same as the width of the concat
4319
4.26k
  // and all but the first operand is undef, just use the widened operand.
4320
4.26k
  unsigned NumOperands = N->getNumOperands();
4321
4.26k
  if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4322
4.20k
    unsigned i;
4323
10.9k
    for (i = 1; i < NumOperands; 
++i6.71k
)
4324
7.68k
      if (!N->getOperand(i).isUndef())
4325
966
        break;
4326
4.20k
4327
4.20k
    if (i == NumOperands)
4328
3.24k
      return GetWidenedVector(N->getOperand(0));
4329
1.01k
  }
4330
1.01k
4331
1.01k
  // Otherwise, fall back to a nasty build vector.
4332
1.01k
  unsigned NumElts = VT.getVectorNumElements();
4333
1.01k
  SmallVector<SDValue, 16> Ops(NumElts);
4334
1.01k
4335
1.01k
  unsigned NumInElts = InVT.getVectorNumElements();
4336
1.01k
4337
1.01k
  unsigned Idx = 0;
4338
3.22k
  for (unsigned i=0; i < NumOperands; 
++i2.20k
) {
4339
2.20k
    SDValue InOp = N->getOperand(i);
4340
2.20k
    assert(getTypeAction(InOp.getValueType()) ==
4341
2.20k
               TargetLowering::TypeWidenVector &&
4342
2.20k
           "Unexpected type action");
4343
2.20k
    InOp = GetWidenedVector(InOp);
4344
8.43k
    for (unsigned j=0; j < NumInElts; 
++j6.23k
)
4345
6.23k
      Ops[Idx++] = DAG.getNode(
4346
6.23k
          ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4347
6.23k
          DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4348
2.20k
  }
4349
1.01k
  return DAG.getBuildVector(VT, dl, Ops);
4350
1.01k
}
4351
4352
2.23k
SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4353
2.23k
  SDValue InOp = GetWidenedVector(N->getOperand(0));
4354
2.23k
  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4355
2.23k
                     N->getValueType(0), InOp, N->getOperand(1));
4356
2.23k
}
4357
4358
10.7k
SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4359
10.7k
  SDValue InOp = GetWidenedVector(N->getOperand(0));
4360
10.7k
  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4361
10.7k
                     N->getValueType(0), InOp, N->getOperand(1));
4362
10.7k
}
4363
4364
983
SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4365
983
  // We have to widen the value, but we want only to store the original
4366
983
  // vector type.
4367
983
  StoreSDNode *ST = cast<StoreSDNode>(N);
4368
983
4369
983
  if (!ST->getMemoryVT().getScalarType().isByteSized())
4370
12
    return TLI.scalarizeVectorStore(ST, DAG);
4371
971
4372
971
  SmallVector<SDValue, 16> StChain;
4373
971
  if (ST->isTruncatingStore())
4374
0
    GenWidenVectorTruncStores(StChain, ST);
4375
971
  else
4376
971
    GenWidenVectorStores(StChain, ST);
4377
971
4378
971
  if (StChain.size() == 1)
4379
142
    return StChain[0];
4380
829
  else
4381
829
    return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4382
971
}
4383
4384
13
SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4385
13
  assert((OpNo == 1 || OpNo == 3) &&
4386
13
         "Can widen only data or mask operand of mstore");
4387
13
  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
4388
13
  SDValue Mask = MST->getMask();
4389
13
  EVT MaskVT = Mask.getValueType();
4390
13
  SDValue StVal = MST->getValue();
4391
13
  SDLoc dl(N);
4392
13
4393
13
  if (OpNo == 1) {
4394
13
    // Widen the value.
4395
13
    StVal = GetWidenedVector(StVal);
4396
13
4397
13
    // The mask should be widened as well.
4398
13
    EVT WideVT = StVal.getValueType();
4399
13
    EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4400
13
                                      MaskVT.getVectorElementType(),
4401
13
                                      WideVT.getVectorNumElements());
4402
13
    Mask = ModifyToType(Mask, WideMaskVT, true);
4403
13
  } else {
4404
0
    // Widen the mask.
4405
0
    EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4406
0
    Mask = ModifyToType(Mask, WideMaskVT, true);
4407
0
4408
0
    EVT ValueVT = StVal.getValueType();
4409
0
    EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4410
0
                                  ValueVT.getVectorElementType(),
4411
0
                                  WideMaskVT.getVectorNumElements());
4412
0
    StVal = ModifyToType(StVal, WideVT);
4413
0
  }
4414
13
4415
13
  assert(Mask.getValueType().getVectorNumElements() ==
4416
13
         StVal.getValueType().getVectorNumElements() &&
4417
13
         "Mask and data vectors should have the same number of elements");
4418
13
  return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4419
13
                            Mask, MST->getMemoryVT(), MST->getMemOperand(),
4420
13
                            false, MST->isCompressingStore());
4421
13
}
4422
4423
3
SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4424
3
  assert(OpNo == 4 && "Can widen only the index of mgather");
4425
3
  auto *MG = cast<MaskedGatherSDNode>(N);
4426
3
  SDValue DataOp = MG->getPassThru();
4427
3
  SDValue Mask = MG->getMask();
4428
3
  SDValue Scale = MG->getScale();
4429
3
4430
3
  // Just widen the index. It's allowed to have extra elements.
4431
3
  SDValue Index = GetWidenedVector(MG->getIndex());
4432
3
4433
3
  SDLoc dl(N);
4434
3
  SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4435
3
                   Scale};
4436
3
  SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4437
3
                                    MG->getMemOperand());
4438
3
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4439
3
  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4440
3
  return SDValue();
4441
3
}
4442
4443
5
SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4444
5
  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
4445
5
  SDValue DataOp = MSC->getValue();
4446
5
  SDValue Mask = MSC->getMask();
4447
5
  SDValue Index = MSC->getIndex();
4448
5
  SDValue Scale = MSC->getScale();
4449
5
4450
5
  if (OpNo == 1) {
4451
3
    DataOp = GetWidenedVector(DataOp);
4452
3
    unsigned NumElts = DataOp.getValueType().getVectorNumElements();
4453
3
4454
3
    // Widen index.
4455
3
    EVT IndexVT = Index.getValueType();
4456
3
    EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4457
3
                                       IndexVT.getVectorElementType(), NumElts);
4458
3
    Index = ModifyToType(Index, WideIndexVT);
4459
3
4460
3
    // The mask should be widened as well.
4461
3
    EVT MaskVT = Mask.getValueType();
4462
3
    EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4463
3
                                      MaskVT.getVectorElementType(), NumElts);
4464
3
    Mask = ModifyToType(Mask, WideMaskVT, true);
4465
3
  } else 
if (2
OpNo == 42
) {
4466
2
    // Just widen the index. It's allowed to have extra elements.
4467
2
    Index = GetWidenedVector(Index);
4468
2
  } else
4469
2
    
llvm_unreachable0
("Can't widen this operand of mscatter");
4470
5
4471
5
  SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4472
5
                   Scale};
4473
5
  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other),
4474
5
                              MSC->getMemoryVT(), SDLoc(N), Ops,
4475
5
                              MSC->getMemOperand());
4476
5
}
4477
4478
17
SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4479
17
  SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4480
17
  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4481
17
  SDLoc dl(N);
4482
17
  EVT VT = N->getValueType(0);
4483
17
4484
17
  // WARNING: In this code we widen the compare instruction with garbage.
4485
17
  // This garbage may contain denormal floats which may be slow. Is this a real
4486
17
  // concern ? Should we zero the unused lanes if this is a float compare ?
4487
17
4488
17
  // Get a new SETCC node to compare the newly widened operands.
4489
17
  // Only some of the compared elements are legal.
4490
17
  EVT SVT = getSetCCResultType(InOp0.getValueType());
4491
17
  // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
4492
17
  if (VT.getScalarType() == MVT::i1)
4493
2
    SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
4494
2
                           SVT.getVectorNumElements());
4495
17
4496
17
  SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
4497
17
                                  SVT, InOp0, InOp1, N->getOperand(2));
4498
17
4499
17
  // Extract the needed results from the result vector.
4500
17
  EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
4501
17
                               SVT.getVectorElementType(),
4502
17
                               VT.getVectorNumElements());
4503
17
  SDValue CC = DAG.getNode(
4504
17
      ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
4505
17
      DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4506
17
4507
17
  return PromoteTargetBoolean(CC, VT);
4508
17
}
4509
4510
11
SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
4511
11
  SDLoc dl(N);
4512
11
  SDValue Op = GetWidenedVector(N->getOperand(0));
4513
11
  EVT OrigVT = N->getOperand(0).getValueType();
4514
11
  EVT WideVT = Op.getValueType();
4515
11
  EVT ElemVT = OrigVT.getVectorElementType();
4516
11
4517
11
  SDValue NeutralElem;
4518
11
  switch (N->getOpcode()) {
4519
11
  case ISD::VECREDUCE_ADD:
4520
6
  case ISD::VECREDUCE_OR:
4521
6
  case ISD::VECREDUCE_XOR:
4522
6
  case ISD::VECREDUCE_UMAX:
4523
6
    NeutralElem = DAG.getConstant(0, dl, ElemVT);
4524
6
    break;
4525
6
  case ISD::VECREDUCE_MUL:
4526
0
    NeutralElem = DAG.getConstant(1, dl, ElemVT);
4527
0
    break;
4528
6
  case ISD::VECREDUCE_AND:
4529
3
  case ISD::VECREDUCE_UMIN:
4530
3
    NeutralElem = DAG.getAllOnesConstant(dl, ElemVT);
4531
3
    break;
4532
3
  case ISD::VECREDUCE_SMAX:
4533
0
    NeutralElem = DAG.getConstant(
4534
0
        APInt::getSignedMinValue(ElemVT.getSizeInBits()), dl, ElemVT);
4535
0
    break;
4536
3
  case ISD::VECREDUCE_SMIN:
4537
0
    NeutralElem = DAG.getConstant(
4538
0
        APInt::getSignedMaxValue(ElemVT.getSizeInBits()), dl, ElemVT);
4539
0
    break;
4540
3
  case ISD::VECREDUCE_FADD:
4541
1
    NeutralElem = DAG.getConstantFP(0.0, dl, ElemVT);
4542
1
    break;
4543
3
  case ISD::VECREDUCE_FMUL:
4544
0
    NeutralElem = DAG.getConstantFP(1.0, dl, ElemVT);
4545
0
    break;
4546
3
  case ISD::VECREDUCE_FMAX:
4547
1
    NeutralElem = DAG.getConstantFP(
4548
1
        std::numeric_limits<double>::infinity(), dl, ElemVT);
4549
1
    break;
4550
3
  case ISD::VECREDUCE_FMIN:
4551
0
    NeutralElem = DAG.getConstantFP(
4552
0
        -std::numeric_limits<double>::infinity(), dl, ElemVT);
4553
0
    break;
4554
11
  }
4555
11
4556
11
  // Pad the vector with the neutral element.
4557
11
  unsigned OrigElts = OrigVT.getVectorNumElements();
4558
11
  unsigned WideElts = WideVT.getVectorNumElements();
4559
40
  for (unsigned Idx = OrigElts; Idx < WideElts; 
Idx++29
)
4560
29
    Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
4561
29
        DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4562
11
4563
11
  return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Op, N->getFlags());
4564
11
}
4565
4566
8
SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
4567
8
  // This only gets called in the case that the left and right inputs and
4568
8
  // result are of a legal odd vector type, and the condition is illegal i1 of
4569
8
  // the same odd width that needs widening.
4570
8
  EVT VT = N->getValueType(0);
4571
8
  assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
4572
8
4573
8
  SDValue Cond = GetWidenedVector(N->getOperand(0));
4574
8
  SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
4575
8
  SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
4576
8
  SDLoc DL(N);
4577
8
4578
8
  SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
4579
8
                               LeftIn, RightIn);
4580
8
  return DAG.getNode(
4581
8
      ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
4582
8
      DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
4583
8
}
4584
4585
//===----------------------------------------------------------------------===//
4586
// Vector Widening Utilities
4587
//===----------------------------------------------------------------------===//
4588
4589
// Utility function to find the type to chop up a widen vector for load/store
4590
//  TLI:       Target lowering used to determine legal types.
4591
//  Width:     Width left need to load/store.
4592
//  WidenVT:   The widen vector type to load to/store from
4593
//  Align:     If 0, don't allow use of a wider type
4594
//  WidenEx:   If Align is not 0, the amount additional we can load/store from.
4595
4596
static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
4597
                       unsigned Width, EVT WidenVT,
4598
6.32k
                       unsigned Align = 0, unsigned WidenEx = 0) {
4599
6.32k
  EVT WidenEltVT = WidenVT.getVectorElementType();
4600
6.32k
  unsigned WidenWidth = WidenVT.getSizeInBits();
4601
6.32k
  unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
4602
6.32k
  unsigned AlignInBits = Align*8;
4603
6.32k
4604
6.32k
  // If we have one element to load/store, return it.
4605
6.32k
  EVT RetVT = WidenEltVT;
4606
6.32k
  if (Width == WidenEltWidth)
4607
547
    return RetVT;
4608
5.78k
4609
5.78k
  // See if there is larger legal integer than the element type to load/store.
4610
5.78k
  unsigned VT;
4611
5.78k
  for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
4612
12.7k
       VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; 
--VT7.00k
) {
4613
12.7k
    EVT MemVT((MVT::SimpleValueType) VT);
4614
12.7k
    unsigned MemVTWidth = MemVT.getSizeInBits();
4615
12.7k
    if (MemVT.getSizeInBits() <= WidenEltWidth)
4616
517
      break;
4617
12.2k
    auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4618
12.2k
    if ((Action == TargetLowering::TypeLegal ||
4619
12.2k
         
Action == TargetLowering::TypePromoteInteger6.25k
) &&
4620
12.2k
        
(WidenWidth % MemVTWidth) == 06.12k
&&
4621
12.2k
        
isPowerOf2_32(WidenWidth / MemVTWidth)5.89k
&&
4622
12.2k
        
(5.89k
MemVTWidth <= Width5.89k
||
4623
5.89k
         
(799
Align!=0799
&&
MemVTWidth<=AlignInBits485
&&
MemVTWidth<=Width+WidenEx165
))) {
4624
5.26k
      if (MemVTWidth == WidenWidth)
4625
155
        return MemVT;
4626
5.10k
      RetVT = MemVT;
4627
5.10k
      break;
4628
5.10k
    }
4629
12.2k
  }
4630
5.78k
4631
5.78k
  // See if there is a larger vector type to load/store that has the same vector
4632
5.78k
  // element type and is evenly divisible with the WidenVT.
4633
5.78k
  for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
4634
535k
       VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 
--VT530k
) {
4635
533k
    EVT MemVT = (MVT::SimpleValueType) VT;
4636
533k
    unsigned MemVTWidth = MemVT.getSizeInBits();
4637
533k
    auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
4638
533k
    if ((Action == TargetLowering::TypeLegal ||
4639
533k
         
Action == TargetLowering::TypePromoteInteger463k
) &&
4640
533k
        
WidenEltVT == MemVT.getVectorElementType()75.5k
&&
4641
533k
        
(WidenWidth % MemVTWidth) == 020.7k
&&
4642
533k
        
isPowerOf2_32(WidenWidth / MemVTWidth)11.5k
&&
4643
533k
        
(11.5k
MemVTWidth <= Width11.5k
||
4644
11.5k
         
(6.77k
Align!=06.77k
&&
MemVTWidth<=AlignInBits5.80k
&&
MemVTWidth<=Width+WidenEx352
))) {
4645
5.12k
      if (RetVT.getSizeInBits() < MemVTWidth || 
MemVT == WidenVT2.20k
)
4646
2.91k
        return MemVT;
4647
5.12k
    }
4648
533k
  }
4649
5.62k
4650
5.62k
  
return RetVT2.70k
;
4651
5.62k
}
4652
4653
// Builds a vector type from scalar loads
4654
//  VecTy: Resulting Vector type
4655
//  LDOps: Load operators to build a vector type
4656
//  [Start,End) the list of loads to use.
4657
static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
4658
                                     SmallVectorImpl<SDValue> &LdOps,
4659
1.78k
                                     unsigned Start, unsigned End) {
4660
1.78k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4661
1.78k
  SDLoc dl(LdOps[Start]);
4662
1.78k
  EVT LdTy = LdOps[Start].getValueType();
4663
1.78k
  unsigned Width = VecTy.getSizeInBits();
4664
1.78k
  unsigned NumElts = Width / LdTy.getSizeInBits();
4665
1.78k
  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
4666
1.78k
4667
1.78k
  unsigned Idx = 1;
4668
1.78k
  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
4669
1.78k
4670
3.00k
  for (unsigned i = Start + 1; i != End; 
++i1.21k
) {
4671
1.21k
    EVT NewLdTy = LdOps[i].getValueType();
4672
1.21k
    if (NewLdTy != LdTy) {
4673
46
      NumElts = Width / NewLdTy.getSizeInBits();
4674
46
      NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
4675
46
      VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
4676
46
      // Readjust position and vector position based on new load type.
4677
46
      Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
4678
46
      LdTy = NewLdTy;
4679
46
    }
4680
1.21k
    VecOp = DAG.getNode(
4681
1.21k
        ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
4682
1.21k
        DAG.getConstant(Idx++, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4683
1.21k
  }
4684
1.78k
  return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
4685
1.78k
}
4686
4687
SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
4688
2.75k
                                              LoadSDNode *LD) {
4689
2.75k
  // The strategy assumes that we can efficiently load power-of-two widths.
4690
2.75k
  // The routine chops the vector into the largest vector loads with the same
4691
2.75k
  // element type or scalar loads and then recombines it to the widen vector
4692
2.75k
  // type.
4693
2.75k
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4694
2.75k
  unsigned WidenWidth = WidenVT.getSizeInBits();
4695
2.75k
  EVT LdVT    = LD->getMemoryVT();
4696
2.75k
  SDLoc dl(LD);
4697
2.75k
  assert(LdVT.isVector() && WidenVT.isVector());
4698
2.75k
  assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
4699
2.75k
4700
2.75k
  // Load information
4701
2.75k
  SDValue Chain = LD->getChain();
4702
2.75k
  SDValue BasePtr = LD->getBasePtr();
4703
2.75k
  unsigned Align = LD->getAlignment();
4704
2.75k
  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4705
2.75k
  AAMDNodes AAInfo = LD->getAAInfo();
4706
2.75k
4707
2.75k
  int LdWidth = LdVT.getSizeInBits();
4708
2.75k
  int WidthDiff = WidenWidth - LdWidth;
4709
2.75k
  unsigned LdAlign = LD->isVolatile() ? 
03
:
Align2.74k
; // Allow wider loads.
4710
2.75k
4711
2.75k
  // Find the vector type that can load from.
4712
2.75k
  EVT NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4713
2.75k
  int NewVTWidth = NewVT.getSizeInBits();
4714
2.75k
  SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
4715
2.75k
                             Align, MMOFlags, AAInfo);
4716
2.75k
  LdChain.push_back(LdOp.getValue(1));
4717
2.75k
4718
2.75k
  // Check if we can load the element with one instruction.
4719
2.75k
  if (LdWidth <= NewVTWidth) {
4720
635
    if (!NewVT.isVector()) {
4721
481
      unsigned NumElts = WidenWidth / NewVTWidth;
4722
481
      EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4723
481
      SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
4724
481
      return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
4725
481
    }
4726
154
    if (NewVT == WidenVT)
4727
154
      return LdOp;
4728
0
4729
0
    assert(WidenWidth % NewVTWidth == 0);
4730
0
    unsigned NumConcat = WidenWidth / NewVTWidth;
4731
0
    SmallVector<SDValue, 16> ConcatOps(NumConcat);
4732
0
    SDValue UndefVal = DAG.getUNDEF(NewVT);
4733
0
    ConcatOps[0] = LdOp;
4734
0
    for (unsigned i = 1; i != NumConcat; ++i)
4735
0
      ConcatOps[i] = UndefVal;
4736
0
    return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
4737
0
  }
4738
2.11k
4739
2.11k
  // Load vector by using multiple loads from largest vector to scalar.
4740
2.11k
  SmallVector<SDValue, 16> LdOps;
4741
2.11k
  LdOps.push_back(LdOp);
4742
2.11k
4743
2.11k
  LdWidth -= NewVTWidth;
4744
2.11k
  unsigned Offset = 0;
4745
2.11k
4746
5.86k
  while (LdWidth > 0) {
4747
3.74k
    unsigned Increment = NewVTWidth / 8;
4748
3.74k
    Offset += Increment;
4749
3.74k
    BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4750
3.74k
4751
3.74k
    SDValue L;
4752
3.74k
    if (LdWidth < NewVTWidth) {
4753
1.93k
      // The current type we are using is too large. Find a better size.
4754
1.93k
      NewVT = FindMemType(DAG, TLI, LdWidth, WidenVT, LdAlign, WidthDiff);
4755
1.93k
      NewVTWidth = NewVT.getSizeInBits();
4756
1.93k
      L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4757
1.93k
                      LD->getPointerInfo().getWithOffset(Offset),
4758
1.93k
                      MinAlign(Align, Increment), MMOFlags, AAInfo);
4759
1.93k
      LdChain.push_back(L.getValue(1));
4760
1.93k
      if (L->getValueType(0).isVector() && 
NewVTWidth >= LdWidth223
) {
4761
212
        // Later code assumes the vector loads produced will be mergeable, so we
4762
212
        // must pad the final entry up to the previous width. Scalars are
4763
212
        // combined separately.
4764
212
        SmallVector<SDValue, 16> Loads;
4765
212
        Loads.push_back(L);
4766
212
        unsigned size = L->getValueSizeInBits(0);
4767
229
        while (size < LdOp->getValueSizeInBits(0)) {
4768
17
          Loads.push_back(DAG.getUNDEF(L->getValueType(0)));
4769
17
          size += L->getValueSizeInBits(0);
4770
17
        }
4771
212
        L = DAG.getNode(ISD::CONCAT_VECTORS, dl, LdOp->getValueType(0), Loads);
4772
212
      }
4773
1.93k
    } else {
4774
1.81k
      L = DAG.getLoad(NewVT, dl, Chain, BasePtr,
4775
1.81k
                      LD->getPointerInfo().getWithOffset(Offset),
4776
1.81k
                      MinAlign(Align, Increment), MMOFlags, AAInfo);
4777
1.81k
      LdChain.push_back(L.getValue(1));
4778
1.81k
    }
4779
3.74k
4780
3.74k
    LdOps.push_back(L);
4781
3.74k
    LdOp = L;
4782
3.74k
4783
3.74k
    LdWidth -= NewVTWidth;
4784
3.74k
  }
4785
2.11k
4786
2.11k
  // Build the vector from the load operations.
4787
2.11k
  unsigned End = LdOps.size();
4788
2.11k
  if (!LdOps[0].getValueType().isVector())
4789
123
    // All the loads are scalar loads.
4790
123
    return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
4791
1.99k
4792
1.99k
  // If the load contains vectors, build the vector using concat vector.
4793
1.99k
  // All of the vectors used to load are power-of-2, and the scalar loads can be
4794
1.99k
  // combined to make a power-of-2 vector.
4795
1.99k
  SmallVector<SDValue, 16> ConcatOps(End);
4796
1.99k
  int i = End - 1;
4797
1.99k
  int Idx = End;
4798
1.99k
  EVT LdTy = LdOps[i].getValueType();
4799
1.99k
  // First, combine the scalar loads to a vector.
4800
1.99k
  if (!LdTy.isVector())  {
4801
1.66k
    for (--i; i >= 0; 
--i0
) {
4802
1.66k
      LdTy = LdOps[i].getValueType();
4803
1.66k
      if (LdTy.isVector())
4804
1.66k
        break;
4805
1.66k
    }
4806
1.66k
    ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
4807
1.66k
  }
4808
1.99k
  ConcatOps[--Idx] = LdOps[i];
4809
2.85k
  for (--i; i >= 0; 
--i863
) {
4810
863
    EVT NewLdTy = LdOps[i].getValueType();
4811
863
    if (NewLdTy != LdTy) {
4812
11
      // Create a larger vector.
4813
11
      ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
4814
11
                                     makeArrayRef(&ConcatOps[Idx], End - Idx));
4815
11
      Idx = End - 1;
4816
11
      LdTy = NewLdTy;
4817
11
    }
4818
863
    ConcatOps[--Idx] = LdOps[i];
4819
863
  }
4820
1.99k
4821
1.99k
  if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
4822
1.85k
    return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
4823
1.85k
                       makeArrayRef(&ConcatOps[Idx], End - Idx));
4824
142
4825
142
  // We need to fill the rest with undefs to build the vector.
4826
142
  unsigned NumOps = WidenWidth / LdTy.getSizeInBits();
4827
142
  SmallVector<SDValue, 16> WidenOps(NumOps);
4828
142
  SDValue UndefVal = DAG.getUNDEF(LdTy);
4829
142
  {
4830
142
    unsigned i = 0;
4831
941
    for (; i != End-Idx; 
++i799
)
4832
799
      WidenOps[i] = ConcatOps[Idx+i];
4833
447
    for (; i != NumOps; 
++i305
)
4834
305
      WidenOps[i] = UndefVal;
4835
142
  }
4836
142
  return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
4837
142
}
4838
4839
SDValue
4840
DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
4841
                                         LoadSDNode *LD,
4842
0
                                         ISD::LoadExtType ExtType) {
4843
0
  // For extension loads, it may not be more efficient to chop up the vector
4844
0
  // and then extend it. Instead, we unroll the load and build a new vector.
4845
0
  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
4846
0
  EVT LdVT    = LD->getMemoryVT();
4847
0
  SDLoc dl(LD);
4848
0
  assert(LdVT.isVector() && WidenVT.isVector());
4849
0
4850
0
  // Load information
4851
0
  SDValue Chain = LD->getChain();
4852
0
  SDValue BasePtr = LD->getBasePtr();
4853
0
  unsigned Align = LD->getAlignment();
4854
0
  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
4855
0
  AAMDNodes AAInfo = LD->getAAInfo();
4856
0
4857
0
  EVT EltVT = WidenVT.getVectorElementType();
4858
0
  EVT LdEltVT = LdVT.getVectorElementType();
4859
0
  unsigned NumElts = LdVT.getVectorNumElements();
4860
0
4861
0
  // Load each element and widen.
4862
0
  unsigned WidenNumElts = WidenVT.getVectorNumElements();
4863
0
  SmallVector<SDValue, 16> Ops(WidenNumElts);
4864
0
  unsigned Increment = LdEltVT.getSizeInBits() / 8;
4865
0
  Ops[0] =
4866
0
      DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
4867
0
                     LdEltVT, Align, MMOFlags, AAInfo);
4868
0
  LdChain.push_back(Ops[0].getValue(1));
4869
0
  unsigned i = 0, Offset = Increment;
4870
0
  for (i=1; i < NumElts; ++i, Offset += Increment) {
4871
0
    SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4872
0
    Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
4873
0
                            LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
4874
0
                            Align, MMOFlags, AAInfo);
4875
0
    LdChain.push_back(Ops[i].getValue(1));
4876
0
  }
4877
0
4878
0
  // Fill the rest with undefs.
4879
0
  SDValue UndefVal = DAG.getUNDEF(EltVT);
4880
0
  for (; i != WidenNumElts; ++i)
4881
0
    Ops[i] = UndefVal;
4882
0
4883
0
  return DAG.getBuildVector(WidenVT, dl, Ops);
4884
0
}
4885
4886
void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
4887
971
                                            StoreSDNode *ST) {
4888
971
  // The strategy assumes that we can efficiently store power-of-two widths.
4889
971
  // The routine chops the vector into the largest vector stores with the same
4890
971
  // element type or scalar stores.
4891
971
  SDValue  Chain = ST->getChain();
4892
971
  SDValue  BasePtr = ST->getBasePtr();
4893
971
  unsigned Align = ST->getAlignment();
4894
971
  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4895
971
  AAMDNodes AAInfo = ST->getAAInfo();
4896
971
  SDValue  ValOp = GetWidenedVector(ST->getValue());
4897
971
  SDLoc dl(ST);
4898
971
4899
971
  EVT StVT = ST->getMemoryVT();
4900
971
  unsigned StWidth = StVT.getSizeInBits();
4901
971
  EVT ValVT = ValOp.getValueType();
4902
971
  unsigned ValWidth = ValVT.getSizeInBits();
4903
971
  EVT ValEltVT = ValVT.getVectorElementType();
4904
971
  unsigned ValEltWidth = ValEltVT.getSizeInBits();
4905
971
  assert(StVT.getVectorElementType() == ValEltVT);
4906
971
4907
971
  int Idx = 0;          // current index to store
4908
971
  unsigned Offset = 0;  // offset from base to store
4909
2.61k
  while (StWidth != 0) {
4910
1.64k
    // Find the largest vector type we can store with.
4911
1.64k
    EVT NewVT = FindMemType(DAG, TLI, StWidth, ValVT);
4912
1.64k
    unsigned NewVTWidth = NewVT.getSizeInBits();
4913
1.64k
    unsigned Increment = NewVTWidth / 8;
4914
1.64k
    if (NewVT.isVector()) {
4915
545
      unsigned NumVTElts = NewVT.getVectorNumElements();
4916
1.45k
      do {
4917
1.45k
        SDValue EOp = DAG.getNode(
4918
1.45k
            ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
4919
1.45k
            DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4920
1.45k
        StChain.push_back(DAG.getStore(
4921
1.45k
            Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4922
1.45k
            MinAlign(Align, Offset), MMOFlags, AAInfo));
4923
1.45k
        StWidth -= NewVTWidth;
4924
1.45k
        Offset += Increment;
4925
1.45k
        Idx += NumVTElts;
4926
1.45k
4927
1.45k
        BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4928
1.45k
      } while (StWidth != 0 && 
StWidth >= NewVTWidth1.29k
);
4929
1.09k
    } else {
4930
1.09k
      // Cast the vector to the scalar type we can store.
4931
1.09k
      unsigned NumElts = ValWidth / NewVTWidth;
4932
1.09k
      EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
4933
1.09k
      SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
4934
1.09k
      // Readjust index position based on new vector type.
4935
1.09k
      Idx = Idx * ValEltWidth / NewVTWidth;
4936
2.29k
      do {
4937
2.29k
        SDValue EOp = DAG.getNode(
4938
2.29k
            ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
4939
2.29k
            DAG.getConstant(Idx++, dl,
4940
2.29k
                            TLI.getVectorIdxTy(DAG.getDataLayout())));
4941
2.29k
        StChain.push_back(DAG.getStore(
4942
2.29k
            Chain, dl, EOp, BasePtr, ST->getPointerInfo().getWithOffset(Offset),
4943
2.29k
            MinAlign(Align, Offset), MMOFlags, AAInfo));
4944
2.29k
        StWidth -= NewVTWidth;
4945
2.29k
        Offset += Increment;
4946
2.29k
        BasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Increment);
4947
2.29k
      } while (StWidth != 0 && 
StWidth >= NewVTWidth1.48k
);
4948
1.09k
      // Restore index back to be relative to the original widen element type.
4949
1.09k
      Idx = Idx * NewVTWidth / ValEltWidth;
4950
1.09k
    }
4951
1.64k
  }
4952
971
}
4953
4954
void
4955
DAGTypeLegalizer::GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
4956
0
                                            StoreSDNode *ST) {
4957
0
  // For extension loads, it may not be more efficient to truncate the vector
4958
0
  // and then store it. Instead, we extract each element and then store it.
4959
0
  SDValue Chain = ST->getChain();
4960
0
  SDValue BasePtr = ST->getBasePtr();
4961
0
  unsigned Align = ST->getAlignment();
4962
0
  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
4963
0
  AAMDNodes AAInfo = ST->getAAInfo();
4964
0
  SDValue ValOp = GetWidenedVector(ST->getValue());
4965
0
  SDLoc dl(ST);
4966
0
4967
0
  EVT StVT = ST->getMemoryVT();
4968
0
  EVT ValVT = ValOp.getValueType();
4969
0
4970
0
  // It must be true that the wide vector type is bigger than where we need to
4971
0
  // store.
4972
0
  assert(StVT.isVector() && ValOp.getValueType().isVector());
4973
0
  assert(StVT.bitsLT(ValOp.getValueType()));
4974
0
4975
0
  // For truncating stores, we can not play the tricks of chopping legal vector
4976
0
  // types and bitcast it to the right type. Instead, we unroll the store.
4977
0
  EVT StEltVT  = StVT.getVectorElementType();
4978
0
  EVT ValEltVT = ValVT.getVectorElementType();
4979
0
  unsigned Increment = ValEltVT.getSizeInBits() / 8;
4980
0
  unsigned NumElts = StVT.getVectorNumElements();
4981
0
  SDValue EOp = DAG.getNode(
4982
0
      ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4983
0
      DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4984
0
  StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr,
4985
0
                                      ST->getPointerInfo(), StEltVT, Align,
4986
0
                                      MMOFlags, AAInfo));
4987
0
  unsigned Offset = Increment;
4988
0
  for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
4989
0
    SDValue NewBasePtr = DAG.getObjectPtrOffset(dl, BasePtr, Offset);
4990
0
    SDValue EOp = DAG.getNode(
4991
0
        ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
4992
0
        DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4993
0
    StChain.push_back(DAG.getTruncStore(
4994
0
        Chain, dl, EOp, NewBasePtr, ST->getPointerInfo().getWithOffset(Offset),
4995
0
        StEltVT, MinAlign(Align, Offset), MMOFlags, AAInfo));
4996
0
  }
4997
0
}
4998
4999
/// Modifies a vector input (widen or narrows) to a vector of NVT.  The
5000
/// input vector must have the same element type as NVT.
5001
/// FillWithZeroes specifies that the vector should be widened with zeroes.
5002
SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5003
84
                                       bool FillWithZeroes) {
5004
84
  // Note that InOp might have been widened so it might already have
5005
84
  // the right width or it might need be narrowed.
5006
84
  EVT InVT = InOp.getValueType();
5007
84
  assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
5008
84
         "input and widen element type must match");
5009
84
  SDLoc dl(InOp);
5010
84
5011
84
  // Check if InOp already has the right width.
5012
84
  if (InVT == NVT)
5013
0
    return InOp;
5014
84
5015
84
  unsigned InNumElts = InVT.getVectorNumElements();
5016
84
  unsigned WidenNumElts = NVT.getVectorNumElements();
5017
84
  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
5018
79
    unsigned NumConcat = WidenNumElts / InNumElts;
5019
79
    SmallVector<SDValue, 16> Ops(NumConcat);
5020
79
    SDValue FillVal = FillWithZeroes ? 
DAG.getConstant(0, dl, InVT)46
:
5021
79
      
DAG.getUNDEF(InVT)33
;
5022
79
    Ops[0] = InOp;
5023
158
    for (unsigned i = 1; i != NumConcat; 
++i79
)
5024
79
      Ops[i] = FillVal;
5025
79
5026
79
    return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
5027
79
  }
5028
5
5029
5
  if (WidenNumElts < InNumElts && 
InNumElts % WidenNumElts0
)
5030
0
    return DAG.getNode(
5031
0
        ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
5032
0
        DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5033
5
5034
5
  // Fall back to extract and build.
5035
5
  SmallVector<SDValue, 16> Ops(WidenNumElts);
5036
5
  EVT EltVT = NVT.getVectorElementType();
5037
5
  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
5038
5
  unsigned Idx;
5039
20
  for (Idx = 0; Idx < MinNumElts; 
++Idx15
)
5040
15
    Ops[Idx] = DAG.getNode(
5041
15
        ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5042
15
        DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
5043
5
5044
5
  SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
5045
5
    
DAG.getUNDEF(EltVT)0
;
5046
10
  for ( ; Idx < WidenNumElts; 
++Idx5
)
5047
5
    Ops[Idx] = FillVal;
5048
5
  return DAG.getBuildVector(NVT, dl, Ops);
5049
5
}