Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
Line
Count
Source (jump to first uncovered line)
1
//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// This file implements integer type expansion and promotion for LegalizeTypes.
11
// Promotion is the act of changing a computation in an illegal type into a
12
// computation in a larger type.  For example, implementing i8 arithmetic in an
13
// i32 register (often needed on powerpc).
14
// Expansion is the act of changing a computation in an illegal type into a
15
// computation in two identical registers of a smaller type.  For example,
16
// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17
// targets).
18
//
19
//===----------------------------------------------------------------------===//
20
21
#include "LegalizeTypes.h"
22
#include "llvm/IR/DerivedTypes.h"
23
#include "llvm/Support/ErrorHandling.h"
24
#include "llvm/Support/KnownBits.h"
25
#include "llvm/Support/raw_ostream.h"
26
using namespace llvm;
27
28
#define DEBUG_TYPE "legalize-types"
29
30
//===----------------------------------------------------------------------===//
31
//  Integer Result Promotion
32
//===----------------------------------------------------------------------===//
33
34
/// PromoteIntegerResult - This method is called when a result of a node is
35
/// found to be in need of promotion to a larger type.  At this point, the node
36
/// may also have invalid operands or may have other results that need
37
/// expansion, we just know that (at least) one result needs promotion.
38
1.38M
void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39
1.38M
  DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
40
1.38M
  SDValue Res = SDValue();
41
1.38M
42
1.38M
  // See if the target wants to custom expand this node.
43
1.38M
  if (CustomLowerNode(N, N->getValueType(ResNo), true))
44
181
    return;
45
1.38M
46
1.38M
  switch (N->getOpcode()) {
47
0
  default:
48
#ifndef NDEBUG
49
    dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
50
    N->dump(&DAG); dbgs() << "\n";
51
#endif
52
0
    llvm_unreachable("Do not know how to promote this operator!");
53
0
  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
54
63
  case ISD::AssertSext:  Res = PromoteIntRes_AssertSext(N); break;
55
63
  case ISD::AssertZext:  Res = PromoteIntRes_AssertZext(N); break;
56
4.88k
  case ISD::BITCAST:     Res = PromoteIntRes_BITCAST(N); break;
57
15
  case ISD::BITREVERSE:  Res = PromoteIntRes_BITREVERSE(N); break;
58
88
  case ISD::BSWAP:       Res = PromoteIntRes_BSWAP(N); break;
59
0
  case ISD::BUILD_PAIR:  Res = PromoteIntRes_BUILD_PAIR(N); break;
60
458k
  case ISD::Constant:    Res = PromoteIntRes_Constant(N); break;
61
8
  case ISD::CTLZ_ZERO_UNDEF:
62
8
  case ISD::CTLZ:        Res = PromoteIntRes_CTLZ(N); break;
63
30
  case ISD::CTPOP:       Res = PromoteIntRes_CTPOP(N); break;
64
20
  case ISD::CTTZ_ZERO_UNDEF:
65
20
  case ISD::CTTZ:        Res = PromoteIntRes_CTTZ(N); break;
66
80.4k
  case ISD::EXTRACT_VECTOR_ELT:
67
80.4k
                         Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
68
224k
  case ISD::LOAD:        Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
69
4
  case ISD::MLOAD:       Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
70
4
    break;
71
10
  case ISD::MGATHER:     Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
72
10
    break;
73
13.6k
  case ISD::SELECT:      Res = PromoteIntRes_SELECT(N); break;
74
78
  case ISD::VSELECT:     Res = PromoteIntRes_VSELECT(N); break;
75
0
  case ISD::SELECT_CC:   Res = PromoteIntRes_SELECT_CC(N); break;
76
339k
  case ISD::SETCC:       Res = PromoteIntRes_SETCC(N); break;
77
92
  case ISD::SMIN:
78
92
  case ISD::SMAX:        Res = PromoteIntRes_SExtIntBinOp(N); break;
79
50
  case ISD::UMIN:
80
50
  case ISD::UMAX:        Res = PromoteIntRes_ZExtIntBinOp(N); break;
81
50
82
1.62k
  case ISD::SHL:         Res = PromoteIntRes_SHL(N); break;
83
60
  case ISD::SIGN_EXTEND_INREG:
84
60
                         Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
85
298
  case ISD::SRA:         Res = PromoteIntRes_SRA(N); break;
86
6.11k
  case ISD::SRL:         Res = PromoteIntRes_SRL(N); break;
87
148k
  case ISD::TRUNCATE:    Res = PromoteIntRes_TRUNCATE(N); break;
88
2.63k
  case ISD::UNDEF:       Res = PromoteIntRes_UNDEF(N); break;
89
36
  case ISD::VAARG:       Res = PromoteIntRes_VAARG(N); break;
90
50
91
23.6k
  case ISD::EXTRACT_SUBVECTOR:
92
23.6k
                         Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
93
104
  case ISD::VECTOR_SHUFFLE:
94
104
                         Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
95
184
  case ISD::INSERT_VECTOR_ELT:
96
184
                         Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
97
1.80k
  case ISD::BUILD_VECTOR:
98
1.80k
                         Res = PromoteIntRes_BUILD_VECTOR(N); break;
99
26
  case ISD::SCALAR_TO_VECTOR:
100
26
                         Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
101
1.83k
  case ISD::CONCAT_VECTORS:
102
1.83k
                         Res = PromoteIntRes_CONCAT_VECTORS(N); break;
103
50
104
3
  case ISD::ANY_EXTEND_VECTOR_INREG:
105
3
  case ISD::SIGN_EXTEND_VECTOR_INREG:
106
3
  case ISD::ZERO_EXTEND_VECTOR_INREG:
107
3
                         Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
108
3
109
28.8k
  case ISD::SIGN_EXTEND:
110
28.8k
  case ISD::ZERO_EXTEND:
111
28.8k
  case ISD::ANY_EXTEND:  Res = PromoteIntRes_INT_EXTEND(N); break;
112
28.8k
113
216
  case ISD::FP_TO_SINT:
114
216
  case ISD::FP_TO_UINT:  Res = PromoteIntRes_FP_TO_XINT(N); break;
115
216
116
803
  case ISD::FP_TO_FP16:  Res = PromoteIntRes_FP_TO_FP16(N); break;
117
216
118
38.5k
  case ISD::AND:
119
38.5k
  case ISD::OR:
120
38.5k
  case ISD::XOR:
121
38.5k
  case ISD::ADD:
122
38.5k
  case ISD::SUB:
123
38.5k
  case ISD::MUL:         Res = PromoteIntRes_SimpleIntBinOp(N); break;
124
38.5k
125
303
  case ISD::SDIV:
126
303
  case ISD::SREM:        Res = PromoteIntRes_SExtIntBinOp(N); break;
127
303
128
332
  case ISD::UDIV:
129
332
  case ISD::UREM:        Res = PromoteIntRes_ZExtIntBinOp(N); break;
130
332
131
105
  case ISD::SADDO:
132
105
  case ISD::SSUBO:       Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
133
266
  case ISD::UADDO:
134
266
  case ISD::USUBO:       Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
135
1.19k
  case ISD::SMULO:
136
1.19k
  case ISD::UMULO:       Res = PromoteIntRes_XMULO(N, ResNo); break;
137
1.19k
138
5
  case ISD::ADDCARRY:
139
5
  case ISD::SUBCARRY:    Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
140
5
141
335
  case ISD::ATOMIC_LOAD:
142
335
    Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
143
5
144
973
  case ISD::ATOMIC_LOAD_ADD:
145
973
  case ISD::ATOMIC_LOAD_SUB:
146
973
  case ISD::ATOMIC_LOAD_AND:
147
973
  case ISD::ATOMIC_LOAD_OR:
148
973
  case ISD::ATOMIC_LOAD_XOR:
149
973
  case ISD::ATOMIC_LOAD_NAND:
150
973
  case ISD::ATOMIC_LOAD_MIN:
151
973
  case ISD::ATOMIC_LOAD_MAX:
152
973
  case ISD::ATOMIC_LOAD_UMIN:
153
973
  case ISD::ATOMIC_LOAD_UMAX:
154
973
  case ISD::ATOMIC_SWAP:
155
973
    Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
156
973
157
1.80k
  case ISD::ATOMIC_CMP_SWAP:
158
1.80k
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
159
1.80k
    Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
160
1.80k
    break;
161
1.38M
  }
162
1.38M
163
1.38M
  // If the result is null then the sub-method took care of registering it.
164
1.38M
  
if (1.38M
Res.getNode()1.38M
)
165
1.38M
    SetPromotedInteger(SDValue(N, ResNo), Res);
166
1.38M
}
167
168
SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
169
0
                                                     unsigned ResNo) {
170
0
  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
171
0
  return GetPromotedInteger(Op);
172
0
}
173
174
63
SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
175
63
  // Sign-extend the new bits, and continue the assertion.
176
63
  SDValue Op = SExtPromotedInteger(N->getOperand(0));
177
63
  return DAG.getNode(ISD::AssertSext, SDLoc(N),
178
63
                     Op.getValueType(), Op, N->getOperand(1));
179
63
}
180
181
63
SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
182
63
  // Zero the new bits, and continue the assertion.
183
63
  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
184
63
  return DAG.getNode(ISD::AssertZext, SDLoc(N),
185
63
                     Op.getValueType(), Op, N->getOperand(1));
186
63
}
187
188
335
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
189
335
  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
190
335
  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
191
335
                              N->getMemoryVT(), ResVT,
192
335
                              N->getChain(), N->getBasePtr(),
193
335
                              N->getMemOperand());
194
335
  // Legalize the chain result - switch anything that used the old chain to
195
335
  // use the new one.
196
335
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
197
335
  return Res;
198
335
}
199
200
973
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
201
973
  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
202
973
  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
203
973
                              N->getMemoryVT(),
204
973
                              N->getChain(), N->getBasePtr(),
205
973
                              Op2, N->getMemOperand());
206
973
  // Legalize the chain result - switch anything that used the old chain to
207
973
  // use the new one.
208
973
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
209
973
  return Res;
210
973
}
211
212
SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
213
1.80k
                                                      unsigned ResNo) {
214
1.80k
  if (
ResNo == 11.80k
) {
215
1.67k
    assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
216
1.67k
    EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
217
1.67k
    EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
218
1.67k
219
1.67k
    // Only use the result of getSetCCResultType if it is legal,
220
1.67k
    // otherwise just use the promoted result type (NVT).
221
1.67k
    if (!TLI.isTypeLegal(SVT))
222
0
      SVT = NVT;
223
1.67k
224
1.67k
    SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
225
1.67k
    SDValue Res = DAG.getAtomicCmpSwap(
226
1.67k
        ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
227
1.67k
        N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
228
1.67k
        N->getMemOperand());
229
1.67k
    ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
230
1.67k
    ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
231
1.67k
    return Res.getValue(1);
232
1.67k
  }
233
137
234
137
  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
235
137
  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
236
137
  SDVTList VTs =
237
137
      DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
238
137
  SDValue Res = DAG.getAtomicCmpSwap(
239
137
      N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
240
137
      N->getBasePtr(), Op2, Op3, N->getMemOperand());
241
137
  // Update the use to N with the newly created Res.
242
411
  for (unsigned i = 1, NumResults = N->getNumValues(); 
i < NumResults411
;
++i274
)
243
274
    ReplaceValueWith(SDValue(N, i), Res.getValue(i));
244
1.80k
  return Res;
245
1.80k
}
246
247
4.88k
SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
248
4.88k
  SDValue InOp = N->getOperand(0);
249
4.88k
  EVT InVT = InOp.getValueType();
250
4.88k
  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
251
4.88k
  EVT OutVT = N->getValueType(0);
252
4.88k
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
253
4.88k
  SDLoc dl(N);
254
4.88k
255
4.88k
  switch (getTypeAction(InVT)) {
256
774
  case TargetLowering::TypeLegal:
257
774
    break;
258
542
  case TargetLowering::TypePromoteInteger:
259
542
    if (
NOutVT.bitsEq(NInVT) && 542
!NOutVT.isVector()58
&&
!NInVT.isVector()0
)
260
542
      // The input promotes to the same size.  Convert the promoted value.
261
0
      return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
262
542
    break;
263
0
  case TargetLowering::TypeSoftenFloat:
264
0
    // Promote the integer operand by hand.
265
0
    return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
266
434
  case TargetLowering::TypePromoteFloat: {
267
434
    // Convert the promoted float by hand.
268
434
    SDValue PromotedOp = GetPromotedFloat(InOp);
269
434
    return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
270
0
    break;
271
542
  }
272
357
  case TargetLowering::TypeExpandInteger:
273
357
  case TargetLowering::TypeExpandFloat:
274
357
    break;
275
2.31k
  case TargetLowering::TypeScalarizeVector:
276
2.31k
    // Convert the element to an integer and promote it by hand.
277
2.31k
    if (!NOutVT.isVector())
278
2.27k
      return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
279
2.27k
                         BitConvertToInteger(GetScalarizedVector(InOp)));
280
37
    break;
281
457
  case TargetLowering::TypeSplitVector: {
282
457
    // For example, i32 = BITCAST v2i16 on alpha.  Convert the split
283
457
    // pieces of the input into integers and reassemble in the final type.
284
457
    SDValue Lo, Hi;
285
457
    GetSplitVector(N->getOperand(0), Lo, Hi);
286
457
    Lo = BitConvertToInteger(Lo);
287
457
    Hi = BitConvertToInteger(Hi);
288
457
289
457
    if (DAG.getDataLayout().isBigEndian())
290
45
      std::swap(Lo, Hi);
291
457
292
457
    InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
293
457
                       EVT::getIntegerVT(*DAG.getContext(),
294
457
                                         NOutVT.getSizeInBits()),
295
457
                       JoinIntegers(Lo, Hi));
296
457
    return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
297
37
  }
298
7
  case TargetLowering::TypeWidenVector:
299
7
    // The input is widened to the same size. Convert to the widened value.
300
7
    // Make sure that the outgoing value is not a vector, because this would
301
7
    // make us bitcast between two vectors which are legalized in different ways.
302
7
    if (
NOutVT.bitsEq(NInVT) && 7
!NOutVT.isVector()7
)
303
5
      return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
304
1.71k
  }
305
1.71k
306
1.71k
  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
307
1.71k
                     CreateStackStoreLoad(InOp, OutVT));
308
1.71k
}
309
310
88
SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
311
88
  SDValue Op = GetPromotedInteger(N->getOperand(0));
312
88
  EVT OVT = N->getValueType(0);
313
88
  EVT NVT = Op.getValueType();
314
88
  SDLoc dl(N);
315
88
316
88
  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
317
88
  return DAG.getNode(
318
88
      ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
319
88
      DAG.getConstant(DiffBits, dl,
320
88
                      TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
321
88
}
322
323
15
SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
324
15
  SDValue Op = GetPromotedInteger(N->getOperand(0));
325
15
  EVT OVT = N->getValueType(0);
326
15
  EVT NVT = Op.getValueType();
327
15
  SDLoc dl(N);
328
15
329
15
  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
330
15
  return DAG.getNode(
331
15
      ISD::SRL, dl, NVT, DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
332
15
      DAG.getConstant(DiffBits, dl,
333
15
                      TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
334
15
}
335
336
0
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
337
0
  // The pair element type may be legal, or may not promote to the same type as
338
0
  // the result, for example i14 = BUILD_PAIR (i7, i7).  Handle all cases.
339
0
  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
340
0
                     TLI.getTypeToTransformTo(*DAG.getContext(),
341
0
                     N->getValueType(0)), JoinIntegers(N->getOperand(0),
342
0
                     N->getOperand(1)));
343
0
}
344
345
458k
SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
346
458k
  EVT VT = N->getValueType(0);
347
458k
  // FIXME there is no actual debug info here
348
458k
  SDLoc dl(N);
349
458k
  // Zero extend things like i1, sign extend everything else.  It shouldn't
350
458k
  // matter in theory which one we pick, but this tends to give better code?
351
458k
  unsigned Opc = VT.isByteSized() ? 
ISD::SIGN_EXTEND386k
:
ISD::ZERO_EXTEND71.6k
;
352
458k
  SDValue Result = DAG.getNode(Opc, dl,
353
458k
                               TLI.getTypeToTransformTo(*DAG.getContext(), VT),
354
458k
                               SDValue(N, 0));
355
458k
  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
356
458k
  return Result;
357
458k
}
358
359
8
SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
360
8
  // Zero extend to the promoted type and do the count there.
361
8
  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
362
8
  SDLoc dl(N);
363
8
  EVT OVT = N->getValueType(0);
364
8
  EVT NVT = Op.getValueType();
365
8
  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
366
8
  // Subtract off the extra leading bits in the bigger type.
367
8
  return DAG.getNode(
368
8
      ISD::SUB, dl, NVT, Op,
369
8
      DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
370
8
                      NVT));
371
8
}
372
373
30
SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
374
30
  // Zero extend to the promoted type and do the count there.
375
30
  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
376
30
  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
377
30
}
378
379
20
SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
380
20
  SDValue Op = GetPromotedInteger(N->getOperand(0));
381
20
  EVT OVT = N->getValueType(0);
382
20
  EVT NVT = Op.getValueType();
383
20
  SDLoc dl(N);
384
20
  if (
N->getOpcode() == ISD::CTTZ20
) {
385
10
    // The count is the same in the promoted type except if the original
386
10
    // value was zero.  This can be handled by setting the bit just off
387
10
    // the top of the original type.
388
10
    auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
389
10
                                      OVT.getScalarSizeInBits());
390
10
    Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
391
10
  }
392
20
  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
393
20
}
394
395
80.4k
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
396
80.4k
  SDLoc dl(N);
397
80.4k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
398
80.4k
  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
399
80.4k
                     N->getOperand(1));
400
80.4k
}
401
402
216
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
403
216
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
404
216
  unsigned NewOpc = N->getOpcode();
405
216
  SDLoc dl(N);
406
216
407
216
  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
408
216
  // not Legal, check to see if we can use FP_TO_SINT instead.  (If both UINT
409
216
  // and SINT conversions are Custom, there is no way to tell which is
410
216
  // preferable. We choose SINT because that's the right thing on PPC.)
411
216
  if (N->getOpcode() == ISD::FP_TO_UINT &&
412
136
      !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
413
125
      TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
414
94
    NewOpc = ISD::FP_TO_SINT;
415
216
416
216
  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
417
216
418
216
  // Assert that the converted value fits in the original type.  If it doesn't
419
216
  // (eg: because the value being converted is too big), then the result of the
420
216
  // original operation was undefined anyway, so the assert is still correct.
421
216
  //
422
216
  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
423
216
  //   before legalization: fp-to-uint16, 65534. -> 0xfffe
424
216
  //   after legalization: fp-to-sint32, 65534. -> 0x0000fffe
425
216
  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
426
216
                     
ISD::AssertZext136
:
ISD::AssertSext80
, dl, NVT, Res,
427
216
                     DAG.getValueType(N->getValueType(0).getScalarType()));
428
216
}
429
430
803
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
431
803
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
432
803
  SDLoc dl(N);
433
803
434
803
  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
435
803
}
436
437
28.8k
SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
438
28.8k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
439
28.8k
  SDLoc dl(N);
440
28.8k
441
28.8k
  if (getTypeAction(N->getOperand(0).getValueType())
442
28.8k
      == TargetLowering::TypePromoteInteger) {
443
28.0k
    SDValue Res = GetPromotedInteger(N->getOperand(0));
444
28.0k
    assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
445
28.0k
446
28.0k
    // If the result and operand types are the same after promotion, simplify
447
28.0k
    // to an in-register extension.
448
28.0k
    if (
NVT == Res.getValueType()28.0k
) {
449
28.0k
      // The high bits are not guaranteed to be anything.  Insert an extend.
450
28.0k
      if (N->getOpcode() == ISD::SIGN_EXTEND)
451
469
        return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
452
469
                           DAG.getValueType(N->getOperand(0).getValueType()));
453
27.5k
      
if (27.5k
N->getOpcode() == ISD::ZERO_EXTEND27.5k
)
454
1.31k
        return DAG.getZeroExtendInReg(Res, dl,
455
1.31k
                      N->getOperand(0).getValueType().getScalarType());
456
0
      assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
457
26.2k
      return Res;
458
26.2k
    }
459
28.0k
  }
460
801
461
801
  // Otherwise, just extend the original operand all the way to the larger type.
462
801
  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
463
801
}
464
465
224k
SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
466
224k
  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
467
224k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
468
224k
  ISD::LoadExtType ExtType =
469
224k
    ISD::isNON_EXTLoad(N) ? 
ISD::EXTLOAD223k
:
N->getExtensionType()1.17k
;
470
224k
  SDLoc dl(N);
471
224k
  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
472
224k
                               N->getMemoryVT(), N->getMemOperand());
473
224k
474
224k
  // Legalize the chain result - switch anything that used the old chain to
475
224k
  // use the new one.
476
224k
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
477
224k
  return Res;
478
224k
}
479
480
4
SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
481
4
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
482
4
  SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
483
4
484
4
  SDLoc dl(N);
485
4
  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
486
4
                                  N->getMask(), ExtSrc0, N->getMemoryVT(),
487
4
                                  N->getMemOperand(), ISD::SEXTLOAD);
488
4
  // Legalize the chain result - switch anything that used the old chain to
489
4
  // use the new one.
490
4
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
491
4
  return Res;
492
4
}
493
494
10
SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
495
10
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
496
10
  SDValue ExtSrc0 = GetPromotedInteger(N->getValue());
497
10
  assert(NVT == ExtSrc0.getValueType() &&
498
10
      "Gather result type and the passThru agrument type should be the same");
499
10
500
10
  SDLoc dl(N);
501
10
  SDValue Ops[] = {N->getChain(), ExtSrc0, N->getMask(), N->getBasePtr(),
502
10
                   N->getIndex()};
503
10
  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
504
10
                                    N->getMemoryVT(), dl, Ops,
505
10
                                    N->getMemOperand());
506
10
  // Legalize the chain result - switch anything that used the old chain to
507
10
  // use the new one.
508
10
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
509
10
  return Res;
510
10
}
511
512
/// Promote the overflow flag of an overflowing arithmetic node.
513
1.54k
SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
514
1.54k
  // Simply change the return type of the boolean result.
515
1.54k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
516
1.54k
  EVT ValueVTs[] = { N->getValueType(0), NVT };
517
1.54k
  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
518
1.54k
  unsigned NumOps = N->getNumOperands();
519
1.54k
  assert(NumOps <= 3 && "Too many operands");
520
1.54k
  if (NumOps == 3)
521
5
    Ops[2] = N->getOperand(2);
522
1.54k
523
1.54k
  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
524
1.54k
                            DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps));
525
1.54k
526
1.54k
  // Modified the sum result - switch anything that used the old sum to use
527
1.54k
  // the new one.
528
1.54k
  ReplaceValueWith(SDValue(N, 0), Res);
529
1.54k
530
1.54k
  return SDValue(Res.getNode(), 1);
531
1.54k
}
532
533
105
SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
534
105
  if (ResNo == 1)
535
103
    return PromoteIntRes_Overflow(N);
536
2
537
2
  // The operation overflowed iff the result in the larger type is not the
538
2
  // sign extension of its truncation to the original type.
539
2
  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
540
2
  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
541
2
  EVT OVT = N->getOperand(0).getValueType();
542
2
  EVT NVT = LHS.getValueType();
543
2
  SDLoc dl(N);
544
2
545
2
  // Do the arithmetic in the larger type.
546
2
  unsigned Opcode = N->getOpcode() == ISD::SADDO ? 
ISD::ADD2
:
ISD::SUB0
;
547
105
  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
548
105
549
105
  // Calculate the overflow flag: sign extend the arithmetic result from
550
105
  // the original type.
551
105
  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
552
105
                            DAG.getValueType(OVT));
553
105
  // Overflowed if and only if this is not equal to Res.
554
105
  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
555
105
556
105
  // Use the calculated overflow everywhere.
557
105
  ReplaceValueWith(SDValue(N, 1), Ofl);
558
105
559
105
  return Res;
560
105
}
561
562
13.6k
SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
563
13.6k
  SDValue LHS = GetPromotedInteger(N->getOperand(1));
564
13.6k
  SDValue RHS = GetPromotedInteger(N->getOperand(2));
565
13.6k
  return DAG.getSelect(SDLoc(N),
566
13.6k
                       LHS.getValueType(), N->getOperand(0), LHS, RHS);
567
13.6k
}
568
569
78
SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
570
78
  SDValue Mask = N->getOperand(0);
571
78
  EVT OpTy = N->getOperand(1).getValueType();
572
78
573
78
  // Promote all the way up to the canonical SetCC type.
574
78
  Mask = PromoteTargetBoolean(Mask, OpTy);
575
78
  SDValue LHS = GetPromotedInteger(N->getOperand(1));
576
78
  SDValue RHS = GetPromotedInteger(N->getOperand(2));
577
78
  return DAG.getNode(ISD::VSELECT, SDLoc(N),
578
78
                     LHS.getValueType(), Mask, LHS, RHS);
579
78
}
580
581
0
SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
582
0
  SDValue LHS = GetPromotedInteger(N->getOperand(2));
583
0
  SDValue RHS = GetPromotedInteger(N->getOperand(3));
584
0
  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
585
0
                     LHS.getValueType(), N->getOperand(0),
586
0
                     N->getOperand(1), LHS, RHS, N->getOperand(4));
587
0
}
588
589
339k
SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
590
339k
  EVT SVT = getSetCCResultType(N->getOperand(0).getValueType());
591
339k
592
339k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
593
339k
594
339k
  // Only use the result of getSetCCResultType if it is legal,
595
339k
  // otherwise just use the promoted result type (NVT).
596
339k
  if (!TLI.isTypeLegal(SVT))
597
1.63k
    SVT = NVT;
598
339k
599
339k
  SDLoc dl(N);
600
339k
  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
601
339k
         "Vector compare must return a vector result!");
602
339k
603
339k
  SDValue LHS = N->getOperand(0);
604
339k
  SDValue RHS = N->getOperand(1);
605
339k
  if (
LHS.getValueType() != RHS.getValueType()339k
) {
606
1
    if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger &&
607
0
        !LHS.getValueType().isVector())
608
0
      LHS = GetPromotedInteger(LHS);
609
1
    if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger &&
610
1
        !RHS.getValueType().isVector())
611
1
      RHS = GetPromotedInteger(RHS);
612
1
  }
613
339k
614
339k
  // Get the SETCC result using the canonical SETCC type.
615
339k
  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS,
616
339k
                              N->getOperand(2));
617
339k
618
339k
  // Convert to the expected type.
619
339k
  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
620
339k
}
621
622
1.62k
SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
623
1.62k
  SDValue LHS = N->getOperand(0);
624
1.62k
  SDValue RHS = N->getOperand(1);
625
1.62k
  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
626
1.62k
    LHS = GetPromotedInteger(LHS);
627
1.62k
  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
628
51
    RHS = ZExtPromotedInteger(RHS);
629
1.62k
  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
630
1.62k
}
631
632
60
SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
633
60
  SDValue Op = GetPromotedInteger(N->getOperand(0));
634
60
  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
635
60
                     Op.getValueType(), Op, N->getOperand(1));
636
60
}
637
638
38.5k
SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
639
38.5k
  // The input may have strange things in the top bits of the registers, but
640
38.5k
  // these operations don't care.  They may have weird bits going out, but
641
38.5k
  // that too is okay if they are integer operations.
642
38.5k
  SDValue LHS = GetPromotedInteger(N->getOperand(0));
643
38.5k
  SDValue RHS = GetPromotedInteger(N->getOperand(1));
644
38.5k
  return DAG.getNode(N->getOpcode(), SDLoc(N),
645
38.5k
                     LHS.getValueType(), LHS, RHS);
646
38.5k
}
647
648
395
SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
649
395
  // Sign extend the input.
650
395
  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
651
395
  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
652
395
  return DAG.getNode(N->getOpcode(), SDLoc(N),
653
395
                     LHS.getValueType(), LHS, RHS);
654
395
}
655
656
382
SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
657
382
  // Zero extend the input.
658
382
  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
659
382
  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
660
382
  return DAG.getNode(N->getOpcode(), SDLoc(N),
661
382
                     LHS.getValueType(), LHS, RHS);
662
382
}
663
664
298
SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
665
298
  SDValue LHS = N->getOperand(0);
666
298
  SDValue RHS = N->getOperand(1);
667
298
  // The input value must be properly sign extended.
668
298
  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
669
298
    LHS = SExtPromotedInteger(LHS);
670
298
  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
671
42
    RHS = ZExtPromotedInteger(RHS);
672
298
  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
673
298
}
674
675
6.11k
SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
676
6.11k
  SDValue LHS = N->getOperand(0);
677
6.11k
  SDValue RHS = N->getOperand(1);
678
6.11k
  // The input value must be properly zero extended.
679
6.11k
  if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger)
680
6.11k
    LHS = ZExtPromotedInteger(LHS);
681
6.11k
  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
682
56
    RHS = ZExtPromotedInteger(RHS);
683
6.11k
  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
684
6.11k
}
685
686
148k
SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
687
148k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
688
148k
  SDValue Res;
689
148k
  SDValue InOp = N->getOperand(0);
690
148k
  SDLoc dl(N);
691
148k
692
148k
  switch (getTypeAction(InOp.getValueType())) {
693
0
  
default: 0
llvm_unreachable0
("Unknown type action!");
694
142k
  case TargetLowering::TypeLegal:
695
142k
  case TargetLowering::TypeExpandInteger:
696
142k
    Res = InOp;
697
142k
    break;
698
5.43k
  case TargetLowering::TypePromoteInteger:
699
5.43k
    Res = GetPromotedInteger(InOp);
700
5.43k
    break;
701
13
  case TargetLowering::TypeSplitVector: {
702
13
    EVT InVT = InOp.getValueType();
703
13
    assert(InVT.isVector() && "Cannot split scalar types");
704
13
    unsigned NumElts = InVT.getVectorNumElements();
705
13
    assert(NumElts == NVT.getVectorNumElements() &&
706
13
           "Dst and Src must have the same number of elements");
707
13
    assert(isPowerOf2_32(NumElts) &&
708
13
           "Promoted vector type must be a power of two");
709
13
710
13
    SDValue EOp1, EOp2;
711
13
    GetSplitVector(InOp, EOp1, EOp2);
712
13
713
13
    EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
714
13
                                   NumElts/2);
715
13
    EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
716
13
    EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
717
13
718
13
    return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
719
142k
  }
720
1
  case TargetLowering::TypeWidenVector: {
721
1
    SDValue WideInOp = GetWidenedVector(InOp);
722
1
723
1
    // Truncate widened InOp.
724
1
    unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
725
1
    EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
726
1
                                   N->getValueType(0).getScalarType(), NumElem);
727
1
    SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
728
1
729
1
    // Zero extend so that the elements are of same type as those of NVT
730
1
    EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
731
1
                                 NumElem);
732
1
    SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
733
1
734
1
    // Extract the low NVT subvector.
735
1
    MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
736
1
    SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
737
1
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
738
148k
  }
739
148k
  }
740
148k
741
148k
  // Truncate to NVT instead of VT
742
148k
  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
743
148k
}
744
745
266
SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
746
266
  if (ResNo == 1)
747
239
    return PromoteIntRes_Overflow(N);
748
27
749
27
  // The operation overflowed iff the result in the larger type is not the
750
27
  // zero extension of its truncation to the original type.
751
27
  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
752
27
  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
753
27
  EVT OVT = N->getOperand(0).getValueType();
754
27
  EVT NVT = LHS.getValueType();
755
27
  SDLoc dl(N);
756
27
757
27
  // Do the arithmetic in the larger type.
758
27
  unsigned Opcode = N->getOpcode() == ISD::UADDO ? 
ISD::ADD25
:
ISD::SUB2
;
759
266
  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
760
266
761
266
  // Calculate the overflow flag: zero extend the arithmetic result from
762
266
  // the original type.
763
266
  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
764
266
  // Overflowed if and only if this is not equal to Res.
765
266
  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
766
266
767
266
  // Use the calculated overflow everywhere.
768
266
  ReplaceValueWith(SDValue(N, 1), Ofl);
769
266
770
266
  return Res;
771
266
}
772
773
5
SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
774
5
  if (ResNo == 1)
775
5
    return PromoteIntRes_Overflow(N);
776
0
  
llvm_unreachable0
("Not implemented");
777
0
}
778
779
1.19k
SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
780
1.19k
  // Promote the overflow bit trivially.
781
1.19k
  if (ResNo == 1)
782
1.19k
    return PromoteIntRes_Overflow(N);
783
1
784
1
  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
785
1
  SDLoc DL(N);
786
1
  EVT SmallVT = LHS.getValueType();
787
1
788
1
  // To determine if the result overflowed in a larger type, we extend the
789
1
  // input to the larger type, do the multiply (checking if it overflows),
790
1
  // then also check the high bits of the result to see if overflow happened
791
1
  // there.
792
1
  if (
N->getOpcode() == ISD::SMULO1
) {
793
1
    LHS = SExtPromotedInteger(LHS);
794
1
    RHS = SExtPromotedInteger(RHS);
795
1
  } else {
796
0
    LHS = ZExtPromotedInteger(LHS);
797
0
    RHS = ZExtPromotedInteger(RHS);
798
0
  }
799
1
  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
800
1
  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
801
1
802
1
  // Overflow occurred if it occurred in the larger type, or if the high part
803
1
  // of the result does not zero/sign-extend the low part.  Check this second
804
1
  // possibility first.
805
1
  SDValue Overflow;
806
1
  if (
N->getOpcode() == ISD::UMULO1
) {
807
0
    // Unsigned overflow occurred if the high part is non-zero.
808
0
    SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
809
0
                             DAG.getIntPtrConstant(SmallVT.getSizeInBits(),
810
0
                                                   DL));
811
0
    Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
812
0
                            DAG.getConstant(0, DL, Hi.getValueType()),
813
0
                            ISD::SETNE);
814
1
  } else {
815
1
    // Signed overflow occurred if the high part does not sign extend the low.
816
1
    SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
817
1
                               Mul, DAG.getValueType(SmallVT));
818
1
    Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
819
1
  }
820
1.19k
821
1.19k
  // The only other way for overflow to occur is if the multiplication in the
822
1.19k
  // larger type itself overflowed.
823
1.19k
  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
824
1.19k
                         SDValue(Mul.getNode(), 1));
825
1.19k
826
1.19k
  // Use the calculated overflow everywhere.
827
1.19k
  ReplaceValueWith(SDValue(N, 1), Overflow);
828
1.19k
  return Mul;
829
1.19k
}
830
831
2.63k
SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
832
2.63k
  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
833
2.63k
                                               N->getValueType(0)));
834
2.63k
}
835
836
36
SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
837
36
  SDValue Chain = N->getOperand(0); // Get the chain.
838
36
  SDValue Ptr = N->getOperand(1); // Get the pointer.
839
36
  EVT VT = N->getValueType(0);
840
36
  SDLoc dl(N);
841
36
842
36
  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
843
36
  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
844
36
  // The argument is passed as NumRegs registers of type RegVT.
845
36
846
36
  SmallVector<SDValue, 8> Parts(NumRegs);
847
72
  for (unsigned i = 0; 
i < NumRegs72
;
++i36
) {
848
36
    Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
849
36
                            N->getConstantOperandVal(3));
850
36
    Chain = Parts[i].getValue(1);
851
36
  }
852
36
853
36
  // Handle endianness of the load.
854
36
  if (DAG.getDataLayout().isBigEndian())
855
18
    std::reverse(Parts.begin(), Parts.end());
856
36
857
36
  // Assemble the parts in the promoted type.
858
36
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
859
36
  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
860
36
  for (unsigned i = 1; 
i < NumRegs36
;
++i0
) {
861
0
    SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
862
0
    // Shift it to the right position and "or" it in.
863
0
    Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
864
0
                       DAG.getConstant(i * RegVT.getSizeInBits(), dl,
865
0
                                       TLI.getPointerTy(DAG.getDataLayout())));
866
0
    Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
867
0
  }
868
36
869
36
  // Modified the chain result - switch anything that used the old chain to
870
36
  // use the new one.
871
36
  ReplaceValueWith(SDValue(N, 1), Chain);
872
36
873
36
  return Res;
874
36
}
875
876
//===----------------------------------------------------------------------===//
877
//  Integer Operand Promotion
878
//===----------------------------------------------------------------------===//
879
880
/// PromoteIntegerOperand - This method is called when the specified operand of
881
/// the specified node is found to need promotion.  At this point, all of the
882
/// result types of the node are known to be legal, but other operands of the
883
/// node may need promotion or expansion as well as the specified one.
884
1.41M
bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
885
1.41M
  DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
886
1.41M
  SDValue Res = SDValue();
887
1.41M
888
1.41M
  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
889
385
    return false;
890
1.41M
891
1.41M
  switch (N->getOpcode()) {
892
0
    default:
893
  #ifndef NDEBUG
894
    dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
895
    N->dump(&DAG); dbgs() << "\n";
896
  #endif
897
0
    llvm_unreachable("Do not know how to promote this operator's operand!");
898
1.41M
899
108k
  case ISD::ANY_EXTEND:   Res = PromoteIntOp_ANY_EXTEND(N); break;
900
257
  case ISD::ATOMIC_STORE:
901
257
    Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
902
257
    break;
903
1.26k
  case ISD::BITCAST:      Res = PromoteIntOp_BITCAST(N); break;
904
10.3k
  case ISD::BR_CC:        Res = PromoteIntOp_BR_CC(N, OpNo); break;
905
287k
  case ISD::BRCOND:       Res = PromoteIntOp_BRCOND(N, OpNo); break;
906
0
  case ISD::BUILD_PAIR:   Res = PromoteIntOp_BUILD_PAIR(N); break;
907
18.8k
  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
908
1.84k
  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
909
38.0k
  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
910
173
  case ISD::INSERT_VECTOR_ELT:
911
173
                          Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
912
21
  case ISD::SCALAR_TO_VECTOR:
913
21
                          Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
914
30.9k
  case ISD::VSELECT:
915
30.9k
  case ISD::SELECT:       Res = PromoteIntOp_SELECT(N, OpNo); break;
916
14.2k
  case ISD::SELECT_CC:    Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
917
236k
  case ISD::SETCC:        Res = PromoteIntOp_SETCC(N, OpNo); break;
918
11.8k
  case ISD::SIGN_EXTEND:  Res = PromoteIntOp_SIGN_EXTEND(N); break;
919
8.91k
  case ISD::SINT_TO_FP:   Res = PromoteIntOp_SINT_TO_FP(N); break;
920
286k
  case ISD::STORE:        Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
921
286k
                                                   OpNo); break;
922
22
  case ISD::MSTORE:       Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
923
22
                                                    OpNo); break;
924
45
  case ISD::MLOAD:        Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
925
45
                                                    OpNo); break;
926
42
  case ISD::MGATHER:  Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
927
42
                                                 OpNo); break;
928
22
  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
929
22
                                                  OpNo); break;
930
2.88k
  case ISD::TRUNCATE:     Res = PromoteIntOp_TRUNCATE(N); break;
931
12.1k
  case ISD::FP16_TO_FP:
932
12.1k
  case ISD::UINT_TO_FP:   Res = PromoteIntOp_UINT_TO_FP(N); break;
933
347k
  case ISD::ZERO_EXTEND:  Res = PromoteIntOp_ZERO_EXTEND(N); break;
934
42
  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
935
12.1k
936
0
  case ISD::SHL:
937
0
  case ISD::SRA:
938
0
  case ISD::SRL:
939
0
  case ISD::ROTL:
940
0
  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
941
0
942
5
  case ISD::ADDCARRY:
943
5
  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
944
1.41M
  }
945
1.41M
946
1.41M
  // If the result is null, the sub-method took care of registering results etc.
947
1.41M
  
if (1.41M
!Res.getNode()1.41M
)
return false2
;
948
1.41M
949
1.41M
  // If the result is N, the sub-method updated N in place.  Tell the legalizer
950
1.41M
  // core about this.
951
1.41M
  
if (1.41M
Res.getNode() == N1.41M
)
952
618k
    return true;
953
800k
954
1.41M
  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
955
800k
         "Invalid operand expansion");
956
800k
957
800k
  ReplaceValueWith(SDValue(N, 0), Res);
958
800k
  return false;
959
800k
}
960
961
/// PromoteSetCCOperands - Promote the operands of a comparison.  This code is
962
/// shared among BR_CC, SELECT_CC, and SETCC handlers.
963
void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
964
261k
                                            ISD::CondCode CCCode) {
965
261k
  // We have to insert explicit sign or zero extends.  Note that we could
966
261k
  // insert sign extends for ALL conditions, but zero extend is cheaper on
967
261k
  // many machines (an AND instead of two shifts), so prefer it.
968
261k
  switch (CCCode) {
969
0
  
default: 0
llvm_unreachable0
("Unknown integer comparison!");
970
226k
  case ISD::SETEQ:
971
226k
  case ISD::SETNE: {
972
226k
    SDValue OpL = GetPromotedInteger(NewLHS);
973
226k
    SDValue OpR = GetPromotedInteger(NewRHS);
974
226k
975
226k
    // We would prefer to promote the comparison operand with sign extension.
976
226k
    // If the width of OpL/OpR excluding the duplicated sign bits is no greater
977
226k
    // than the width of NewLHS/NewRH, we can avoid inserting real truncate
978
226k
    // instruction, which is redudant eventually.
979
226k
    unsigned OpLEffectiveBits =
980
226k
        OpL.getValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
981
226k
    unsigned OpREffectiveBits =
982
226k
        OpR.getValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
983
226k
    if (OpLEffectiveBits <= NewLHS.getValueSizeInBits() &&
984
226k
        
OpREffectiveBits <= NewRHS.getValueSizeInBits()7.99k
) {
985
7.96k
      NewLHS = OpL;
986
7.96k
      NewRHS = OpR;
987
226k
    } else {
988
219k
      NewLHS = ZExtPromotedInteger(NewLHS);
989
219k
      NewRHS = ZExtPromotedInteger(NewRHS);
990
219k
    }
991
226k
    break;
992
226k
  }
993
23.5k
  case ISD::SETUGE:
994
23.5k
  case ISD::SETUGT:
995
23.5k
  case ISD::SETULE:
996
23.5k
  case ISD::SETULT:
997
23.5k
    // ALL of these operations will work if we either sign or zero extend
998
23.5k
    // the operands (including the unsigned comparisons!).  Zero extend is
999
23.5k
    // usually a simpler/cheaper operation, so prefer it.
1000
23.5k
    NewLHS = ZExtPromotedInteger(NewLHS);
1001
23.5k
    NewRHS = ZExtPromotedInteger(NewRHS);
1002
23.5k
    break;
1003
10.8k
  case ISD::SETGE:
1004
10.8k
  case ISD::SETGT:
1005
10.8k
  case ISD::SETLT:
1006
10.8k
  case ISD::SETLE:
1007
10.8k
    NewLHS = SExtPromotedInteger(NewLHS);
1008
10.8k
    NewRHS = SExtPromotedInteger(NewRHS);
1009
10.8k
    break;
1010
261k
  }
1011
261k
}
1012
1013
108k
SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1014
108k
  SDValue Op = GetPromotedInteger(N->getOperand(0));
1015
108k
  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1016
108k
}
1017
1018
257
SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1019
257
  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1020
257
  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1021
257
                       N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1022
257
}
1023
1024
1.26k
SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1025
1.26k
  // This should only occur in unusual situations like bitcasting to an
1026
1.26k
  // x86_fp80, so just turn it into a store+load
1027
1.26k
  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1028
1.26k
}
1029
1030
10.3k
SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1031
10.3k
  assert(OpNo == 2 && "Don't know how to promote this operand!");
1032
10.3k
1033
10.3k
  SDValue LHS = N->getOperand(2);
1034
10.3k
  SDValue RHS = N->getOperand(3);
1035
10.3k
  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1036
10.3k
1037
10.3k
  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1038
10.3k
  // legal types.
1039
10.3k
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1040
10.3k
                                N->getOperand(1), LHS, RHS, N->getOperand(4)),
1041
10.3k
                 0);
1042
10.3k
}
1043
1044
287k
SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1045
287k
  assert(OpNo == 1 && "only know how to promote condition");
1046
287k
1047
287k
  // Promote all the way up to the canonical SetCC type.
1048
287k
  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1049
287k
1050
287k
  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1051
287k
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1052
287k
                                        N->getOperand(2)), 0);
1053
287k
}
1054
1055
0
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1056
0
  // Since the result type is legal, the operands must promote to it.
1057
0
  EVT OVT = N->getOperand(0).getValueType();
1058
0
  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1059
0
  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1060
0
  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1061
0
  SDLoc dl(N);
1062
0
1063
0
  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1064
0
                   DAG.getConstant(OVT.getSizeInBits(), dl,
1065
0
                                   TLI.getPointerTy(DAG.getDataLayout())));
1066
0
  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1067
0
}
1068
1069
18.8k
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1070
18.8k
  // The vector type is legal but the element type is not.  This implies
1071
18.8k
  // that the vector is a power-of-two in length and that the element
1072
18.8k
  // type does not have a strange size (eg: it is not i1).
1073
18.8k
  EVT VecVT = N->getValueType(0);
1074
18.8k
  unsigned NumElts = VecVT.getVectorNumElements();
1075
18.8k
  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1076
18.8k
         "Legal vector of one illegal element?");
1077
18.8k
1078
18.8k
  // Promote the inserted value.  The type does not need to match the
1079
18.8k
  // vector element type.  Check that any extra bits introduced will be
1080
18.8k
  // truncated away.
1081
18.8k
  assert(N->getOperand(0).getValueSizeInBits() >=
1082
18.8k
         N->getValueType(0).getScalarSizeInBits() &&
1083
18.8k
         "Type of inserted value narrower than vector element type!");
1084
18.8k
1085
18.8k
  SmallVector<SDValue, 16> NewOps;
1086
169k
  for (unsigned i = 0; 
i < NumElts169k
;
++i150k
)
1087
150k
    NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1088
18.8k
1089
18.8k
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1090
18.8k
}
1091
1092
SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1093
173
                                                         unsigned OpNo) {
1094
173
  if (
OpNo == 1173
) {
1095
173
    // Promote the inserted value.  This is valid because the type does not
1096
173
    // have to match the vector element type.
1097
173
1098
173
    // Check that any extra bits introduced will be truncated away.
1099
173
    assert(N->getOperand(1).getValueSizeInBits() >=
1100
173
           N->getValueType(0).getScalarSizeInBits() &&
1101
173
           "Type of inserted value narrower than vector element type!");
1102
173
    return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1103
173
                                  GetPromotedInteger(N->getOperand(1)),
1104
173
                                  N->getOperand(2)),
1105
173
                   0);
1106
173
  }
1107
0
1108
173
  assert(OpNo == 2 && "Different operand and result vector types?");
1109
0
1110
0
  // Promote the index.
1111
0
  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1112
0
                                   TLI.getVectorIdxTy(DAG.getDataLayout()));
1113
0
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1114
0
                                N->getOperand(1), Idx), 0);
1115
0
}
1116
1117
21
SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1118
21
  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1119
21
  // the operand in place.
1120
21
  return SDValue(DAG.UpdateNodeOperands(N,
1121
21
                                GetPromotedInteger(N->getOperand(0))), 0);
1122
21
}
1123
1124
30.9k
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1125
30.9k
  assert(OpNo == 0 && "Only know how to promote the condition!");
1126
30.9k
  SDValue Cond = N->getOperand(0);
1127
30.9k
  EVT OpTy = N->getOperand(1).getValueType();
1128
30.9k
1129
30.9k
  if (N->getOpcode() == ISD::VSELECT)
1130
2.12k
    
if (SDValue 2.12k
Res2.12k
= WidenVSELECTAndMask(N))
1131
1.64k
      return Res;
1132
29.2k
1133
29.2k
  // Promote all the way up to the canonical SetCC type.
1134
29.2k
  
EVT OpVT = N->getOpcode() == ISD::SELECT ? 29.2k
OpTy.getScalarType()28.8k
:
OpTy484
;
1135
30.9k
  Cond = PromoteTargetBoolean(Cond, OpVT);
1136
30.9k
1137
30.9k
  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1138
30.9k
                                        N->getOperand(2)), 0);
1139
30.9k
}
1140
1141
14.2k
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1142
14.2k
  assert(OpNo == 0 && "Don't know how to promote this operand!");
1143
14.2k
1144
14.2k
  SDValue LHS = N->getOperand(0);
1145
14.2k
  SDValue RHS = N->getOperand(1);
1146
14.2k
  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1147
14.2k
1148
14.2k
  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1149
14.2k
  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1150
14.2k
                                N->getOperand(3), N->getOperand(4)), 0);
1151
14.2k
}
1152
1153
236k
SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1154
236k
  assert(OpNo == 0 && "Don't know how to promote this operand!");
1155
236k
1156
236k
  SDValue LHS = N->getOperand(0);
1157
236k
  SDValue RHS = N->getOperand(1);
1158
236k
  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1159
236k
1160
236k
  // The CC (#2) is always legal.
1161
236k
  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1162
236k
}
1163
1164
0
SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1165
0
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1166
0
                                ZExtPromotedInteger(N->getOperand(1))), 0);
1167
0
}
1168
1169
11.8k
SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1170
11.8k
  SDValue Op = GetPromotedInteger(N->getOperand(0));
1171
11.8k
  SDLoc dl(N);
1172
11.8k
  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1173
11.8k
  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1174
11.8k
                     Op, DAG.getValueType(N->getOperand(0).getValueType()));
1175
11.8k
}
1176
1177
8.91k
SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1178
8.91k
  return SDValue(DAG.UpdateNodeOperands(N,
1179
8.91k
                                SExtPromotedInteger(N->getOperand(0))), 0);
1180
8.91k
}
1181
1182
286k
SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1183
286k
  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1184
286k
  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1185
286k
  SDLoc dl(N);
1186
286k
1187
286k
  SDValue Val = GetPromotedInteger(N->getValue());  // Get promoted value.
1188
286k
1189
286k
  // Truncate the value and store the result.
1190
286k
  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1191
286k
                           N->getMemoryVT(), N->getMemOperand());
1192
286k
}
1193
1194
SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1195
25
                                              unsigned OpNo) {
1196
25
1197
25
  SDValue DataOp = N->getValue();
1198
25
  EVT DataVT = DataOp.getValueType();
1199
25
  SDValue Mask = N->getMask();
1200
25
  SDLoc dl(N);
1201
25
1202
25
  bool TruncateStore = false;
1203
25
  if (
OpNo == 225
) {
1204
21
    // Mask comes before the data operand. If the data operand is legal, we just
1205
21
    // promote the mask.
1206
21
    // When the data operand has illegal type, we should legalize the data
1207
21
    // operand first. The mask will be promoted/splitted/widened according to
1208
21
    // the data operand type.
1209
21
    if (TLI.isTypeLegal(DataVT))
1210
14
      Mask = PromoteTargetBoolean(Mask, DataVT);
1211
7
    else {
1212
7
      if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger)
1213
3
        return PromoteIntOp_MSTORE(N, 3);
1214
7
1215
4
      else 
if (4
getTypeAction(DataVT) == TargetLowering::TypeWidenVector4
)
1216
4
        return WidenVecOp_MSTORE(N, 3);
1217
4
1218
0
      else {
1219
0
        assert (getTypeAction(DataVT) == TargetLowering::TypeSplitVector);
1220
0
        return SplitVecOp_MSTORE(N, 3);
1221
0
      }
1222
25
    }
1223
0
  } else { // Data operand
1224
4
    assert(OpNo == 3 && "Unexpected operand for promotion");
1225
4
    DataOp = GetPromotedInteger(DataOp);
1226
4
    Mask = PromoteTargetBoolean(Mask, DataOp.getValueType());
1227
4
    TruncateStore = true;
1228
4
  }
1229
25
1230
18
  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1231
18
                            N->getMemoryVT(), N->getMemOperand(),
1232
18
                            TruncateStore, N->isCompressingStore());
1233
25
}
1234
1235
SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1236
45
                                             unsigned OpNo) {
1237
45
  assert(OpNo == 2 && "Only know how to promote the mask!");
1238
45
  EVT DataVT = N->getValueType(0);
1239
45
  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1240
45
  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1241
45
  NewOps[OpNo] = Mask;
1242
45
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1243
45
}
1244
1245
SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1246
42
                                               unsigned OpNo) {
1247
42
1248
42
  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1249
42
  if (
OpNo == 242
) {
1250
20
    // The Mask
1251
20
    EVT DataVT = N->getValueType(0);
1252
20
    NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1253
20
  } else
1254
22
    NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1255
42
1256
42
  SDValue Res = SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1257
42
  // updated in place.
1258
42
  if (Res.getNode() == N)
1259
40
    return Res;
1260
2
1261
2
  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
1262
2
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1263
2
  return SDValue();
1264
2
}
1265
1266
SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1267
22
                                                unsigned OpNo) {
1268
22
  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1269
22
  if (
OpNo == 222
) {
1270
8
    // The Mask
1271
8
    EVT DataVT = N->getValue().getValueType();
1272
8
    NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1273
8
  } else
1274
14
    NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1275
22
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1276
22
}
1277
1278
2.88k
SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1279
2.88k
  SDValue Op = GetPromotedInteger(N->getOperand(0));
1280
2.88k
  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1281
2.88k
}
1282
1283
12.1k
SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1284
12.1k
  return SDValue(DAG.UpdateNodeOperands(N,
1285
12.1k
                                ZExtPromotedInteger(N->getOperand(0))), 0);
1286
12.1k
}
1287
1288
347k
SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1289
347k
  SDLoc dl(N);
1290
347k
  SDValue Op = GetPromotedInteger(N->getOperand(0));
1291
347k
  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1292
347k
  return DAG.getZeroExtendInReg(Op, dl,
1293
347k
                                N->getOperand(0).getValueType().getScalarType());
1294
347k
}
1295
1296
5
SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1297
5
  assert(OpNo == 2 && "Don't know how to promote this operand!");
1298
5
1299
5
  SDValue LHS = N->getOperand(0);
1300
5
  SDValue RHS = N->getOperand(1);
1301
5
  SDValue Carry = N->getOperand(2);
1302
5
  SDLoc DL(N);
1303
5
1304
5
  auto VT = getSetCCResultType(LHS.getValueType());
1305
5
  TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT);
1306
5
  switch (BoolType) {
1307
0
  case TargetLoweringBase::UndefinedBooleanContent:
1308
0
    Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT);
1309
0
    break;
1310
5
  case TargetLoweringBase::ZeroOrOneBooleanContent:
1311
5
    Carry = DAG.getZExtOrTrunc(Carry, DL, VT);
1312
5
    break;
1313
0
  case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
1314
0
    Carry = DAG.getSExtOrTrunc(Carry, DL, VT);
1315
0
    break;
1316
5
  }
1317
5
1318
5
  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1319
5
}
1320
1321
//===----------------------------------------------------------------------===//
1322
//  Integer Result Expansion
1323
//===----------------------------------------------------------------------===//
1324
1325
/// ExpandIntegerResult - This method is called when the specified result of the
1326
/// specified node is found to need expansion.  At this point, the node may also
1327
/// have invalid operands or may have other results that need promotion, we just
1328
/// know that (at least) one result needs expansion.
1329
182k
void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1330
182k
  DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
1331
182k
  SDValue Lo, Hi;
1332
182k
  Lo = Hi = SDValue();
1333
182k
1334
182k
  // See if the target wants to custom expand this node.
1335
182k
  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1336
1.21k
    return;
1337
181k
1338
181k
  switch (N->getOpcode()) {
1339
0
  default:
1340
#ifndef NDEBUG
1341
    dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1342
    N->dump(&DAG); dbgs() << "\n";
1343
#endif
1344
0
    llvm_unreachable("Do not know how to expand the result of this operator!");
1345
181k
1346
40
  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1347
749
  case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
1348
620
  case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
1349
1.63k
  case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
1350
181k
1351
9.88k
  case ISD::BITCAST:            ExpandRes_BITCAST(N, Lo, Hi); break;
1352
21.8k
  case ISD::BUILD_PAIR:         ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1353
2.76k
  case ISD::EXTRACT_ELEMENT:    ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1354
1.81k
  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1355
14
  case ISD::VAARG:              ExpandRes_VAARG(N, Lo, Hi); break;
1356
181k
1357
4.47k
  case ISD::ANY_EXTEND:  ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1358
317
  case ISD::AssertSext:  ExpandIntRes_AssertSext(N, Lo, Hi); break;
1359
315
  case ISD::AssertZext:  ExpandIntRes_AssertZext(N, Lo, Hi); break;
1360
1
  case ISD::BITREVERSE:  ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1361
47
  case ISD::BSWAP:       ExpandIntRes_BSWAP(N, Lo, Hi); break;
1362
11.7k
  case ISD::Constant:    ExpandIntRes_Constant(N, Lo, Hi); break;
1363
100
  case ISD::CTLZ_ZERO_UNDEF:
1364
100
  case ISD::CTLZ:        ExpandIntRes_CTLZ(N, Lo, Hi); break;
1365
11
  case ISD::CTPOP:       ExpandIntRes_CTPOP(N, Lo, Hi); break;
1366
8
  case ISD::CTTZ_ZERO_UNDEF:
1367
8
  case ISD::CTTZ:        ExpandIntRes_CTTZ(N, Lo, Hi); break;
1368
1
  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1369
89
  case ISD::FP_TO_SINT:  ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1370
81
  case ISD::FP_TO_UINT:  ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1371
7.26k
  case ISD::LOAD:        ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1372
1.67k
  case ISD::MUL:         ExpandIntRes_MUL(N, Lo, Hi); break;
1373
2
  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1374
107
  case ISD::SDIV:        ExpandIntRes_SDIV(N, Lo, Hi); break;
1375
1.82k
  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1376
156
  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1377
73
  case ISD::SREM:        ExpandIntRes_SREM(N, Lo, Hi); break;
1378
12.3k
  case ISD::TRUNCATE:    ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1379
111
  case ISD::UDIV:        ExpandIntRes_UDIV(N, Lo, Hi); break;
1380
73
  case ISD::UREM:        ExpandIntRes_UREM(N, Lo, Hi); break;
1381
9.29k
  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1382
11
  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1383
8
1384
41
  case ISD::ATOMIC_LOAD_ADD:
1385
41
  case ISD::ATOMIC_LOAD_SUB:
1386
41
  case ISD::ATOMIC_LOAD_AND:
1387
41
  case ISD::ATOMIC_LOAD_OR:
1388
41
  case ISD::ATOMIC_LOAD_XOR:
1389
41
  case ISD::ATOMIC_LOAD_NAND:
1390
41
  case ISD::ATOMIC_LOAD_MIN:
1391
41
  case ISD::ATOMIC_LOAD_MAX:
1392
41
  case ISD::ATOMIC_LOAD_UMIN:
1393
41
  case ISD::ATOMIC_LOAD_UMAX:
1394
41
  case ISD::ATOMIC_SWAP:
1395
41
  case ISD::ATOMIC_CMP_SWAP: {
1396
41
    std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1397
41
    SplitInteger(Tmp.first, Lo, Hi);
1398
41
    ReplaceValueWith(SDValue(N, 1), Tmp.second);
1399
41
    break;
1400
41
  }
1401
25
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
1402
25
    AtomicSDNode *AN = cast<AtomicSDNode>(N);
1403
25
    SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1404
25
    SDValue Tmp = DAG.getAtomicCmpSwap(
1405
25
        ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1406
25
        N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1407
25
        AN->getMemOperand());
1408
25
1409
25
    // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1410
25
    // success simply by comparing the loaded value against the ingoing
1411
25
    // comparison.
1412
25
    SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1413
25
                                   N->getOperand(2), ISD::SETEQ);
1414
25
1415
25
    SplitInteger(Tmp, Lo, Hi);
1416
25
    ReplaceValueWith(SDValue(N, 1), Success);
1417
25
    ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1418
25
    break;
1419
41
  }
1420
41
1421
20.2k
  case ISD::AND:
1422
20.2k
  case ISD::OR:
1423
20.2k
  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1424
20.2k
1425
9
  case ISD::UMAX:
1426
9
  case ISD::SMAX:
1427
9
  case ISD::UMIN:
1428
9
  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1429
9
1430
5.38k
  case ISD::ADD:
1431
5.38k
  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1432
5.38k
1433
7
  case ISD::ADDC:
1434
7
  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1435
7
1436
7
  case ISD::ADDE:
1437
7
  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1438
7
1439
760
  case ISD::ADDCARRY:
1440
760
  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1441
760
1442
64.5k
  case ISD::SHL:
1443
64.5k
  case ISD::SRA:
1444
64.5k
  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1445
64.5k
1446
20
  case ISD::SADDO:
1447
20
  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1448
547
  case ISD::UADDO:
1449
547
  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1450
11
  case ISD::UMULO:
1451
11
  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1452
181k
  }
1453
181k
1454
181k
  // If Lo/Hi is null, the sub-method took care of registering results etc.
1455
181k
  
if (181k
Lo.getNode()181k
)
1456
181k
    SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1457
182k
}
1458
1459
/// Lower an atomic node to the appropriate builtin call.
1460
41
std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1461
41
  unsigned Opc = Node->getOpcode();
1462
41
  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1463
41
  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1464
41
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1465
41
1466
41
  return ExpandChainLibCall(LC, Node, false);
1467
41
}
1468
1469
/// N is a shift by a value that needs to be expanded,
1470
/// and the shift amount is a constant 'Amt'.  Expand the operation.
1471
void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1472
63.1k
                                             SDValue &Lo, SDValue &Hi) {
1473
63.1k
  SDLoc DL(N);
1474
63.1k
  // Expand the incoming operand to be shifted, so that we have its parts
1475
63.1k
  SDValue InL, InH;
1476
63.1k
  GetExpandedInteger(N->getOperand(0), InL, InH);
1477
63.1k
1478
63.1k
  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1479
63.1k
  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1480
63.1k
  if (
!Amt63.1k
) {
1481
0
    Lo = InL;
1482
0
    Hi = InH;
1483
0
    return;
1484
0
  }
1485
63.1k
1486
63.1k
  EVT NVT = InL.getValueType();
1487
63.1k
  unsigned VTBits = N->getValueType(0).getSizeInBits();
1488
63.1k
  unsigned NVTBits = NVT.getSizeInBits();
1489
63.1k
  EVT ShTy = N->getOperand(1).getValueType();
1490
63.1k
1491
63.1k
  if (
N->getOpcode() == ISD::SHL63.1k
) {
1492
14.0k
    if (
Amt.ugt(VTBits)14.0k
) {
1493
2
      Lo = Hi = DAG.getConstant(0, DL, NVT);
1494
14.0k
    } else 
if (14.0k
Amt.ugt(NVTBits)14.0k
) {
1495
2.77k
      Lo = DAG.getConstant(0, DL, NVT);
1496
2.77k
      Hi = DAG.getNode(ISD::SHL, DL,
1497
2.77k
                       NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1498
14.0k
    } else 
if (11.2k
Amt == NVTBits11.2k
) {
1499
10.2k
      Lo = DAG.getConstant(0, DL, NVT);
1500
10.2k
      Hi = InL;
1501
11.2k
    } else {
1502
964
      Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1503
964
      Hi = DAG.getNode(ISD::OR, DL, NVT,
1504
964
                       DAG.getNode(ISD::SHL, DL, NVT, InH,
1505
964
                                   DAG.getConstant(Amt, DL, ShTy)),
1506
964
                       DAG.getNode(ISD::SRL, DL, NVT, InL,
1507
964
                                   DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1508
964
    }
1509
14.0k
    return;
1510
14.0k
  }
1511
49.1k
1512
49.1k
  
if (49.1k
N->getOpcode() == ISD::SRL49.1k
) {
1513
48.7k
    if (
Amt.ugt(VTBits)48.7k
) {
1514
2
      Lo = Hi = DAG.getConstant(0, DL, NVT);
1515
48.7k
    } else 
if (48.7k
Amt.ugt(NVTBits)48.7k
) {
1516
447
      Lo = DAG.getNode(ISD::SRL, DL,
1517
447
                       NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1518
447
      Hi = DAG.getConstant(0, DL, NVT);
1519
48.7k
    } else 
if (48.2k
Amt == NVTBits48.2k
) {
1520
30.2k
      Lo = InH;
1521
30.2k
      Hi = DAG.getConstant(0, DL, NVT);
1522
48.2k
    } else {
1523
18.0k
      Lo = DAG.getNode(ISD::OR, DL, NVT,
1524
18.0k
                       DAG.getNode(ISD::SRL, DL, NVT, InL,
1525
18.0k
                                   DAG.getConstant(Amt, DL, ShTy)),
1526
18.0k
                       DAG.getNode(ISD::SHL, DL, NVT, InH,
1527
18.0k
                                   DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1528
18.0k
      Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1529
18.0k
    }
1530
48.7k
    return;
1531
48.7k
  }
1532
411
1533
49.1k
  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1534
411
  if (
Amt.ugt(VTBits)411
) {
1535
2
    Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1536
2
                          DAG.getConstant(NVTBits - 1, DL, ShTy));
1537
411
  } else 
if (409
Amt.ugt(NVTBits)409
) {
1538
354
    Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1539
354
                     DAG.getConstant(Amt - NVTBits, DL, ShTy));
1540
354
    Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1541
354
                     DAG.getConstant(NVTBits - 1, DL, ShTy));
1542
409
  } else 
if (55
Amt == NVTBits55
) {
1543
3
    Lo = InH;
1544
3
    Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1545
3
                     DAG.getConstant(NVTBits - 1, DL, ShTy));
1546
55
  } else {
1547
52
    Lo = DAG.getNode(ISD::OR, DL, NVT,
1548
52
                     DAG.getNode(ISD::SRL, DL, NVT, InL,
1549
52
                                 DAG.getConstant(Amt, DL, ShTy)),
1550
52
                     DAG.getNode(ISD::SHL, DL, NVT, InH,
1551
52
                                 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1552
52
    Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1553
52
  }
1554
63.1k
}
1555
1556
/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1557
/// this shift based on knowledge of the high bit of the shift amount.  If we
1558
/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1559
/// shift amount.
1560
bool DAGTypeLegalizer::
1561
1.37k
ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1562
1.37k
  SDValue Amt = N->getOperand(1);
1563
1.37k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1564
1.37k
  EVT ShTy = Amt.getValueType();
1565
1.37k
  unsigned ShBits = ShTy.getScalarSizeInBits();
1566
1.37k
  unsigned NVTBits = NVT.getScalarSizeInBits();
1567
1.37k
  assert(isPowerOf2_32(NVTBits) &&
1568
1.37k
         "Expanded integer type size not a power of two!");
1569
1.37k
  SDLoc dl(N);
1570
1.37k
1571
1.37k
  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1572
1.37k
  KnownBits Known;
1573
1.37k
  DAG.computeKnownBits(N->getOperand(1), Known);
1574
1.37k
1575
1.37k
  // If we don't know anything about the high bits, exit.
1576
1.37k
  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1577
1.26k
    return false;
1578
113
1579
113
  // Get the incoming operand to be shifted.
1580
113
  SDValue InL, InH;
1581
113
  GetExpandedInteger(N->getOperand(0), InL, InH);
1582
113
1583
113
  // If we know that any of the high bits of the shift amount are one, then we
1584
113
  // can do this as a couple of simple shifts.
1585
113
  if (
Known.One.intersects(HighBitMask)113
) {
1586
2
    // Mask out the high bit, which we know is set.
1587
2
    Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1588
2
                      DAG.getConstant(~HighBitMask, dl, ShTy));
1589
2
1590
2
    switch (N->getOpcode()) {
1591
0
    
default: 0
llvm_unreachable0
("Unknown shift");
1592
0
    case ISD::SHL:
1593
0
      Lo = DAG.getConstant(0, dl, NVT);              // Low part is zero.
1594
0
      Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1595
0
      return true;
1596
2
    case ISD::SRL:
1597
2
      Hi = DAG.getConstant(0, dl, NVT);              // Hi part is zero.
1598
2
      Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1599
2
      return true;
1600
0
    case ISD::SRA:
1601
0
      Hi = DAG.getNode(ISD::SRA, dl, NVT, InH,       // Sign extend high part.
1602
0
                       DAG.getConstant(NVTBits - 1, dl, ShTy));
1603
0
      Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1604
0
      return true;
1605
111
    }
1606
111
  }
1607
111
1608
111
  // If we know that all of the high bits of the shift amount are zero, then we
1609
111
  // can do this as a couple of simple shifts.
1610
111
  
if (111
HighBitMask.isSubsetOf(Known.Zero)111
) {
1611
11
    // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1612
11
    // shift if x is zero.  We can use XOR here because x is known to be smaller
1613
11
    // than 32.
1614
11
    SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1615
11
                               DAG.getConstant(NVTBits - 1, dl, ShTy));
1616
11
1617
11
    unsigned Op1, Op2;
1618
11
    switch (N->getOpcode()) {
1619
0
    
default: 0
llvm_unreachable0
("Unknown shift");
1620
5
    case ISD::SHL:  Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1621
6
    case ISD::SRL:
1622
6
    case ISD::SRA:  Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1623
11
    }
1624
11
1625
11
    // When shifting right the arithmetic for Lo and Hi is swapped.
1626
11
    
if (11
N->getOpcode() != ISD::SHL11
)
1627
6
      std::swap(InL, InH);
1628
11
1629
11
    // Use a little trick to get the bits that move from Lo to Hi. First
1630
11
    // shift by one bit.
1631
11
    SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1632
11
    // Then compute the remaining shift with amount-1.
1633
11
    SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1634
11
1635
11
    Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1636
11
    Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1637
11
1638
11
    if (N->getOpcode() != ISD::SHL)
1639
6
      std::swap(Hi, Lo);
1640
11
    return true;
1641
11
  }
1642
100
1643
100
  return false;
1644
100
}
1645
1646
/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1647
/// of any size.
1648
bool DAGTypeLegalizer::
1649
64
ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1650
64
  SDValue Amt = N->getOperand(1);
1651
64
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1652
64
  EVT ShTy = Amt.getValueType();
1653
64
  unsigned NVTBits = NVT.getSizeInBits();
1654
64
  assert(isPowerOf2_32(NVTBits) &&
1655
64
         "Expanded integer type size not a power of two!");
1656
64
  SDLoc dl(N);
1657
64
1658
64
  // Get the incoming operand to be shifted.
1659
64
  SDValue InL, InH;
1660
64
  GetExpandedInteger(N->getOperand(0), InL, InH);
1661
64
1662
64
  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1663
64
  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1664
64
  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1665
64
  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1666
64
                                 Amt, NVBitsNode, ISD::SETULT);
1667
64
  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1668
64
                                Amt, DAG.getConstant(0, dl, ShTy),
1669
64
                                ISD::SETEQ);
1670
64
1671
64
  SDValue LoS, HiS, LoL, HiL;
1672
64
  switch (N->getOpcode()) {
1673
0
  
default: 0
llvm_unreachable0
("Unknown shift");
1674
26
  case ISD::SHL:
1675
26
    // Short: ShAmt < NVTBits
1676
26
    LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1677
26
    HiS = DAG.getNode(ISD::OR, dl, NVT,
1678
26
                      DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1679
26
                      DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1680
26
1681
26
    // Long: ShAmt >= NVTBits
1682
26
    LoL = DAG.getConstant(0, dl, NVT);                    // Lo part is zero.
1683
26
    HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1684
26
1685
26
    Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1686
26
    Hi = DAG.getSelect(dl, NVT, isZero, InH,
1687
26
                       DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1688
26
    return true;
1689
21
  case ISD::SRL:
1690
21
    // Short: ShAmt < NVTBits
1691
21
    HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1692
21
    LoS = DAG.getNode(ISD::OR, dl, NVT,
1693
21
                      DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1694
21
    // FIXME: If Amt is zero, the following shift generates an undefined result
1695
21
    // on some architectures.
1696
21
                      DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1697
21
1698
21
    // Long: ShAmt >= NVTBits
1699
21
    HiL = DAG.getConstant(0, dl, NVT);                    // Hi part is zero.
1700
21
    LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1701
21
1702
21
    Lo = DAG.getSelect(dl, NVT, isZero, InL,
1703
21
                       DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1704
21
    Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1705
21
    return true;
1706
17
  case ISD::SRA:
1707
17
    // Short: ShAmt < NVTBits
1708
17
    HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1709
17
    LoS = DAG.getNode(ISD::OR, dl, NVT,
1710
17
                      DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1711
17
                      DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1712
17
1713
17
    // Long: ShAmt >= NVTBits
1714
17
    HiL = DAG.getNode(ISD::SRA, dl, NVT, InH,             // Sign of Hi part.
1715
17
                      DAG.getConstant(NVTBits - 1, dl, ShTy));
1716
17
    LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1717
17
1718
17
    Lo = DAG.getSelect(dl, NVT, isZero, InL,
1719
17
                       DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1720
17
    Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1721
17
    return true;
1722
0
  }
1723
0
}
1724
1725
9
static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
1726
9
1727
9
  switch (Op) {
1728
0
    
default: 0
llvm_unreachable0
("invalid min/max opcode");
1729
2
    case ISD::SMAX:
1730
2
      return std::make_pair(ISD::SETGT, ISD::UMAX);
1731
3
    case ISD::UMAX:
1732
3
      return std::make_pair(ISD::SETUGT, ISD::UMAX);
1733
2
    case ISD::SMIN:
1734
2
      return std::make_pair(ISD::SETLT, ISD::UMIN);
1735
2
    case ISD::UMIN:
1736
2
      return std::make_pair(ISD::SETULT, ISD::UMIN);
1737
0
  }
1738
0
}
1739
1740
void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
1741
9
                                           SDValue &Lo, SDValue &Hi) {
1742
9
  SDLoc DL(N);
1743
9
  ISD::NodeType LoOpc;
1744
9
  ISD::CondCode CondC;
1745
9
  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
1746
9
1747
9
  // Expand the subcomponents.
1748
9
  SDValue LHSL, LHSH, RHSL, RHSH;
1749
9
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1750
9
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1751
9
1752
9
  // Value types
1753
9
  EVT NVT = LHSL.getValueType();
1754
9
  EVT CCT = getSetCCResultType(NVT);
1755
9
1756
9
  // Hi part is always the same op
1757
9
  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
1758
9
1759
9
  // We need to know whether to select Lo part that corresponds to 'winning'
1760
9
  // Hi part or if Hi parts are equal.
1761
9
  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
1762
9
  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
1763
9
1764
9
  // Lo part corresponding to the 'winning' Hi part
1765
9
  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
1766
9
1767
9
  // Recursed Lo part if Hi parts are equal, this uses unsigned version
1768
9
  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
1769
9
1770
9
  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
1771
9
}
1772
1773
void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1774
5.38k
                                           SDValue &Lo, SDValue &Hi) {
1775
5.38k
  SDLoc dl(N);
1776
5.38k
  // Expand the subcomponents.
1777
5.38k
  SDValue LHSL, LHSH, RHSL, RHSH;
1778
5.38k
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1779
5.38k
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1780
5.38k
1781
5.38k
  EVT NVT = LHSL.getValueType();
1782
5.38k
  SDValue LoOps[2] = { LHSL, RHSL };
1783
5.38k
  SDValue HiOps[3] = { LHSH, RHSH };
1784
5.38k
1785
5.38k
  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1786
5.38k
      N->getOpcode() == ISD::ADD ? 
ISD::ADDCARRY3.78k
:
ISD::SUBCARRY1.60k
,
1787
5.38k
      TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1788
5.38k
  if (
HasOpCarry5.38k
) {
1789
4.18k
    SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
1790
4.18k
    if (
N->getOpcode() == ISD::ADD4.18k
) {
1791
3.36k
      Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1792
3.36k
      HiOps[2] = Lo.getValue(1);
1793
3.36k
      Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
1794
4.18k
    } else {
1795
821
      Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1796
821
      HiOps[2] = Lo.getValue(1);
1797
821
      Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
1798
821
    }
1799
4.18k
    return;
1800
4.18k
  }
1801
1.19k
1802
1.19k
  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1803
1.19k
  // them.  TODO: Teach operation legalization how to expand unsupported
1804
1.19k
  // ADDC/ADDE/SUBC/SUBE.  The problem is that these operations generate
1805
1.19k
  // a carry of type MVT::Glue, but there doesn't seem to be any way to
1806
1.19k
  // generate a value of this type in the expanded code sequence.
1807
1.19k
  bool hasCarry =
1808
1.19k
    TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
1809
1.19k
                                   
ISD::ADDC418
:
ISD::SUBC779
,
1810
1.19k
                                 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1811
1.19k
1812
1.19k
  if (
hasCarry1.19k
) {
1813
253
    SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1814
253
    if (
N->getOpcode() == ISD::ADD253
) {
1815
166
      Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1816
166
      HiOps[2] = Lo.getValue(1);
1817
166
      Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1818
253
    } else {
1819
87
      Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1820
87
      HiOps[2] = Lo.getValue(1);
1821
87
      Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1822
87
    }
1823
253
    return;
1824
253
  }
1825
944
1826
944
  bool hasOVF =
1827
944
    TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
1828
944
                                   
ISD::UADDO252
:
ISD::USUBO692
,
1829
944
                                 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1830
944
  TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
1831
944
1832
944
  if (
hasOVF944
) {
1833
686
    EVT OvfVT = getSetCCResultType(NVT);
1834
686
    SDVTList VTList = DAG.getVTList(NVT, OvfVT);
1835
686
    int RevOpc;
1836
686
    if (
N->getOpcode() == ISD::ADD686
) {
1837
59
      RevOpc = ISD::SUB;
1838
59
      Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
1839
59
      Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1840
686
    } else {
1841
627
      RevOpc = ISD::ADD;
1842
627
      Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
1843
627
      Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1844
627
    }
1845
686
    SDValue OVF = Lo.getValue(1);
1846
686
1847
686
    switch (BoolType) {
1848
0
    case TargetLoweringBase::UndefinedBooleanContent:
1849
0
      OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
1850
0
      LLVM_FALLTHROUGH;
1851
0
    case TargetLoweringBase::ZeroOrOneBooleanContent:
1852
0
      OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
1853
0
      Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
1854
0
      break;
1855
686
    case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
1856
686
      OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
1857
686
      Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
1858
686
    }
1859
686
    return;
1860
258
  }
1861
258
1862
258
  
if (258
N->getOpcode() == ISD::ADD258
) {
1863
193
    Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1864
193
    Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1865
193
    SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1866
193
                                ISD::SETULT);
1867
193
1868
193
    if (
BoolType == TargetLoweringBase::ZeroOrOneBooleanContent193
) {
1869
191
      SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
1870
191
      Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
1871
191
      return;
1872
191
    }
1873
2
1874
2
    SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1875
2
                                   DAG.getConstant(1, dl, NVT),
1876
2
                                   DAG.getConstant(0, dl, NVT));
1877
2
    SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1878
2
                                ISD::SETULT);
1879
2
    SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1880
2
                                   DAG.getConstant(1, dl, NVT), Carry1);
1881
2
    Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1882
258
  } else {
1883
65
    Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1884
65
    Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1885
65
    SDValue Cmp =
1886
65
      DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1887
65
                   LoOps[0], LoOps[1], ISD::SETULT);
1888
65
1889
65
    SDValue Borrow;
1890
65
    if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
1891
64
      Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
1892
65
    else
1893
1
      Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
1894
1
                             DAG.getConstant(0, dl, NVT));
1895
65
1896
65
    Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1897
65
  }
1898
5.38k
}
1899
1900
void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1901
7
                                            SDValue &Lo, SDValue &Hi) {
1902
7
  // Expand the subcomponents.
1903
7
  SDValue LHSL, LHSH, RHSL, RHSH;
1904
7
  SDLoc dl(N);
1905
7
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1906
7
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1907
7
  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1908
7
  SDValue LoOps[2] = { LHSL, RHSL };
1909
7
  SDValue HiOps[3] = { LHSH, RHSH };
1910
7
1911
7
  if (
N->getOpcode() == ISD::ADDC7
) {
1912
5
    Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1913
5
    HiOps[2] = Lo.getValue(1);
1914
5
    Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1915
7
  } else {
1916
2
    Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1917
2
    HiOps[2] = Lo.getValue(1);
1918
2
    Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1919
2
  }
1920
7
1921
7
  // Legalized the flag result - switch anything that used the old flag to
1922
7
  // use the new one.
1923
7
  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1924
7
}
1925
1926
void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1927
7
                                            SDValue &Lo, SDValue &Hi) {
1928
7
  // Expand the subcomponents.
1929
7
  SDValue LHSL, LHSH, RHSL, RHSH;
1930
7
  SDLoc dl(N);
1931
7
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1932
7
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1933
7
  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1934
7
  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1935
7
  SDValue HiOps[3] = { LHSH, RHSH };
1936
7
1937
7
  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1938
7
  HiOps[2] = Lo.getValue(1);
1939
7
  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1940
7
1941
7
  // Legalized the flag result - switch anything that used the old flag to
1942
7
  // use the new one.
1943
7
  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1944
7
}
1945
1946
void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
1947
547
                                             SDValue &Lo, SDValue &Hi) {
1948
547
  SDValue LHS = N->getOperand(0);
1949
547
  SDValue RHS = N->getOperand(1);
1950
547
  SDLoc dl(N);
1951
547
1952
547
  SDValue Ovf;
1953
547
1954
547
  bool HasOpCarry = TLI.isOperationLegalOrCustom(
1955
547
      N->getOpcode() == ISD::ADD ? 
ISD::ADDCARRY0
:
ISD::SUBCARRY547
,
1956
547
      TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
1957
547
1958
547
  if (
HasOpCarry547
) {
1959
541
    // Expand the subcomponents.
1960
541
    SDValue LHSL, LHSH, RHSL, RHSH;
1961
541
    GetExpandedInteger(LHS, LHSL, LHSH);
1962
541
    GetExpandedInteger(RHS, RHSL, RHSH);
1963
541
    SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
1964
541
    SDValue LoOps[2] = { LHSL, RHSL };
1965
541
    SDValue HiOps[3] = { LHSH, RHSH };
1966
541
1967
541
    unsigned Opc = N->getOpcode() == ISD::UADDO ? 
ISD::ADDCARRY497
:
ISD::SUBCARRY44
;
1968
541
    Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1969
541
    HiOps[2] = Lo.getValue(1);
1970
541
    Hi = DAG.getNode(Opc, dl, VTList, HiOps);
1971
541
1972
541
    Ovf = Hi.getValue(1);
1973
547
  } else {
1974
6
    // Expand the result by simply replacing it with the equivalent
1975
6
    // non-overflow-checking operation.
1976
6
    auto Opc = N->getOpcode() == ISD::UADDO ? 
ISD::ADD3
:
ISD::SUB3
;
1977
6
    SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
1978
6
    SplitInteger(Sum, Lo, Hi);
1979
6
1980
6
    // Calculate the overflow: addition overflows iff a + b < a, and subtraction
1981
6
    // overflows iff a - b > a.
1982
6
    auto Cond = N->getOpcode() == ISD::UADDO ? 
ISD::SETULT3
:
ISD::SETUGT3
;
1983
6
    Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
1984
6
  }
1985
547
1986
547
  // Legalized the flag result - switch anything that used the old flag to
1987
547
  // use the new one.
1988
547
  ReplaceValueWith(SDValue(N, 1), Ovf);
1989
547
}
1990
1991
void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
1992
760
                                                SDValue &Lo, SDValue &Hi) {
1993
760
  // Expand the subcomponents.
1994
760
  SDValue LHSL, LHSH, RHSL, RHSH;
1995
760
  SDLoc dl(N);
1996
760
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1997
760
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1998
760
  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
1999
760
  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2000
760
  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2001
760
2002
760
  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2003
760
  HiOps[2] = Lo.getValue(1);
2004
760
  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2005
760
2006
760
  // Legalized the flag result - switch anything that used the old flag to
2007
760
  // use the new one.
2008
760
  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2009
760
}
2010
2011
void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2012
4.47k
                                               SDValue &Lo, SDValue &Hi) {
2013
4.47k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2014
4.47k
  SDLoc dl(N);
2015
4.47k
  SDValue Op = N->getOperand(0);
2016
4.47k
  if (
Op.getValueType().bitsLE(NVT)4.47k
) {
2017
4.41k
    // The low part is any extension of the input (which degenerates to a copy).
2018
4.41k
    Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2019
4.41k
    Hi = DAG.getUNDEF(NVT);   // The high part is undefined.
2020
4.47k
  } else {
2021
63
    // For example, extension of an i48 to an i64.  The operand type necessarily
2022
63
    // promotes to the result type, so will end up being expanded too.
2023
63
    assert(getTypeAction(Op.getValueType()) ==
2024
63
           TargetLowering::TypePromoteInteger &&
2025
63
           "Only know how to promote this result!");
2026
63
    SDValue Res = GetPromotedInteger(Op);
2027
63
    assert(Res.getValueType() == N->getValueType(0) &&
2028
63
           "Operand over promoted?");
2029
63
    // Split the promoted operand.  This will simplify when it is expanded.
2030
63
    SplitInteger(Res, Lo, Hi);
2031
63
  }
2032
4.47k
}
2033
2034
void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2035
317
                                               SDValue &Lo, SDValue &Hi) {
2036
317
  SDLoc dl(N);
2037
317
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2038
317
  EVT NVT = Lo.getValueType();
2039
317
  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2040
317
  unsigned NVTBits = NVT.getSizeInBits();
2041
317
  unsigned EVTBits = EVT.getSizeInBits();
2042
317
2043
317
  if (
NVTBits < EVTBits317
) {
2044
254
    Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2045
254
                     DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2046
254
                                                        EVTBits - NVTBits)));
2047
317
  } else {
2048
63
    Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2049
63
    // The high part replicates the sign bit of Lo, make it explicit.
2050
63
    Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2051
63
                     DAG.getConstant(NVTBits - 1, dl,
2052
63
                                     TLI.getPointerTy(DAG.getDataLayout())));
2053
63
  }
2054
317
}
2055
2056
void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2057
315
                                               SDValue &Lo, SDValue &Hi) {
2058
315
  SDLoc dl(N);
2059
315
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2060
315
  EVT NVT = Lo.getValueType();
2061
315
  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2062
315
  unsigned NVTBits = NVT.getSizeInBits();
2063
315
  unsigned EVTBits = EVT.getSizeInBits();
2064
315
2065
315
  if (
NVTBits < EVTBits315
) {
2066
252
    Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2067
252
                     DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2068
252
                                                        EVTBits - NVTBits)));
2069
315
  } else {
2070
63
    Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2071
63
    // The high part must be zero, make it explicit.
2072
63
    Hi = DAG.getConstant(0, dl, NVT);
2073
63
  }
2074
315
}
2075
2076
void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2077
1
                                               SDValue &Lo, SDValue &Hi) {
2078
1
  SDLoc dl(N);
2079
1
  GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
2080
1
  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2081
1
  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2082
1
}
2083
2084
void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2085
47
                                          SDValue &Lo, SDValue &Hi) {
2086
47
  SDLoc dl(N);
2087
47
  GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
2088
47
  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2089
47
  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2090
47
}
2091
2092
void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2093
11.7k
                                             SDValue &Lo, SDValue &Hi) {
2094
11.7k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2095
11.7k
  unsigned NBitWidth = NVT.getSizeInBits();
2096
11.7k
  auto Constant = cast<ConstantSDNode>(N);
2097
11.7k
  const APInt &Cst = Constant->getAPIntValue();
2098
11.7k
  bool IsTarget = Constant->isTargetOpcode();
2099
11.7k
  bool IsOpaque = Constant->isOpaque();
2100
11.7k
  SDLoc dl(N);
2101
11.7k
  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2102
11.7k
  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2103
11.7k
                       IsOpaque);
2104
11.7k
}
2105
2106
void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2107
100
                                         SDValue &Lo, SDValue &Hi) {
2108
100
  SDLoc dl(N);
2109
100
  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2110
100
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2111
100
  EVT NVT = Lo.getValueType();
2112
100
2113
100
  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2114
100
                                   DAG.getConstant(0, dl, NVT), ISD::SETNE);
2115
100
2116
100
  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2117
100
  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2118
100
2119
100
  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2120
100
                     DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2121
100
                                 DAG.getConstant(NVT.getSizeInBits(), dl,
2122
100
                                                 NVT)));
2123
100
  Hi = DAG.getConstant(0, dl, NVT);
2124
100
}
2125
2126
void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2127
11
                                          SDValue &Lo, SDValue &Hi) {
2128
11
  SDLoc dl(N);
2129
11
  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2130
11
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2131
11
  EVT NVT = Lo.getValueType();
2132
11
  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2133
11
                   DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2134
11
  Hi = DAG.getConstant(0, dl, NVT);
2135
11
}
2136
2137
void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2138
8
                                         SDValue &Lo, SDValue &Hi) {
2139
8
  SDLoc dl(N);
2140
8
  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2141
8
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2142
8
  EVT NVT = Lo.getValueType();
2143
8
2144
8
  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2145
8
                                   DAG.getConstant(0, dl, NVT), ISD::SETNE);
2146
8
2147
8
  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2148
8
  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2149
8
2150
8
  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2151
8
                     DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2152
8
                                 DAG.getConstant(NVT.getSizeInBits(), dl,
2153
8
                                                 NVT)));
2154
8
  Hi = DAG.getConstant(0, dl, NVT);
2155
8
}
2156
2157
void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2158
1
                                               SDValue &Hi) {
2159
1
  SDLoc dl(N);
2160
1
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2161
1
  unsigned NBitWidth = NVT.getSizeInBits();
2162
1
2163
1
  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2164
1
  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2165
1
  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2166
1
  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2167
1
                   DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2168
1
}
2169
2170
void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2171
89
                                               SDValue &Hi) {
2172
89
  SDLoc dl(N);
2173
89
  EVT VT = N->getValueType(0);
2174
89
2175
89
  SDValue Op = N->getOperand(0);
2176
89
  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2177
6
    Op = GetPromotedFloat(Op);
2178
89
2179
89
  RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
2180
89
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2181
89
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2182
89
               Lo, Hi);
2183
89
}
2184
2185
void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2186
81
                                               SDValue &Hi) {
2187
81
  SDLoc dl(N);
2188
81
  EVT VT = N->getValueType(0);
2189
81
2190
81
  SDValue Op = N->getOperand(0);
2191
81
  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2192
4
    Op = GetPromotedFloat(Op);
2193
81
2194
81
  RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
2195
81
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2196
81
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2197
81
               Lo, Hi);
2198
81
}
2199
2200
void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2201
7.26k
                                         SDValue &Lo, SDValue &Hi) {
2202
7.26k
  if (
ISD::isNormalLoad(N)7.26k
) {
2203
6.49k
    ExpandRes_NormalLoad(N, Lo, Hi);
2204
6.49k
    return;
2205
6.49k
  }
2206
778
2207
7.26k
  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2208
778
2209
778
  EVT VT = N->getValueType(0);
2210
778
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2211
778
  SDValue Ch  = N->getChain();
2212
778
  SDValue Ptr = N->getBasePtr();
2213
778
  ISD::LoadExtType ExtType = N->getExtensionType();
2214
778
  unsigned Alignment = N->getAlignment();
2215
778
  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2216
778
  AAMDNodes AAInfo = N->getAAInfo();
2217
778
  SDLoc dl(N);
2218
778
2219
778
  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2220
778
2221
778
  if (
N->getMemoryVT().bitsLE(NVT)778
) {
2222
454
    EVT MemVT = N->getMemoryVT();
2223
454
2224
454
    Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2225
454
                        Alignment, MMOFlags, AAInfo);
2226
454
2227
454
    // Remember the chain.
2228
454
    Ch = Lo.getValue(1);
2229
454
2230
454
    if (
ExtType == ISD::SEXTLOAD454
) {
2231
86
      // The high part is obtained by SRA'ing all but one of the bits of the
2232
86
      // lo part.
2233
86
      unsigned LoSize = Lo.getValueSizeInBits();
2234
86
      Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2235
86
                       DAG.getConstant(LoSize - 1, dl,
2236
86
                                       TLI.getPointerTy(DAG.getDataLayout())));
2237
454
    } else 
if (368
ExtType == ISD::ZEXTLOAD368
) {
2238
236
      // The high part is just a zero.
2239
236
      Hi = DAG.getConstant(0, dl, NVT);
2240
368
    } else {
2241
132
      assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2242
132
      // The high part is undefined.
2243
132
      Hi = DAG.getUNDEF(NVT);
2244
132
    }
2245
778
  } else 
if (324
DAG.getDataLayout().isLittleEndian()324
) {
2246
316
    // Little-endian - low bits are at low addresses.
2247
316
    Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2248
316
                     AAInfo);
2249
316
2250
316
    unsigned ExcessBits =
2251
316
      N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2252
316
    EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2253
316
2254
316
    // Increment the pointer to the other half.
2255
316
    unsigned IncrementSize = NVT.getSizeInBits()/8;
2256
316
    Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2257
316
                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2258
316
    Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2259
316
                        N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2260
316
                        MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2261
316
2262
316
    // Build a factor node to remember that this load is independent of the
2263
316
    // other one.
2264
316
    Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2265
316
                     Hi.getValue(1));
2266
324
  } else {
2267
8
    // Big-endian - high bits are at low addresses.  Favor aligned loads at
2268
8
    // the cost of some bit-fiddling.
2269
8
    EVT MemVT = N->getMemoryVT();
2270
8
    unsigned EBytes = MemVT.getStoreSize();
2271
8
    unsigned IncrementSize = NVT.getSizeInBits()/8;
2272
8
    unsigned ExcessBits = (EBytes - IncrementSize)*8;
2273
8
2274
8
    // Load both the high bits and maybe some of the low bits.
2275
8
    Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2276
8
                        EVT::getIntegerVT(*DAG.getContext(),
2277
8
                                          MemVT.getSizeInBits() - ExcessBits),
2278
8
                        Alignment, MMOFlags, AAInfo);
2279
8
2280
8
    // Increment the pointer to the other half.
2281
8
    Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2282
8
                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2283
8
    // Load the rest of the low bits.
2284
8
    Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2285
8
                        N->getPointerInfo().getWithOffset(IncrementSize),
2286
8
                        EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2287
8
                        MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2288
8
2289
8
    // Build a factor node to remember that this load is independent of the
2290
8
    // other one.
2291
8
    Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2292
8
                     Hi.getValue(1));
2293
8
2294
8
    if (
ExcessBits < NVT.getSizeInBits()8
) {
2295
7
      // Transfer low bits from the bottom of Hi to the top of Lo.
2296
7
      Lo = DAG.getNode(
2297
7
          ISD::OR, dl, NVT, Lo,
2298
7
          DAG.getNode(ISD::SHL, dl, NVT, Hi,
2299
7
                      DAG.getConstant(ExcessBits, dl,
2300
7
                                      TLI.getPointerTy(DAG.getDataLayout()))));
2301
7
      // Move high bits to the right position in Hi.
2302
7
      Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? 
ISD::SRA0
:
ISD::SRL7
, dl, NVT,
2303
7
                       Hi,
2304
7
                       DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2305
7
                                       TLI.getPointerTy(DAG.getDataLayout())));
2306
7
    }
2307
324
  }
2308
7.26k
2309
7.26k
  // Legalize the chain result - switch anything that used the old chain to
2310
7.26k
  // use the new one.
2311
7.26k
  ReplaceValueWith(SDValue(N, 1), Ch);
2312
7.26k
}
2313
2314
void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2315
20.2k
                                            SDValue &Lo, SDValue &Hi) {
2316
20.2k
  SDLoc dl(N);
2317
20.2k
  SDValue LL, LH, RL, RH;
2318
20.2k
  GetExpandedInteger(N->getOperand(0), LL, LH);
2319
20.2k
  GetExpandedInteger(N->getOperand(1), RL, RH);
2320
20.2k
  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2321
20.2k
  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2322
20.2k
}
2323
2324
void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2325
1.67k
                                        SDValue &Lo, SDValue &Hi) {
2326
1.67k
  EVT VT = N->getValueType(0);
2327
1.67k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2328
1.67k
  SDLoc dl(N);
2329
1.67k
2330
1.67k
  SDValue LL, LH, RL, RH;
2331
1.67k
  GetExpandedInteger(N->getOperand(0), LL, LH);
2332
1.67k
  GetExpandedInteger(N->getOperand(1), RL, RH);
2333
1.67k
2334
1.67k
  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2335
1.67k
                    TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
2336
1.67k
                    LL, LH, RL, RH))
2337
1.27k
    return;
2338
400
2339
400
  // If nothing else, we can make a libcall.
2340
400
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2341
400
  if (VT == MVT::i16)
2342
0
    LC = RTLIB::MUL_I16;
2343
400
  else 
if (400
VT == MVT::i32400
)
2344
7
    LC = RTLIB::MUL_I32;
2345
393
  else 
if (393
VT == MVT::i64393
)
2346
105
    LC = RTLIB::MUL_I64;
2347
288
  else 
if (288
VT == MVT::i128288
)
2348
194
    LC = RTLIB::MUL_I128;
2349
400
2350
400
  if (
LC == RTLIB::UNKNOWN_LIBCALL || 400
!TLI.getLibcallName(LC)306
) {
2351
100
    // We'll expand the multiplication by brute force because we have no other
2352
100
    // options. This is a trivially-generalized version of the code from
2353
100
    // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2354
100
    // 4.3.1).
2355
100
    unsigned Bits = NVT.getSizeInBits();
2356
100
    unsigned HalfBits = Bits >> 1;
2357
100
    SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2358
100
                                   NVT);
2359
100
    SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2360
100
    SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2361
100
2362
100
    SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2363
100
    SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2364
100
2365
100
    EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2366
100
    if (
APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)100
) {
2367
2
      // The type from TLI is too small to fit the shift amount we want.
2368
2
      // Override it with i32. The shift will have to be legalized.
2369
2
      ShiftAmtTy = MVT::i32;
2370
2
    }
2371
100
    SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2372
100
    SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2373
100
    SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2374
100
    SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2375
100
2376
100
    SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2377
100
                            DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2378
100
    SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2379
100
    SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2380
100
2381
100
    SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2382
100
                            DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2383
100
    SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2384
100
2385
100
    SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2386
100
                            DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2387
100
                            DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2388
100
    Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2389
100
                     DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2390
100
2391
100
    Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2392
100
                     DAG.getNode(ISD::ADD, dl, NVT,
2393
100
                                 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2394
100
                                 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2395
100
    return;
2396
100
  }
2397
300
2398
300
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2399
300
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2400
300
               Lo, Hi);
2401
300
}
2402
2403
void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2404
2
                                                     SDValue &Hi) {
2405
2
  SDLoc DL(N);
2406
2
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2407
2
  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2408
2
  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2409
2
  Lo = R.getValue(0);
2410
2
  Hi = R.getValue(1);
2411
2
  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2412
2
}
2413
2414
void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2415
20
                                             SDValue &Lo, SDValue &Hi) {
2416
20
  SDValue LHS = Node->getOperand(0);
2417
20
  SDValue RHS = Node->getOperand(1);
2418
20
  SDLoc dl(Node);
2419
20
2420
20
  // Expand the result by simply replacing it with the equivalent
2421
20
  // non-overflow-checking operation.
2422
20
  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2423
20
                            
ISD::ADD15
:
ISD::SUB5
, dl, LHS.getValueType(),
2424
20
                            LHS, RHS);
2425
20
  SplitInteger(Sum, Lo, Hi);
2426
20
2427
20
  // Compute the overflow.
2428
20
  //
2429
20
  //   LHSSign -> LHS >= 0
2430
20
  //   RHSSign -> RHS >= 0
2431
20
  //   SumSign -> Sum >= 0
2432
20
  //
2433
20
  //   Add:
2434
20
  //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2435
20
  //   Sub:
2436
20
  //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2437
20
  //
2438
20
  EVT OType = Node->getValueType(1);
2439
20
  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2440
20
2441
20
  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2442
20
  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2443
20
  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2444
20
                                    Node->getOpcode() == ISD::SADDO ?
2445
20
                                    
ISD::SETEQ15
:
ISD::SETNE5
);
2446
20
2447
20
  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2448
20
  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2449
20
2450
20
  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2451
20
2452
20
  // Use the calculated overflow everywhere.
2453
20
  ReplaceValueWith(SDValue(Node, 1), Cmp);
2454
20
}
2455
2456
void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2457
107
                                         SDValue &Lo, SDValue &Hi) {
2458
107
  EVT VT = N->getValueType(0);
2459
107
  SDLoc dl(N);
2460
107
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2461
107
2462
107
  if (
TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom107
) {
2463
3
    SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2464
3
    SplitInteger(Res.getValue(0), Lo, Hi);
2465
3
    return;
2466
3
  }
2467
104
2468
104
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2469
104
  if (VT == MVT::i16)
2470
0
    LC = RTLIB::SDIV_I16;
2471
104
  else 
if (104
VT == MVT::i32104
)
2472
1
    LC = RTLIB::SDIV_I32;
2473
103
  else 
if (103
VT == MVT::i64103
)
2474
72
    LC = RTLIB::SDIV_I64;
2475
31
  else 
if (31
VT == MVT::i12831
)
2476
31
    LC = RTLIB::SDIV_I128;
2477
107
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2478
107
2479
107
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2480
107
}
2481
2482
void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2483
64.5k
                                          SDValue &Lo, SDValue &Hi) {
2484
64.5k
  EVT VT = N->getValueType(0);
2485
64.5k
  SDLoc dl(N);
2486
64.5k
2487
64.5k
  // If we can emit an efficient shift operation, do so now.  Check to see if
2488
64.5k
  // the RHS is a constant.
2489
64.5k
  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2490
63.1k
    return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
2491
1.37k
2492
1.37k
  // If we can determine that the high bit of the shift is zero or one, even if
2493
1.37k
  // the low bits are variable, emit this shift in an optimized form.
2494
1.37k
  
if (1.37k
ExpandShiftWithKnownAmountBit(N, Lo, Hi)1.37k
)
2495
13
    return;
2496
1.36k
2497
1.36k
  // If this target supports shift_PARTS, use it.  First, map to the _PARTS opc.
2498
1.36k
  unsigned PartsOpc;
2499
1.36k
  if (
N->getOpcode() == ISD::SHL1.36k
) {
2500
761
    PartsOpc = ISD::SHL_PARTS;
2501
1.36k
  } else 
if (604
N->getOpcode() == ISD::SRL604
) {
2502
515
    PartsOpc = ISD::SRL_PARTS;
2503
604
  } else {
2504
89
    assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2505
89
    PartsOpc = ISD::SRA_PARTS;
2506
89
  }
2507
1.36k
2508
1.36k
  // Next check to see if the target supports this SHL_PARTS operation or if it
2509
1.36k
  // will custom expand it.
2510
1.36k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2511
1.36k
  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2512
1.36k
  if (
(Action == TargetLowering::Legal && 1.36k
TLI.isTypeLegal(NVT)64
) ||
2513
1.36k
      
Action == TargetLowering::Custom1.36k
) {
2514
1.30k
    // Expand the subcomponents.
2515
1.30k
    SDValue LHSL, LHSH;
2516
1.30k
    GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2517
1.30k
    EVT VT = LHSL.getValueType();
2518
1.30k
2519
1.30k
    // If the shift amount operand is coming from a vector legalization it may
2520
1.30k
    // have an illegal type.  Fix that first by casting the operand, otherwise
2521
1.30k
    // the new SHL_PARTS operation would need further legalization.
2522
1.30k
    SDValue ShiftOp = N->getOperand(1);
2523
1.30k
    EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2524
1.30k
    assert(ShiftTy.getScalarSizeInBits() >=
2525
1.30k
           Log2_32_Ceil(VT.getScalarSizeInBits()) &&
2526
1.30k
           "ShiftAmountTy is too small to cover the range of this type!");
2527
1.30k
    if (ShiftOp.getValueType() != ShiftTy)
2528
55
      ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2529
1.30k
2530
1.30k
    SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2531
1.30k
    Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2532
1.30k
    Hi = Lo.getValue(1);
2533
1.30k
    return;
2534
1.30k
  }
2535
65
2536
65
  // Otherwise, emit a libcall.
2537
65
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2538
65
  bool isSigned;
2539
65
  if (
N->getOpcode() == ISD::SHL65
) {
2540
27
    isSigned = false; /*sign irrelevant*/
2541
27
    if (VT == MVT::i16)
2542
0
      LC = RTLIB::SHL_I16;
2543
27
    else 
if (27
VT == MVT::i3227
)
2544
0
      LC = RTLIB::SHL_I32;
2545
27
    else 
if (27
VT == MVT::i6427
)
2546
0
      LC = RTLIB::SHL_I64;
2547
27
    else 
if (27
VT == MVT::i12827
)
2548
18
      LC = RTLIB::SHL_I128;
2549
65
  } else 
if (38
N->getOpcode() == ISD::SRL38
) {
2550
21
    isSigned = false;
2551
21
    if (VT == MVT::i16)
2552
0
      LC = RTLIB::SRL_I16;
2553
21
    else 
if (21
VT == MVT::i3221
)
2554
0
      LC = RTLIB::SRL_I32;
2555
21
    else 
if (21
VT == MVT::i6421
)
2556
0
      LC = RTLIB::SRL_I64;
2557
21
    else 
if (21
VT == MVT::i12821
)
2558
15
      LC = RTLIB::SRL_I128;
2559
38
  } else {
2560
17
    assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2561
17
    isSigned = true;
2562
17
    if (VT == MVT::i16)
2563
0
      LC = RTLIB::SRA_I16;
2564
17
    else 
if (17
VT == MVT::i3217
)
2565
0
      LC = RTLIB::SRA_I32;
2566
17
    else 
if (17
VT == MVT::i6417
)
2567
0
      LC = RTLIB::SRA_I64;
2568
17
    else 
if (17
VT == MVT::i12817
)
2569
14
      LC = RTLIB::SRA_I128;
2570
38
  }
2571
65
2572
65
  if (
LC != RTLIB::UNKNOWN_LIBCALL && 65
TLI.getLibcallName(LC)47
) {
2573
1
    SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2574
1
    SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
2575
1
    return;
2576
1
  }
2577
64
2578
64
  
if (64
!ExpandShiftWithUnknownAmountBit(N, Lo, Hi)64
)
2579
0
    llvm_unreachable("Unsupported shift!");
2580
64.5k
}
2581
2582
void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2583
1.82k
                                                SDValue &Lo, SDValue &Hi) {
2584
1.82k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2585
1.82k
  SDLoc dl(N);
2586
1.82k
  SDValue Op = N->getOperand(0);
2587
1.82k
  if (
Op.getValueType().bitsLE(NVT)1.82k
) {
2588
1.82k
    // The low part is sign extension of the input (degenerates to a copy).
2589
1.82k
    Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2590
1.82k
    // The high part is obtained by SRA'ing all but one of the bits of low part.
2591
1.82k
    unsigned LoSize = NVT.getSizeInBits();
2592
1.82k
    Hi = DAG.getNode(
2593
1.82k
        ISD::SRA, dl, NVT, Lo,
2594
1.82k
        DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
2595
1.82k
  } else {
2596
1
    // For example, extension of an i48 to an i64.  The operand type necessarily
2597
1
    // promotes to the result type, so will end up being expanded too.
2598
1
    assert(getTypeAction(Op.getValueType()) ==
2599
1
           TargetLowering::TypePromoteInteger &&
2600
1
           "Only know how to promote this result!");
2601
1
    SDValue Res = GetPromotedInteger(Op);
2602
1
    assert(Res.getValueType() == N->getValueType(0) &&
2603
1
           "Operand over promoted?");
2604
1
    // Split the promoted operand.  This will simplify when it is expanded.
2605
1
    SplitInteger(Res, Lo, Hi);
2606
1
    unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2607
1
    Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2608
1
                     DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2609
1
                                                        ExcessBits)));
2610
1
  }
2611
1.82k
}
2612
2613
void DAGTypeLegalizer::
2614
156
ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2615
156
  SDLoc dl(N);
2616
156
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2617
156
  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2618
156
2619
156
  if (
EVT.bitsLE(Lo.getValueType())156
) {
2620
87
    // sext_inreg the low part if needed.
2621
87
    Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2622
87
                     N->getOperand(1));
2623
87
2624
87
    // The high part gets the sign extension from the lo-part.  This handles
2625
87
    // things like sextinreg V:i64 from i8.
2626
87
    Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2627
87
                     DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
2628
87
                                     TLI.getPointerTy(DAG.getDataLayout())));
2629
156
  } else {
2630
69
    // For example, extension of an i48 to an i64.  Leave the low part alone,
2631
69
    // sext_inreg the high part.
2632
69
    unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
2633
69
    Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2634
69
                     DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2635
69
                                                        ExcessBits)));
2636
69
  }
2637
156
}
2638
2639
void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2640
73
                                         SDValue &Lo, SDValue &Hi) {
2641
73
  EVT VT = N->getValueType(0);
2642
73
  SDLoc dl(N);
2643
73
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2644
73
2645
73
  if (
TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom73
) {
2646
9
    SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2647
9
    SplitInteger(Res.getValue(1), Lo, Hi);
2648
9
    return;
2649
9
  }
2650
64
2651
64
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2652
64
  if (VT == MVT::i16)
2653
0
    LC = RTLIB::SREM_I16;
2654
64
  else 
if (64
VT == MVT::i3264
)
2655
1
    LC = RTLIB::SREM_I32;
2656
63
  else 
if (63
VT == MVT::i6463
)
2657
46
    LC = RTLIB::SREM_I64;
2658
17
  else 
if (17
VT == MVT::i12817
)
2659
17
    LC = RTLIB::SREM_I128;
2660
73
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2661
73
2662
73
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
2663
73
}
2664
2665
void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2666
12.3k
                                             SDValue &Lo, SDValue &Hi) {
2667
12.3k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2668
12.3k
  SDLoc dl(N);
2669
12.3k
  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2670
12.3k
  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
2671
12.3k
                   N->getOperand(0),
2672
12.3k
                   DAG.getConstant(NVT.getSizeInBits(), dl,
2673
12.3k
                                   TLI.getPointerTy(DAG.getDataLayout())));
2674
12.3k
  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2675
12.3k
}
2676
2677
void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2678
11
                                          SDValue &Lo, SDValue &Hi) {
2679
11
  EVT VT = N->getValueType(0);
2680
11
  SDLoc dl(N);
2681
11
2682
11
  // A divide for UMULO should be faster than a function call.
2683
11
  if (
N->getOpcode() == ISD::UMULO11
) {
2684
2
    SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2685
2
2686
2
    SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
2687
2
    SplitInteger(MUL, Lo, Hi);
2688
2
2689
2
    // A divide for UMULO will be faster than a function call. Select to
2690
2
    // make sure we aren't using 0.
2691
2
    SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
2692
2
                                  RHS, DAG.getConstant(0, dl, VT), ISD::SETEQ);
2693
2
    SDValue NotZero = DAG.getSelect(dl, VT, isZero,
2694
2
                                    DAG.getConstant(1, dl, VT), RHS);
2695
2
    SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
2696
2
    SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
2697
2
                                    ISD::SETNE);
2698
2
    Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
2699
2
                             DAG.getConstant(0, dl, N->getValueType(1)),
2700
2
                             Overflow);
2701
2
    ReplaceValueWith(SDValue(N, 1), Overflow);
2702
2
    return;
2703
2
  }
2704
9
2705
9
  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2706
9
  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
2707
9
  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2708
9
2709
9
  // Replace this with a libcall that will check overflow.
2710
9
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2711
9
  if (VT == MVT::i32)
2712
0
    LC = RTLIB::MULO_I32;
2713
9
  else 
if (9
VT == MVT::i649
)
2714
7
    LC = RTLIB::MULO_I64;
2715
2
  else 
if (2
VT == MVT::i1282
)
2716
2
    LC = RTLIB::MULO_I128;
2717
9
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2718
9
2719
9
  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2720
9
  // Temporary for the overflow value, default it to zero.
2721
9
  SDValue Chain =
2722
9
      DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
2723
9
                   MachinePointerInfo());
2724
9
2725
9
  TargetLowering::ArgListTy Args;
2726
9
  TargetLowering::ArgListEntry Entry;
2727
18
  for (const SDValue &Op : N->op_values()) {
2728
18
    EVT ArgVT = Op.getValueType();
2729
18
    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2730
18
    Entry.Node = Op;
2731
18
    Entry.Ty = ArgTy;
2732
18
    Entry.IsSExt = true;
2733
18
    Entry.IsZExt = false;
2734
18
    Args.push_back(Entry);
2735
18
  }
2736
11
2737
11
  // Also pass the address of the overflow check.
2738
11
  Entry.Node = Temp;
2739
11
  Entry.Ty = PtrTy->getPointerTo();
2740
11
  Entry.IsSExt = true;
2741
11
  Entry.IsZExt = false;
2742
11
  Args.push_back(Entry);
2743
11
2744
11
  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2745
11
2746
11
  TargetLowering::CallLoweringInfo CLI(DAG);
2747
11
  CLI.setDebugLoc(dl)
2748
11
      .setChain(Chain)
2749
11
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
2750
11
      .setSExtResult();
2751
11
2752
11
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2753
11
2754
11
  SplitInteger(CallInfo.first, Lo, Hi);
2755
11
  SDValue Temp2 =
2756
11
      DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
2757
11
  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2758
11
                             DAG.getConstant(0, dl, PtrVT),
2759
11
                             ISD::SETNE);
2760
11
  // Use the overflow from the libcall everywhere.
2761
11
  ReplaceValueWith(SDValue(N, 1), Ofl);
2762
11
}
2763
2764
void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2765
111
                                         SDValue &Lo, SDValue &Hi) {
2766
111
  EVT VT = N->getValueType(0);
2767
111
  SDLoc dl(N);
2768
111
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2769
111
2770
111
  if (
TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom111
) {
2771
5
    SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2772
5
    SplitInteger(Res.getValue(0), Lo, Hi);
2773
5
    return;
2774
5
  }
2775
106
2776
106
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2777
106
  if (VT == MVT::i16)
2778
0
    LC = RTLIB::UDIV_I16;
2779
106
  else 
if (106
VT == MVT::i32106
)
2780
1
    LC = RTLIB::UDIV_I32;
2781
105
  else 
if (105
VT == MVT::i64105
)
2782
73
    LC = RTLIB::UDIV_I64;
2783
32
  else 
if (32
VT == MVT::i12832
)
2784
32
    LC = RTLIB::UDIV_I128;
2785
111
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2786
111
2787
111
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2788
111
}
2789
2790
void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2791
73
                                         SDValue &Lo, SDValue &Hi) {
2792
73
  EVT VT = N->getValueType(0);
2793
73
  SDLoc dl(N);
2794
73
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2795
73
2796
73
  if (
TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom73
) {
2797
11
    SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
2798
11
    SplitInteger(Res.getValue(1), Lo, Hi);
2799
11
    return;
2800
11
  }
2801
62
2802
62
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2803
62
  if (VT == MVT::i16)
2804
0
    LC = RTLIB::UREM_I16;
2805
62
  else 
if (62
VT == MVT::i3262
)
2806
1
    LC = RTLIB::UREM_I32;
2807
61
  else 
if (61
VT == MVT::i6461
)
2808
45
    LC = RTLIB::UREM_I64;
2809
16
  else 
if (16
VT == MVT::i12816
)
2810
16
    LC = RTLIB::UREM_I128;
2811
73
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2812
73
2813
73
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
2814
73
}
2815
2816
void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2817
9.29k
                                                SDValue &Lo, SDValue &Hi) {
2818
9.29k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2819
9.29k
  SDLoc dl(N);
2820
9.29k
  SDValue Op = N->getOperand(0);
2821
9.29k
  if (
Op.getValueType().bitsLE(NVT)9.29k
) {
2822
9.29k
    // The low part is zero extension of the input (degenerates to a copy).
2823
9.29k
    Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2824
9.29k
    Hi = DAG.getConstant(0, dl, NVT);   // The high part is just a zero.
2825
9.29k
  } else {
2826
0
    // For example, extension of an i48 to an i64.  The operand type necessarily
2827
0
    // promotes to the result type, so will end up being expanded too.
2828
0
    assert(getTypeAction(Op.getValueType()) ==
2829
0
           TargetLowering::TypePromoteInteger &&
2830
0
           "Only know how to promote this result!");
2831
0
    SDValue Res = GetPromotedInteger(Op);
2832
0
    assert(Res.getValueType() == N->getValueType(0) &&
2833
0
           "Operand over promoted?");
2834
0
    // Split the promoted operand.  This will simplify when it is expanded.
2835
0
    SplitInteger(Res, Lo, Hi);
2836
0
    unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
2837
0
    Hi = DAG.getZeroExtendInReg(Hi, dl,
2838
0
                                EVT::getIntegerVT(*DAG.getContext(),
2839
0
                                                  ExcessBits));
2840
0
  }
2841
9.29k
}
2842
2843
void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2844
11
                                                SDValue &Lo, SDValue &Hi) {
2845
11
  SDLoc dl(N);
2846
11
  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2847
11
  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2848
11
  SDValue Zero = DAG.getConstant(0, dl, VT);
2849
11
  SDValue Swap = DAG.getAtomicCmpSwap(
2850
11
      ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
2851
11
      cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2852
11
      N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
2853
11
2854
11
  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2855
11
  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2856
11
}
2857
2858
//===----------------------------------------------------------------------===//
2859
//  Integer Operand Expansion
2860
//===----------------------------------------------------------------------===//
2861
2862
/// ExpandIntegerOperand - This method is called when the specified operand of
2863
/// the specified node is found to need expansion.  At this point, all of the
2864
/// result types of the node are known to be legal, but other operands of the
2865
/// node may need promotion or expansion as well as the specified one.
2866
148k
bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2867
148k
  DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
2868
148k
  SDValue Res = SDValue();
2869
148k
2870
148k
  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2871
782
    return false;
2872
148k
2873
148k
  switch (N->getOpcode()) {
2874
0
  default:
2875
  #ifndef NDEBUG
2876
    dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2877
    N->dump(&DAG); dbgs() << "\n";
2878
  #endif
2879
0
    llvm_unreachable("Do not know how to expand this operator's operand!");
2880
148k
2881
3.77k
  case ISD::BITCAST:           Res = ExpandOp_BITCAST(N); break;
2882
0
  case ISD::BR_CC:             Res = ExpandIntOp_BR_CC(N); break;
2883
1.03k
  case ISD::BUILD_VECTOR:      Res = ExpandOp_BUILD_VECTOR(N); break;
2884
32.4k
  case ISD::EXTRACT_ELEMENT:   Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2885
35
  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2886
39
  case ISD::SCALAR_TO_VECTOR:  Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2887
1.84k
  case ISD::SELECT_CC:         Res = ExpandIntOp_SELECT_CC(N); break;
2888
4.23k
  case ISD::SETCC:             Res = ExpandIntOp_SETCC(N); break;
2889
0
  case ISD::SETCCE:            Res = ExpandIntOp_SETCCE(N); break;
2890
24
  case ISD::SETCCCARRY:        Res = ExpandIntOp_SETCCCARRY(N); break;
2891
42
  case ISD::SINT_TO_FP:        Res = ExpandIntOp_SINT_TO_FP(N); break;
2892
7.89k
  case ISD::STORE:   Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2893
94.7k
  case ISD::TRUNCATE:          Res = ExpandIntOp_TRUNCATE(N); break;
2894
39
  case ISD::UINT_TO_FP:        Res = ExpandIntOp_UINT_TO_FP(N); break;
2895
148k
2896
1.93k
  case ISD::SHL:
2897
1.93k
  case ISD::SRA:
2898
1.93k
  case ISD::SRL:
2899
1.93k
  case ISD::ROTL:
2900
1.93k
  case ISD::ROTR:              Res = ExpandIntOp_Shift(N); break;
2901
1
  case ISD::RETURNADDR:
2902
1
  case ISD::FRAMEADDR:         Res = ExpandIntOp_RETURNADDR(N); break;
2903
1
2904
11
  case ISD::ATOMIC_STORE:      Res = ExpandIntOp_ATOMIC_STORE(N); break;
2905
148k
  }
2906
148k
2907
148k
  // If the result is null, the sub-method took care of registering results etc.
2908
148k
  
if (148k
!Res.getNode()148k
)
return false0
;
2909
148k
2910
148k
  // If the result is N, the sub-method updated N in place.  Tell the legalizer
2911
148k
  // core about this.
2912
148k
  
if (148k
Res.getNode() == N148k
)
2913
6.01k
    return true;
2914
142k
2915
148k
  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2916
142k
         "Invalid operand expansion");
2917
142k
2918
142k
  ReplaceValueWith(SDValue(N, 0), Res);
2919
142k
  return false;
2920
142k
}
2921
2922
/// IntegerExpandSetCCOperands - Expand the operands of a comparison.  This code
2923
/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
2924
void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2925
                                                  SDValue &NewRHS,
2926
                                                  ISD::CondCode &CCCode,
2927
6.07k
                                                  const SDLoc &dl) {
2928
6.07k
  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2929
6.07k
  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2930
6.07k
  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2931
6.07k
2932
6.07k
  if (
CCCode == ISD::SETEQ || 6.07k
CCCode == ISD::SETNE5.31k
) {
2933
1.93k
    if (
RHSLo == RHSHi1.93k
) {
2934
946
      if (ConstantSDNode *
RHSCST946
= dyn_cast<ConstantSDNode>(RHSLo)) {
2935
943
        if (
RHSCST->isAllOnesValue()943
) {
2936
0
          // Equality comparison to -1.
2937
0
          NewLHS = DAG.getNode(ISD::AND, dl,
2938
0
                               LHSLo.getValueType(), LHSLo, LHSHi);
2939
0
          NewRHS = RHSLo;
2940
0
          return;
2941
0
        }
2942
1.93k
      }
2943
946
    }
2944
1.93k
2945
1.93k
    NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2946
1.93k
    NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2947
1.93k
    NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2948
1.93k
    NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
2949
1.93k
    return;
2950
1.93k
  }
2951
4.14k
2952
4.14k
  // If this is a comparison of the sign bit, just look at the top part.
2953
4.14k
  // X > -1,  x < 0
2954
4.14k
  
if (ConstantSDNode *4.14k
CST4.14k
= dyn_cast<ConstantSDNode>(NewRHS))
2955
1.30k
    
if (1.30k
(CCCode == ISD::SETLT && 1.30k
CST->isNullValue()403
) || // X < 0
2956
1.30k
        
(CCCode == ISD::SETGT && 996
CST->isAllOnesValue()65
)) { // X > -1
2957
337
      NewLHS = LHSHi;
2958
337
      NewRHS = RHSHi;
2959
337
      return;
2960
337
    }
2961
3.80k
2962
3.80k
  // FIXME: This generated code sucks.
2963
3.80k
  ISD::CondCode LowCC;
2964
3.80k
  switch (CCCode) {
2965
0
  
default: 0
llvm_unreachable0
("Unknown integer setcc!");
2966
731
  case ISD::SETLT:
2967
731
  case ISD::SETULT: LowCC = ISD::SETULT; break;
2968
780
  case ISD::SETGT:
2969
780
  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
2970
365
  case ISD::SETLE:
2971
365
  case ISD::SETULE: LowCC = ISD::SETULE; break;
2972
1.93k
  case ISD::SETGE:
2973
1.93k
  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
2974
3.80k
  }
2975
3.80k
2976
3.80k
  // LoCmp = lo(op1) < lo(op2)   // Always unsigned comparison
2977
3.80k
  // HiCmp = hi(op1) < hi(op2)   // Signedness depends on operands
2978
3.80k
  // dest  = hi(op1) == hi(op2) ? LoCmp : HiCmp;
2979
3.80k
2980
3.80k
  // NOTE: on targets without efficient SELECT of bools, we can always use
2981
3.80k
  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
2982
3.80k
  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
2983
3.80k
                                                 nullptr);
2984
3.80k
  SDValue LoCmp, HiCmp;
2985
3.80k
  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
2986
3.78k
      TLI.isTypeLegal(RHSLo.getValueType()))
2987
3.78k
    LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
2988
3.78k
                              RHSLo, LowCC, false, DagCombineInfo, dl);
2989
3.80k
  if (!LoCmp.getNode())
2990
2.99k
    LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2991
2.99k
                         RHSLo, LowCC);
2992
3.80k
  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
2993
3.78k
      TLI.isTypeLegal(RHSHi.getValueType()))
2994
3.78k
    HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
2995
3.78k
                              RHSHi, CCCode, false, DagCombineInfo, dl);
2996
3.80k
  if (!HiCmp.getNode())
2997
3.25k
    HiCmp =
2998
3.25k
        DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
2999
3.25k
                    LHSHi, RHSHi, DAG.getCondCode(CCCode));
3000
3.80k
3001
3.80k
  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3002
3.80k
  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3003
3.80k
3004
3.67k
  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3005
3.80k
                    
CCCode == ISD::SETUGE3.59k
||
CCCode == ISD::SETULE1.74k
);
3006
3.80k
3007
3.80k
  if (
(EqAllowed && 3.80k
(HiCmpC && 2.29k
HiCmpC->isNullValue()27
)) ||
3008
3.80k
      
(!EqAllowed && 3.80k
((HiCmpC && 1.51k
(HiCmpC->getAPIntValue() == 1)229
) ||
3009
3.80k
                      
(LoCmpC && 1.51k
LoCmpC->isNullValue()188
)))) {
3010
190
    // For LE / GE, if high part is known false, ignore the low part.
3011
190
    // For LT / GT: if low part is known false, return the high part.
3012
190
    //              if high part is known true, ignore the low part.
3013
190
    NewLHS = HiCmp;
3014
190
    NewRHS = SDValue();
3015
190
    return;
3016
190
  }
3017
3.61k
3018
3.61k
  
if (3.61k
LHSHi == RHSHi3.61k
) {
3019
69
    // Comparing the low bits is enough.
3020
69
    NewLHS = LoCmp;
3021
69
    NewRHS = SDValue();
3022
69
    return;
3023
69
  }
3024
3.54k
3025
3.54k
  // Lower with SETCCE or SETCCCARRY if the target supports it.
3026
3.54k
  EVT HiVT = LHSHi.getValueType();
3027
3.54k
  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3028
3.54k
  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3029
3.54k
3030
3.54k
  // FIXME: Make all targets support this, then remove the other lowering.
3031
3.54k
  if (HasSETCCCARRY ||
3032
3.54k
      
TLI.getOperationAction(ISD::SETCCE, ExpandVT) == TargetLowering::Custom2.96k
) {
3033
1.57k
    // SETCCE/SETCCCARRY can detect < and >= directly. For > and <=, flip
3034
1.57k
    // operands and condition code.
3035
1.57k
    bool FlipOperands = false;
3036
1.57k
    switch (CCCode) {
3037
135
    case ISD::SETGT:  CCCode = ISD::SETLT;  FlipOperands = true; break;
3038
381
    case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3039
108
    case ISD::SETLE:  CCCode = ISD::SETGE;  FlipOperands = true; break;
3040
210
    case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3041
744
    default: break;
3042
1.57k
    }
3043
1.57k
    
if (1.57k
FlipOperands1.57k
) {
3044
834
      std::swap(LHSLo, RHSLo);
3045
834
      std::swap(LHSHi, RHSHi);
3046
834
    }
3047
1.57k
    // Perform a wide subtraction, feeding the carry from the low part into
3048
1.57k
    // SETCCE/SETCCCARRY. The SETCCE/SETCCCARRY operation is essentially
3049
1.57k
    // looking at the high part of the result of LHS - RHS. It is negative
3050
1.57k
    // iff LHS < RHS. It is zero or positive iff LHS >= RHS.
3051
1.57k
    EVT LoVT = LHSLo.getValueType();
3052
1.57k
    SDVTList VTList = DAG.getVTList(
3053
1.57k
        LoVT, HasSETCCCARRY ? 
getSetCCResultType(LoVT)579
:
MVT::Glue999
);
3054
1.57k
    SDValue LowCmp = DAG.getNode(HasSETCCCARRY ? 
ISD::USUBO579
:
ISD::SUBC999
, dl,
3055
1.57k
                                 VTList, LHSLo, RHSLo);
3056
1.57k
    SDValue Res = DAG.getNode(HasSETCCCARRY ? 
ISD::SETCCCARRY579
:
ISD::SETCCE999
, dl,
3057
1.57k
                              getSetCCResultType(HiVT), LHSHi, RHSHi,
3058
1.57k
                              LowCmp.getValue(1), DAG.getCondCode(CCCode));
3059
1.57k
    NewLHS = Res;
3060
1.57k
    NewRHS = SDValue();
3061
1.57k
    return;
3062
1.57k
  }
3063
1.97k
3064
1.97k
  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3065
1.97k
                             false, DagCombineInfo, dl);
3066
1.97k
  if (!NewLHS.getNode())
3067
1.94k
    NewLHS =
3068
1.94k
        DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3069
6.07k
  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3070
6.07k
  NewRHS = SDValue();
3071
6.07k
}
3072
3073
0
SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3074
0
  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3075
0
  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3076
0
  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3077
0
3078
0
  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3079
0
  // against zero to select between true and false values.
3080
0
  if (
!NewRHS.getNode()0
) {
3081
0
    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3082
0
    CCCode = ISD::SETNE;
3083
0
  }
3084
0
3085
0
  // Update N to have the operands specified.
3086
0
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3087
0
                                DAG.getCondCode(CCCode), NewLHS, NewRHS,
3088
0
                                N->getOperand(4)), 0);
3089
0
}
3090
3091
1.84k
SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3092
1.84k
  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3093
1.84k
  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3094
1.84k
  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3095
1.84k
3096
1.84k
  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3097
1.84k
  // against zero to select between true and false values.
3098
1.84k
  if (
!NewRHS.getNode()1.84k
) {
3099
1.81k
    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3100
1.81k
    CCCode = ISD::SETNE;
3101
1.81k
  }
3102
1.84k
3103
1.84k
  // Update N to have the operands specified.
3104
1.84k
  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3105
1.84k
                                N->getOperand(2), N->getOperand(3),
3106
1.84k
                                DAG.getCondCode(CCCode)), 0);
3107
1.84k
}
3108
3109
4.23k
SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3110
4.23k
  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3111
4.23k
  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3112
4.23k
  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3113
4.23k
3114
4.23k
  // If ExpandSetCCOperands returned a scalar, use it.
3115
4.23k
  if (
!NewRHS.getNode()4.23k
) {
3116
1.99k
    assert(NewLHS.getValueType() == N->getValueType(0) &&
3117
1.99k
           "Unexpected setcc expansion!");
3118
1.99k
    return NewLHS;
3119
1.99k
  }
3120
2.23k
3121
2.23k
  // Otherwise, update N to have the operands specified.
3122
2.23k
  return SDValue(
3123
2.23k
      DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3124
2.23k
}
3125
3126
0
SDValue DAGTypeLegalizer::ExpandIntOp_SETCCE(SDNode *N) {
3127
0
  SDValue LHS = N->getOperand(0);
3128
0
  SDValue RHS = N->getOperand(1);
3129
0
  SDValue Carry = N->getOperand(2);
3130
0
  SDValue Cond = N->getOperand(3);
3131
0
  SDLoc dl = SDLoc(N);
3132
0
3133
0
  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3134
0
  GetExpandedInteger(LHS, LHSLo, LHSHi);
3135
0
  GetExpandedInteger(RHS, RHSLo, RHSHi);
3136
0
3137
0
  // Expand to a SUBE for the low part and a smaller SETCCE for the high.
3138
0
  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), MVT::Glue);
3139
0
  SDValue LowCmp = DAG.getNode(ISD::SUBE, dl, VTList, LHSLo, RHSLo, Carry);
3140
0
  return DAG.getNode(ISD::SETCCE, dl, N->getValueType(0), LHSHi, RHSHi,
3141
0
                     LowCmp.getValue(1), Cond);
3142
0
}
3143
3144
24
SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3145
24
  SDValue LHS = N->getOperand(0);
3146
24
  SDValue RHS = N->getOperand(1);
3147
24
  SDValue Carry = N->getOperand(2);
3148
24
  SDValue Cond = N->getOperand(3);
3149
24
  SDLoc dl = SDLoc(N);
3150
24
3151
24
  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3152
24
  GetExpandedInteger(LHS, LHSLo, LHSHi);
3153
24
  GetExpandedInteger(RHS, RHSLo, RHSHi);
3154
24
3155
24
  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3156
24
  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3157
24
  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3158
24
  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3159
24
                     LowCmp.getValue(1), Cond);
3160
24
}
3161
3162
1.93k
SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3163
1.93k
  // The value being shifted is legal, but the shift amount is too big.
3164
1.93k
  // It follows that either the result of the shift is undefined, or the
3165
1.93k
  // upper half of the shift amount is zero.  Just use the lower half.
3166
1.93k
  SDValue Lo, Hi;
3167
1.93k
  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3168
1.93k
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3169
1.93k
}
3170
3171
1
SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3172
1
  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant.  This
3173
1
  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3174
1
  // constant to valid type.
3175
1
  SDValue Lo, Hi;
3176
1
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3177
1
  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3178
1
}
3179
3180
42
SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3181
42
  SDValue Op = N->getOperand(0);
3182
42
  EVT DstVT = N->getValueType(0);
3183
42
  RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
3184
42
  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3185
42
         "Don't know how to expand this SINT_TO_FP!");
3186
42
  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3187
42
}
3188
3189
7.89k
SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3190
7.89k
  if (ISD::isNormalStore(N))
3191
6.50k
    return ExpandOp_NormalStore(N, OpNo);
3192
1.38k
3193
7.89k
  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3194
1.38k
  assert(OpNo == 1 && "Can only expand the stored value so far");
3195
1.38k
3196
1.38k
  EVT VT = N->getOperand(1).getValueType();
3197
1.38k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3198
1.38k
  SDValue Ch  = N->getChain();
3199
1.38k
  SDValue Ptr = N->getBasePtr();
3200
1.38k
  unsigned Alignment = N->getAlignment();
3201
1.38k
  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3202
1.38k
  AAMDNodes AAInfo = N->getAAInfo();
3203
1.38k
  SDLoc dl(N);
3204
1.38k
  SDValue Lo, Hi;
3205
1.38k
3206
1.38k
  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3207
1.38k
3208
1.38k
  if (
N->getMemoryVT().bitsLE(NVT)1.38k
) {
3209
336
    GetExpandedInteger(N->getValue(), Lo, Hi);
3210
336
    return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3211
336
                             N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3212
336
  }
3213
1.04k
3214
1.04k
  
if (1.04k
DAG.getDataLayout().isLittleEndian()1.04k
) {
3215
1.04k
    // Little-endian - low bits are at low addresses.
3216
1.04k
    GetExpandedInteger(N->getValue(), Lo, Hi);
3217
1.04k
3218
1.04k
    Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3219
1.04k
                      AAInfo);
3220
1.04k
3221
1.04k
    unsigned ExcessBits =
3222
1.04k
      N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3223
1.04k
    EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3224
1.04k
3225
1.04k
    // Increment the pointer to the other half.
3226
1.04k
    unsigned IncrementSize = NVT.getSizeInBits()/8;
3227
1.04k
    Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3228
1.04k
                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3229
1.04k
    Hi = DAG.getTruncStore(
3230
1.04k
        Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3231
1.04k
        MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3232
1.04k
    return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3233
1.04k
  }
3234
6
3235
6
  // Big-endian - high bits are at low addresses.  Favor aligned stores at
3236
6
  // the cost of some bit-fiddling.
3237
6
  GetExpandedInteger(N->getValue(), Lo, Hi);
3238
6
3239
6
  EVT ExtVT = N->getMemoryVT();
3240
6
  unsigned EBytes = ExtVT.getStoreSize();
3241
6
  unsigned IncrementSize = NVT.getSizeInBits()/8;
3242
6
  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3243
6
  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3244
6
                               ExtVT.getSizeInBits() - ExcessBits);
3245
6
3246
6
  if (
ExcessBits < NVT.getSizeInBits()6
) {
3247
5
    // Transfer high bits from the top of Lo to the bottom of Hi.
3248
5
    Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3249
5
                     DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3250
5
                                     TLI.getPointerTy(DAG.getDataLayout())));
3251
5
    Hi = DAG.getNode(
3252
5
        ISD::OR, dl, NVT, Hi,
3253
5
        DAG.getNode(ISD::SRL, dl, NVT, Lo,
3254
5
                    DAG.getConstant(ExcessBits, dl,
3255
5
                                    TLI.getPointerTy(DAG.getDataLayout()))));
3256
5
  }
3257
7.89k
3258
7.89k
  // Store both the high bits and maybe some of the low bits.
3259
7.89k
  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3260
7.89k
                         MMOFlags, AAInfo);
3261
7.89k
3262
7.89k
  // Increment the pointer to the other half.
3263
7.89k
  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
3264
7.89k
                    DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
3265
7.89k
  // Store the lowest ExcessBits bits in the second half.
3266
7.89k
  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3267
7.89k
                         N->getPointerInfo().getWithOffset(IncrementSize),
3268
7.89k
                         EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3269
7.89k
                         MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3270
7.89k
  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3271
7.89k
}
3272
3273
94.7k
SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3274
94.7k
  SDValue InL, InH;
3275
94.7k
  GetExpandedInteger(N->getOperand(0), InL, InH);
3276
94.7k
  // Just truncate the low part of the source.
3277
94.7k
  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3278
94.7k
}
3279
3280
39
SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3281
39
  SDValue Op = N->getOperand(0);
3282
39
  EVT SrcVT = Op.getValueType();
3283
39
  EVT DstVT = N->getValueType(0);
3284
39
  SDLoc dl(N);
3285
39
3286
39
  // The following optimization is valid only if every value in SrcVT (when
3287
39
  // treated as signed) is representable in DstVT.  Check that the mantissa
3288
39
  // size of DstVT is >= than the number of bits in SrcVT -1.
3289
39
  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3290
39
  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3291
39
      
TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom0
){
3292
0
    // Do a signed conversion then adjust the result.
3293
0
    SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3294
0
    SignedConv = TLI.LowerOperation(SignedConv, DAG);
3295
0
3296
0
    // The result of the signed conversion needs adjusting if the 'sign bit' of
3297
0
    // the incoming integer was set.  To handle this, we dynamically test to see
3298
0
    // if it is set, and, if so, add a fudge factor.
3299
0
3300
0
    const uint64_t F32TwoE32  = 0x4F800000ULL;
3301
0
    const uint64_t F32TwoE64  = 0x5F800000ULL;
3302
0
    const uint64_t F32TwoE128 = 0x7F800000ULL;
3303
0
3304
0
    APInt FF(32, 0);
3305
0
    if (SrcVT == MVT::i32)
3306
0
      FF = APInt(32, F32TwoE32);
3307
0
    else 
if (0
SrcVT == MVT::i640
)
3308
0
      FF = APInt(32, F32TwoE64);
3309
0
    else 
if (0
SrcVT == MVT::i1280
)
3310
0
      FF = APInt(32, F32TwoE128);
3311
0
    else
3312
0
      llvm_unreachable("Unsupported UINT_TO_FP!");
3313
0
3314
0
    // Check whether the sign bit is set.
3315
0
    SDValue Lo, Hi;
3316
0
    GetExpandedInteger(Op, Lo, Hi);
3317
0
    SDValue SignSet = DAG.getSetCC(dl,
3318
0
                                   getSetCCResultType(Hi.getValueType()),
3319
0
                                   Hi,
3320
0
                                   DAG.getConstant(0, dl, Hi.getValueType()),
3321
0
                                   ISD::SETLT);
3322
0
3323
0
    // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3324
0
    SDValue FudgePtr =
3325
0
        DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3326
0
                            TLI.getPointerTy(DAG.getDataLayout()));
3327
0
3328
0
    // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3329
0
    SDValue Zero = DAG.getIntPtrConstant(0, dl);
3330
0
    SDValue Four = DAG.getIntPtrConstant(4, dl);
3331
0
    if (DAG.getDataLayout().isBigEndian())
3332
0
      std::swap(Zero, Four);
3333
0
    SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3334
0
                                   Zero, Four);
3335
0
    unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3336
0
    FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3337
0
                           FudgePtr, Offset);
3338
0
    Alignment = std::min(Alignment, 4u);
3339
0
3340
0
    // Load the value out, extending it from f32 to the destination float type.
3341
0
    // FIXME: Avoid the extend by constructing the right constant pool?
3342
0
    SDValue Fudge = DAG.getExtLoad(
3343
0
        ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3344
0
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
3345
0
        Alignment);
3346
0
    return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3347
39
  }
3348
39
3349
39
  // Otherwise, use a libcall.
3350
39
  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3351
39
  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3352
39
         "Don't know how to expand this UINT_TO_FP!");
3353
39
  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3354
39
}
3355
3356
11
SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3357
11
  SDLoc dl(N);
3358
11
  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3359
11
                               cast<AtomicSDNode>(N)->getMemoryVT(),
3360
11
                               N->getOperand(0),
3361
11
                               N->getOperand(1), N->getOperand(2),
3362
11
                               cast<AtomicSDNode>(N)->getMemOperand());
3363
11
  return Swap.getValue(1);
3364
11
}
3365
3366
3367
23.6k
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3368
23.6k
  SDValue InOp0 = N->getOperand(0);
3369
23.6k
  EVT InVT = InOp0.getValueType();
3370
23.6k
3371
23.6k
  EVT OutVT = N->getValueType(0);
3372
23.6k
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3373
23.6k
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3374
23.6k
  unsigned OutNumElems = OutVT.getVectorNumElements();
3375
23.6k
  EVT NOutVTElem = NOutVT.getVectorElementType();
3376
23.6k
3377
23.6k
  SDLoc dl(N);
3378
23.6k
  SDValue BaseIdx = N->getOperand(1);
3379
23.6k
3380
23.6k
  SmallVector<SDValue, 8> Ops;
3381
23.6k
  Ops.reserve(OutNumElems);
3382
84.0k
  for (unsigned i = 0; 
i != OutNumElems84.0k
;
++i60.4k
) {
3383
60.4k
3384
60.4k
    // Extract the element from the original vector.
3385
60.4k
    SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3386
60.4k
      BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3387
60.4k
    SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3388
60.4k
      InVT.getVectorElementType(), N->getOperand(0), Index);
3389
60.4k
3390
60.4k
    SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
3391
60.4k
    // Insert the converted element to the new vector.
3392
60.4k
    Ops.push_back(Op);
3393
60.4k
  }
3394
23.6k
3395
23.6k
  return DAG.getBuildVector(NOutVT, dl, Ops);
3396
23.6k
}
3397
3398
3399
104
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3400
104
  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3401
104
  EVT VT = N->getValueType(0);
3402
104
  SDLoc dl(N);
3403
104
3404
104
  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3405
104
3406
104
  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3407
104
  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3408
104
  EVT OutVT = V0.getValueType();
3409
104
3410
104
  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3411
104
}
3412
3413
3414
1.80k
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3415
1.80k
  EVT OutVT = N->getValueType(0);
3416
1.80k
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3417
1.80k
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3418
1.80k
  unsigned NumElems = N->getNumOperands();
3419
1.80k
  EVT NOutVTElem = NOutVT.getVectorElementType();
3420
1.80k
3421
1.80k
  SDLoc dl(N);
3422
1.80k
3423
1.80k
  SmallVector<SDValue, 8> Ops;
3424
1.80k
  Ops.reserve(NumElems);
3425
16.3k
  for (unsigned i = 0; 
i != NumElems16.3k
;
++i14.5k
) {
3426
14.5k
    SDValue Op;
3427
14.5k
    // BUILD_VECTOR integer operand types are allowed to be larger than the
3428
14.5k
    // result's element type. This may still be true after the promotion. For
3429
14.5k
    // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
3430
14.5k
    // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
3431
14.5k
    if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
3432
10.1k
      Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
3433
14.5k
    else
3434
4.42k
      Op = N->getOperand(i);
3435
14.5k
    Ops.push_back(Op);
3436
14.5k
  }
3437
1.80k
3438
1.80k
  return DAG.getBuildVector(NOutVT, dl, Ops);
3439
1.80k
}
3440
3441
26
SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
3442
26
3443
26
  SDLoc dl(N);
3444
26
3445
26
  assert(!N->getOperand(0).getValueType().isVector() &&
3446
26
         "Input must be a scalar");
3447
26
3448
26
  EVT OutVT = N->getValueType(0);
3449
26
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3450
26
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3451
26
  EVT NOutVTElem = NOutVT.getVectorElementType();
3452
26
3453
26
  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3454
26
3455
26
  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3456
26
}
3457
3458
1.83k
SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3459
1.83k
  SDLoc dl(N);
3460
1.83k
3461
1.83k
  EVT OutVT = N->getValueType(0);
3462
1.83k
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3463
1.83k
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3464
1.83k
3465
1.83k
  EVT InElemTy = OutVT.getVectorElementType();
3466
1.83k
  EVT OutElemTy = NOutVT.getVectorElementType();
3467
1.83k
3468
1.83k
  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3469
1.83k
  unsigned NumOutElem = NOutVT.getVectorNumElements();
3470
1.83k
  unsigned NumOperands = N->getNumOperands();
3471
1.83k
  assert(NumElem * NumOperands == NumOutElem &&
3472
1.83k
         "Unexpected number of elements");
3473
1.83k
3474
1.83k
  // Take the elements from the first vector.
3475
1.83k
  SmallVector<SDValue, 8> Ops(NumOutElem);
3476
7.43k
  for (unsigned i = 0; 
i < NumOperands7.43k
;
++i5.59k
) {
3477
5.59k
    SDValue Op = N->getOperand(i);
3478
32.2k
    for (unsigned j = 0; 
j < NumElem32.2k
;
++j26.6k
) {
3479
26.6k
      SDValue Ext = DAG.getNode(
3480
26.6k
          ISD::EXTRACT_VECTOR_ELT, dl, InElemTy, Op,
3481
26.6k
          DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3482
26.6k
      Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
3483
26.6k
    }
3484
5.59k
  }
3485
1.83k
3486
1.83k
  return DAG.getBuildVector(NOutVT, dl, Ops);
3487
1.83k
}
3488
3489
3
SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
3490
3
  EVT VT = N->getValueType(0);
3491
3
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3492
3
  assert(NVT.isVector() && "This type must be promoted to a vector type");
3493
3
3494
3
  SDLoc dl(N);
3495
3
3496
3
  // For operands whose TypeAction is to promote, extend the promoted node
3497
3
  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
3498
3
  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
3499
3
  // type..
3500
3
  if (getTypeAction(N->getOperand(0).getValueType())
3501
3
      == TargetLowering::TypePromoteInteger) {
3502
3
    SDValue Promoted;
3503
3
3504
3
    switch(N->getOpcode()) {
3505
1
      case ISD::SIGN_EXTEND_VECTOR_INREG:
3506
1
        Promoted = SExtPromotedInteger(N->getOperand(0));
3507
1
        break;
3508
1
      case ISD::ZERO_EXTEND_VECTOR_INREG:
3509
1
        Promoted = ZExtPromotedInteger(N->getOperand(0));
3510
1
        break;
3511
1
      case ISD::ANY_EXTEND_VECTOR_INREG:
3512
1
        Promoted = GetPromotedInteger(N->getOperand(0));
3513
1
        break;
3514
0
      default:
3515
0
        llvm_unreachable("Node has unexpected Opcode");
3516
3
    }
3517
3
    return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
3518
3
  }
3519
0
3520
0
  // Directly extend to the appropriate transform-to type.
3521
0
  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3522
0
}
3523
3524
184
SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3525
184
  EVT OutVT = N->getValueType(0);
3526
184
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3527
184
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3528
184
3529
184
  EVT NOutVTElem = NOutVT.getVectorElementType();
3530
184
3531
184
  SDLoc dl(N);
3532
184
  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3533
184
3534
184
  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3535
184
    NOutVTElem, N->getOperand(1));
3536
184
  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3537
184
    V0, ConvElem, N->getOperand(2));
3538
184
}
3539
3540
38.0k
SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3541
38.0k
  SDLoc dl(N);
3542
38.0k
  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3543
38.0k
  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
3544
38.0k
                                  TLI.getVectorIdxTy(DAG.getDataLayout()));
3545
38.0k
  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3546
38.0k
    V0->getValueType(0).getScalarType(), V0, V1);
3547
38.0k
3548
38.0k
  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3549
38.0k
  // element types. If this is the case then we need to expand the outgoing
3550
38.0k
  // value and not truncate it.
3551
38.0k
  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3552
38.0k
}
3553
3554
42
SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
3555
42
  SDLoc dl(N);
3556
42
  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3557
42
  MVT InVT = V0.getValueType().getSimpleVT();
3558
42
  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
3559
42
                               N->getValueType(0).getVectorNumElements());
3560
42
  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
3561
42
  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
3562
42
}
3563
3564
1.84k
SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3565
1.84k
  SDLoc dl(N);
3566
1.84k
  unsigned NumElems = N->getNumOperands();
3567
1.84k
3568
1.84k
  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3569
1.84k
3570
1.84k
  SmallVector<SDValue, 8> NewOps;
3571
1.84k
  NewOps.reserve(NumElems);
3572
1.84k
3573
1.84k
  // For each incoming vector
3574
7.63k
  for (unsigned VecIdx = 0; 
VecIdx != NumElems7.63k
;
++VecIdx5.78k
) {
3575
5.78k
    SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3576
5.78k
    EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3577
5.78k
    unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3578
5.78k
3579
27.5k
    for (unsigned i=0; 
i<NumElem27.5k
;
++i21.7k
) {
3580
21.7k
      // Extract element from incoming vector
3581
21.7k
      SDValue Ex = DAG.getNode(
3582
21.7k
          ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
3583
21.7k
          DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3584
21.7k
      SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3585
21.7k
      NewOps.push_back(Tr);
3586
21.7k
    }
3587
5.78k
  }
3588
1.84k
3589
1.84k
  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
3590
1.84k
}