Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
Line
Count
Source (jump to first uncovered line)
1
//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements integer type expansion and promotion for LegalizeTypes.
10
// Promotion is the act of changing a computation in an illegal type into a
11
// computation in a larger type.  For example, implementing i8 arithmetic in an
12
// i32 register (often needed on powerpc).
13
// Expansion is the act of changing a computation in an illegal type into a
14
// computation in two identical registers of a smaller type.  For example,
15
// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16
// targets).
17
//
18
//===----------------------------------------------------------------------===//
19
20
#include "LegalizeTypes.h"
21
#include "llvm/IR/DerivedTypes.h"
22
#include "llvm/Support/ErrorHandling.h"
23
#include "llvm/Support/KnownBits.h"
24
#include "llvm/Support/raw_ostream.h"
25
using namespace llvm;
26
27
#define DEBUG_TYPE "legalize-types"
28
29
//===----------------------------------------------------------------------===//
30
//  Integer Result Promotion
31
//===----------------------------------------------------------------------===//
32
33
/// PromoteIntegerResult - This method is called when a result of a node is
34
/// found to be in need of promotion to a larger type.  At this point, the node
35
/// may also have invalid operands or may have other results that need
36
/// expansion, we just know that (at least) one result needs promotion.
37
736k
void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38
736k
  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39
736k
             dbgs() << "\n");
40
736k
  SDValue Res = SDValue();
41
736k
42
736k
  // See if the target wants to custom expand this node.
43
736k
  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44
1.32k
    LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45
1.32k
    return;
46
1.32k
  }
47
735k
48
735k
  switch (N->getOpcode()) {
49
735k
  default:
50
#ifndef NDEBUG
51
    dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52
    N->dump(&DAG); dbgs() << "\n";
53
#endif
54
0
    llvm_unreachable("Do not know how to promote this operator!");
55
735k
  
case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break34
;
56
735k
  
case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break90
;
57
735k
  
case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break90
;
58
735k
  
case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break5.64k
;
59
735k
  
case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break18
;
60
735k
  
case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break92
;
61
735k
  
case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break0
;
62
735k
  
case ISD::Constant: Res = PromoteIntRes_Constant(N); break146k
;
63
735k
  case ISD::CTLZ_ZERO_UNDEF:
64
60
  case ISD::CTLZ:        Res = PromoteIntRes_CTLZ(N); break;
65
81
  case ISD::CTPOP:       Res = PromoteIntRes_CTPOP(N); break;
66
87
  case ISD::CTTZ_ZERO_UNDEF:
67
87
  case ISD::CTTZ:        Res = PromoteIntRes_CTTZ(N); break;
68
58.5k
  case ISD::EXTRACT_VECTOR_ELT:
69
58.5k
                         Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70
93.0k
  case ISD::LOAD:        Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71
87
  case ISD::MLOAD:       Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72
5
    break;
73
87
  case ISD::MGATHER:     Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74
0
    break;
75
3.95k
  case ISD::SELECT:      Res = PromoteIntRes_SELECT(N); break;
76
143
  case ISD::VSELECT:     Res = PromoteIntRes_VSELECT(N); break;
77
87
  
case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break63
;
78
256k
  case ISD::SETCC:       Res = PromoteIntRes_SETCC(N); break;
79
290
  case ISD::SMIN:
80
290
  case ISD::SMAX:        Res = PromoteIntRes_SExtIntBinOp(N); break;
81
290
  case ISD::UMIN:
82
235
  case ISD::UMAX:        Res = PromoteIntRes_ZExtIntBinOp(N); break;
83
235
84
2.53k
  case ISD::SHL:         Res = PromoteIntRes_SHL(N); break;
85
235
  case ISD::SIGN_EXTEND_INREG:
86
91
                         Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87
1.04k
  case ISD::SRA:         Res = PromoteIntRes_SRA(N); break;
88
6.94k
  case ISD::SRL:         Res = PromoteIntRes_SRL(N); break;
89
79.7k
  case ISD::TRUNCATE:    Res = PromoteIntRes_TRUNCATE(N); break;
90
3.88k
  case ISD::UNDEF:       Res = PromoteIntRes_UNDEF(N); break;
91
235
  
case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break65
;
92
235
93
19.3k
  case ISD::EXTRACT_SUBVECTOR:
94
19.3k
                         Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
95
1.02k
  case ISD::VECTOR_SHUFFLE:
96
1.02k
                         Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
97
235
  case ISD::INSERT_VECTOR_ELT:
98
180
                         Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99
2.15k
  case ISD::BUILD_VECTOR:
100
2.15k
                         Res = PromoteIntRes_BUILD_VECTOR(N); break;
101
235
  case ISD::SCALAR_TO_VECTOR:
102
17
                         Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103
348
  case ISD::CONCAT_VECTORS:
104
348
                         Res = PromoteIntRes_CONCAT_VECTORS(N); break;
105
235
106
235
  case ISD::ANY_EXTEND_VECTOR_INREG:
107
2
  case ISD::SIGN_EXTEND_VECTOR_INREG:
108
2
  case ISD::ZERO_EXTEND_VECTOR_INREG:
109
2
                         Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
110
2
111
21.9k
  case ISD::SIGN_EXTEND:
112
21.9k
  case ISD::ZERO_EXTEND:
113
21.9k
  case ISD::ANY_EXTEND:  Res = PromoteIntRes_INT_EXTEND(N); break;
114
21.9k
115
21.9k
  case ISD::FP_TO_SINT:
116
1.28k
  case ISD::FP_TO_UINT:  Res = PromoteIntRes_FP_TO_XINT(N); break;
117
1.28k
118
1.28k
  
case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break1.12k
;
119
1.28k
120
1.28k
  
case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break1
;
121
1.28k
122
18.1k
  case ISD::AND:
123
18.1k
  case ISD::OR:
124
18.1k
  case ISD::XOR:
125
18.1k
  case ISD::ADD:
126
18.1k
  case ISD::SUB:
127
18.1k
  case ISD::MUL:         Res = PromoteIntRes_SimpleIntBinOp(N); break;
128
18.1k
129
18.1k
  case ISD::SDIV:
130
519
  case ISD::SREM:        Res = PromoteIntRes_SExtIntBinOp(N); break;
131
519
132
519
  case ISD::UDIV:
133
485
  case ISD::UREM:        Res = PromoteIntRes_ZExtIntBinOp(N); break;
134
485
135
771
  case ISD::SADDO:
136
771
  case ISD::SSUBO:       Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
137
3.15k
  case ISD::UADDO:
138
3.15k
  case ISD::USUBO:       Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
139
3.15k
  case ISD::SMULO:
140
717
  case ISD::UMULO:       Res = PromoteIntRes_XMULO(N, ResNo); break;
141
717
142
717
  case ISD::ADDE:
143
148
  case ISD::SUBE:
144
148
  case ISD::ADDCARRY:
145
148
  case ISD::SUBCARRY:    Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
146
148
147
148
  case ISD::SADDSAT:
148
116
  case ISD::UADDSAT:
149
116
  case ISD::SSUBSAT:
150
116
  case ISD::USUBSAT:     Res = PromoteIntRes_ADDSUBSAT(N); break;
151
116
  case ISD::SMULFIX:
152
12
  case ISD::SMULFIXSAT:
153
12
  case ISD::UMULFIX:     Res = PromoteIntRes_MULFIX(N); break;
154
13
  case ISD::ABS:         Res = PromoteIntRes_ABS(N); break;
155
12
156
522
  case ISD::ATOMIC_LOAD:
157
522
    Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
158
12
159
1.30k
  case ISD::ATOMIC_LOAD_ADD:
160
1.30k
  case ISD::ATOMIC_LOAD_SUB:
161
1.30k
  case ISD::ATOMIC_LOAD_AND:
162
1.30k
  case ISD::ATOMIC_LOAD_CLR:
163
1.30k
  case ISD::ATOMIC_LOAD_OR:
164
1.30k
  case ISD::ATOMIC_LOAD_XOR:
165
1.30k
  case ISD::ATOMIC_LOAD_NAND:
166
1.30k
  case ISD::ATOMIC_LOAD_MIN:
167
1.30k
  case ISD::ATOMIC_LOAD_MAX:
168
1.30k
  case ISD::ATOMIC_LOAD_UMIN:
169
1.30k
  case ISD::ATOMIC_LOAD_UMAX:
170
1.30k
  case ISD::ATOMIC_SWAP:
171
1.30k
    Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
172
1.30k
173
1.59k
  case ISD::ATOMIC_CMP_SWAP:
174
1.59k
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
175
1.59k
    Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
176
1.59k
    break;
177
1.59k
178
1.59k
  case ISD::VECREDUCE_ADD:
179
190
  case ISD::VECREDUCE_MUL:
180
190
  case ISD::VECREDUCE_AND:
181
190
  case ISD::VECREDUCE_OR:
182
190
  case ISD::VECREDUCE_XOR:
183
190
  case ISD::VECREDUCE_SMAX:
184
190
  case ISD::VECREDUCE_SMIN:
185
190
  case ISD::VECREDUCE_UMAX:
186
190
  case ISD::VECREDUCE_UMIN:
187
190
    Res = PromoteIntRes_VECREDUCE(N);
188
190
    break;
189
735k
  }
190
735k
191
735k
  // If the result is null then the sub-method took care of registering it.
192
735k
  if (Res.getNode())
193
735k
    SetPromotedInteger(SDValue(N, ResNo), Res);
194
735k
}
195
196
SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
197
34
                                                     unsigned ResNo) {
198
34
  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
199
34
  return GetPromotedInteger(Op);
200
34
}
201
202
90
SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
203
90
  // Sign-extend the new bits, and continue the assertion.
204
90
  SDValue Op = SExtPromotedInteger(N->getOperand(0));
205
90
  return DAG.getNode(ISD::AssertSext, SDLoc(N),
206
90
                     Op.getValueType(), Op, N->getOperand(1));
207
90
}
208
209
90
SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
210
90
  // Zero the new bits, and continue the assertion.
211
90
  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
212
90
  return DAG.getNode(ISD::AssertZext, SDLoc(N),
213
90
                     Op.getValueType(), Op, N->getOperand(1));
214
90
}
215
216
522
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
217
522
  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
218
522
  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
219
522
                              N->getMemoryVT(), ResVT,
220
522
                              N->getChain(), N->getBasePtr(),
221
522
                              N->getMemOperand());
222
522
  // Legalize the chain result - switch anything that used the old chain to
223
522
  // use the new one.
224
522
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
225
522
  return Res;
226
522
}
227
228
1.30k
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
229
1.30k
  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
230
1.30k
  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
231
1.30k
                              N->getMemoryVT(),
232
1.30k
                              N->getChain(), N->getBasePtr(),
233
1.30k
                              Op2, N->getMemOperand());
234
1.30k
  // Legalize the chain result - switch anything that used the old chain to
235
1.30k
  // use the new one.
236
1.30k
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
237
1.30k
  return Res;
238
1.30k
}
239
240
SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
241
1.59k
                                                      unsigned ResNo) {
242
1.59k
  if (ResNo == 1) {
243
1.37k
    assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
244
1.37k
    EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
245
1.37k
    EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
246
1.37k
247
1.37k
    // Only use the result of getSetCCResultType if it is legal,
248
1.37k
    // otherwise just use the promoted result type (NVT).
249
1.37k
    if (!TLI.isTypeLegal(SVT))
250
0
      SVT = NVT;
251
1.37k
252
1.37k
    SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
253
1.37k
    SDValue Res = DAG.getAtomicCmpSwap(
254
1.37k
        ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
255
1.37k
        N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
256
1.37k
        N->getMemOperand());
257
1.37k
    ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
258
1.37k
    ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
259
1.37k
    return Res.getValue(1);
260
1.37k
  }
261
218
262
218
  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
263
218
  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
264
218
  SDVTList VTs =
265
218
      DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
266
218
  SDValue Res = DAG.getAtomicCmpSwap(
267
218
      N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
268
218
      N->getBasePtr(), Op2, Op3, N->getMemOperand());
269
218
  // Update the use to N with the newly created Res.
270
654
  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; 
++i436
)
271
436
    ReplaceValueWith(SDValue(N, i), Res.getValue(i));
272
218
  return Res;
273
218
}
274
275
5.64k
SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
276
5.64k
  SDValue InOp = N->getOperand(0);
277
5.64k
  EVT InVT = InOp.getValueType();
278
5.64k
  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
279
5.64k
  EVT OutVT = N->getValueType(0);
280
5.64k
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
281
5.64k
  SDLoc dl(N);
282
5.64k
283
5.64k
  switch (getTypeAction(InVT)) {
284
5.64k
  case TargetLowering::TypeLegal:
285
936
    break;
286
5.64k
  case TargetLowering::TypePromoteInteger:
287
530
    if (NOutVT.bitsEq(NInVT) && 
!NOutVT.isVector()96
&&
!NInVT.isVector()0
)
288
0
      // The input promotes to the same size.  Convert the promoted value.
289
0
      return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
290
530
    break;
291
530
  case TargetLowering::TypeSoftenFloat:
292
2
    // Promote the integer operand by hand.
293
2
    return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
294
749
  case TargetLowering::TypePromoteFloat: {
295
749
    // Convert the promoted float by hand.
296
749
    if (!NOutVT.isVector())
297
747
      return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
298
2
    break;
299
2
  }
300
363
  case TargetLowering::TypeExpandInteger:
301
363
  case TargetLowering::TypeExpandFloat:
302
363
    break;
303
2.57k
  case TargetLowering::TypeScalarizeVector:
304
2.57k
    // Convert the element to an integer and promote it by hand.
305
2.57k
    if (!NOutVT.isVector())
306
2.53k
      return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
307
2.53k
                         BitConvertToInteger(GetScalarizedVector(InOp)));
308
45
    break;
309
457
  case TargetLowering::TypeSplitVector: {
310
457
    if (!NOutVT.isVector()) {
311
398
      // For example, i32 = BITCAST v2i16 on alpha.  Convert the split
312
398
      // pieces of the input into integers and reassemble in the final type.
313
398
      SDValue Lo, Hi;
314
398
      GetSplitVector(N->getOperand(0), Lo, Hi);
315
398
      Lo = BitConvertToInteger(Lo);
316
398
      Hi = BitConvertToInteger(Hi);
317
398
318
398
      if (DAG.getDataLayout().isBigEndian())
319
45
        std::swap(Lo, Hi);
320
398
321
398
      InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
322
398
                         EVT::getIntegerVT(*DAG.getContext(),
323
398
                                           NOutVT.getSizeInBits()),
324
398
                         JoinIntegers(Lo, Hi));
325
398
      return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
326
398
    }
327
59
    break;
328
59
  }
329
59
  case TargetLowering::TypeWidenVector:
330
30
    // The input is widened to the same size. Convert to the widened value.
331
30
    // Make sure that the outgoing value is not a vector, because this would
332
30
    // make us bitcast between two vectors which are legalized in different ways.
333
30
    if (NOutVT.bitsEq(NInVT) && 
!NOutVT.isVector()18
)
334
9
      return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
335
21
    // If the output type is also a vector and widening it to the same size
336
21
    // as the widened input type would be a legal type, we can widen the bitcast
337
21
    // and handle the promotion after.
338
21
    if (NOutVT.isVector()) {
339
9
      unsigned WidenInSize = NInVT.getSizeInBits();
340
9
      unsigned OutSize = OutVT.getSizeInBits();
341
9
      if (WidenInSize % OutSize == 0) {
342
9
        unsigned Scale = WidenInSize / OutSize;
343
9
        EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
344
9
                                         OutVT.getVectorElementType(),
345
9
                                         OutVT.getVectorNumElements() * Scale);
346
9
        if (isTypeLegal(WideOutVT)) {
347
9
          InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
348
9
          MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
349
9
          InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
350
9
                             DAG.getConstant(0, dl, IdxTy));
351
9
          return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
352
9
        }
353
1.94k
      }
354
9
    }
355
1.94k
  }
356
1.94k
357
1.94k
  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
358
1.94k
                     CreateStackStoreLoad(InOp, OutVT));
359
1.94k
}
360
361
// Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
362
// in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
363
static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
364
                                       const TargetLowering &TLI,
365
136
                                       SelectionDAG &DAG) {
366
136
  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
367
136
  // If the value won't fit in the prefered type, just use something safe. It
368
136
  // will be legalized when the shift is expanded.
369
136
  if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
370
6
    ShiftVT = MVT::i32;
371
136
  return ShiftVT;
372
136
}
373
374
92
SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
375
92
  SDValue Op = GetPromotedInteger(N->getOperand(0));
376
92
  EVT OVT = N->getValueType(0);
377
92
  EVT NVT = Op.getValueType();
378
92
  SDLoc dl(N);
379
92
380
92
  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
381
92
  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
382
92
  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
383
92
                     DAG.getConstant(DiffBits, dl, ShiftVT));
384
92
}
385
386
18
SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
387
18
  SDValue Op = GetPromotedInteger(N->getOperand(0));
388
18
  EVT OVT = N->getValueType(0);
389
18
  EVT NVT = Op.getValueType();
390
18
  SDLoc dl(N);
391
18
392
18
  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
393
18
  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
394
18
  return DAG.getNode(ISD::SRL, dl, NVT,
395
18
                     DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
396
18
                     DAG.getConstant(DiffBits, dl, ShiftVT));
397
18
}
398
399
0
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
400
0
  // The pair element type may be legal, or may not promote to the same type as
401
0
  // the result, for example i14 = BUILD_PAIR (i7, i7).  Handle all cases.
402
0
  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
403
0
                     TLI.getTypeToTransformTo(*DAG.getContext(),
404
0
                     N->getValueType(0)), JoinIntegers(N->getOperand(0),
405
0
                     N->getOperand(1)));
406
0
}
407
408
146k
SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
409
146k
  EVT VT = N->getValueType(0);
410
146k
  // FIXME there is no actual debug info here
411
146k
  SDLoc dl(N);
412
146k
  // Zero extend things like i1, sign extend everything else.  It shouldn't
413
146k
  // matter in theory which one we pick, but this tends to give better code?
414
146k
  unsigned Opc = VT.isByteSized() ? 
ISD::SIGN_EXTEND125k
:
ISD::ZERO_EXTEND21.6k
;
415
146k
  SDValue Result = DAG.getNode(Opc, dl,
416
146k
                               TLI.getTypeToTransformTo(*DAG.getContext(), VT),
417
146k
                               SDValue(N, 0));
418
146k
  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
419
146k
  return Result;
420
146k
}
421
422
60
SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
423
60
  // Zero extend to the promoted type and do the count there.
424
60
  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
425
60
  SDLoc dl(N);
426
60
  EVT OVT = N->getValueType(0);
427
60
  EVT NVT = Op.getValueType();
428
60
  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
429
60
  // Subtract off the extra leading bits in the bigger type.
430
60
  return DAG.getNode(
431
60
      ISD::SUB, dl, NVT, Op,
432
60
      DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
433
60
                      NVT));
434
60
}
435
436
81
SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
437
81
  // Zero extend to the promoted type and do the count there.
438
81
  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
439
81
  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
440
81
}
441
442
87
SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
443
87
  SDValue Op = GetPromotedInteger(N->getOperand(0));
444
87
  EVT OVT = N->getValueType(0);
445
87
  EVT NVT = Op.getValueType();
446
87
  SDLoc dl(N);
447
87
  if (N->getOpcode() == ISD::CTTZ) {
448
39
    // The count is the same in the promoted type except if the original
449
39
    // value was zero.  This can be handled by setting the bit just off
450
39
    // the top of the original type.
451
39
    auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
452
39
                                      OVT.getScalarSizeInBits());
453
39
    Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
454
39
  }
455
87
  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
456
87
}
457
458
58.5k
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
459
58.5k
  SDLoc dl(N);
460
58.5k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
461
58.5k
462
58.5k
  SDValue Op0 = N->getOperand(0);
463
58.5k
  SDValue Op1 = N->getOperand(1);
464
58.5k
465
58.5k
  // If the input also needs to be promoted, do that first so we can get a
466
58.5k
  // get a good idea for the output type.
467
58.5k
  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
468
58.5k
      == TargetLowering::TypePromoteInteger) {
469
24.3k
    SDValue In = GetPromotedInteger(Op0);
470
24.3k
471
24.3k
    // If the new type is larger than NVT, use it. We probably won't need to
472
24.3k
    // promote it again.
473
24.3k
    EVT SVT = In.getValueType().getScalarType();
474
24.3k
    if (SVT.bitsGE(NVT)) {
475
6.46k
      SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
476
6.46k
      return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
477
6.46k
    }
478
52.1k
  }
479
52.1k
480
52.1k
  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
481
52.1k
}
482
483
1.28k
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
484
1.28k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
485
1.28k
  unsigned NewOpc = N->getOpcode();
486
1.28k
  SDLoc dl(N);
487
1.28k
488
1.28k
  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
489
1.28k
  // not Legal, check to see if we can use FP_TO_SINT instead.  (If both UINT
490
1.28k
  // and SINT conversions are Custom, there is no way to tell which is
491
1.28k
  // preferable. We choose SINT because that's the right thing on PPC.)
492
1.28k
  if (N->getOpcode() == ISD::FP_TO_UINT &&
493
1.28k
      
!TLI.isOperationLegal(ISD::FP_TO_UINT, NVT)652
&&
494
1.28k
      
TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT)624
)
495
590
    NewOpc = ISD::FP_TO_SINT;
496
1.28k
497
1.28k
  SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
498
1.28k
499
1.28k
  // Assert that the converted value fits in the original type.  If it doesn't
500
1.28k
  // (eg: because the value being converted is too big), then the result of the
501
1.28k
  // original operation was undefined anyway, so the assert is still correct.
502
1.28k
  //
503
1.28k
  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
504
1.28k
  //   before legalization: fp-to-uint16, 65534. -> 0xfffe
505
1.28k
  //   after legalization: fp-to-sint32, 65534. -> 0x0000fffe
506
1.28k
  return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
507
652
                     ISD::AssertZext : 
ISD::AssertSext635
, dl, NVT, Res,
508
1.28k
                     DAG.getValueType(N->getValueType(0).getScalarType()));
509
1.28k
}
510
511
1.12k
SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
512
1.12k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
513
1.12k
  SDLoc dl(N);
514
1.12k
515
1.12k
  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
516
1.12k
}
517
518
1
SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
519
1
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
520
1
  SDLoc dl(N);
521
1
522
1
  return DAG.getNode(N->getOpcode(), dl, NVT);
523
1
}
524
525
21.9k
SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
526
21.9k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
527
21.9k
  SDLoc dl(N);
528
21.9k
529
21.9k
  if (getTypeAction(N->getOperand(0).getValueType())
530
21.9k
      == TargetLowering::TypePromoteInteger) {
531
20.8k
    SDValue Res = GetPromotedInteger(N->getOperand(0));
532
20.8k
    assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
533
20.8k
534
20.8k
    // If the result and operand types are the same after promotion, simplify
535
20.8k
    // to an in-register extension.
536
20.8k
    if (NVT == Res.getValueType()) {
537
20.8k
      // The high bits are not guaranteed to be anything.  Insert an extend.
538
20.8k
      if (N->getOpcode() == ISD::SIGN_EXTEND)
539
864
        return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
540
864
                           DAG.getValueType(N->getOperand(0).getValueType()));
541
19.9k
      if (N->getOpcode() == ISD::ZERO_EXTEND)
542
1.77k
        return DAG.getZeroExtendInReg(Res, dl,
543
1.77k
                      N->getOperand(0).getValueType().getScalarType());
544
18.2k
      assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
545
18.2k
      return Res;
546
18.2k
    }
547
20.8k
  }
548
1.13k
549
1.13k
  // Otherwise, just extend the original operand all the way to the larger type.
550
1.13k
  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
551
1.13k
}
552
553
93.0k
SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
554
93.0k
  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
555
93.0k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
556
93.0k
  ISD::LoadExtType ExtType =
557
93.0k
    ISD::isNON_EXTLoad(N) ? 
ISD::EXTLOAD92.4k
:
N->getExtensionType()527
;
558
93.0k
  SDLoc dl(N);
559
93.0k
  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
560
93.0k
                               N->getMemoryVT(), N->getMemOperand());
561
93.0k
562
93.0k
  // Legalize the chain result - switch anything that used the old chain to
563
93.0k
  // use the new one.
564
93.0k
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
565
93.0k
  return Res;
566
93.0k
}
567
568
5
SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
569
5
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
570
5
  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
571
5
572
5
  SDLoc dl(N);
573
5
  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
574
5
                                  N->getMask(), ExtPassThru, N->getMemoryVT(),
575
5
                                  N->getMemOperand(), ISD::EXTLOAD);
576
5
  // Legalize the chain result - switch anything that used the old chain to
577
5
  // use the new one.
578
5
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
579
5
  return Res;
580
5
}
581
582
0
SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
583
0
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
584
0
  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
585
0
  assert(NVT == ExtPassThru.getValueType() &&
586
0
      "Gather result type and the passThru agrument type should be the same");
587
0
588
0
  SDLoc dl(N);
589
0
  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
590
0
                   N->getIndex(), N->getScale() };
591
0
  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
592
0
                                    N->getMemoryVT(), dl, Ops,
593
0
                                    N->getMemOperand());
594
0
  // Legalize the chain result - switch anything that used the old chain to
595
0
  // use the new one.
596
0
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
597
0
  return Res;
598
0
}
599
600
/// Promote the overflow flag of an overflowing arithmetic node.
601
4.65k
SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
602
4.65k
  // Change the return type of the boolean result while obeying
603
4.65k
  // getSetCCResultType.
604
4.65k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
605
4.65k
  EVT VT = N->getValueType(0);
606
4.65k
  EVT SVT = getSetCCResultType(VT);
607
4.65k
  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
608
4.65k
  unsigned NumOps = N->getNumOperands();
609
4.65k
  assert(NumOps <= 3 && "Too many operands");
610
4.65k
  if (NumOps == 3)
611
148
    Ops[2] = N->getOperand(2);
612
4.65k
613
4.65k
  SDLoc dl(N);
614
4.65k
  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
615
4.65k
                            makeArrayRef(Ops, NumOps));
616
4.65k
617
4.65k
  // Modified the sum result - switch anything that used the old sum to use
618
4.65k
  // the new one.
619
4.65k
  ReplaceValueWith(SDValue(N, 0), Res);
620
4.65k
621
4.65k
  // Convert to the expected type.
622
4.65k
  return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
623
4.65k
}
624
625
116
SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
626
116
  // For promoting iN -> iM, this can be expanded by
627
116
  // 1. ANY_EXTEND iN to iM
628
116
  // 2. SHL by M-N
629
116
  // 3. [US][ADD|SUB]SAT
630
116
  // 4. L/ASHR by M-N
631
116
  SDLoc dl(N);
632
116
  SDValue Op1 = N->getOperand(0);
633
116
  SDValue Op2 = N->getOperand(1);
634
116
  unsigned OldBits = Op1.getScalarValueSizeInBits();
635
116
636
116
  unsigned Opcode = N->getOpcode();
637
116
  unsigned ShiftOp;
638
116
  switch (Opcode) {
639
116
  case ISD::SADDSAT:
640
50
  case ISD::SSUBSAT:
641
50
    ShiftOp = ISD::SRA;
642
50
    break;
643
66
  case ISD::UADDSAT:
644
66
  case ISD::USUBSAT:
645
66
    ShiftOp = ISD::SRL;
646
66
    break;
647
66
  default:
648
0
    llvm_unreachable("Expected opcode to be signed or unsigned saturation "
649
116
                     "addition or subtraction");
650
116
  }
651
116
652
116
  SDValue Op1Promoted = GetPromotedInteger(Op1);
653
116
  SDValue Op2Promoted = GetPromotedInteger(Op2);
654
116
655
116
  EVT PromotedType = Op1Promoted.getValueType();
656
116
  unsigned NewBits = PromotedType.getScalarSizeInBits();
657
116
  unsigned SHLAmount = NewBits - OldBits;
658
116
  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
659
116
  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
660
116
  Op1Promoted =
661
116
      DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
662
116
  Op2Promoted =
663
116
      DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
664
116
665
116
  SDValue Result =
666
116
      DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
667
116
  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
668
116
}
669
670
12
SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
671
12
  // Can just promote the operands then continue with operation.
672
12
  SDLoc dl(N);
673
12
  SDValue Op1Promoted, Op2Promoted;
674
12
  bool Signed =
675
12
      N->getOpcode() == ISD::SMULFIX || 
N->getOpcode() == ISD::SMULFIXSAT8
;
676
12
  if (Signed) {
677
8
    Op1Promoted = SExtPromotedInteger(N->getOperand(0));
678
8
    Op2Promoted = SExtPromotedInteger(N->getOperand(1));
679
8
  } else {
680
4
    Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
681
4
    Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
682
4
  }
683
12
  EVT OldType = N->getOperand(0).getValueType();
684
12
  EVT PromotedType = Op1Promoted.getValueType();
685
12
  unsigned DiffSize =
686
12
      PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
687
12
688
12
  bool Saturating = N->getOpcode() == ISD::SMULFIXSAT;
689
12
  if (Saturating) {
690
4
    // Promoting the operand and result values changes the saturation width,
691
4
    // which is extends the values that we clamp to on saturation. This could be
692
4
    // resolved by shifting one of the operands the same amount, which would
693
4
    // also shift the result we compare against, then shifting back.
694
4
    EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
695
4
    Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
696
4
                              DAG.getConstant(DiffSize, dl, ShiftTy));
697
4
    SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
698
4
                                 Op2Promoted, N->getOperand(2));
699
4
    unsigned ShiftOp = Signed ? ISD::SRA : 
ISD::SRL0
;
700
4
    return DAG.getNode(ShiftOp, dl, PromotedType, Result,
701
4
                       DAG.getConstant(DiffSize, dl, ShiftTy));
702
4
  }
703
8
  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
704
8
                     N->getOperand(2));
705
8
}
706
707
771
SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
708
771
  if (ResNo == 1)
709
735
    return PromoteIntRes_Overflow(N);
710
36
711
36
  // The operation overflowed iff the result in the larger type is not the
712
36
  // sign extension of its truncation to the original type.
713
36
  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
714
36
  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
715
36
  EVT OVT = N->getOperand(0).getValueType();
716
36
  EVT NVT = LHS.getValueType();
717
36
  SDLoc dl(N);
718
36
719
36
  // Do the arithmetic in the larger type.
720
36
  unsigned Opcode = N->getOpcode() == ISD::SADDO ? 
ISD::ADD19
:
ISD::SUB17
;
721
36
  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
722
36
723
36
  // Calculate the overflow flag: sign extend the arithmetic result from
724
36
  // the original type.
725
36
  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
726
36
                            DAG.getValueType(OVT));
727
36
  // Overflowed if and only if this is not equal to Res.
728
36
  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
729
36
730
36
  // Use the calculated overflow everywhere.
731
36
  ReplaceValueWith(SDValue(N, 1), Ofl);
732
36
733
36
  return Res;
734
36
}
735
736
3.95k
SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
737
3.95k
  SDValue LHS = GetPromotedInteger(N->getOperand(1));
738
3.95k
  SDValue RHS = GetPromotedInteger(N->getOperand(2));
739
3.95k
  return DAG.getSelect(SDLoc(N),
740
3.95k
                       LHS.getValueType(), N->getOperand(0), LHS, RHS);
741
3.95k
}
742
743
143
SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
744
143
  SDValue Mask = N->getOperand(0);
745
143
746
143
  SDValue LHS = GetPromotedInteger(N->getOperand(1));
747
143
  SDValue RHS = GetPromotedInteger(N->getOperand(2));
748
143
  return DAG.getNode(ISD::VSELECT, SDLoc(N),
749
143
                     LHS.getValueType(), Mask, LHS, RHS);
750
143
}
751
752
63
SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
753
63
  SDValue LHS = GetPromotedInteger(N->getOperand(2));
754
63
  SDValue RHS = GetPromotedInteger(N->getOperand(3));
755
63
  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
756
63
                     LHS.getValueType(), N->getOperand(0),
757
63
                     N->getOperand(1), LHS, RHS, N->getOperand(4));
758
63
}
759
760
256k
SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
761
256k
  EVT InVT = N->getOperand(0).getValueType();
762
256k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
763
256k
764
256k
  EVT SVT = getSetCCResultType(InVT);
765
256k
766
256k
  // If we got back a type that needs to be promoted, this likely means the
767
256k
  // the input type also needs to be promoted. So get the promoted type for
768
256k
  // the input and try the query again.
769
256k
  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
770
494
    if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
771
479
      InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
772
479
      SVT = getSetCCResultType(InVT);
773
479
    } else {
774
15
      // Input type isn't promoted, just use the default promoted type.
775
15
      SVT = NVT;
776
15
    }
777
494
  }
778
256k
779
256k
  SDLoc dl(N);
780
256k
  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
781
256k
         "Vector compare must return a vector result!");
782
256k
783
256k
  // Get the SETCC result using the canonical SETCC type.
784
256k
  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
785
256k
                              N->getOperand(1), N->getOperand(2));
786
256k
787
256k
  // Convert to the expected type.
788
256k
  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
789
256k
}
790
791
2.53k
SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
792
2.53k
  SDValue LHS = GetPromotedInteger(N->getOperand(0));
793
2.53k
  SDValue RHS = N->getOperand(1);
794
2.53k
  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
795
1.33k
    RHS = ZExtPromotedInteger(RHS);
796
2.53k
  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
797
2.53k
}
798
799
91
SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
800
91
  SDValue Op = GetPromotedInteger(N->getOperand(0));
801
91
  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
802
91
                     Op.getValueType(), Op, N->getOperand(1));
803
91
}
804
805
18.1k
SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
806
18.1k
  // The input may have strange things in the top bits of the registers, but
807
18.1k
  // these operations don't care.  They may have weird bits going out, but
808
18.1k
  // that too is okay if they are integer operations.
809
18.1k
  SDValue LHS = GetPromotedInteger(N->getOperand(0));
810
18.1k
  SDValue RHS = GetPromotedInteger(N->getOperand(1));
811
18.1k
  return DAG.getNode(N->getOpcode(), SDLoc(N),
812
18.1k
                     LHS.getValueType(), LHS, RHS);
813
18.1k
}
814
815
809
SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
816
809
  // Sign extend the input.
817
809
  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
818
809
  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
819
809
  return DAG.getNode(N->getOpcode(), SDLoc(N),
820
809
                     LHS.getValueType(), LHS, RHS);
821
809
}
822
823
720
SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
824
720
  // Zero extend the input.
825
720
  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
826
720
  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
827
720
  return DAG.getNode(N->getOpcode(), SDLoc(N),
828
720
                     LHS.getValueType(), LHS, RHS);
829
720
}
830
831
1.04k
SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
832
1.04k
  // The input value must be properly sign extended.
833
1.04k
  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
834
1.04k
  SDValue RHS = N->getOperand(1);
835
1.04k
  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
836
646
    RHS = ZExtPromotedInteger(RHS);
837
1.04k
  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
838
1.04k
}
839
840
6.94k
SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
841
6.94k
  // The input value must be properly zero extended.
842
6.94k
  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
843
6.94k
  SDValue RHS = N->getOperand(1);
844
6.94k
  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
845
2.38k
    RHS = ZExtPromotedInteger(RHS);
846
6.94k
  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
847
6.94k
}
848
849
79.7k
SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
850
79.7k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
851
79.7k
  SDValue Res;
852
79.7k
  SDValue InOp = N->getOperand(0);
853
79.7k
  SDLoc dl(N);
854
79.7k
855
79.7k
  switch (getTypeAction(InOp.getValueType())) {
856
79.7k
  
default: 0
llvm_unreachable0
("Unknown type action!");
857
79.7k
  case TargetLowering::TypeLegal:
858
74.4k
  case TargetLowering::TypeExpandInteger:
859
74.4k
    Res = InOp;
860
74.4k
    break;
861
74.4k
  case TargetLowering::TypePromoteInteger:
862
5.17k
    Res = GetPromotedInteger(InOp);
863
5.17k
    break;
864
74.4k
  case TargetLowering::TypeSplitVector: {
865
105
    EVT InVT = InOp.getValueType();
866
105
    assert(InVT.isVector() && "Cannot split scalar types");
867
105
    unsigned NumElts = InVT.getVectorNumElements();
868
105
    assert(NumElts == NVT.getVectorNumElements() &&
869
105
           "Dst and Src must have the same number of elements");
870
105
    assert(isPowerOf2_32(NumElts) &&
871
105
           "Promoted vector type must be a power of two");
872
105
873
105
    SDValue EOp1, EOp2;
874
105
    GetSplitVector(InOp, EOp1, EOp2);
875
105
876
105
    EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
877
105
                                   NumElts/2);
878
105
    EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
879
105
    EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
880
105
881
105
    return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
882
74.4k
  }
883
74.4k
  case TargetLowering::TypeWidenVector: {
884
1
    SDValue WideInOp = GetWidenedVector(InOp);
885
1
886
1
    // Truncate widened InOp.
887
1
    unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
888
1
    EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
889
1
                                   N->getValueType(0).getScalarType(), NumElem);
890
1
    SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
891
1
892
1
    // Zero extend so that the elements are of same type as those of NVT
893
1
    EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
894
1
                                 NumElem);
895
1
    SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
896
1
897
1
    // Extract the low NVT subvector.
898
1
    MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
899
1
    SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
900
1
    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
901
79.6k
  }
902
79.6k
  }
903
79.6k
904
79.6k
  // Truncate to NVT instead of VT
905
79.6k
  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
906
79.6k
}
907
908
3.15k
SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
909
3.15k
  if (ResNo == 1)
910
3.10k
    return PromoteIntRes_Overflow(N);
911
54
912
54
  // The operation overflowed iff the result in the larger type is not the
913
54
  // zero extension of its truncation to the original type.
914
54
  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
915
54
  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
916
54
  EVT OVT = N->getOperand(0).getValueType();
917
54
  EVT NVT = LHS.getValueType();
918
54
  SDLoc dl(N);
919
54
920
54
  // Do the arithmetic in the larger type.
921
54
  unsigned Opcode = N->getOpcode() == ISD::UADDO ? 
ISD::ADD36
:
ISD::SUB18
;
922
54
  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
923
54
924
54
  // Calculate the overflow flag: zero extend the arithmetic result from
925
54
  // the original type.
926
54
  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
927
54
  // Overflowed if and only if this is not equal to Res.
928
54
  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
929
54
930
54
  // Use the calculated overflow everywhere.
931
54
  ReplaceValueWith(SDValue(N, 1), Ofl);
932
54
933
54
  return Res;
934
54
}
935
936
// Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
937
// the third operand of ADDE/SUBE nodes is carry flag, which differs from 
938
// the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
939
148
SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
940
148
  if (ResNo == 1)
941
148
    return PromoteIntRes_Overflow(N);
942
0
943
0
  // We need to sign-extend the operands so the carry value computed by the
944
0
  // wide operation will be equivalent to the carry value computed by the
945
0
  // narrow operation.
946
0
  // An ADDCARRY can generate carry only if any of the operands has its
947
0
  // most significant bit set. Sign extension propagates the most significant
948
0
  // bit into the higher bits which means the extra bit that the narrow
949
0
  // addition would need (i.e. the carry) will be propagated through the higher
950
0
  // bits of the wide addition.
951
0
  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
952
0
  // preserved by sign extension.
953
0
  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
954
0
  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
955
0
956
0
  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
957
0
958
0
  // Do the arithmetic in the wide type.
959
0
  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
960
0
                            LHS, RHS, N->getOperand(2));
961
0
962
0
  // Update the users of the original carry/borrow value.
963
0
  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
964
0
965
0
  return SDValue(Res.getNode(), 0);
966
0
}
967
968
13
SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
969
13
  SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
970
13
  return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
971
13
}
972
973
717
SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
974
717
  // Promote the overflow bit trivially.
975
717
  if (ResNo == 1)
976
669
    return PromoteIntRes_Overflow(N);
977
48
978
48
  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
979
48
  SDLoc DL(N);
980
48
  EVT SmallVT = LHS.getValueType();
981
48
982
48
  // To determine if the result overflowed in a larger type, we extend the
983
48
  // input to the larger type, do the multiply (checking if it overflows),
984
48
  // then also check the high bits of the result to see if overflow happened
985
48
  // there.
986
48
  if (N->getOpcode() == ISD::SMULO) {
987
22
    LHS = SExtPromotedInteger(LHS);
988
22
    RHS = SExtPromotedInteger(RHS);
989
26
  } else {
990
26
    LHS = ZExtPromotedInteger(LHS);
991
26
    RHS = ZExtPromotedInteger(RHS);
992
26
  }
993
48
  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
994
48
  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
995
48
996
48
  // Overflow occurred if it occurred in the larger type, or if the high part
997
48
  // of the result does not zero/sign-extend the low part.  Check this second
998
48
  // possibility first.
999
48
  SDValue Overflow;
1000
48
  if (N->getOpcode() == ISD::UMULO) {
1001
26
    // Unsigned overflow occurred if the high part is non-zero.
1002
26
    unsigned Shift = SmallVT.getScalarSizeInBits();
1003
26
    EVT ShiftTy = getShiftAmountTyForConstant(Shift, Mul.getValueType(),
1004
26
                                              TLI, DAG);
1005
26
    SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1006
26
                             DAG.getConstant(Shift, DL, ShiftTy));
1007
26
    Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1008
26
                            DAG.getConstant(0, DL, Hi.getValueType()),
1009
26
                            ISD::SETNE);
1010
26
  } else {
1011
22
    // Signed overflow occurred if the high part does not sign extend the low.
1012
22
    SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1013
22
                               Mul, DAG.getValueType(SmallVT));
1014
22
    Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1015
22
  }
1016
48
1017
48
  // The only other way for overflow to occur is if the multiplication in the
1018
48
  // larger type itself overflowed.
1019
48
  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1020
48
                         SDValue(Mul.getNode(), 1));
1021
48
1022
48
  // Use the calculated overflow everywhere.
1023
48
  ReplaceValueWith(SDValue(N, 1), Overflow);
1024
48
  return Mul;
1025
48
}
1026
1027
3.88k
SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1028
3.88k
  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1029
3.88k
                                               N->getValueType(0)));
1030
3.88k
}
1031
1032
65
SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1033
65
  SDValue Chain = N->getOperand(0); // Get the chain.
1034
65
  SDValue Ptr = N->getOperand(1); // Get the pointer.
1035
65
  EVT VT = N->getValueType(0);
1036
65
  SDLoc dl(N);
1037
65
1038
65
  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1039
65
  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1040
65
  // The argument is passed as NumRegs registers of type RegVT.
1041
65
1042
65
  SmallVector<SDValue, 8> Parts(NumRegs);
1043
130
  for (unsigned i = 0; i < NumRegs; 
++i65
) {
1044
65
    Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1045
65
                            N->getConstantOperandVal(3));
1046
65
    Chain = Parts[i].getValue(1);
1047
65
  }
1048
65
1049
65
  // Handle endianness of the load.
1050
65
  if (DAG.getDataLayout().isBigEndian())
1051
18
    std::reverse(Parts.begin(), Parts.end());
1052
65
1053
65
  // Assemble the parts in the promoted type.
1054
65
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1055
65
  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1056
65
  for (unsigned i = 1; i < NumRegs; 
++i0
) {
1057
0
    SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1058
0
    // Shift it to the right position and "or" it in.
1059
0
    Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1060
0
                       DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1061
0
                                       TLI.getPointerTy(DAG.getDataLayout())));
1062
0
    Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1063
0
  }
1064
65
1065
65
  // Modified the chain result - switch anything that used the old chain to
1066
65
  // use the new one.
1067
65
  ReplaceValueWith(SDValue(N, 1), Chain);
1068
65
1069
65
  return Res;
1070
65
}
1071
1072
//===----------------------------------------------------------------------===//
1073
//  Integer Operand Promotion
1074
//===----------------------------------------------------------------------===//
1075
1076
/// PromoteIntegerOperand - This method is called when the specified operand of
1077
/// the specified node is found to need promotion.  At this point, all of the
1078
/// result types of the node are known to be legal, but other operands of the
1079
/// node may need promotion or expansion as well as the specified one.
1080
851k
bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1081
851k
  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1082
851k
             dbgs() << "\n");
1083
851k
  SDValue Res = SDValue();
1084
851k
1085
851k
  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1086
1.49k
    LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1087
1.49k
    return false;
1088
1.49k
  }
1089
850k
1090
850k
  switch (N->getOpcode()) {
1091
850k
    default:
1092
  #ifndef NDEBUG
1093
    dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1094
    N->dump(&DAG); dbgs() << "\n";
1095
  #endif
1096
0
    llvm_unreachable("Do not know how to promote this operator's operand!");
1097
850k
1098
850k
  
case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break65.1k
;
1099
850k
  case ISD::ATOMIC_STORE:
1100
226
    Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1101
226
    break;
1102
850k
  
case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break334
;
1103
850k
  
case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break2.09k
;
1104
850k
  
case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break234k
;
1105
850k
  
case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break0
;
1106
850k
  
case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break13.3k
;
1107
850k
  
case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break5.50k
;
1108
850k
  
case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break25.7k
;
1109
850k
  case ISD::INSERT_VECTOR_ELT:
1110
402
                          Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1111
850k
  case ISD::SCALAR_TO_VECTOR:
1112
93
                          Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1113
850k
  case ISD::VSELECT:
1114
19.3k
  case ISD::SELECT:       Res = PromoteIntOp_SELECT(N, OpNo); break;
1115
19.3k
  
case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break5.86k
;
1116
88.6k
  case ISD::SETCC:        Res = PromoteIntOp_SETCC(N, OpNo); break;
1117
19.3k
  
case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break10.9k
;
1118
19.3k
  
case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break5.76k
;
1119
86.2k
  case ISD::STORE:        Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1120
86.2k
                                                   OpNo); break;
1121
19.3k
  case ISD::MSTORE:       Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1122
129
                                                    OpNo); break;
1123
19.3k
  case ISD::MLOAD:        Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1124
62
                                                    OpNo); break;
1125
19.3k
  case ISD::MGATHER:  Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1126
52
                                                 OpNo); break;
1127
19.3k
  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1128
8
                                                  OpNo); break;
1129
19.3k
  
case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break3.92k
;
1130
19.3k
  case ISD::FP16_TO_FP:
1131
9.94k
  case ISD::UINT_TO_FP:   Res = PromoteIntOp_UINT_TO_FP(N); break;
1132
271k
  case ISD::ZERO_EXTEND:  Res = PromoteIntOp_ZERO_EXTEND(N); break;
1133
9.94k
  
case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break0
;
1134
9.94k
1135
9.94k
  case ISD::SHL:
1136
0
  case ISD::SRA:
1137
0
  case ISD::SRL:
1138
0
  case ISD::ROTL:
1139
0
  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1140
0
1141
147
  case ISD::ADDCARRY:
1142
147
  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1143
147
1144
147
  case ISD::FRAMEADDR:
1145
5
  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1146
5
1147
5
  
case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break1
;
1148
5
1149
5
  case ISD::SMULFIX:
1150
0
  case ISD::SMULFIXSAT:
1151
0
  case ISD::UMULFIX: Res = PromoteIntOp_MULFIX(N); break;
1152
0
1153
3
  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1154
0
1155
19
  case ISD::VECREDUCE_ADD:
1156
19
  case ISD::VECREDUCE_MUL:
1157
19
  case ISD::VECREDUCE_AND:
1158
19
  case ISD::VECREDUCE_OR:
1159
19
  case ISD::VECREDUCE_XOR:
1160
19
  case ISD::VECREDUCE_SMAX:
1161
19
  case ISD::VECREDUCE_SMIN:
1162
19
  case ISD::VECREDUCE_UMAX:
1163
19
  case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1164
850k
  }
1165
850k
1166
850k
  // If the result is null, the sub-method took care of registering results etc.
1167
850k
  if (!Res.getNode()) 
return false0
;
1168
850k
1169
850k
  // If the result is N, the sub-method updated N in place.  Tell the legalizer
1170
850k
  // core about this.
1171
850k
  if (Res.getNode() == N)
1172
378k
    return true;
1173
472k
1174
472k
  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1175
472k
         "Invalid operand expansion");
1176
472k
1177
472k
  ReplaceValueWith(SDValue(N, 0), Res);
1178
472k
  return false;
1179
472k
}
1180
1181
/// PromoteSetCCOperands - Promote the operands of a comparison.  This code is
1182
/// shared among BR_CC, SELECT_CC, and SETCC handlers.
1183
void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1184
96.6k
                                            ISD::CondCode CCCode) {
1185
96.6k
  // We have to insert explicit sign or zero extends. Note that we could
1186
96.6k
  // insert sign extends for ALL conditions. For those operations where either
1187
96.6k
  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1188
96.6k
  // which will choose the cheapest for the target.
1189
96.6k
  switch (CCCode) {
1190
96.6k
  
default: 0
llvm_unreachable0
("Unknown integer comparison!");
1191
96.6k
  case ISD::SETEQ:
1192
81.0k
  case ISD::SETNE: {
1193
81.0k
    SDValue OpL = GetPromotedInteger(NewLHS);
1194
81.0k
    SDValue OpR = GetPromotedInteger(NewRHS);
1195
81.0k
1196
81.0k
    // We would prefer to promote the comparison operand with sign extension.
1197
81.0k
    // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1198
81.0k
    // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1199
81.0k
    // instruction, which is redundant eventually.
1200
81.0k
    unsigned OpLEffectiveBits =
1201
81.0k
        OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1202
81.0k
    unsigned OpREffectiveBits =
1203
81.0k
        OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1204
81.0k
    if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1205
81.0k
        
OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()6.34k
) {
1206
3.37k
      NewLHS = OpL;
1207
3.37k
      NewRHS = OpR;
1208
77.6k
    } else {
1209
77.6k
      NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1210
77.6k
      NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1211
77.6k
    }
1212
81.0k
    break;
1213
81.0k
  }
1214
81.0k
  case ISD::SETUGE:
1215
10.1k
  case ISD::SETUGT:
1216
10.1k
  case ISD::SETULE:
1217
10.1k
  case ISD::SETULT:
1218
10.1k
    NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1219
10.1k
    NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1220
10.1k
    break;
1221
10.1k
  case ISD::SETGE:
1222
5.44k
  case ISD::SETGT:
1223
5.44k
  case ISD::SETLT:
1224
5.44k
  case ISD::SETLE:
1225
5.44k
    NewLHS = SExtPromotedInteger(NewLHS);
1226
5.44k
    NewRHS = SExtPromotedInteger(NewRHS);
1227
5.44k
    break;
1228
96.6k
  }
1229
96.6k
}
1230
1231
65.1k
SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1232
65.1k
  SDValue Op = GetPromotedInteger(N->getOperand(0));
1233
65.1k
  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1234
65.1k
}
1235
1236
226
SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1237
226
  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1238
226
  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1239
226
                       N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1240
226
}
1241
1242
334
SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1243
334
  // This should only occur in unusual situations like bitcasting to an
1244
334
  // x86_fp80, so just turn it into a store+load
1245
334
  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1246
334
}
1247
1248
2.09k
SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1249
2.09k
  assert(OpNo == 2 && "Don't know how to promote this operand!");
1250
2.09k
1251
2.09k
  SDValue LHS = N->getOperand(2);
1252
2.09k
  SDValue RHS = N->getOperand(3);
1253
2.09k
  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1254
2.09k
1255
2.09k
  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1256
2.09k
  // legal types.
1257
2.09k
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1258
2.09k
                                N->getOperand(1), LHS, RHS, N->getOperand(4)),
1259
2.09k
                 0);
1260
2.09k
}
1261
1262
234k
SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1263
234k
  assert(OpNo == 1 && "only know how to promote condition");
1264
234k
1265
234k
  // Promote all the way up to the canonical SetCC type.
1266
234k
  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1267
234k
1268
234k
  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1269
234k
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1270
234k
                                        N->getOperand(2)), 0);
1271
234k
}
1272
1273
0
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1274
0
  // Since the result type is legal, the operands must promote to it.
1275
0
  EVT OVT = N->getOperand(0).getValueType();
1276
0
  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1277
0
  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1278
0
  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1279
0
  SDLoc dl(N);
1280
0
1281
0
  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1282
0
                   DAG.getConstant(OVT.getSizeInBits(), dl,
1283
0
                                   TLI.getPointerTy(DAG.getDataLayout())));
1284
0
  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1285
0
}
1286
1287
13.3k
SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1288
13.3k
  // The vector type is legal but the element type is not.  This implies
1289
13.3k
  // that the vector is a power-of-two in length and that the element
1290
13.3k
  // type does not have a strange size (eg: it is not i1).
1291
13.3k
  EVT VecVT = N->getValueType(0);
1292
13.3k
  unsigned NumElts = VecVT.getVectorNumElements();
1293
13.3k
  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1294
13.3k
         "Legal vector of one illegal element?");
1295
13.3k
1296
13.3k
  // Promote the inserted value.  The type does not need to match the
1297
13.3k
  // vector element type.  Check that any extra bits introduced will be
1298
13.3k
  // truncated away.
1299
13.3k
  assert(N->getOperand(0).getValueSizeInBits() >=
1300
13.3k
         N->getValueType(0).getScalarSizeInBits() &&
1301
13.3k
         "Type of inserted value narrower than vector element type!");
1302
13.3k
1303
13.3k
  SmallVector<SDValue, 16> NewOps;
1304
120k
  for (unsigned i = 0; i < NumElts; 
++i107k
)
1305
107k
    NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1306
13.3k
1307
13.3k
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1308
13.3k
}
1309
1310
SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1311
402
                                                         unsigned OpNo) {
1312
402
  if (OpNo == 1) {
1313
402
    // Promote the inserted value.  This is valid because the type does not
1314
402
    // have to match the vector element type.
1315
402
1316
402
    // Check that any extra bits introduced will be truncated away.
1317
402
    assert(N->getOperand(1).getValueSizeInBits() >=
1318
402
           N->getValueType(0).getScalarSizeInBits() &&
1319
402
           "Type of inserted value narrower than vector element type!");
1320
402
    return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1321
402
                                  GetPromotedInteger(N->getOperand(1)),
1322
402
                                  N->getOperand(2)),
1323
402
                   0);
1324
402
  }
1325
0
1326
0
  assert(OpNo == 2 && "Different operand and result vector types?");
1327
0
1328
0
  // Promote the index.
1329
0
  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1330
0
                                   TLI.getVectorIdxTy(DAG.getDataLayout()));
1331
0
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1332
0
                                N->getOperand(1), Idx), 0);
1333
0
}
1334
1335
93
SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1336
93
  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1337
93
  // the operand in place.
1338
93
  return SDValue(DAG.UpdateNodeOperands(N,
1339
93
                                GetPromotedInteger(N->getOperand(0))), 0);
1340
93
}
1341
1342
19.3k
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1343
19.3k
  assert(OpNo == 0 && "Only know how to promote the condition!");
1344
19.3k
  SDValue Cond = N->getOperand(0);
1345
19.3k
  EVT OpTy = N->getOperand(1).getValueType();
1346
19.3k
1347
19.3k
  if (N->getOpcode() == ISD::VSELECT)
1348
2.00k
    if (SDValue Res = WidenVSELECTAndMask(N))
1349
1.90k
      return Res;
1350
17.4k
1351
17.4k
  // Promote all the way up to the canonical SetCC type.
1352
17.4k
  EVT OpVT = N->getOpcode() == ISD::SELECT ? 
OpTy.getScalarType()17.3k
:
OpTy100
;
1353
17.4k
  Cond = PromoteTargetBoolean(Cond, OpVT);
1354
17.4k
1355
17.4k
  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1356
17.4k
                                        N->getOperand(2)), 0);
1357
17.4k
}
1358
1359
5.86k
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1360
5.86k
  assert(OpNo == 0 && "Don't know how to promote this operand!");
1361
5.86k
1362
5.86k
  SDValue LHS = N->getOperand(0);
1363
5.86k
  SDValue RHS = N->getOperand(1);
1364
5.86k
  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1365
5.86k
1366
5.86k
  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1367
5.86k
  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1368
5.86k
                                N->getOperand(3), N->getOperand(4)), 0);
1369
5.86k
}
1370
1371
88.6k
SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1372
88.6k
  assert(OpNo == 0 && "Don't know how to promote this operand!");
1373
88.6k
1374
88.6k
  SDValue LHS = N->getOperand(0);
1375
88.6k
  SDValue RHS = N->getOperand(1);
1376
88.6k
  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1377
88.6k
1378
88.6k
  // The CC (#2) is always legal.
1379
88.6k
  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1380
88.6k
}
1381
1382
0
SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1383
0
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1384
0
                                ZExtPromotedInteger(N->getOperand(1))), 0);
1385
0
}
1386
1387
10.9k
SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1388
10.9k
  SDValue Op = GetPromotedInteger(N->getOperand(0));
1389
10.9k
  SDLoc dl(N);
1390
10.9k
  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1391
10.9k
  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1392
10.9k
                     Op, DAG.getValueType(N->getOperand(0).getValueType()));
1393
10.9k
}
1394
1395
5.76k
SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1396
5.76k
  return SDValue(DAG.UpdateNodeOperands(N,
1397
5.76k
                                SExtPromotedInteger(N->getOperand(0))), 0);
1398
5.76k
}
1399
1400
86.2k
SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1401
86.2k
  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1402
86.2k
  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1403
86.2k
  SDLoc dl(N);
1404
86.2k
1405
86.2k
  SDValue Val = GetPromotedInteger(N->getValue());  // Get promoted value.
1406
86.2k
1407
86.2k
  // Truncate the value and store the result.
1408
86.2k
  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1409
86.2k
                           N->getMemoryVT(), N->getMemOperand());
1410
86.2k
}
1411
1412
SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1413
129
                                              unsigned OpNo) {
1414
129
1415
129
  SDValue DataOp = N->getValue();
1416
129
  EVT DataVT = DataOp.getValueType();
1417
129
  SDValue Mask = N->getMask();
1418
129
  SDLoc dl(N);
1419
129
1420
129
  bool TruncateStore = false;
1421
129
  if (OpNo == 3) {
1422
90
    Mask = PromoteTargetBoolean(Mask, DataVT);
1423
90
    // Update in place.
1424
90
    SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1425
90
    NewOps[3] = Mask;
1426
90
    return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1427
90
  } else { // Data operand
1428
39
    assert(OpNo == 1 && "Unexpected operand for promotion");
1429
39
    DataOp = GetPromotedInteger(DataOp);
1430
39
    TruncateStore = true;
1431
39
  }
1432
129
1433
129
  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1434
39
                            N->getMemoryVT(), N->getMemOperand(),
1435
39
                            TruncateStore, N->isCompressingStore());
1436
129
}
1437
1438
SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1439
62
                                             unsigned OpNo) {
1440
62
  assert(OpNo == 2 && "Only know how to promote the mask!");
1441
62
  EVT DataVT = N->getValueType(0);
1442
62
  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1443
62
  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1444
62
  NewOps[OpNo] = Mask;
1445
62
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1446
62
}
1447
1448
SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1449
52
                                               unsigned OpNo) {
1450
52
1451
52
  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1452
52
  if (OpNo == 2) {
1453
32
    // The Mask
1454
32
    EVT DataVT = N->getValueType(0);
1455
32
    NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1456
32
  } else 
if (20
OpNo == 420
) {
1457
20
    // Need to sign extend the index since the bits will likely be used.
1458
20
    NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1459
20
  } else
1460
0
    NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1461
52
1462
52
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1463
52
}
1464
1465
SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1466
8
                                                unsigned OpNo) {
1467
8
  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1468
8
  if (OpNo == 2) {
1469
0
    // The Mask
1470
0
    EVT DataVT = N->getValue().getValueType();
1471
0
    NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1472
8
  } else if (OpNo == 4) {
1473
8
    // Need to sign extend the index since the bits will likely be used.
1474
8
    NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1475
8
  } else
1476
0
    NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1477
8
  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1478
8
}
1479
1480
3.92k
SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1481
3.92k
  SDValue Op = GetPromotedInteger(N->getOperand(0));
1482
3.92k
  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1483
3.92k
}
1484
1485
9.94k
SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1486
9.94k
  return SDValue(DAG.UpdateNodeOperands(N,
1487
9.94k
                                ZExtPromotedInteger(N->getOperand(0))), 0);
1488
9.94k
}
1489
1490
271k
SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1491
271k
  SDLoc dl(N);
1492
271k
  SDValue Op = GetPromotedInteger(N->getOperand(0));
1493
271k
  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1494
271k
  return DAG.getZeroExtendInReg(Op, dl,
1495
271k
                                N->getOperand(0).getValueType().getScalarType());
1496
271k
}
1497
1498
147
SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1499
147
  assert(OpNo == 2 && "Don't know how to promote this operand!");
1500
147
1501
147
  SDValue LHS = N->getOperand(0);
1502
147
  SDValue RHS = N->getOperand(1);
1503
147
  SDValue Carry = N->getOperand(2);
1504
147
  SDLoc DL(N);
1505
147
1506
147
  Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1507
147
1508
147
  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1509
147
}
1510
1511
0
SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
1512
0
  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1513
0
  return SDValue(
1514
0
      DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1515
0
}
1516
1517
5
SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1518
5
  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1519
5
  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1520
5
  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1521
5
}
1522
1523
1
SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1524
1
  assert(OpNo > 1 && "Don't know how to promote this operand!");
1525
1
  // Promote the rw, locality, and cache type arguments to a supported integer
1526
1
  // width.
1527
1
  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1528
1
  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1529
1
  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1530
1
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1531
1
                                        Op2, Op3, Op4),
1532
1
                 0);
1533
1
}
1534
1535
3
SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1536
3
  SDValue Op = SExtPromotedInteger(N->getOperand(1));
1537
3
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1538
3
}
1539
1540
19
SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1541
19
  SDLoc dl(N);
1542
19
  SDValue Op;
1543
19
  switch (N->getOpcode()) {
1544
19
  
default: 0
llvm_unreachable0
("Expected integer vector reduction");
1545
19
  case ISD::VECREDUCE_ADD:
1546
16
  case ISD::VECREDUCE_MUL:
1547
16
  case ISD::VECREDUCE_AND:
1548
16
  case ISD::VECREDUCE_OR:
1549
16
  case ISD::VECREDUCE_XOR:
1550
16
    Op = GetPromotedInteger(N->getOperand(0));
1551
16
    break;
1552
16
  case ISD::VECREDUCE_SMAX:
1553
0
  case ISD::VECREDUCE_SMIN:
1554
0
    Op = SExtPromotedInteger(N->getOperand(0));
1555
0
    break;
1556
3
  case ISD::VECREDUCE_UMAX:
1557
3
  case ISD::VECREDUCE_UMIN:
1558
3
    Op = ZExtPromotedInteger(N->getOperand(0));
1559
3
    break;
1560
19
  }
1561
19
1562
19
  EVT EltVT = Op.getValueType().getVectorElementType();
1563
19
  EVT VT = N->getValueType(0);
1564
19
  if (VT.bitsGE(EltVT))
1565
19
    return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1566
0
1567
0
  // Result size must be >= element size. If this is not the case after
1568
0
  // promotion, also promote the result type and then truncate.
1569
0
  SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1570
0
  return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1571
0
}
1572
1573
//===----------------------------------------------------------------------===//
1574
//  Integer Result Expansion
1575
//===----------------------------------------------------------------------===//
1576
1577
/// ExpandIntegerResult - This method is called when the specified result of the
1578
/// specified node is found to need expansion.  At this point, the node may also
1579
/// have invalid operands or may have other results that need promotion, we just
1580
/// know that (at least) one result needs expansion.
1581
262k
void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1582
262k
  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1583
262k
             dbgs() << "\n");
1584
262k
  SDValue Lo, Hi;
1585
262k
  Lo = Hi = SDValue();
1586
262k
1587
262k
  // See if the target wants to custom expand this node.
1588
262k
  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1589
1.41k
    return;
1590
261k
1591
261k
  switch (N->getOpcode()) {
1592
261k
  default:
1593
#ifndef NDEBUG
1594
    dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1595
    N->dump(&DAG); dbgs() << "\n";
1596
#endif
1597
    report_fatal_error("Do not know how to expand the result of this "
1598
1
                       "operator!");
1599
261k
1600
261k
  
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break48
;
1601
261k
  
case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break1.30k
;
1602
261k
  
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break666
;
1603
261k
  
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break2.16k
;
1604
261k
1605
261k
  
case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break14.1k
;
1606
261k
  
case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break29.7k
;
1607
261k
  
case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break2.85k
;
1608
261k
  
case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break2.29k
;
1609
261k
  
case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break20
;
1610
261k
1611
261k
  
case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break5.01k
;
1612
261k
  
case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break317
;
1613
261k
  
case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break315
;
1614
261k
  
case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break32
;
1615
261k
  
case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break77
;
1616
261k
  
case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break18.7k
;
1617
261k
  
case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break5
;
1618
261k
  case ISD::CTLZ_ZERO_UNDEF:
1619
103
  case ISD::CTLZ:        ExpandIntRes_CTLZ(N, Lo, Hi); break;
1620
103
  
case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break35
;
1621
103
  case ISD::CTTZ_ZERO_UNDEF:
1622
16
  case ISD::CTTZ:        ExpandIntRes_CTTZ(N, Lo, Hi); break;
1623
16
  
case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break1
;
1624
103
  case ISD::FP_TO_SINT:  ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1625
95
  case ISD::FP_TO_UINT:  ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1626
16
  
case ISD::LLROUND: ExpandIntRes_LLROUND(N, Lo, Hi); break8
;
1627
16
  
case ISD::LLRINT: ExpandIntRes_LLRINT(N, Lo, Hi); break8
;
1628
9.87k
  case ISD::LOAD:        ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1629
2.78k
  case ISD::MUL:         ExpandIntRes_MUL(N, Lo, Hi); break;
1630
16
  
case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break2
;
1631
116
  case ISD::SDIV:        ExpandIntRes_SDIV(N, Lo, Hi); break;
1632
2.02k
  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1633
200
  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1634
74
  case ISD::SREM:        ExpandIntRes_SREM(N, Lo, Hi); break;
1635
20.0k
  case ISD::TRUNCATE:    ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1636
136
  case ISD::UDIV:        ExpandIntRes_UDIV(N, Lo, Hi); break;
1637
79
  case ISD::UREM:        ExpandIntRes_UREM(N, Lo, Hi); break;
1638
11.6k
  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1639
22
  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1640
16
1641
86
  case ISD::ATOMIC_LOAD_ADD:
1642
86
  case ISD::ATOMIC_LOAD_SUB:
1643
86
  case ISD::ATOMIC_LOAD_AND:
1644
86
  case ISD::ATOMIC_LOAD_CLR:
1645
86
  case ISD::ATOMIC_LOAD_OR:
1646
86
  case ISD::ATOMIC_LOAD_XOR:
1647
86
  case ISD::ATOMIC_LOAD_NAND:
1648
86
  case ISD::ATOMIC_LOAD_MIN:
1649
86
  case ISD::ATOMIC_LOAD_MAX:
1650
86
  case ISD::ATOMIC_LOAD_UMIN:
1651
86
  case ISD::ATOMIC_LOAD_UMAX:
1652
86
  case ISD::ATOMIC_SWAP:
1653
86
  case ISD::ATOMIC_CMP_SWAP: {
1654
86
    std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1655
86
    SplitInteger(Tmp.first, Lo, Hi);
1656
86
    ReplaceValueWith(SDValue(N, 1), Tmp.second);
1657
86
    break;
1658
86
  }
1659
86
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
1660
55
    AtomicSDNode *AN = cast<AtomicSDNode>(N);
1661
55
    SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1662
55
    SDValue Tmp = DAG.getAtomicCmpSwap(
1663
55
        ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1664
55
        N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1665
55
        AN->getMemOperand());
1666
55
1667
55
    // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1668
55
    // success simply by comparing the loaded value against the ingoing
1669
55
    // comparison.
1670
55
    SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1671
55
                                   N->getOperand(2), ISD::SETEQ);
1672
55
1673
55
    SplitInteger(Tmp, Lo, Hi);
1674
55
    ReplaceValueWith(SDValue(N, 1), Success);
1675
55
    ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1676
55
    break;
1677
86
  }
1678
86
1679
27.6k
  case ISD::AND:
1680
27.6k
  case ISD::OR:
1681
27.6k
  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1682
27.6k
1683
27.6k
  case ISD::UMAX:
1684
10
  case ISD::SMAX:
1685
10
  case ISD::UMIN:
1686
10
  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1687
10
1688
7.33k
  case ISD::ADD:
1689
7.33k
  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1690
7.33k
1691
7.33k
  case ISD::ADDC:
1692
12
  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1693
12
1694
12
  case ISD::ADDE:
1695
12
  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1696
12
1697
985
  case ISD::ADDCARRY:
1698
985
  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1699
985
1700
99.2k
  case ISD::SHL:
1701
99.2k
  case ISD::SRA:
1702
99.2k
  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1703
99.2k
1704
99.2k
  case ISD::SADDO:
1705
56
  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1706
839
  case ISD::UADDO:
1707
839
  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1708
839
  case ISD::UMULO:
1709
90
  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1710
90
1711
90
  case ISD::SADDSAT:
1712
84
  case ISD::UADDSAT:
1713
84
  case ISD::SSUBSAT:
1714
84
  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1715
84
1716
84
  case ISD::SMULFIX:
1717
14
  case ISD::SMULFIXSAT:
1718
14
  case ISD::UMULFIX: ExpandIntRes_MULFIX(N, Lo, Hi); break;
1719
14
1720
14
  case ISD::VECREDUCE_ADD:
1721
3
  case ISD::VECREDUCE_MUL:
1722
3
  case ISD::VECREDUCE_AND:
1723
3
  case ISD::VECREDUCE_OR:
1724
3
  case ISD::VECREDUCE_XOR:
1725
3
  case ISD::VECREDUCE_SMAX:
1726
3
  case ISD::VECREDUCE_SMIN:
1727
3
  case ISD::VECREDUCE_UMAX:
1728
3
  case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
1729
261k
  }
1730
261k
1731
261k
  // If Lo/Hi is null, the sub-method took care of registering results etc.
1732
261k
  if (Lo.getNode())
1733
261k
    SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1734
261k
}
1735
1736
/// Lower an atomic node to the appropriate builtin call.
1737
86
std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1738
86
  unsigned Opc = Node->getOpcode();
1739
86
  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1740
86
  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1741
86
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1742
86
1743
86
  return ExpandChainLibCall(LC, Node, false);
1744
86
}
1745
1746
/// N is a shift by a value that needs to be expanded,
1747
/// and the shift amount is a constant 'Amt'.  Expand the operation.
1748
void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1749
96.9k
                                             SDValue &Lo, SDValue &Hi) {
1750
96.9k
  SDLoc DL(N);
1751
96.9k
  // Expand the incoming operand to be shifted, so that we have its parts
1752
96.9k
  SDValue InL, InH;
1753
96.9k
  GetExpandedInteger(N->getOperand(0), InL, InH);
1754
96.9k
1755
96.9k
  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1756
96.9k
  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1757
96.9k
  if (!Amt) {
1758
0
    Lo = InL;
1759
0
    Hi = InH;
1760
0
    return;
1761
0
  }
1762
96.9k
1763
96.9k
  EVT NVT = InL.getValueType();
1764
96.9k
  unsigned VTBits = N->getValueType(0).getSizeInBits();
1765
96.9k
  unsigned NVTBits = NVT.getSizeInBits();
1766
96.9k
  EVT ShTy = N->getOperand(1).getValueType();
1767
96.9k
1768
96.9k
  if (N->getOpcode() == ISD::SHL) {
1769
19.2k
    if (Amt.ugt(VTBits)) {
1770
0
      Lo = Hi = DAG.getConstant(0, DL, NVT);
1771
19.2k
    } else if (Amt.ugt(NVTBits)) {
1772
3.51k
      Lo = DAG.getConstant(0, DL, NVT);
1773
3.51k
      Hi = DAG.getNode(ISD::SHL, DL,
1774
3.51k
                       NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1775
15.7k
    } else if (Amt == NVTBits) {
1776
14.5k
      Lo = DAG.getConstant(0, DL, NVT);
1777
14.5k
      Hi = InL;
1778
14.5k
    } else {
1779
1.21k
      Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1780
1.21k
      Hi = DAG.getNode(ISD::OR, DL, NVT,
1781
1.21k
                       DAG.getNode(ISD::SHL, DL, NVT, InH,
1782
1.21k
                                   DAG.getConstant(Amt, DL, ShTy)),
1783
1.21k
                       DAG.getNode(ISD::SRL, DL, NVT, InL,
1784
1.21k
                                   DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1785
1.21k
    }
1786
19.2k
    return;
1787
19.2k
  }
1788
77.6k
1789
77.6k
  if (N->getOpcode() == ISD::SRL) {
1790
77.2k
    if (Amt.ugt(VTBits)) {
1791
0
      Lo = Hi = DAG.getConstant(0, DL, NVT);
1792
77.2k
    } else if (Amt.ugt(NVTBits)) {
1793
725
      Lo = DAG.getNode(ISD::SRL, DL,
1794
725
                       NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1795
725
      Hi = DAG.getConstant(0, DL, NVT);
1796
76.5k
    } else if (Amt == NVTBits) {
1797
49.3k
      Lo = InH;
1798
49.3k
      Hi = DAG.getConstant(0, DL, NVT);
1799
49.3k
    } else {
1800
27.1k
      Lo = DAG.getNode(ISD::OR, DL, NVT,
1801
27.1k
                       DAG.getNode(ISD::SRL, DL, NVT, InL,
1802
27.1k
                                   DAG.getConstant(Amt, DL, ShTy)),
1803
27.1k
                       DAG.getNode(ISD::SHL, DL, NVT, InH,
1804
27.1k
                                   DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1805
27.1k
      Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1806
27.1k
    }
1807
77.2k
    return;
1808
77.2k
  }
1809
367
1810
367
  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1811
367
  if (Amt.ugt(VTBits)) {
1812
0
    Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1813
0
                          DAG.getConstant(NVTBits - 1, DL, ShTy));
1814
367
  } else if (Amt.ugt(NVTBits)) {
1815
309
    Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1816
309
                     DAG.getConstant(Amt - NVTBits, DL, ShTy));
1817
309
    Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1818
309
                     DAG.getConstant(NVTBits - 1, DL, ShTy));
1819
309
  } else 
if (58
Amt == NVTBits58
) {
1820
4
    Lo = InH;
1821
4
    Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1822
4
                     DAG.getConstant(NVTBits - 1, DL, ShTy));
1823
54
  } else {
1824
54
    Lo = DAG.getNode(ISD::OR, DL, NVT,
1825
54
                     DAG.getNode(ISD::SRL, DL, NVT, InL,
1826
54
                                 DAG.getConstant(Amt, DL, ShTy)),
1827
54
                     DAG.getNode(ISD::SHL, DL, NVT, InH,
1828
54
                                 DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1829
54
    Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1830
54
  }
1831
367
}
1832
1833
/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1834
/// this shift based on knowledge of the high bit of the shift amount.  If we
1835
/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1836
/// shift amount.
1837
bool DAGTypeLegalizer::
1838
2.31k
ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1839
2.31k
  SDValue Amt = N->getOperand(1);
1840
2.31k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1841
2.31k
  EVT ShTy = Amt.getValueType();
1842
2.31k
  unsigned ShBits = ShTy.getScalarSizeInBits();
1843
2.31k
  unsigned NVTBits = NVT.getScalarSizeInBits();
1844
2.31k
  assert(isPowerOf2_32(NVTBits) &&
1845
2.31k
         "Expanded integer type size not a power of two!");
1846
2.31k
  SDLoc dl(N);
1847
2.31k
1848
2.31k
  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1849
2.31k
  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
1850
2.31k
1851
2.31k
  // If we don't know anything about the high bits, exit.
1852
2.31k
  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1853
2.10k
    return false;
1854
213
1855
213
  // Get the incoming operand to be shifted.
1856
213
  SDValue InL, InH;
1857
213
  GetExpandedInteger(N->getOperand(0), InL, InH);
1858
213
1859
213
  // If we know that any of the high bits of the shift amount are one, then we
1860
213
  // can do this as a couple of simple shifts.
1861
213
  if (Known.One.intersects(HighBitMask)) {
1862
19
    // Mask out the high bit, which we know is set.
1863
19
    Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1864
19
                      DAG.getConstant(~HighBitMask, dl, ShTy));
1865
19
1866
19
    switch (N->getOpcode()) {
1867
19
    
default: 0
llvm_unreachable0
("Unknown shift");
1868
19
    case ISD::SHL:
1869
0
      Lo = DAG.getConstant(0, dl, NVT);              // Low part is zero.
1870
0
      Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1871
0
      return true;
1872
19
    case ISD::SRL:
1873
17
      Hi = DAG.getConstant(0, dl, NVT);              // Hi part is zero.
1874
17
      Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1875
17
      return true;
1876
19
    case ISD::SRA:
1877
2
      Hi = DAG.getNode(ISD::SRA, dl, NVT, InH,       // Sign extend high part.
1878
2
                       DAG.getConstant(NVTBits - 1, dl, ShTy));
1879
2
      Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1880
2
      return true;
1881
194
    }
1882
194
  }
1883
194
1884
194
  // If we know that all of the high bits of the shift amount are zero, then we
1885
194
  // can do this as a couple of simple shifts.
1886
194
  if (HighBitMask.isSubsetOf(Known.Zero)) {
1887
15
    // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1888
15
    // shift if x is zero.  We can use XOR here because x is known to be smaller
1889
15
    // than 32.
1890
15
    SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1891
15
                               DAG.getConstant(NVTBits - 1, dl, ShTy));
1892
15
1893
15
    unsigned Op1, Op2;
1894
15
    switch (N->getOpcode()) {
1895
15
    
default: 0
llvm_unreachable0
("Unknown shift");
1896
15
    
case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break8
;
1897
15
    case ISD::SRL:
1898
7
    case ISD::SRA:  Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1899
15
    }
1900
15
1901
15
    // When shifting right the arithmetic for Lo and Hi is swapped.
1902
15
    if (N->getOpcode() != ISD::SHL)
1903
7
      std::swap(InL, InH);
1904
15
1905
15
    // Use a little trick to get the bits that move from Lo to Hi. First
1906
15
    // shift by one bit.
1907
15
    SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1908
15
    // Then compute the remaining shift with amount-1.
1909
15
    SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1910
15
1911
15
    Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1912
15
    Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1913
15
1914
15
    if (N->getOpcode() != ISD::SHL)
1915
7
      std::swap(Hi, Lo);
1916
15
    return true;
1917
15
  }
1918
179
1919
179
  return false;
1920
179
}
1921
1922
/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1923
/// of any size.
1924
bool DAGTypeLegalizer::
1925
97
ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1926
97
  SDValue Amt = N->getOperand(1);
1927
97
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1928
97
  EVT ShTy = Amt.getValueType();
1929
97
  unsigned NVTBits = NVT.getSizeInBits();
1930
97
  assert(isPowerOf2_32(NVTBits) &&
1931
97
         "Expanded integer type size not a power of two!");
1932
97
  SDLoc dl(N);
1933
97
1934
97
  // Get the incoming operand to be shifted.
1935
97
  SDValue InL, InH;
1936
97
  GetExpandedInteger(N->getOperand(0), InL, InH);
1937
97
1938
97
  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
1939
97
  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1940
97
  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1941
97
  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1942
97
                                 Amt, NVBitsNode, ISD::SETULT);
1943
97
  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1944
97
                                Amt, DAG.getConstant(0, dl, ShTy),
1945
97
                                ISD::SETEQ);
1946
97
1947
97
  SDValue LoS, HiS, LoL, HiL;
1948
97
  switch (N->getOpcode()) {
1949
97
  
default: 0
llvm_unreachable0
("Unknown shift");
1950
97
  case ISD::SHL:
1951
45
    // Short: ShAmt < NVTBits
1952
45
    LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1953
45
    HiS = DAG.getNode(ISD::OR, dl, NVT,
1954
45
                      DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1955
45
                      DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1956
45
1957
45
    // Long: ShAmt >= NVTBits
1958
45
    LoL = DAG.getConstant(0, dl, NVT);                    // Lo part is zero.
1959
45
    HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1960
45
1961
45
    Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1962
45
    Hi = DAG.getSelect(dl, NVT, isZero, InH,
1963
45
                       DAG.getSelect(dl, NVT, isShort, HiS, HiL));
1964
45
    return true;
1965
97
  case ISD::SRL:
1966
29
    // Short: ShAmt < NVTBits
1967
29
    HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1968
29
    LoS = DAG.getNode(ISD::OR, dl, NVT,
1969
29
                      DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1970
29
    // FIXME: If Amt is zero, the following shift generates an undefined result
1971
29
    // on some architectures.
1972
29
                      DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1973
29
1974
29
    // Long: ShAmt >= NVTBits
1975
29
    HiL = DAG.getConstant(0, dl, NVT);                    // Hi part is zero.
1976
29
    LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1977
29
1978
29
    Lo = DAG.getSelect(dl, NVT, isZero, InL,
1979
29
                       DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1980
29
    Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1981
29
    return true;
1982
97
  case ISD::SRA:
1983
23
    // Short: ShAmt < NVTBits
1984
23
    HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1985
23
    LoS = DAG.getNode(ISD::OR, dl, NVT,
1986
23
                      DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1987
23
                      DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1988
23
1989
23
    // Long: ShAmt >= NVTBits
1990
23
    HiL = DAG.getNode(ISD::SRA, dl, NVT, InH,             // Sign of Hi part.
1991
23
                      DAG.getConstant(NVTBits - 1, dl, ShTy));
1992
23
    LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1993
23
1994
23
    Lo = DAG.getSelect(dl, NVT, isZero, InL,
1995
23
                       DAG.getSelect(dl, NVT, isShort, LoS, LoL));
1996
23
    Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1997
23
    return true;
1998
97
  }
1999
97
}
2000
2001
10
static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2002
10
2003
10
  switch (Op) {
2004
10
    
default: 0
llvm_unreachable0
("invalid min/max opcode");
2005
10
    case ISD::SMAX:
2006
2
      return std::make_pair(ISD::SETGT, ISD::UMAX);
2007
10
    case ISD::UMAX:
2008
4
      return std::make_pair(ISD::SETUGT, ISD::UMAX);
2009
10
    case ISD::SMIN:
2010
2
      return std::make_pair(ISD::SETLT, ISD::UMIN);
2011
10
    case ISD::UMIN:
2012
2
      return std::make_pair(ISD::SETULT, ISD::UMIN);
2013
10
  }
2014
10
}
2015
2016
void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2017
10
                                           SDValue &Lo, SDValue &Hi) {
2018
10
  SDLoc DL(N);
2019
10
  ISD::NodeType LoOpc;
2020
10
  ISD::CondCode CondC;
2021
10
  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2022
10
2023
10
  // Expand the subcomponents.
2024
10
  SDValue LHSL, LHSH, RHSL, RHSH;
2025
10
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2026
10
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2027
10
2028
10
  // Value types
2029
10
  EVT NVT = LHSL.getValueType();
2030
10
  EVT CCT = getSetCCResultType(NVT);
2031
10
2032
10
  // Hi part is always the same op
2033
10
  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2034
10
2035
10
  // We need to know whether to select Lo part that corresponds to 'winning'
2036
10
  // Hi part or if Hi parts are equal.
2037
10
  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2038
10
  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2039
10
2040
10
  // Lo part corresponding to the 'winning' Hi part
2041
10
  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2042
10
2043
10
  // Recursed Lo part if Hi parts are equal, this uses unsigned version
2044
10
  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2045
10
2046
10
  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2047
10
}
2048
2049
void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2050
7.33k
                                           SDValue &Lo, SDValue &Hi) {
2051
7.33k
  SDLoc dl(N);
2052
7.33k
  // Expand the subcomponents.
2053
7.33k
  SDValue LHSL, LHSH, RHSL, RHSH;
2054
7.33k
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2055
7.33k
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2056
7.33k
2057
7.33k
  EVT NVT = LHSL.getValueType();
2058
7.33k
  SDValue LoOps[2] = { LHSL, RHSL };
2059
7.33k
  SDValue HiOps[3] = { LHSH, RHSH };
2060
7.33k
2061
7.33k
  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2062
7.33k
      N->getOpcode() == ISD::ADD ? 
ISD::ADDCARRY5.71k
:
ISD::SUBCARRY1.62k
,
2063
7.33k
      TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2064
7.33k
  if (HasOpCarry) {
2065
6.04k
    SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2066
6.04k
    if (N->getOpcode() == ISD::ADD) {
2067
5.23k
      Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2068
5.23k
      HiOps[2] = Lo.getValue(1);
2069
5.23k
      Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2070
5.23k
    } else {
2071
807
      Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2072
807
      HiOps[2] = Lo.getValue(1);
2073
807
      Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2074
807
    }
2075
6.04k
    return;
2076
6.04k
  }
2077
1.28k
2078
1.28k
  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2079
1.28k
  // them.  TODO: Teach operation legalization how to expand unsupported
2080
1.28k
  // ADDC/ADDE/SUBC/SUBE.  The problem is that these operations generate
2081
1.28k
  // a carry of type MVT::Glue, but there doesn't seem to be any way to
2082
1.28k
  // generate a value of this type in the expanded code sequence.
2083
1.28k
  bool hasCarry =
2084
1.28k
    TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2085
815
                                   
ISD::ADDC472
: ISD::SUBC,
2086
1.28k
                                 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2087
1.28k
2088
1.28k
  if (hasCarry) {
2089
262
    SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2090
262
    if (N->getOpcode() == ISD::ADD) {
2091
173
      Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2092
173
      HiOps[2] = Lo.getValue(1);
2093
173
      Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2094
173
    } else {
2095
89
      Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2096
89
      HiOps[2] = Lo.getValue(1);
2097
89
      Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2098
89
    }
2099
262
    return;
2100
262
  }
2101
1.02k
2102
1.02k
  bool hasOVF =
2103
1.02k
    TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2104
726
                                   
ISD::UADDO299
: ISD::USUBO,
2105
1.02k
                                 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2106
1.02k
  TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(NVT);
2107
1.02k
2108
1.02k
  if (hasOVF) {
2109
706
    EVT OvfVT = getSetCCResultType(NVT);
2110
706
    SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2111
706
    int RevOpc;
2112
706
    if (N->getOpcode() == ISD::ADD) {
2113
53
      RevOpc = ISD::SUB;
2114
53
      Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2115
53
      Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2116
653
    } else {
2117
653
      RevOpc = ISD::ADD;
2118
653
      Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2119
653
      Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2120
653
    }
2121
706
    SDValue OVF = Lo.getValue(1);
2122
706
2123
706
    switch (BoolType) {
2124
706
    case TargetLoweringBase::UndefinedBooleanContent:
2125
0
      OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2126
0
      LLVM_FALLTHROUGH;
2127
0
    case TargetLoweringBase::ZeroOrOneBooleanContent:
2128
0
      OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2129
0
      Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2130
0
      break;
2131
706
    case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
2132
706
      OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2133
706
      Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2134
706
    }
2135
706
    return;
2136
319
  }
2137
319
2138
319
  if (N->getOpcode() == ISD::ADD) {
2139
246
    Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2140
246
    Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2141
246
    SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2142
246
                                ISD::SETULT);
2143
246
2144
246
    if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent) {
2145
245
      SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2146
245
      Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2147
245
      return;
2148
245
    }
2149
1
2150
1
    SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2151
1
                                   DAG.getConstant(1, dl, NVT),
2152
1
                                   DAG.getConstant(0, dl, NVT));
2153
1
    SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2154
1
                                ISD::SETULT);
2155
1
    SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2156
1
                                   DAG.getConstant(1, dl, NVT), Carry1);
2157
1
    Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2158
73
  } else {
2159
73
    Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2160
73
    Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2161
73
    SDValue Cmp =
2162
73
      DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2163
73
                   LoOps[0], LoOps[1], ISD::SETULT);
2164
73
2165
73
    SDValue Borrow;
2166
73
    if (BoolType == TargetLoweringBase::ZeroOrOneBooleanContent)
2167
73
      Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2168
0
    else
2169
0
      Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2170
0
                             DAG.getConstant(0, dl, NVT));
2171
73
2172
73
    Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2173
73
  }
2174
319
}
2175
2176
void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2177
12
                                            SDValue &Lo, SDValue &Hi) {
2178
12
  // Expand the subcomponents.
2179
12
  SDValue LHSL, LHSH, RHSL, RHSH;
2180
12
  SDLoc dl(N);
2181
12
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2182
12
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2183
12
  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2184
12
  SDValue LoOps[2] = { LHSL, RHSL };
2185
12
  SDValue HiOps[3] = { LHSH, RHSH };
2186
12
2187
12
  if (N->getOpcode() == ISD::ADDC) {
2188
10
    Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2189
10
    HiOps[2] = Lo.getValue(1);
2190
10
    Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2191
10
  } else {
2192
2
    Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2193
2
    HiOps[2] = Lo.getValue(1);
2194
2
    Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2195
2
  }
2196
12
2197
12
  // Legalized the flag result - switch anything that used the old flag to
2198
12
  // use the new one.
2199
12
  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2200
12
}
2201
2202
void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2203
12
                                            SDValue &Lo, SDValue &Hi) {
2204
12
  // Expand the subcomponents.
2205
12
  SDValue LHSL, LHSH, RHSL, RHSH;
2206
12
  SDLoc dl(N);
2207
12
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2208
12
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2209
12
  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2210
12
  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2211
12
  SDValue HiOps[3] = { LHSH, RHSH };
2212
12
2213
12
  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2214
12
  HiOps[2] = Lo.getValue(1);
2215
12
  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2216
12
2217
12
  // Legalized the flag result - switch anything that used the old flag to
2218
12
  // use the new one.
2219
12
  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2220
12
}
2221
2222
void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2223
839
                                             SDValue &Lo, SDValue &Hi) {
2224
839
  SDValue LHS = N->getOperand(0);
2225
839
  SDValue RHS = N->getOperand(1);
2226
839
  SDLoc dl(N);
2227
839
2228
839
  SDValue Ovf;
2229
839
2230
839
  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2231
839
      N->getOpcode() == ISD::ADD ? 
ISD::ADDCARRY0
: ISD::SUBCARRY,
2232
839
      TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2233
839
2234
839
  if (HasOpCarry) {
2235
817
    // Expand the subcomponents.
2236
817
    SDValue LHSL, LHSH, RHSL, RHSH;
2237
817
    GetExpandedInteger(LHS, LHSL, LHSH);
2238
817
    GetExpandedInteger(RHS, RHSL, RHSH);
2239
817
    SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2240
817
    SDValue LoOps[2] = { LHSL, RHSL };
2241
817
    SDValue HiOps[3] = { LHSH, RHSH };
2242
817
2243
817
    unsigned Opc = N->getOpcode() == ISD::UADDO ? 
ISD::ADDCARRY740
:
ISD::SUBCARRY77
;
2244
817
    Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2245
817
    HiOps[2] = Lo.getValue(1);
2246
817
    Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2247
817
2248
817
    Ovf = Hi.getValue(1);
2249
817
  } else {
2250
22
    // Expand the result by simply replacing it with the equivalent
2251
22
    // non-overflow-checking operation.
2252
22
    auto Opc = N->getOpcode() == ISD::UADDO ? 
ISD::ADD20
:
ISD::SUB2
;
2253
22
    SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2254
22
    SplitInteger(Sum, Lo, Hi);
2255
22
2256
22
    // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2257
22
    // overflows iff a - b > a.
2258
22
    auto Cond = N->getOpcode() == ISD::UADDO ? 
ISD::SETULT20
:
ISD::SETUGT2
;
2259
22
    Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2260
22
  }
2261
839
2262
839
  // Legalized the flag result - switch anything that used the old flag to
2263
839
  // use the new one.
2264
839
  ReplaceValueWith(SDValue(N, 1), Ovf);
2265
839
}
2266
2267
void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2268
985
                                                SDValue &Lo, SDValue &Hi) {
2269
985
  // Expand the subcomponents.
2270
985
  SDValue LHSL, LHSH, RHSL, RHSH;
2271
985
  SDLoc dl(N);
2272
985
  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2273
985
  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2274
985
  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2275
985
  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2276
985
  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2277
985
2278
985
  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2279
985
  HiOps[2] = Lo.getValue(1);
2280
985
  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2281
985
2282
985
  // Legalized the flag result - switch anything that used the old flag to
2283
985
  // use the new one.
2284
985
  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2285
985
}
2286
2287
void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2288
5.01k
                                               SDValue &Lo, SDValue &Hi) {
2289
5.01k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2290
5.01k
  SDLoc dl(N);
2291
5.01k
  SDValue Op = N->getOperand(0);
2292
5.01k
  if (Op.getValueType().bitsLE(NVT)) {
2293
4.88k
    // The low part is any extension of the input (which degenerates to a copy).
2294
4.88k
    Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2295
4.88k
    Hi = DAG.getUNDEF(NVT);   // The high part is undefined.
2296
4.88k
  } else {
2297
130
    // For example, extension of an i48 to an i64.  The operand type necessarily
2298
130
    // promotes to the result type, so will end up being expanded too.
2299
130
    assert(getTypeAction(Op.getValueType()) ==
2300
130
           TargetLowering::TypePromoteInteger &&
2301
130
           "Only know how to promote this result!");
2302
130
    SDValue Res = GetPromotedInteger(Op);
2303
130
    assert(Res.getValueType() == N->getValueType(0) &&
2304
130
           "Operand over promoted?");
2305
130
    // Split the promoted operand.  This will simplify when it is expanded.
2306
130
    SplitInteger(Res, Lo, Hi);
2307
130
  }
2308
5.01k
}
2309
2310
void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2311
317
                                               SDValue &Lo, SDValue &Hi) {
2312
317
  SDLoc dl(N);
2313
317
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2314
317
  EVT NVT = Lo.getValueType();
2315
317
  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2316
317
  unsigned NVTBits = NVT.getSizeInBits();
2317
317
  unsigned EVTBits = EVT.getSizeInBits();
2318
317
2319
317
  if (NVTBits < EVTBits) {
2320
254
    Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2321
254
                     DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2322
254
                                                        EVTBits - NVTBits)));
2323
254
  } else {
2324
63
    Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2325
63
    // The high part replicates the sign bit of Lo, make it explicit.
2326
63
    Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2327
63
                     DAG.getConstant(NVTBits - 1, dl,
2328
63
                                     TLI.getPointerTy(DAG.getDataLayout())));
2329
63
  }
2330
317
}
2331
2332
void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2333
315
                                               SDValue &Lo, SDValue &Hi) {
2334
315
  SDLoc dl(N);
2335
315
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2336
315
  EVT NVT = Lo.getValueType();
2337
315
  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2338
315
  unsigned NVTBits = NVT.getSizeInBits();
2339
315
  unsigned EVTBits = EVT.getSizeInBits();
2340
315
2341
315
  if (NVTBits < EVTBits) {
2342
252
    Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2343
252
                     DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2344
252
                                                        EVTBits - NVTBits)));
2345
252
  } else {
2346
63
    Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2347
63
    // The high part must be zero, make it explicit.
2348
63
    Hi = DAG.getConstant(0, dl, NVT);
2349
63
  }
2350
315
}
2351
2352
void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2353
32
                                               SDValue &Lo, SDValue &Hi) {
2354
32
  SDLoc dl(N);
2355
32
  GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
2356
32
  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2357
32
  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2358
32
}
2359
2360
void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2361
77
                                          SDValue &Lo, SDValue &Hi) {
2362
77
  SDLoc dl(N);
2363
77
  GetExpandedInteger(N->getOperand(0), Hi, Lo);  // Note swapped operands.
2364
77
  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2365
77
  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2366
77
}
2367
2368
void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2369
18.7k
                                             SDValue &Lo, SDValue &Hi) {
2370
18.7k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2371
18.7k
  unsigned NBitWidth = NVT.getSizeInBits();
2372
18.7k
  auto Constant = cast<ConstantSDNode>(N);
2373
18.7k
  const APInt &Cst = Constant->getAPIntValue();
2374
18.7k
  bool IsTarget = Constant->isTargetOpcode();
2375
18.7k
  bool IsOpaque = Constant->isOpaque();
2376
18.7k
  SDLoc dl(N);
2377
18.7k
  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2378
18.7k
  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2379
18.7k
                       IsOpaque);
2380
18.7k
}
2381
2382
5
void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2383
5
  SDLoc dl(N);
2384
5
2385
5
  // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2386
5
  EVT VT = N->getValueType(0);
2387
5
  SDValue N0 = N->getOperand(0);
2388
5
  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2389
5
                            DAG.getConstant(0, dl, VT), N0);
2390
5
  SDValue NegLo, NegHi;
2391
5
  SplitInteger(Neg, NegLo, NegHi);
2392
5
2393
5
  GetExpandedInteger(N0, Lo, Hi);
2394
5
  EVT NVT = Lo.getValueType();
2395
5
  SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2396
5
                                 DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2397
5
  Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2398
5
  Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2399
5
}
2400
2401
void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2402
103
                                         SDValue &Lo, SDValue &Hi) {
2403
103
  SDLoc dl(N);
2404
103
  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2405
103
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2406
103
  EVT NVT = Lo.getValueType();
2407
103
2408
103
  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2409
103
                                   DAG.getConstant(0, dl, NVT), ISD::SETNE);
2410
103
2411
103
  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2412
103
  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2413
103
2414
103
  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2415
103
                     DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2416
103
                                 DAG.getConstant(NVT.getSizeInBits(), dl,
2417
103
                                                 NVT)));
2418
103
  Hi = DAG.getConstant(0, dl, NVT);
2419
103
}
2420
2421
void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2422
35
                                          SDValue &Lo, SDValue &Hi) {
2423
35
  SDLoc dl(N);
2424
35
  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2425
35
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2426
35
  EVT NVT = Lo.getValueType();
2427
35
  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2428
35
                   DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2429
35
  Hi = DAG.getConstant(0, dl, NVT);
2430
35
}
2431
2432
void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2433
16
                                         SDValue &Lo, SDValue &Hi) {
2434
16
  SDLoc dl(N);
2435
16
  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2436
16
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2437
16
  EVT NVT = Lo.getValueType();
2438
16
2439
16
  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2440
16
                                   DAG.getConstant(0, dl, NVT), ISD::SETNE);
2441
16
2442
16
  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2443
16
  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2444
16
2445
16
  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2446
16
                     DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2447
16
                                 DAG.getConstant(NVT.getSizeInBits(), dl,
2448
16
                                                 NVT)));
2449
16
  Hi = DAG.getConstant(0, dl, NVT);
2450
16
}
2451
2452
void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2453
1
                                               SDValue &Hi) {
2454
1
  SDLoc dl(N);
2455
1
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2456
1
  unsigned NBitWidth = NVT.getSizeInBits();
2457
1
2458
1
  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2459
1
  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2460
1
  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2461
1
  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2462
1
                   DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2463
1
}
2464
2465
void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2466
103
                                               SDValue &Hi) {
2467
103
  SDLoc dl(N);
2468
103
  EVT VT = N->getValueType(0);
2469
103
2470
103
  SDValue Op = N->getOperand(0);
2471
103
  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2472
6
    Op = GetPromotedFloat(Op);
2473
103
2474
103
  RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
2475
103
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2476
103
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, true/*irrelevant*/, dl).first,
2477
103
               Lo, Hi);
2478
103
}
2479
2480
void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2481
95
                                               SDValue &Hi) {
2482
95
  SDLoc dl(N);
2483
95
  EVT VT = N->getValueType(0);
2484
95
2485
95
  SDValue Op = N->getOperand(0);
2486
95
  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2487
4
    Op = GetPromotedFloat(Op);
2488
95
2489
95
  RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
2490
95
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2491
95
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, false/*irrelevant*/, dl).first,
2492
95
               Lo, Hi);
2493
95
}
2494
2495
void DAGTypeLegalizer::ExpandIntRes_LLROUND(SDNode *N, SDValue &Lo,
2496
8
                                            SDValue &Hi) {
2497
8
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2498
8
  EVT VT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy;
2499
8
  if (VT == MVT::f32)
2500
4
    LC = RTLIB::LLROUND_F32;
2501
4
  else if (VT == MVT::f64)
2502
4
    LC = RTLIB::LLROUND_F64;
2503
0
  else if (VT == MVT::f80)
2504
0
    LC = RTLIB::LLROUND_F80;
2505
0
  else if (VT == MVT::f128)
2506
0
    LC = RTLIB::LLROUND_F128;
2507
0
  else if (VT == MVT::ppcf128)
2508
0
    LC = RTLIB::LLROUND_PPCF128;
2509
8
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
2510
8
2511
8
  SDValue Op = N->getOperand(0);
2512
8
  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2513
0
    Op = GetPromotedFloat(Op);
2514
8
2515
8
  SDLoc dl(N);
2516
8
  EVT RetVT = N->getValueType(0);
2517
8
  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, true/*irrelevant*/, dl).first,
2518
8
               Lo, Hi);
2519
8
}
2520
2521
void DAGTypeLegalizer::ExpandIntRes_LLRINT(SDNode *N, SDValue &Lo,
2522
8
                                            SDValue &Hi) {
2523
8
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2524
8
  EVT VT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy;
2525
8
  if (VT == MVT::f32)
2526
4
    LC = RTLIB::LLRINT_F32;
2527
4
  else if (VT == MVT::f64)
2528
4
    LC = RTLIB::LLRINT_F64;
2529
0
  else if (VT == MVT::f80)
2530
0
    LC = RTLIB::LLRINT_F80;
2531
0
  else if (VT == MVT::f128)
2532
0
    LC = RTLIB::LLRINT_F128;
2533
0
  else if (VT == MVT::ppcf128)
2534
0
    LC = RTLIB::LLRINT_PPCF128;
2535
8
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
2536
8
2537
8
  SDValue Op = N->getOperand(0);
2538
8
  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2539
0
    Op = GetPromotedFloat(Op);
2540
8
2541
8
  SDLoc dl(N);
2542
8
  EVT RetVT = N->getValueType(0);
2543
8
  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, true/*irrelevant*/, dl).first,
2544
8
               Lo, Hi);
2545
8
}
2546
2547
void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2548
9.87k
                                         SDValue &Lo, SDValue &Hi) {
2549
9.87k
  if (ISD::isNormalLoad(N)) {
2550
8.50k
    ExpandRes_NormalLoad(N, Lo, Hi);
2551
8.50k
    return;
2552
8.50k
  }
2553
1.37k
2554
1.37k
  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2555
1.37k
2556
1.37k
  EVT VT = N->getValueType(0);
2557
1.37k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2558
1.37k
  SDValue Ch  = N->getChain();
2559
1.37k
  SDValue Ptr = N->getBasePtr();
2560
1.37k
  ISD::LoadExtType ExtType = N->getExtensionType();
2561
1.37k
  unsigned Alignment = N->getAlignment();
2562
1.37k
  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2563
1.37k
  AAMDNodes AAInfo = N->getAAInfo();
2564
1.37k
  SDLoc dl(N);
2565
1.37k
2566
1.37k
  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2567
1.37k
2568
1.37k
  if (N->getMemoryVT().bitsLE(NVT)) {
2569
801
    EVT MemVT = N->getMemoryVT();
2570
801
2571
801
    Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2572
801
                        Alignment, MMOFlags, AAInfo);
2573
801
2574
801
    // Remember the chain.
2575
801
    Ch = Lo.getValue(1);
2576
801
2577
801
    if (ExtType == ISD::SEXTLOAD) {
2578
112
      // The high part is obtained by SRA'ing all but one of the bits of the
2579
112
      // lo part.
2580
112
      unsigned LoSize = Lo.getValueSizeInBits();
2581
112
      Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2582
112
                       DAG.getConstant(LoSize - 1, dl,
2583
112
                                       TLI.getPointerTy(DAG.getDataLayout())));
2584
689
    } else if (ExtType == ISD::ZEXTLOAD) {
2585
416
      // The high part is just a zero.
2586
416
      Hi = DAG.getConstant(0, dl, NVT);
2587
416
    } else {
2588
273
      assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2589
273
      // The high part is undefined.
2590
273
      Hi = DAG.getUNDEF(NVT);
2591
273
    }
2592
801
  } else 
if (576
DAG.getDataLayout().isLittleEndian()576
) {
2593
568
    // Little-endian - low bits are at low addresses.
2594
568
    Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2595
568
                     AAInfo);
2596
568
2597
568
    unsigned ExcessBits =
2598
568
      N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2599
568
    EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2600
568
2601
568
    // Increment the pointer to the other half.
2602
568
    unsigned IncrementSize = NVT.getSizeInBits()/8;
2603
568
    Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2604
568
                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2605
568
    Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2606
568
                        N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2607
568
                        MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2608
568
2609
568
    // Build a factor node to remember that this load is independent of the
2610
568
    // other one.
2611
568
    Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2612
568
                     Hi.getValue(1));
2613
568
  } else {
2614
8
    // Big-endian - high bits are at low addresses.  Favor aligned loads at
2615
8
    // the cost of some bit-fiddling.
2616
8
    EVT MemVT = N->getMemoryVT();
2617
8
    unsigned EBytes = MemVT.getStoreSize();
2618
8
    unsigned IncrementSize = NVT.getSizeInBits()/8;
2619
8
    unsigned ExcessBits = (EBytes - IncrementSize)*8;
2620
8
2621
8
    // Load both the high bits and maybe some of the low bits.
2622
8
    Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2623
8
                        EVT::getIntegerVT(*DAG.getContext(),
2624
8
                                          MemVT.getSizeInBits() - ExcessBits),
2625
8
                        Alignment, MMOFlags, AAInfo);
2626
8
2627
8
    // Increment the pointer to the other half.
2628
8
    Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2629
8
                      DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2630
8
    // Load the rest of the low bits.
2631
8
    Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2632
8
                        N->getPointerInfo().getWithOffset(IncrementSize),
2633
8
                        EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2634
8
                        MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2635
8
2636
8
    // Build a factor node to remember that this load is independent of the
2637
8
    // other one.
2638
8
    Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2639
8
                     Hi.getValue(1));
2640
8
2641
8
    if (ExcessBits < NVT.getSizeInBits()) {
2642
7
      // Transfer low bits from the bottom of Hi to the top of Lo.
2643
7
      Lo = DAG.getNode(
2644
7
          ISD::OR, dl, NVT, Lo,
2645
7
          DAG.getNode(ISD::SHL, dl, NVT, Hi,
2646
7
                      DAG.getConstant(ExcessBits, dl,
2647
7
                                      TLI.getPointerTy(DAG.getDataLayout()))));
2648
7
      // Move high bits to the right position in Hi.
2649
7
      Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? 
ISD::SRA0
: ISD::SRL, dl, NVT,
2650
7
                       Hi,
2651
7
                       DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2652
7
                                       TLI.getPointerTy(DAG.getDataLayout())));
2653
7
    }
2654
8
  }
2655
1.37k
2656
1.37k
  // Legalize the chain result - switch anything that used the old chain to
2657
1.37k
  // use the new one.
2658
1.37k
  ReplaceValueWith(SDValue(N, 1), Ch);
2659
1.37k
}
2660
2661
void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2662
27.6k
                                            SDValue &Lo, SDValue &Hi) {
2663
27.6k
  SDLoc dl(N);
2664
27.6k
  SDValue LL, LH, RL, RH;
2665
27.6k
  GetExpandedInteger(N->getOperand(0), LL, LH);
2666
27.6k
  GetExpandedInteger(N->getOperand(1), RL, RH);
2667
27.6k
  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2668
27.6k
  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2669
27.6k
}
2670
2671
void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2672
2.78k
                                        SDValue &Lo, SDValue &Hi) {
2673
2.78k
  EVT VT = N->getValueType(0);
2674
2.78k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2675
2.78k
  SDLoc dl(N);
2676
2.78k
2677
2.78k
  SDValue LL, LH, RL, RH;
2678
2.78k
  GetExpandedInteger(N->getOperand(0), LL, LH);
2679
2.78k
  GetExpandedInteger(N->getOperand(1), RL, RH);
2680
2.78k
2681
2.78k
  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2682
2.78k
                    TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
2683
2.78k
                    LL, LH, RL, RH))
2684
2.30k
    return;
2685
484
2686
484
  // If nothing else, we can make a libcall.
2687
484
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2688
484
  if (VT == MVT::i16)
2689
0
    LC = RTLIB::MUL_I16;
2690
484
  else if (VT == MVT::i32)
2691
7
    LC = RTLIB::MUL_I32;
2692
477
  else if (VT == MVT::i64)
2693
109
    LC = RTLIB::MUL_I64;
2694
368
  else if (VT == MVT::i128)
2695
256
    LC = RTLIB::MUL_I128;
2696
484
2697
484
  if (LC == RTLIB::UNKNOWN_LIBCALL || 
!TLI.getLibcallName(LC)372
) {
2698
348
    // We'll expand the multiplication by brute force because we have no other
2699
348
    // options. This is a trivially-generalized version of the code from
2700
348
    // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2701
348
    // 4.3.1).
2702
348
    unsigned Bits = NVT.getSizeInBits();
2703
348
    unsigned HalfBits = Bits >> 1;
2704
348
    SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2705
348
                                   NVT);
2706
348
    SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2707
348
    SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2708
348
2709
348
    SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2710
348
    SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2711
348
2712
348
    EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2713
348
    if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2714
2
      // The type from TLI is too small to fit the shift amount we want.
2715
2
      // Override it with i32. The shift will have to be legalized.
2716
2
      ShiftAmtTy = MVT::i32;
2717
2
    }
2718
348
    SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2719
348
    SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2720
348
    SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2721
348
    SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2722
348
2723
348
    SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2724
348
                            DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2725
348
    SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2726
348
    SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2727
348
2728
348
    SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2729
348
                            DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2730
348
    SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2731
348
2732
348
    SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2733
348
                            DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2734
348
                            DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2735
348
    Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2736
348
                     DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2737
348
2738
348
    Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2739
348
                     DAG.getNode(ISD::ADD, dl, NVT,
2740
348
                                 DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2741
348
                                 DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2742
348
    return;
2743
348
  }
2744
136
2745
136
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2746
136
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true/*irrelevant*/, dl).first,
2747
136
               Lo, Hi);
2748
136
}
2749
2750
void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2751
2
                                                     SDValue &Hi) {
2752
2
  SDLoc DL(N);
2753
2
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2754
2
  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2755
2
  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2756
2
  Lo = R.getValue(0);
2757
2
  Hi = R.getValue(1);
2758
2
  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2759
2
}
2760
2761
void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2762
84
                                              SDValue &Hi) {
2763
84
  SDValue Result = TLI.expandAddSubSat(N, DAG);
2764
84
  SplitInteger(Result, Lo, Hi);
2765
84
}
2766
2767
/// This performs an expansion of the integer result for a fixed point
2768
/// multiplication. The default expansion performs rounding down towards
2769
/// negative infinity, though targets that do care about rounding should specify
2770
/// a target hook for rounding and provide their own expansion or lowering of
2771
/// fixed point multiplication to be consistent with rounding.
2772
void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
2773
14
                                           SDValue &Hi) {
2774
14
  SDLoc dl(N);
2775
14
  EVT VT = N->getValueType(0);
2776
14
  unsigned VTSize = VT.getScalarSizeInBits();
2777
14
  SDValue LHS = N->getOperand(0);
2778
14
  SDValue RHS = N->getOperand(1);
2779
14
  uint64_t Scale = N->getConstantOperandVal(2);
2780
14
  bool Saturating = N->getOpcode() == ISD::SMULFIXSAT;
2781
14
  EVT BoolVT = getSetCCResultType(VT);
2782
14
  SDValue Zero = DAG.getConstant(0, dl, VT);
2783
14
  if (!Scale) {
2784
3
    SDValue Result;
2785
3
    if (!Saturating) {
2786
2
      Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2787
2
    } else {
2788
1
      Result = DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
2789
1
      SDValue Product = Result.getValue(0);
2790
1
      SDValue Overflow = Result.getValue(1);
2791
1
2792
1
      APInt MinVal = APInt::getSignedMinValue(VTSize);
2793
1
      APInt MaxVal = APInt::getSignedMaxValue(VTSize);
2794
1
      SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
2795
1
      SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2796
1
      SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
2797
1
      Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
2798
1
      Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
2799
1
    }
2800
3
    SplitInteger(Result, Lo, Hi);
2801
3
    return;
2802
3
  }
2803
11
2804
11
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2805
11
  SDValue LL, LH, RL, RH;
2806
11
  GetExpandedInteger(LHS, LL, LH);
2807
11
  GetExpandedInteger(RHS, RL, RH);
2808
11
  SmallVector<SDValue, 4> Result;
2809
11
2810
11
  bool Signed = (N->getOpcode() == ISD::SMULFIX ||
2811
11
                 
N->getOpcode() == ISD::SMULFIXSAT7
);
2812
11
  unsigned LoHiOp = Signed ? 
ISD::SMUL_LOHI7
:
ISD::UMUL_LOHI4
;
2813
11
  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
2814
11
                          TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
2815
11
                          LL, LH, RL, RH)) {
2816
0
    report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
2817
0
    return;
2818
11
  }
2819
11
2820
11
  unsigned NVTSize = NVT.getScalarSizeInBits();
2821
11
  assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
2822
11
                                    "the size of the current value type");
2823
11
  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2824
11
2825
11
  // Shift whole amount by scale.
2826
11
  SDValue ResultLL = Result[0];
2827
11
  SDValue ResultLH = Result[1];
2828
11
  SDValue ResultHL = Result[2];
2829
11
  SDValue ResultHH = Result[3];
2830
11
2831
11
  SDValue SatMax, SatMin;
2832
11
  SDValue NVTZero = DAG.getConstant(0, dl, NVT);
2833
11
  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
2834
11
  EVT BoolNVT = getSetCCResultType(NVT);
2835
11
2836
11
  // After getting the multplication result in 4 parts, we need to perform a
2837
11
  // shift right by the amount of the scale to get the result in that scale.
2838
11
  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
2839
11
  // 128 bits that are cut into 4 32-bit parts:
2840
11
  //
2841
11
  //      HH       HL       LH       LL
2842
11
  //  |---32---|---32---|---32---|---32---|
2843
11
  // 128      96       64       32        0
2844
11
  //
2845
11
  //                    |------VTSize-----|
2846
11
  //
2847
11
  //                             |NVTSize-|
2848
11
  //
2849
11
  // The resulting Lo and Hi will only need to be one of these 32-bit parts
2850
11
  // after shifting.
2851
11
  if (Scale < NVTSize) {
2852
4
    // If the scale is less than the size of the VT we expand to, the Hi and
2853
4
    // Lo of the result will be in the first 2 parts of the result after
2854
4
    // shifting right. This only requires shifting by the scale as far as the
2855
4
    // third part in the result (ResultHL).
2856
4
    SDValue SRLAmnt = DAG.getConstant(Scale, dl, ShiftTy);
2857
4
    SDValue SHLAmnt = DAG.getConstant(NVTSize - Scale, dl, ShiftTy);
2858
4
    Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLL, SRLAmnt);
2859
4
    Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2860
4
                     DAG.getNode(ISD::SHL, dl, NVT, ResultLH, SHLAmnt));
2861
4
    Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2862
4
    Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2863
4
                     DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2864
4
2865
4
    // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
2866
4
    // highest bit of HH determines saturation direction in the event of
2867
4
    // saturation.
2868
4
    // The number of overflow bits we can check are VTSize - Scale + 1 (we
2869
4
    // include the sign bit). If these top bits are > 0, then we overflowed past
2870
4
    // the max value. If these top bits are < -1, then we overflowed past the
2871
4
    // min value. Otherwise, we did not overflow.
2872
4
    if (Saturating) {
2873
1
      unsigned OverflowBits = VTSize - Scale + 1;
2874
1
      assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
2875
1
             "Extent of overflow bits must start within HL");
2876
1
      SDValue HLHiMask = DAG.getConstant(
2877
1
          APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
2878
1
      SDValue HLLoMask = DAG.getConstant(
2879
1
          APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
2880
1
2881
1
      // HH > 0 or HH == 0 && HL > HLLoMask
2882
1
      SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
2883
1
      SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
2884
1
      SDValue HLPos =
2885
1
          DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
2886
1
      SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos,
2887
1
                           DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLPos));
2888
1
2889
1
      // HH < -1 or HH == -1 && HL < HLHiMask
2890
1
      SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
2891
1
      SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
2892
1
      SDValue HLNeg =
2893
1
          DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
2894
1
      SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg,
2895
1
                           DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLNeg));
2896
1
    }
2897
7
  } else if (Scale == NVTSize) {
2898
3
    // If the scales are equal, Lo and Hi are ResultLH and Result HL,
2899
3
    // respectively. Avoid shifting to prevent undefined behavior.
2900
3
    Lo = ResultLH;
2901
3
    Hi = ResultHL;
2902
3
2903
3
    // We overflow max if HH > 0 or HH == 0 && HL sign bit is 1.
2904
3
    // We overflow min if HH < -1 or HH == -1 && HL sign bit is 0.
2905
3
    if (Saturating) {
2906
1
      SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
2907
1
      SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
2908
1
      SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
2909
1
      SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos,
2910
1
                           DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLNeg));
2911
1
2912
1
      SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
2913
1
      SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
2914
1
      SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
2915
1
      SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg,
2916
1
                           DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLPos));
2917
1
    }
2918
4
  } else if (Scale < VTSize) {
2919
3
    // If the scale is instead less than the old VT size, but greater than or
2920
3
    // equal to the expanded VT size, the first part of the result (ResultLL) is
2921
3
    // no longer a part of Lo because it would be scaled out anyway. Instead we
2922
3
    // can start shifting right from the fourth part (ResultHH) to the second
2923
3
    // part (ResultLH), and Result LH will be the new Lo.
2924
3
    SDValue SRLAmnt = DAG.getConstant(Scale - NVTSize, dl, ShiftTy);
2925
3
    SDValue SHLAmnt = DAG.getConstant(VTSize - Scale, dl, ShiftTy);
2926
3
    Lo = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt);
2927
3
    Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
2928
3
                     DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt));
2929
3
    Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt);
2930
3
    Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2931
3
                     DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt));
2932
3
2933
3
    // This is similar to the case when we saturate if Scale < NVTSize, but we
2934
3
    // only need to chech HH.
2935
3
    if (Saturating) {
2936
1
      unsigned OverflowBits = VTSize - Scale + 1;
2937
1
      SDValue HHHiMask = DAG.getConstant(
2938
1
          APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
2939
1
      SDValue HHLoMask = DAG.getConstant(
2940
1
          APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
2941
1
2942
1
      SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
2943
1
      SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
2944
1
    }
2945
3
  } else 
if (1
Scale == VTSize1
) {
2946
1
    assert(
2947
1
        !Signed &&
2948
1
        "Only unsigned types can have a scale equal to the operand bit width");
2949
1
2950
1
    Lo = ResultHL;
2951
1
    Hi = ResultHH;
2952
1
  } else {
2953
0
    llvm_unreachable("Expected the scale to be less than or equal to the width "
2954
0
                     "of the operands");
2955
0
  }
2956
11
2957
11
  if (Saturating) {
2958
3
    APInt LHMax = APInt::getSignedMaxValue(NVTSize);
2959
3
    APInt LLMax = APInt::getAllOnesValue(NVTSize);
2960
3
    APInt LHMin = APInt::getSignedMinValue(NVTSize);
2961
3
    Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LHMax, dl, NVT), Hi);
2962
3
    Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(LHMin, dl, NVT), Hi);
2963
3
    Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LLMax, dl, NVT), Lo);
2964
3
    Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
2965
3
  }
2966
11
}
2967
2968
void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2969
56
                                             SDValue &Lo, SDValue &Hi) {
2970
56
  SDValue LHS = Node->getOperand(0);
2971
56
  SDValue RHS = Node->getOperand(1);
2972
56
  SDLoc dl(Node);
2973
56
2974
56
  // Expand the result by simply replacing it with the equivalent
2975
56
  // non-overflow-checking operation.
2976
56
  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2977
28
                            ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2978
56
                            LHS, RHS);
2979
56
  SplitInteger(Sum, Lo, Hi);
2980
56
2981
56
  // Compute the overflow.
2982
56
  //
2983
56
  //   LHSSign -> LHS >= 0
2984
56
  //   RHSSign -> RHS >= 0
2985
56
  //   SumSign -> Sum >= 0
2986
56
  //
2987
56
  //   Add:
2988
56
  //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2989
56
  //   Sub:
2990
56
  //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2991
56
  //
2992
56
  EVT OType = Node->getValueType(1);
2993
56
  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
2994
56
2995
56
  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2996
56
  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2997
56
  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2998
56
                                    Node->getOpcode() == ISD::SADDO ?
2999
28
                                    ISD::SETEQ : ISD::SETNE);
3000
56
3001
56
  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3002
56
  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3003
56
3004
56
  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3005
56
3006
56
  // Use the calculated overflow everywhere.
3007
56
  ReplaceValueWith(SDValue(Node, 1), Cmp);
3008
56
}
3009
3010
void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3011
116
                                         SDValue &Lo, SDValue &Hi) {
3012
116
  EVT VT = N->getValueType(0);
3013
116
  SDLoc dl(N);
3014
116
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3015
116
3016
116
  if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3017
7
    SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3018
7
    SplitInteger(Res.getValue(0), Lo, Hi);
3019
7
    return;
3020
7
  }
3021
109
3022
109
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3023
109
  if (VT == MVT::i16)
3024
0
    LC = RTLIB::SDIV_I16;
3025
109
  else if (VT == MVT::i32)
3026
1
    LC = RTLIB::SDIV_I32;
3027
108
  else if (VT == MVT::i64)
3028
76
    LC = RTLIB::SDIV_I64;
3029
32
  else if (VT == MVT::i128)
3030
32
    LC = RTLIB::SDIV_I128;
3031
109
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3032
109
3033
109
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
3034
109
}
3035
3036
void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3037
99.2k
                                          SDValue &Lo, SDValue &Hi) {
3038
99.2k
  EVT VT = N->getValueType(0);
3039
99.2k
  SDLoc dl(N);
3040
99.2k
3041
99.2k
  // If we can emit an efficient shift operation, do so now.  Check to see if
3042
99.2k
  // the RHS is a constant.
3043
99.2k
  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3044
96.9k
    return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3045
2.31k
3046
2.31k
  // If we can determine that the high bit of the shift is zero or one, even if
3047
2.31k
  // the low bits are variable, emit this shift in an optimized form.
3048
2.31k
  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3049
34
    return;
3050
2.28k
3051
2.28k
  // If this target supports shift_PARTS, use it.  First, map to the _PARTS opc.
3052
2.28k
  unsigned PartsOpc;
3053
2.28k
  if (N->getOpcode() == ISD::SHL) {
3054
1.23k
    PartsOpc = ISD::SHL_PARTS;
3055
1.23k
  } else 
if (1.05k
N->getOpcode() == ISD::SRL1.05k
) {
3056
924
    PartsOpc = ISD::SRL_PARTS;
3057
924
  } else {
3058
130
    assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3059
130
    PartsOpc = ISD::SRA_PARTS;
3060
130
  }
3061
2.28k
3062
2.28k
  // Next check to see if the target supports this SHL_PARTS operation or if it
3063
2.28k
  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3064
2.28k
  // size, but create a libcall instead.
3065
2.28k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3066
2.28k
  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3067
2.28k
  const bool LegalOrCustom =
3068
2.28k
    (Action == TargetLowering::Legal && 
TLI.isTypeLegal(NVT)74
) ||
3069
2.28k
    Action == TargetLowering::Custom;
3070
2.28k
3071
2.28k
  if (LegalOrCustom && 
TLI.shouldExpandShift(DAG, N)2.03k
) {
3072
1.69k
    // Expand the subcomponents.
3073
1.69k
    SDValue LHSL, LHSH;
3074
1.69k
    GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3075
1.69k
    EVT VT = LHSL.getValueType();
3076
1.69k
3077
1.69k
    // If the shift amount operand is coming from a vector legalization it may
3078
1.69k
    // have an illegal type.  Fix that first by casting the operand, otherwise
3079
1.69k
    // the new SHL_PARTS operation would need further legalization.
3080
1.69k
    SDValue ShiftOp = N->getOperand(1);
3081
1.69k
    EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3082
1.69k
    assert(ShiftTy.getScalarSizeInBits() >=
3083
1.69k
           Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3084
1.69k
           "ShiftAmountTy is too small to cover the range of this type!");
3085
1.69k
    if (ShiftOp.getValueType() != ShiftTy)
3086
68
      ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3087
1.69k
3088
1.69k
    SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3089
1.69k
    Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3090
1.69k
    Hi = Lo.getValue(1);
3091
1.69k
    return;
3092
1.69k
  }
3093
588
3094
588
  // Otherwise, emit a libcall.
3095
588
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3096
588
  bool isSigned;
3097
588
  if (N->getOpcode() == ISD::SHL) {
3098
321
    isSigned = false; /*sign irrelevant*/
3099
321
    if (VT == MVT::i16)
3100
0
      LC = RTLIB::SHL_I16;
3101
321
    else if (VT == MVT::i32)
3102
1
      LC = RTLIB::SHL_I32;
3103
320
    else if (VT == MVT::i64)
3104
229
      LC = RTLIB::SHL_I64;
3105
91
    else if (VT == MVT::i128)
3106
68
      LC = RTLIB::SHL_I128;
3107
321
  } else 
if (267
N->getOpcode() == ISD::SRL267
) {
3108
234
    isSigned = false;
3109
234
    if (VT == MVT::i16)
3110
0
      LC = RTLIB::SRL_I16;
3111
234
    else if (VT == MVT::i32)
3112
1
      LC = RTLIB::SRL_I32;
3113
233
    else if (VT == MVT::i64)
3114
168
      LC = RTLIB::SRL_I64;
3115
65
    else if (VT == MVT::i128)
3116
58
      LC = RTLIB::SRL_I128;
3117
234
  } else {
3118
33
    assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3119
33
    isSigned = true;
3120
33
    if (VT == MVT::i16)
3121
0
      LC = RTLIB::SRA_I16;
3122
33
    else if (VT == MVT::i32)
3123
1
      LC = RTLIB::SRA_I32;
3124
32
    else if (VT == MVT::i64)
3125
5
      LC = RTLIB::SRA_I64;
3126
27
    else if (VT == MVT::i128)
3127
24
      LC = RTLIB::SRA_I128;
3128
33
  }
3129
588
3130
588
  if (LC != RTLIB::UNKNOWN_LIBCALL && 
TLI.getLibcallName(LC)555
) {
3131
491
    SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3132
491
    SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, isSigned, dl).first, Lo, Hi);
3133
491
    return;
3134
491
  }
3135
97
3136
97
  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3137
97
    
llvm_unreachable0
("Unsupported shift!");
3138
97
}
3139
3140
void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3141
2.02k
                                                SDValue &Lo, SDValue &Hi) {
3142
2.02k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3143
2.02k
  SDLoc dl(N);
3144
2.02k
  SDValue Op = N->getOperand(0);
3145
2.02k
  if (Op.getValueType().bitsLE(NVT)) {
3146
2.02k
    // The low part is sign extension of the input (degenerates to a copy).
3147
2.02k
    Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3148
2.02k
    // The high part is obtained by SRA'ing all but one of the bits of low part.
3149
2.02k
    unsigned LoSize = NVT.getSizeInBits();
3150
2.02k
    Hi = DAG.getNode(
3151
2.02k
        ISD::SRA, dl, NVT, Lo,
3152
2.02k
        DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3153
2.02k
  } else {
3154
1
    // For example, extension of an i48 to an i64.  The operand type necessarily
3155
1
    // promotes to the result type, so will end up being expanded too.
3156
1
    assert(getTypeAction(Op.getValueType()) ==
3157
1
           TargetLowering::TypePromoteInteger &&
3158
1
           "Only know how to promote this result!");
3159
1
    SDValue Res = GetPromotedInteger(Op);
3160
1
    assert(Res.getValueType() == N->getValueType(0) &&
3161
1
           "Operand over promoted?");
3162
1
    // Split the promoted operand.  This will simplify when it is expanded.
3163
1
    SplitInteger(Res, Lo, Hi);
3164
1
    unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3165
1
    Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3166
1
                     DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3167
1
                                                        ExcessBits)));
3168
1
  }
3169
2.02k
}
3170
3171
void DAGTypeLegalizer::
3172
200
ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3173
200
  SDLoc dl(N);
3174
200
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3175
200
  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3176
200
3177
200
  if (EVT.bitsLE(Lo.getValueType())) {
3178
131
    // sext_inreg the low part if needed.
3179
131
    Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3180
131
                     N->getOperand(1));
3181
131
3182
131
    // The high part gets the sign extension from the lo-part.  This handles
3183
131
    // things like sextinreg V:i64 from i8.
3184
131
    Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3185
131
                     DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3186
131
                                     TLI.getPointerTy(DAG.getDataLayout())));
3187
131
  } else {
3188
69
    // For example, extension of an i48 to an i64.  Leave the low part alone,
3189
69
    // sext_inreg the high part.
3190
69
    unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3191
69
    Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3192
69
                     DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
3193
69
                                                        ExcessBits)));
3194
69
  }
3195
200
}
3196
3197
void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3198
74
                                         SDValue &Lo, SDValue &Hi) {
3199
74
  EVT VT = N->getValueType(0);
3200
74
  SDLoc dl(N);
3201
74
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3202
74
3203
74
  if (TLI.getOperationAction(ISD::SDIVREM, VT) == TargetLowering::Custom) {
3204
9
    SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3205
9
    SplitInteger(Res.getValue(1), Lo, Hi);
3206
9
    return;
3207
9
  }
3208
65
3209
65
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3210
65
  if (VT == MVT::i16)
3211
0
    LC = RTLIB::SREM_I16;
3212
65
  else if (VT == MVT::i32)
3213
1
    LC = RTLIB::SREM_I32;
3214
64
  else if (VT == MVT::i64)
3215
47
    LC = RTLIB::SREM_I64;
3216
17
  else if (VT == MVT::i128)
3217
17
    LC = RTLIB::SREM_I128;
3218
65
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3219
65
3220
65
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, true, dl).first, Lo, Hi);
3221
65
}
3222
3223
void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3224
20.0k
                                             SDValue &Lo, SDValue &Hi) {
3225
20.0k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3226
20.0k
  SDLoc dl(N);
3227
20.0k
  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3228
20.0k
  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3229
20.0k
                   N->getOperand(0),
3230
20.0k
                   DAG.getConstant(NVT.getSizeInBits(), dl,
3231
20.0k
                                   TLI.getPointerTy(DAG.getDataLayout())));
3232
20.0k
  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3233
20.0k
}
3234
3235
void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3236
90
                                          SDValue &Lo, SDValue &Hi) {
3237
90
  EVT VT = N->getValueType(0);
3238
90
  SDLoc dl(N);
3239
90
3240
90
  if (N->getOpcode() == ISD::UMULO) {
3241
69
    // This section expands the operation into the following sequence of
3242
69
    // instructions. `iNh` here refers to a type which has half the bit width of
3243
69
    // the type the original operation operated on.
3244
69
    //
3245
69
    // %0 = %LHS.HI != 0 && %RHS.HI != 0
3246
69
    // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3247
69
    // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3248
69
    // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3249
69
    // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3250
69
    // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3251
69
    //
3252
69
    // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3253
69
    SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3254
69
    SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3255
69
    SplitInteger(LHS, LHSLow, LHSHigh);
3256
69
    SplitInteger(RHS, RHSLow, RHSHigh);
3257
69
    EVT HalfVT = LHSLow.getValueType()
3258
69
      , BitVT = N->getValueType(1);
3259
69
    SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3260
69
    SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3261
69
3262
69
    SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3263
69
    SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3264
69
      DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3265
69
      DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3266
69
3267
69
    SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3268
69
    Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3269
69
    SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3270
69
                                    One.getValue(0));
3271
69
3272
69
    SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3273
69
    Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3274
69
    SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3275
69
                                    Two.getValue(0));
3276
69
3277
69
    // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3278
69
    // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3279
69
    // operation recursively legalized?).
3280
69
    //
3281
69
    // Many backends understand this pattern and will convert into LOHI
3282
69
    // themselves, if applicable.
3283
69
    SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3284
69
      DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3285
69
      DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3286
69
    SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3287
69
    SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3288
69
    Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3289
69
    SplitInteger(Five, Lo, Hi);
3290
69
    ReplaceValueWith(SDValue(N, 1), Overflow);
3291
69
    return;
3292
69
  }
3293
21
3294
21
  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3295
21
  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3296
21
  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3297
21
3298
21
  // Replace this with a libcall that will check overflow.
3299
21
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3300
21
  if (VT == MVT::i32)
3301
0
    LC = RTLIB::MULO_I32;
3302
21
  else if (VT == MVT::i64)
3303
6
    LC = RTLIB::MULO_I64;
3304
15
  else if (VT == MVT::i128)
3305
15
    LC = RTLIB::MULO_I128;
3306
21
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3307
21
3308
21
  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3309
21
  // Temporary for the overflow value, default it to zero.
3310
21
  SDValue Chain =
3311
21
      DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3312
21
                   MachinePointerInfo());
3313
21
3314
21
  TargetLowering::ArgListTy Args;
3315
21
  TargetLowering::ArgListEntry Entry;
3316
42
  for (const SDValue &Op : N->op_values()) {
3317
42
    EVT ArgVT = Op.getValueType();
3318
42
    Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3319
42
    Entry.Node = Op;
3320
42
    Entry.Ty = ArgTy;
3321
42
    Entry.IsSExt = true;
3322
42
    Entry.IsZExt = false;
3323
42
    Args.push_back(Entry);
3324
42
  }
3325
21
3326
21
  // Also pass the address of the overflow check.
3327
21
  Entry.Node = Temp;
3328
21
  Entry.Ty = PtrTy->getPointerTo();
3329
21
  Entry.IsSExt = true;
3330
21
  Entry.IsZExt = false;
3331
21
  Args.push_back(Entry);
3332
21
3333
21
  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3334
21
3335
21
  TargetLowering::CallLoweringInfo CLI(DAG);
3336
21
  CLI.setDebugLoc(dl)
3337
21
      .setChain(Chain)
3338
21
      .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3339
21
      .setSExtResult();
3340
21
3341
21
  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3342
21
3343
21
  SplitInteger(CallInfo.first, Lo, Hi);
3344
21
  SDValue Temp2 =
3345
21
      DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3346
21
  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3347
21
                             DAG.getConstant(0, dl, PtrVT),
3348
21
                             ISD::SETNE);
3349
21
  // Use the overflow from the libcall everywhere.
3350
21
  ReplaceValueWith(SDValue(N, 1), Ofl);
3351
21
}
3352
3353
void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3354
136
                                         SDValue &Lo, SDValue &Hi) {
3355
136
  EVT VT = N->getValueType(0);
3356
136
  SDLoc dl(N);
3357
136
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3358
136
3359
136
  if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
3360
10
    SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3361
10
    SplitInteger(Res.getValue(0), Lo, Hi);
3362
10
    return;
3363
10
  }
3364
126
3365
126
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3366
126
  if (VT == MVT::i16)
3367
0
    LC = RTLIB::UDIV_I16;
3368
126
  else if (VT == MVT::i32)
3369
1
    LC = RTLIB::UDIV_I32;
3370
125
  else if (VT == MVT::i64)
3371
93
    LC = RTLIB::UDIV_I64;
3372
32
  else if (VT == MVT::i128)
3373
32
    LC = RTLIB::UDIV_I128;
3374
126
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3375
126
3376
126
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3377
126
}
3378
3379
void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3380
79
                                         SDValue &Lo, SDValue &Hi) {
3381
79
  EVT VT = N->getValueType(0);
3382
79
  SDLoc dl(N);
3383
79
  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3384
79
3385
79
  if (TLI.getOperationAction(ISD::UDIVREM, VT) == TargetLowering::Custom) {
3386
11
    SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3387
11
    SplitInteger(Res.getValue(1), Lo, Hi);
3388
11
    return;
3389
11
  }
3390
68
3391
68
  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3392
68
  if (VT == MVT::i16)
3393
0
    LC = RTLIB::UREM_I16;
3394
68
  else if (VT == MVT::i32)
3395
1
    LC = RTLIB::UREM_I32;
3396
67
  else if (VT == MVT::i64)
3397
48
    LC = RTLIB::UREM_I64;
3398
19
  else if (VT == MVT::i128)
3399
19
    LC = RTLIB::UREM_I128;
3400
68
  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3401
68
3402
68
  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, false, dl).first, Lo, Hi);
3403
68
}
3404
3405
void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3406
11.6k
                                                SDValue &Lo, SDValue &Hi) {
3407
11.6k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3408
11.6k
  SDLoc dl(N);
3409
11.6k
  SDValue Op = N->getOperand(0);
3410
11.6k
  if (Op.getValueType().bitsLE(NVT)) {
3411
11.6k
    // The low part is zero extension of the input (degenerates to a copy).
3412
11.6k
    Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3413
11.6k
    Hi = DAG.getConstant(0, dl, NVT);   // The high part is just a zero.
3414
11.6k
  } else {
3415
0
    // For example, extension of an i48 to an i64.  The operand type necessarily
3416
0
    // promotes to the result type, so will end up being expanded too.
3417
0
    assert(getTypeAction(Op.getValueType()) ==
3418
0
           TargetLowering::TypePromoteInteger &&
3419
0
           "Only know how to promote this result!");
3420
0
    SDValue Res = GetPromotedInteger(Op);
3421
0
    assert(Res.getValueType() == N->getValueType(0) &&
3422
0
           "Operand over promoted?");
3423
0
    // Split the promoted operand.  This will simplify when it is expanded.
3424
0
    SplitInteger(Res, Lo, Hi);
3425
0
    unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3426
0
    Hi = DAG.getZeroExtendInReg(Hi, dl,
3427
0
                                EVT::getIntegerVT(*DAG.getContext(),
3428
0
                                                  ExcessBits));
3429
0
  }
3430
11.6k
}
3431
3432
void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3433
22
                                                SDValue &Lo, SDValue &Hi) {
3434
22
  SDLoc dl(N);
3435
22
  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3436
22
  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3437
22
  SDValue Zero = DAG.getConstant(0, dl, VT);
3438
22
  SDValue Swap = DAG.getAtomicCmpSwap(
3439
22
      ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
3440
22
      cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3441
22
      N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3442
22
3443
22
  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3444
22
  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3445
22
}
3446
3447
void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
3448
3
                                              SDValue &Lo, SDValue &Hi) {
3449
3
  // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
3450
3
  // both halves independently.
3451
3
  SDValue Res = TLI.expandVecReduce(N, DAG);
3452
3
  SplitInteger(Res, Lo, Hi);
3453
3
}
3454
3455
//===----------------------------------------------------------------------===//
3456
//  Integer Operand Expansion
3457
//===----------------------------------------------------------------------===//
3458
3459
/// ExpandIntegerOperand - This method is called when the specified operand of
3460
/// the specified node is found to need expansion.  At this point, all of the
3461
/// result types of the node are known to be legal, but other operands of the
3462
/// node may need promotion or expansion as well as the specified one.
3463
209k
bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3464
209k
  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3465
209k
             dbgs() << "\n");
3466
209k
  SDValue Res = SDValue();
3467
209k
3468
209k
  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3469
1.32k
    return false;
3470
207k
3471
207k
  switch (N->getOpcode()) {
3472
207k
  default:
3473
  #ifndef NDEBUG
3474
    dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3475
    N->dump(&DAG); dbgs() << "\n";
3476
  #endif
3477
    report_fatal_error("Do not know how to expand this operator's operand!");
3478
207k
3479
207k
  
case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break9.51k
;
3480
207k
  
case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break0
;
3481
207k
  
case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break2.52k
;
3482
207k
  
case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break38.2k
;
3483
207k
  
case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break22
;
3484
207k
  
case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break270
;
3485
207k
  
case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break2.00k
;
3486
207k
  
case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break5.01k
;
3487
207k
  
case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break24
;
3488
207k
  
case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break51
;
3489
207k
  
case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break9.68k
;
3490
207k
  
case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break137k
;
3491
207k
  
case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break48
;
3492
207k
3493
207k
  case ISD::SHL:
3494
2.67k
  case ISD::SRA:
3495
2.67k
  case ISD::SRL:
3496
2.67k
  case ISD::ROTL:
3497
2.67k
  case ISD::ROTR:              Res = ExpandIntOp_Shift(N); break;
3498
2.67k
  case ISD::RETURNADDR:
3499
1
  case ISD::FRAMEADDR:         Res = ExpandIntOp_RETURNADDR(N); break;
3500
1
3501
24
  case ISD::ATOMIC_STORE:      Res = ExpandIntOp_ATOMIC_STORE(N); break;
3502
207k
  }
3503
207k
3504
207k
  // If the result is null, the sub-method took care of registering results etc.
3505
207k
  if (!Res.getNode()) 
return false0
;
3506
207k
3507
207k
  // If the result is N, the sub-method updated N in place.  Tell the legalizer
3508
207k
  // core about this.
3509
207k
  if (Res.getNode() == N)
3510
7.56k
    return true;
3511
200k
3512
200k
  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3513
200k
         "Invalid operand expansion");
3514
200k
3515
200k
  ReplaceValueWith(SDValue(N, 0), Res);
3516
200k
  return false;
3517
200k
}
3518
3519
/// IntegerExpandSetCCOperands - Expand the operands of a comparison.  This code
3520
/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3521
void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3522
                                                  SDValue &NewRHS,
3523
                                                  ISD::CondCode &CCCode,
3524
7.02k
                                                  const SDLoc &dl) {
3525
7.02k
  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3526
7.02k
  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3527
7.02k
  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3528
7.02k
3529
7.02k
  if (CCCode == ISD::SETEQ || 
CCCode == ISD::SETNE5.75k
) {
3530
2.41k
    if (RHSLo == RHSHi) {
3531
1.46k
      if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3532
1.46k
        if (RHSCST->isAllOnesValue()) {
3533
10
          // Equality comparison to -1.
3534
10
          NewLHS = DAG.getNode(ISD::AND, dl,
3535
10
                               LHSLo.getValueType(), LHSLo, LHSHi);
3536
10
          NewRHS = RHSLo;
3537
10
          return;
3538
10
        }
3539
2.40k
      }
3540
1.46k
    }
3541
2.40k
3542
2.40k
    NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3543
2.40k
    NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3544
2.40k
    NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3545
2.40k
    NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3546
2.40k
    return;
3547
2.40k
  }
3548
4.60k
3549
4.60k
  // If this is a comparison of the sign bit, just look at the top part.
3550
4.60k
  // X > -1,  x < 0
3551
4.60k
  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3552
1.57k
    if ((CCCode == ISD::SETLT && 
CST->isNullValue()549
) || // X < 0
3553
1.57k
        
(1.11k
CCCode == ISD::SETGT1.11k
&&
CST->isAllOnesValue()113
)) { // X > -1
3554
538
      NewLHS = LHSHi;
3555
538
      NewRHS = RHSHi;
3556
538
      return;
3557
538
    }
3558
4.06k
3559
4.06k
  // FIXME: This generated code sucks.
3560
4.06k
  ISD::CondCode LowCC;
3561
4.06k
  switch (CCCode) {
3562
4.06k
  
default: 0
llvm_unreachable0
("Unknown integer setcc!");
3563
4.06k
  case ISD::SETLT:
3564
733
  case ISD::SETULT: LowCC = ISD::SETULT; break;
3565
825
  case ISD::SETGT:
3566
825
  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3567
825
  case ISD::SETLE:
3568
357
  case ISD::SETULE: LowCC = ISD::SETULE; break;
3569
2.15k
  case ISD::SETGE:
3570
2.15k
  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3571
4.06k
  }
3572
4.06k
3573
4.06k
  // LoCmp = lo(op1) < lo(op2)   // Always unsigned comparison
3574
4.06k
  // HiCmp = hi(op1) < hi(op2)   // Signedness depends on operands
3575
4.06k
  // dest  = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3576
4.06k
3577
4.06k
  // NOTE: on targets without efficient SELECT of bools, we can always use
3578
4.06k
  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3579
4.06k
  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3580
4.06k
                                                 nullptr);
3581
4.06k
  SDValue LoCmp, HiCmp;
3582
4.06k
  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3583
4.06k
      
TLI.isTypeLegal(RHSLo.getValueType())4.04k
)
3584
4.04k
    LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3585
4.04k
                              RHSLo, LowCC, false, DagCombineInfo, dl);
3586
4.06k
  if (!LoCmp.getNode())
3587
3.20k
    LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3588
3.20k
                         RHSLo, LowCC);
3589
4.06k
  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3590
4.06k
      
TLI.isTypeLegal(RHSHi.getValueType())4.04k
)
3591
4.04k
    HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3592
4.04k
                              RHSHi, CCCode, false, DagCombineInfo, dl);
3593
4.06k
  if (!HiCmp.getNode())
3594
3.23k
    HiCmp =
3595
3.23k
        DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3596
3.23k
                    LHSHi, RHSHi, DAG.getCondCode(CCCode));
3597
4.06k
3598
4.06k
  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3599
4.06k
  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3600
4.06k
3601
4.06k
  bool EqAllowed = (CCCode == ISD::SETLE || 
CCCode == ISD::SETGE3.93k
||
3602
4.06k
                    
CCCode == ISD::SETUGE3.69k
||
CCCode == ISD::SETULE1.78k
);
3603
4.06k
3604
4.06k
  if ((EqAllowed && 
(2.50k
HiCmpC2.50k
&&
HiCmpC->isNullValue()19
)) ||
3605
4.06k
      
(4.06k
!EqAllowed4.06k
&&
(1.55k
(1.55k
HiCmpC1.55k
&&
(HiCmpC->getAPIntValue() == 1)219
) ||
3606
1.55k
                      (LoCmpC && 
LoCmpC->isNullValue()215
)))) {
3607
217
    // For LE / GE, if high part is known false, ignore the low part.
3608
217
    // For LT / GT: if low part is known false, return the high part.
3609
217
    //              if high part is known true, ignore the low part.
3610
217
    NewLHS = HiCmp;
3611
217
    NewRHS = SDValue();
3612
217
    return;
3613
217
  }
3614
3.84k
3615
3.84k
  if (LHSHi == RHSHi) {
3616
56
    // Comparing the low bits is enough.
3617
56
    NewLHS = LoCmp;
3618
56
    NewRHS = SDValue();
3619
56
    return;
3620
56
  }
3621
3.79k
3622
3.79k
  // Lower with SETCCCARRY if the target supports it.
3623
3.79k
  EVT HiVT = LHSHi.getValueType();
3624
3.79k
  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3625
3.79k
  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3626
3.79k
3627
3.79k
  // FIXME: Make all targets support this, then remove the other lowering.
3628
3.79k
  if (HasSETCCCARRY) {
3629
1.75k
    // SETCCCARRY can detect < and >= directly. For > and <=, flip
3630
1.75k
    // operands and condition code.
3631
1.75k
    bool FlipOperands = false;
3632
1.75k
    switch (CCCode) {
3633
1.75k
    
case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break115
;
3634
1.75k
    
case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break432
;
3635
1.75k
    
case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break108
;
3636
1.75k
    
case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break206
;
3637
1.75k
    
default: break891
;
3638
1.75k
    }
3639
1.75k
    if (FlipOperands) {
3640
861
      std::swap(LHSLo, RHSLo);
3641
861
      std::swap(LHSHi, RHSHi);
3642
861
    }
3643
1.75k
    // Perform a wide subtraction, feeding the carry from the low part into
3644
1.75k
    // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3645
1.75k
    // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3646
1.75k
    // zero or positive iff LHS >= RHS.
3647
1.75k
    EVT LoVT = LHSLo.getValueType();
3648
1.75k
    SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3649
1.75k
    SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3650
1.75k
    SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3651
1.75k
                              LHSHi, RHSHi, LowCmp.getValue(1),
3652
1.75k
                              DAG.getCondCode(CCCode));
3653
1.75k
    NewLHS = Res;
3654
1.75k
    NewRHS = SDValue();
3655
1.75k
    return;
3656
1.75k
  }
3657
2.04k
3658
2.04k
  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3659
2.04k
                             false, DagCombineInfo, dl);
3660
2.04k
  if (!NewLHS.getNode())
3661
2.01k
    NewLHS =
3662
2.01k
        DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3663
2.04k
  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3664
2.04k
  NewRHS = SDValue();
3665
2.04k
}
3666
3667
0
SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3668
0
  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3669
0
  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3670
0
  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3671
0
3672
0
  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3673
0
  // against zero to select between true and false values.
3674
0
  if (!NewRHS.getNode()) {
3675
0
    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3676
0
    CCCode = ISD::SETNE;
3677
0
  }
3678
0
3679
0
  // Update N to have the operands specified.
3680
0
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3681
0
                                DAG.getCondCode(CCCode), NewLHS, NewRHS,
3682
0
                                N->getOperand(4)), 0);
3683
0
}
3684
3685
2.00k
SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3686
2.00k
  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3687
2.00k
  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3688
2.00k
  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3689
2.00k
3690
2.00k
  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3691
2.00k
  // against zero to select between true and false values.
3692
2.00k
  if (!NewRHS.getNode()) {
3693
1.93k
    NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3694
1.93k
    CCCode = ISD::SETNE;
3695
1.93k
  }
3696
2.00k
3697
2.00k
  // Update N to have the operands specified.
3698
2.00k
  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3699
2.00k
                                N->getOperand(2), N->getOperand(3),
3700
2.00k
                                DAG.getCondCode(CCCode)), 0);
3701
2.00k
}
3702
3703
5.01k
SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3704
5.01k
  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3705
5.01k
  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3706
5.01k
  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3707
5.01k
3708
5.01k
  // If ExpandSetCCOperands returned a scalar, use it.
3709
5.01k
  if (!NewRHS.getNode()) {
3710
2.13k
    assert(NewLHS.getValueType() == N->getValueType(0) &&
3711
2.13k
           "Unexpected setcc expansion!");
3712
2.13k
    return NewLHS;
3713
2.13k
  }
3714
2.88k
3715
2.88k
  // Otherwise, update N to have the operands specified.
3716
2.88k
  return SDValue(
3717
2.88k
      DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3718
2.88k
}
3719
3720
24
SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3721
24
  SDValue LHS = N->getOperand(0);
3722
24
  SDValue RHS = N->getOperand(1);
3723
24
  SDValue Carry = N->getOperand(2);
3724
24
  SDValue Cond = N->getOperand(3);
3725
24
  SDLoc dl = SDLoc(N);
3726
24
3727
24
  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3728
24
  GetExpandedInteger(LHS, LHSLo, LHSHi);
3729
24
  GetExpandedInteger(RHS, RHSLo, RHSHi);
3730
24
3731
24
  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3732
24
  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3733
24
  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3734
24
  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3735
24
                     LowCmp.getValue(1), Cond);
3736
24
}
3737
3738
2.67k
SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3739
2.67k
  // The value being shifted is legal, but the shift amount is too big.
3740
2.67k
  // It follows that either the result of the shift is undefined, or the
3741
2.67k
  // upper half of the shift amount is zero.  Just use the lower half.
3742
2.67k
  SDValue Lo, Hi;
3743
2.67k
  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3744
2.67k
  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3745
2.67k
}
3746
3747
1
SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3748
1
  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant.  This
3749
1
  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3750
1
  // constant to valid type.
3751
1
  SDValue Lo, Hi;
3752
1
  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3753
1
  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3754
1
}
3755
3756
51
SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3757
51
  SDValue Op = N->getOperand(0);
3758
51
  EVT DstVT = N->getValueType(0);
3759
51
  RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
3760
51
  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3761
51
         "Don't know how to expand this SINT_TO_FP!");
3762
51
  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, SDLoc(N)).first;
3763
51
}
3764
3765
9.68k
SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3766
9.68k
  if (ISD::isNormalStore(N))
3767
8.11k
    return ExpandOp_NormalStore(N, OpNo);
3768
1.56k
3769
1.56k
  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3770
1.56k
  assert(OpNo == 1 && "Can only expand the stored value so far");
3771
1.56k
3772
1.56k
  EVT VT = N->getOperand(1).getValueType();
3773
1.56k
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3774
1.56k
  SDValue Ch  = N->getChain();
3775
1.56k
  SDValue Ptr = N->getBasePtr();
3776
1.56k
  unsigned Alignment = N->getAlignment();
3777
1.56k
  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3778
1.56k
  AAMDNodes AAInfo = N->getAAInfo();
3779
1.56k
  SDLoc dl(N);
3780
1.56k
  SDValue Lo, Hi;
3781
1.56k
3782
1.56k
  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3783
1.56k
3784
1.56k
  if (N->getMemoryVT().bitsLE(NVT)) {
3785
397
    GetExpandedInteger(N->getValue(), Lo, Hi);
3786
397
    return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3787
397
                             N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3788
397
  }
3789
1.16k
3790
1.16k
  if (DAG.getDataLayout().isLittleEndian()) {
3791
1.15k
    // Little-endian - low bits are at low addresses.
3792
1.15k
    GetExpandedInteger(N->getValue(), Lo, Hi);
3793
1.15k
3794
1.15k
    Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3795
1.15k
                      AAInfo);
3796
1.15k
3797
1.15k
    unsigned ExcessBits =
3798
1.15k
      N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3799
1.15k
    EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3800
1.15k
3801
1.15k
    // Increment the pointer to the other half.
3802
1.15k
    unsigned IncrementSize = NVT.getSizeInBits()/8;
3803
1.15k
    Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3804
1.15k
    Hi = DAG.getTruncStore(
3805
1.15k
        Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3806
1.15k
        MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3807
1.15k
    return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3808
1.15k
  }
3809
10
3810
10
  // Big-endian - high bits are at low addresses.  Favor aligned stores at
3811
10
  // the cost of some bit-fiddling.
3812
10
  GetExpandedInteger(N->getValue(), Lo, Hi);
3813
10
3814
10
  EVT ExtVT = N->getMemoryVT();
3815
10
  unsigned EBytes = ExtVT.getStoreSize();
3816
10
  unsigned IncrementSize = NVT.getSizeInBits()/8;
3817
10
  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3818
10
  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3819
10
                               ExtVT.getSizeInBits() - ExcessBits);
3820
10
3821
10
  if (ExcessBits < NVT.getSizeInBits()) {
3822
8
    // Transfer high bits from the top of Lo to the bottom of Hi.
3823
8
    Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3824
8
                     DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3825
8
                                     TLI.getPointerTy(DAG.getDataLayout())));
3826
8
    Hi = DAG.getNode(
3827
8
        ISD::OR, dl, NVT, Hi,
3828
8
        DAG.getNode(ISD::SRL, dl, NVT, Lo,
3829
8
                    DAG.getConstant(ExcessBits, dl,
3830
8
                                    TLI.getPointerTy(DAG.getDataLayout()))));
3831
8
  }
3832
10
3833
10
  // Store both the high bits and maybe some of the low bits.
3834
10
  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3835
10
                         MMOFlags, AAInfo);
3836
10
3837
10
  // Increment the pointer to the other half.
3838
10
  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3839
10
  // Store the lowest ExcessBits bits in the second half.
3840
10
  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3841
10
                         N->getPointerInfo().getWithOffset(IncrementSize),
3842
10
                         EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3843
10
                         MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3844
10
  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3845
10
}
3846
3847
137k
SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3848
137k
  SDValue InL, InH;
3849
137k
  GetExpandedInteger(N->getOperand(0), InL, InH);
3850
137k
  // Just truncate the low part of the source.
3851
137k
  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3852
137k
}
3853
3854
48
SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3855
48
  SDValue Op = N->getOperand(0);
3856
48
  EVT SrcVT = Op.getValueType();
3857
48
  EVT DstVT = N->getValueType(0);
3858
48
  SDLoc dl(N);
3859
48
3860
48
  // The following optimization is valid only if every value in SrcVT (when
3861
48
  // treated as signed) is representable in DstVT.  Check that the mantissa
3862
48
  // size of DstVT is >= than the number of bits in SrcVT -1.
3863
48
  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3864
48
  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3865
48
      
TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom0
){
3866
0
    // Do a signed conversion then adjust the result.
3867
0
    SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3868
0
    SignedConv = TLI.LowerOperation(SignedConv, DAG);
3869
0
3870
0
    // The result of the signed conversion needs adjusting if the 'sign bit' of
3871
0
    // the incoming integer was set.  To handle this, we dynamically test to see
3872
0
    // if it is set, and, if so, add a fudge factor.
3873
0
3874
0
    const uint64_t F32TwoE32  = 0x4F800000ULL;
3875
0
    const uint64_t F32TwoE64  = 0x5F800000ULL;
3876
0
    const uint64_t F32TwoE128 = 0x7F800000ULL;
3877
0
3878
0
    APInt FF(32, 0);
3879
0
    if (SrcVT == MVT::i32)
3880
0
      FF = APInt(32, F32TwoE32);
3881
0
    else if (SrcVT == MVT::i64)
3882
0
      FF = APInt(32, F32TwoE64);
3883
0
    else if (SrcVT == MVT::i128)
3884
0
      FF = APInt(32, F32TwoE128);
3885
0
    else
3886
0
      llvm_unreachable("Unsupported UINT_TO_FP!");
3887
0
3888
0
    // Check whether the sign bit is set.
3889
0
    SDValue Lo, Hi;
3890
0
    GetExpandedInteger(Op, Lo, Hi);
3891
0
    SDValue SignSet = DAG.getSetCC(dl,
3892
0
                                   getSetCCResultType(Hi.getValueType()),
3893
0
                                   Hi,
3894
0
                                   DAG.getConstant(0, dl, Hi.getValueType()),
3895
0
                                   ISD::SETLT);
3896
0
3897
0
    // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
3898
0
    SDValue FudgePtr =
3899
0
        DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
3900
0
                            TLI.getPointerTy(DAG.getDataLayout()));
3901
0
3902
0
    // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
3903
0
    SDValue Zero = DAG.getIntPtrConstant(0, dl);
3904
0
    SDValue Four = DAG.getIntPtrConstant(4, dl);
3905
0
    if (DAG.getDataLayout().isBigEndian())
3906
0
      std::swap(Zero, Four);
3907
0
    SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
3908
0
                                   Zero, Four);
3909
0
    unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
3910
0
    FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
3911
0
                           FudgePtr, Offset);
3912
0
    Alignment = std::min(Alignment, 4u);
3913
0
3914
0
    // Load the value out, extending it from f32 to the destination float type.
3915
0
    // FIXME: Avoid the extend by constructing the right constant pool?
3916
0
    SDValue Fudge = DAG.getExtLoad(
3917
0
        ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
3918
0
        MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
3919
0
        Alignment);
3920
0
    return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
3921
48
  }
3922
48
3923
48
  // Otherwise, use a libcall.
3924
48
  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
3925
48
  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3926
48
         "Don't know how to expand this UINT_TO_FP!");
3927
48
  return TLI.makeLibCall(DAG, LC, DstVT, Op, true, dl).first;
3928
48
}
3929
3930
24
SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
3931
24
  SDLoc dl(N);
3932
24
  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3933
24
                               cast<AtomicSDNode>(N)->getMemoryVT(),
3934
24
                               N->getOperand(0),
3935
24
                               N->getOperand(1), N->getOperand(2),
3936
24
                               cast<AtomicSDNode>(N)->getMemOperand());
3937
24
  return Swap.getValue(1);
3938
24
}
3939
3940
3941
19.3k
SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
3942
19.3k
3943
19.3k
  EVT OutVT = N->getValueType(0);
3944
19.3k
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3945
19.3k
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3946
19.3k
  unsigned OutNumElems = OutVT.getVectorNumElements();
3947
19.3k
  EVT NOutVTElem = NOutVT.getVectorElementType();
3948
19.3k
3949
19.3k
  SDLoc dl(N);
3950
19.3k
  SDValue BaseIdx = N->getOperand(1);
3951
19.3k
3952
19.3k
  SDValue InOp0 = N->getOperand(0);
3953
19.3k
  if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
3954
10.5k
    InOp0 = GetPromotedInteger(N->getOperand(0));
3955
19.3k
3956
19.3k
  EVT InVT = InOp0.getValueType();
3957
19.3k
3958
19.3k
  SmallVector<SDValue, 8> Ops;
3959
19.3k
  Ops.reserve(OutNumElems);
3960
71.6k
  for (unsigned i = 0; i != OutNumElems; 
++i52.3k
) {
3961
52.3k
3962
52.3k
    // Extract the element from the original vector.
3963
52.3k
    SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
3964
52.3k
      BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
3965
52.3k
    SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3966
52.3k
      InVT.getVectorElementType(), N->getOperand(0), Index);
3967
52.3k
3968
52.3k
    SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
3969
52.3k
    // Insert the converted element to the new vector.
3970
52.3k
    Ops.push_back(Op);
3971
52.3k
  }
3972
19.3k
3973
19.3k
  return DAG.getBuildVector(NOutVT, dl, Ops);
3974
19.3k
}
3975
3976
3977
1.02k
SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
3978
1.02k
  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
3979
1.02k
  EVT VT = N->getValueType(0);
3980
1.02k
  SDLoc dl(N);
3981
1.02k
3982
1.02k
  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
3983
1.02k
3984
1.02k
  SDValue V0 = GetPromotedInteger(N->getOperand(0));
3985
1.02k
  SDValue V1 = GetPromotedInteger(N->getOperand(1));
3986
1.02k
  EVT OutVT = V0.getValueType();
3987
1.02k
3988
1.02k
  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
3989
1.02k
}
3990
3991
3992
2.15k
SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
3993
2.15k
  EVT OutVT = N->getValueType(0);
3994
2.15k
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3995
2.15k
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3996
2.15k
  unsigned NumElems = N->getNumOperands();
3997
2.15k
  EVT NOutVTElem = NOutVT.getVectorElementType();
3998
2.15k
3999
2.15k
  SDLoc dl(N);
4000
2.15k
4001
2.15k
  SmallVector<SDValue, 8> Ops;
4002
2.15k
  Ops.reserve(NumElems);
4003
11.8k
  for (unsigned i = 0; i != NumElems; 
++i9.67k
) {
4004
9.67k
    SDValue Op;
4005
9.67k
    // BUILD_VECTOR integer operand types are allowed to be larger than the
4006
9.67k
    // result's element type. This may still be true after the promotion. For
4007
9.67k
    // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4008
9.67k
    // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4009
9.67k
    if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4010
8.99k
      Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4011
676
    else
4012
676
      Op = N->getOperand(i);
4013
9.67k
    Ops.push_back(Op);
4014
9.67k
  }
4015
2.15k
4016
2.15k
  return DAG.getBuildVector(NOutVT, dl, Ops);
4017
2.15k
}
4018
4019
17
SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4020
17
4021
17
  SDLoc dl(N);
4022
17
4023
17
  assert(!N->getOperand(0).getValueType().isVector() &&
4024
17
         "Input must be a scalar");
4025
17
4026
17
  EVT OutVT = N->getValueType(0);
4027
17
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4028
17
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4029
17
  EVT NOutVTElem = NOutVT.getVectorElementType();
4030
17
4031
17
  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4032
17
4033
17
  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4034
17
}
4035
4036
348
SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4037
348
  SDLoc dl(N);
4038
348
4039
348
  EVT OutVT = N->getValueType(0);
4040
348
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4041
348
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4042
348
4043
348
  EVT OutElemTy = NOutVT.getVectorElementType();
4044
348
4045
348
  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4046
348
  unsigned NumOutElem = NOutVT.getVectorNumElements();
4047
348
  unsigned NumOperands = N->getNumOperands();
4048
348
  assert(NumElem * NumOperands == NumOutElem &&
4049
348
         "Unexpected number of elements");
4050
348
4051
348
  // Take the elements from the first vector.
4052
348
  SmallVector<SDValue, 8> Ops(NumOutElem);
4053
1.11k
  for (unsigned i = 0; i < NumOperands; 
++i768
) {
4054
768
    SDValue Op = N->getOperand(i);
4055
768
    if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4056
700
      Op = GetPromotedInteger(Op);
4057
768
    EVT SclrTy = Op.getValueType().getVectorElementType();
4058
768
    assert(NumElem == Op.getValueType().getVectorNumElements() &&
4059
768
           "Unexpected number of elements");
4060
768
4061
3.19k
    for (unsigned j = 0; j < NumElem; 
++j2.42k
) {
4062
2.42k
      SDValue Ext = DAG.getNode(
4063
2.42k
          ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4064
2.42k
          DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4065
2.42k
      Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4066
2.42k
    }
4067
768
  }
4068
348
4069
348
  return DAG.getBuildVector(NOutVT, dl, Ops);
4070
348
}
4071
4072
2
SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4073
2
  EVT VT = N->getValueType(0);
4074
2
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4075
2
  assert(NVT.isVector() && "This type must be promoted to a vector type");
4076
2
4077
2
  SDLoc dl(N);
4078
2
4079
2
  // For operands whose TypeAction is to promote, extend the promoted node
4080
2
  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4081
2
  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4082
2
  // type..
4083
2
  if (getTypeAction(N->getOperand(0).getValueType())
4084
2
      == TargetLowering::TypePromoteInteger) {
4085
2
    SDValue Promoted;
4086
2
4087
2
    switch(N->getOpcode()) {
4088
2
      case ISD::SIGN_EXTEND_VECTOR_INREG:
4089
1
        Promoted = SExtPromotedInteger(N->getOperand(0));
4090
1
        break;
4091
2
      case ISD::ZERO_EXTEND_VECTOR_INREG:
4092
1
        Promoted = ZExtPromotedInteger(N->getOperand(0));
4093
1
        break;
4094
2
      case ISD::ANY_EXTEND_VECTOR_INREG:
4095
0
        Promoted = GetPromotedInteger(N->getOperand(0));
4096
0
        break;
4097
2
      default:
4098
0
        llvm_unreachable("Node has unexpected Opcode");
4099
2
    }
4100
2
    return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4101
2
  }
4102
0
4103
0
  // Directly extend to the appropriate transform-to type.
4104
0
  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4105
0
}
4106
4107
180
SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4108
180
  EVT OutVT = N->getValueType(0);
4109
180
  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4110
180
  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4111
180
4112
180
  EVT NOutVTElem = NOutVT.getVectorElementType();
4113
180
4114
180
  SDLoc dl(N);
4115
180
  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4116
180
4117
180
  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4118
180
    NOutVTElem, N->getOperand(1));
4119
180
  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4120
180
    V0, ConvElem, N->getOperand(2));
4121
180
}
4122
4123
190
SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4124
190
  // The VECREDUCE result size may be larger than the element size, so
4125
190
  // we can simply change the result type.
4126
190
  SDLoc dl(N);
4127
190
  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4128
190
  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4129
190
}
4130
4131
25.7k
SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4132
25.7k
  SDLoc dl(N);
4133
25.7k
  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4134
25.7k
  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4135
25.7k
                                  TLI.getVectorIdxTy(DAG.getDataLayout()));
4136
25.7k
  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
4137
25.7k
    V0->getValueType(0).getScalarType(), V0, V1);
4138
25.7k
4139
25.7k
  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4140
25.7k
  // element types. If this is the case then we need to expand the outgoing
4141
25.7k
  // value and not truncate it.
4142
25.7k
  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4143
25.7k
}
4144
4145
0
SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4146
0
  SDLoc dl(N);
4147
0
  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4148
0
  MVT InVT = V0.getValueType().getSimpleVT();
4149
0
  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4150
0
                               N->getValueType(0).getVectorNumElements());
4151
0
  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4152
0
  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4153
0
}
4154
4155
5.50k
SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4156
5.50k
  SDLoc dl(N);
4157
5.50k
  unsigned NumElems = N->getNumOperands();
4158
5.50k
4159
5.50k
  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4160
5.50k
4161
5.50k
  SmallVector<SDValue, 8> NewOps;
4162
5.50k
  NewOps.reserve(NumElems);
4163
5.50k
4164
5.50k
  // For each incoming vector
4165
19.4k
  for (unsigned VecIdx = 0; VecIdx != NumElems; 
++VecIdx13.9k
) {
4166
13.9k
    SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4167
13.9k
    EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4168
13.9k
    unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4169
13.9k
4170
71.5k
    for (unsigned i=0; i<NumElem; 
++i57.6k
) {
4171
57.6k
      // Extract element from incoming vector
4172
57.6k
      SDValue Ex = DAG.getNode(
4173
57.6k
          ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4174
57.6k
          DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4175
57.6k
      SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4176
57.6k
      NewOps.push_back(Tr);
4177
57.6k
    }
4178
13.9k
  }
4179
5.50k
4180
5.50k
  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4181
5.50k
}