Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// This implements the SelectionDAG class.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "llvm/CodeGen/SelectionDAG.h"
15
#include "SDNodeDbgValue.h"
16
#include "llvm/ADT/APFloat.h"
17
#include "llvm/ADT/APInt.h"
18
#include "llvm/ADT/APSInt.h"
19
#include "llvm/ADT/ArrayRef.h"
20
#include "llvm/ADT/BitVector.h"
21
#include "llvm/ADT/FoldingSet.h"
22
#include "llvm/ADT/None.h"
23
#include "llvm/ADT/STLExtras.h"
24
#include "llvm/ADT/SmallPtrSet.h"
25
#include "llvm/ADT/SmallVector.h"
26
#include "llvm/ADT/Triple.h"
27
#include "llvm/ADT/Twine.h"
28
#include "llvm/Analysis/ValueTracking.h"
29
#include "llvm/CodeGen/ISDOpcodes.h"
30
#include "llvm/CodeGen/MachineBasicBlock.h"
31
#include "llvm/CodeGen/MachineConstantPool.h"
32
#include "llvm/CodeGen/MachineFrameInfo.h"
33
#include "llvm/CodeGen/MachineFunction.h"
34
#include "llvm/CodeGen/MachineMemOperand.h"
35
#include "llvm/CodeGen/MachineValueType.h"
36
#include "llvm/CodeGen/RuntimeLibcalls.h"
37
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
38
#include "llvm/CodeGen/SelectionDAGNodes.h"
39
#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
40
#include "llvm/CodeGen/ValueTypes.h"
41
#include "llvm/IR/Constant.h"
42
#include "llvm/IR/Constants.h"
43
#include "llvm/IR/DataLayout.h"
44
#include "llvm/IR/DebugInfoMetadata.h"
45
#include "llvm/IR/DebugLoc.h"
46
#include "llvm/IR/DerivedTypes.h"
47
#include "llvm/IR/Function.h"
48
#include "llvm/IR/GlobalValue.h"
49
#include "llvm/IR/Metadata.h"
50
#include "llvm/IR/Type.h"
51
#include "llvm/IR/Value.h"
52
#include "llvm/Support/Casting.h"
53
#include "llvm/Support/CodeGen.h"
54
#include "llvm/Support/Compiler.h"
55
#include "llvm/Support/Debug.h"
56
#include "llvm/Support/ErrorHandling.h"
57
#include "llvm/Support/KnownBits.h"
58
#include "llvm/Support/ManagedStatic.h"
59
#include "llvm/Support/MathExtras.h"
60
#include "llvm/Support/Mutex.h"
61
#include "llvm/Support/raw_ostream.h"
62
#include "llvm/Target/TargetLowering.h"
63
#include "llvm/Target/TargetMachine.h"
64
#include "llvm/Target/TargetOptions.h"
65
#include "llvm/Target/TargetRegisterInfo.h"
66
#include "llvm/Target/TargetSubtargetInfo.h"
67
#include <algorithm>
68
#include <cassert>
69
#include <cstdint>
70
#include <cstdlib>
71
#include <limits>
72
#include <set>
73
#include <string>
74
#include <utility>
75
#include <vector>
76
77
using namespace llvm;
78
79
/// makeVTList - Return an instance of the SDVTList struct initialized with the
80
/// specified members.
81
138M
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
82
138M
  SDVTList Res = {VTs, NumVTs};
83
138M
  return Res;
84
138M
}
85
86
// Default null implementations of the callbacks.
87
0
void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
88
60.3M
void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
89
90
#define DEBUG_TYPE "selectiondag"
91
92
52.9M
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
93
52.9M
  DEBUG(
94
52.9M
    dbgs() << Msg;
95
52.9M
    V.getNode()->dump(G);
96
52.9M
  );
97
52.9M
}
98
99
//===----------------------------------------------------------------------===//
100
//                              ConstantFPSDNode Class
101
//===----------------------------------------------------------------------===//
102
103
/// isExactlyValue - We don't rely on operator== working on double values, as
104
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
105
/// As such, this method can be used to do an exact bit-for-bit comparison of
106
/// two floating point values.
107
172k
bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
108
172k
  return getValueAPF().bitwiseIsEqual(V);
109
172k
}
110
111
bool ConstantFPSDNode::isValueValidForType(EVT VT,
112
21.3k
                                           const APFloat& Val) {
113
21.3k
  assert(VT.isFloatingPoint() && "Can only convert between FP types");
114
21.3k
115
21.3k
  // convert modifies in place, so make a copy.
116
21.3k
  APFloat Val2 = APFloat(Val);
117
21.3k
  bool losesInfo;
118
21.3k
  (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
119
21.3k
                      APFloat::rmNearestTiesToEven,
120
21.3k
                      &losesInfo);
121
21.3k
  return !losesInfo;
122
21.3k
}
123
124
//===----------------------------------------------------------------------===//
125
//                              ISD Namespace
126
//===----------------------------------------------------------------------===//
127
128
72.5k
bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
129
72.5k
  auto *BV = dyn_cast<BuildVectorSDNode>(N);
130
72.5k
  if (!BV)
131
64.2k
    return false;
132
8.26k
133
8.26k
  APInt SplatUndef;
134
8.26k
  unsigned SplatBitSize;
135
8.26k
  bool HasUndefs;
136
8.26k
  unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
137
8.26k
  return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
138
8.26k
                             EltSize) &&
139
7.62k
         EltSize == SplatBitSize;
140
72.5k
}
141
142
// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
143
// specializations of the more general isConstantSplatVector()?
144
145
309k
bool ISD::isBuildVectorAllOnes(const SDNode *N) {
146
309k
  // Look through a bit convert.
147
379k
  while (N->getOpcode() == ISD::BITCAST)
148
70.2k
    N = N->getOperand(0).getNode();
149
309k
150
309k
  if (
N->getOpcode() != ISD::BUILD_VECTOR309k
)
return false196k
;
151
112k
152
112k
  unsigned i = 0, e = N->getNumOperands();
153
112k
154
112k
  // Skip over all of the undef values.
155
113k
  while (
i != e && 113k
N->getOperand(i).isUndef()113k
)
156
1.06k
    ++i;
157
112k
158
112k
  // Do not accept an all-undef vector.
159
112k
  if (
i == e112k
)
return false25
;
160
112k
161
112k
  // Do not accept build_vectors that aren't all constants or which have non-~0
162
112k
  // elements. We have to be a bit careful here, as the type of the constant
163
112k
  // may not be the same as the type of the vector elements due to type
164
112k
  // legalization (the elements are promoted to a legal type for the target and
165
112k
  // a vector of a type may be legal when the base element type is not).
166
112k
  // We only want to check enough bits to cover the vector elements, because
167
112k
  // we care if the resultant vector is all ones, not whether the individual
168
112k
  // constants are.
169
112k
  SDValue NotZero = N->getOperand(i);
170
112k
  unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
171
112k
  if (ConstantSDNode *
CN112k
= dyn_cast<ConstantSDNode>(NotZero)) {
172
81.5k
    if (CN->getAPIntValue().countTrailingOnes() < EltSize)
173
51.4k
      return false;
174
31.3k
  } else 
if (ConstantFPSDNode *31.3k
CFPN31.3k
= dyn_cast<ConstantFPSDNode>(NotZero)) {
175
5.16k
    if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
176
5.06k
      return false;
177
31.3k
  } else
178
26.1k
    return false;
179
30.2k
180
30.2k
  // Okay, we have at least one ~0 value, check to see if the rest match or are
181
30.2k
  // undefs. Even with the above element type twiddling, this should be OK, as
182
30.2k
  // the same type legalization should have applied to all the elements.
183
133k
  
for (++i; 30.2k
i != e133k
;
++i103k
)
184
108k
    
if (108k
N->getOperand(i) != NotZero && 108k
!N->getOperand(i).isUndef()5.24k
)
185
4.74k
      return false;
186
25.4k
  return true;
187
309k
}
188
189
894k
bool ISD::isBuildVectorAllZeros(const SDNode *N) {
190
894k
  // Look through a bit convert.
191
1.01M
  while (N->getOpcode() == ISD::BITCAST)
192
118k
    N = N->getOperand(0).getNode();
193
894k
194
894k
  if (
N->getOpcode() != ISD::BUILD_VECTOR894k
)
return false678k
;
195
215k
196
215k
  bool IsAllUndef = true;
197
911k
  for (const SDValue &Op : N->op_values()) {
198
911k
    if (Op.isUndef())
199
5.71k
      continue;
200
905k
    IsAllUndef = false;
201
905k
    // Do not accept build_vectors that aren't all constants or which have non-0
202
905k
    // elements. We have to be a bit careful here, as the type of the constant
203
905k
    // may not be the same as the type of the vector elements due to type
204
905k
    // legalization (the elements are promoted to a legal type for the target
205
905k
    // and a vector of a type may be legal when the base element type is not).
206
905k
    // We only want to check enough bits to cover the vector elements, because
207
905k
    // we care if the resultant vector is all zeros, not whether the individual
208
905k
    // constants are.
209
905k
    unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
210
905k
    if (ConstantSDNode *
CN905k
= dyn_cast<ConstantSDNode>(Op)) {
211
853k
      if (CN->getAPIntValue().countTrailingZeros() < EltSize)
212
104k
        return false;
213
52.2k
    } else 
if (ConstantFPSDNode *52.2k
CFPN52.2k
= dyn_cast<ConstantFPSDNode>(Op)) {
214
21.4k
      if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
215
5.42k
        return false;
216
52.2k
    } else
217
30.7k
      return false;
218
75.5k
  }
219
75.5k
220
75.5k
  // Do not accept an all-undef vector.
221
75.5k
  
if (75.5k
IsAllUndef75.5k
)
222
25
    return false;
223
75.5k
  return true;
224
75.5k
}
225
226
19.1M
bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
227
19.1M
  if (N->getOpcode() != ISD::BUILD_VECTOR)
228
19.0M
    return false;
229
85.6k
230
85.6k
  
for (const SDValue &Op : N->op_values()) 85.6k
{
231
505k
    if (Op.isUndef())
232
3.01k
      continue;
233
502k
    
if (502k
!isa<ConstantSDNode>(Op)502k
)
234
41.7k
      return false;
235
43.8k
  }
236
43.8k
  return true;
237
43.8k
}
238
239
3.60M
bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
240
3.60M
  if (N->getOpcode() != ISD::BUILD_VECTOR)
241
3.57M
    return false;
242
24.4k
243
24.4k
  
for (const SDValue &Op : N->op_values()) 24.4k
{
244
66.8k
    if (Op.isUndef())
245
657
      continue;
246
66.1k
    
if (66.1k
!isa<ConstantFPSDNode>(Op)66.1k
)
247
9.68k
      return false;
248
14.7k
  }
249
14.7k
  return true;
250
14.7k
}
251
252
397k
bool ISD::allOperandsUndef(const SDNode *N) {
253
397k
  // Return false if the node has no operands.
254
397k
  // This is "logically inconsistent" with the definition of "all" but
255
397k
  // is probably the desired behavior.
256
397k
  if (N->getNumOperands() == 0)
257
0
    return false;
258
397k
259
397k
  for (const SDValue &Op : N->op_values())
260
401k
    
if (401k
!Op.isUndef()401k
)
261
397k
      return false;
262
75
263
75
  return true;
264
75
}
265
266
308
ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
267
308
  switch (ExtType) {
268
0
  case ISD::EXTLOAD:
269
0
    return IsFP ? 
ISD::FP_EXTEND0
:
ISD::ANY_EXTEND0
;
270
113
  case ISD::SEXTLOAD:
271
113
    return ISD::SIGN_EXTEND;
272
195
  case ISD::ZEXTLOAD:
273
195
    return ISD::ZERO_EXTEND;
274
0
  default:
275
0
    break;
276
0
  }
277
0
278
0
  
llvm_unreachable0
("Invalid LoadExtType");
279
0
}
280
281
3.36M
ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
282
3.36M
  // To perform this operation, we just need to swap the L and G bits of the
283
3.36M
  // operation.
284
3.36M
  unsigned OldL = (Operation >> 2) & 1;
285
3.36M
  unsigned OldG = (Operation >> 1) & 1;
286
3.36M
  return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
287
3.36M
                       (OldL << 1) |       // New G bit
288
3.36M
                       (OldG << 2));       // New L bit.
289
3.36M
}
290
291
834k
ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
292
834k
  unsigned Operation = Op;
293
834k
  if (isInteger)
294
817k
    Operation ^= 7;   // Flip L, G, E bits, but not U.
295
834k
  else
296
16.2k
    Operation ^= 15;  // Flip all of the condition bits.
297
834k
298
834k
  if (Operation > ISD::SETTRUE2)
299
295
    Operation &= ~8;  // Don't let N and U bits get set.
300
834k
301
834k
  return ISD::CondCode(Operation);
302
834k
}
303
304
/// For an integer comparison, return 1 if the comparison is a signed operation
305
/// and 2 if the result is an unsigned comparison. Return zero if the operation
306
/// does not depend on the sign of the input (setne and seteq).
307
40
static int isSignedOp(ISD::CondCode Opcode) {
308
40
  switch (Opcode) {
309
0
  
default: 0
llvm_unreachable0
("Illegal integer setcc operation!");
310
24
  case ISD::SETEQ:
311
24
  case ISD::SETNE: return 0;
312
7
  case ISD::SETLT:
313
7
  case ISD::SETLE:
314
7
  case ISD::SETGT:
315
7
  case ISD::SETGE: return 1;
316
9
  case ISD::SETULT:
317
9
  case ISD::SETULE:
318
9
  case ISD::SETUGT:
319
9
  case ISD::SETUGE: return 2;
320
0
  }
321
0
}
322
323
ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
324
37
                                       bool IsInteger) {
325
37
  if (
IsInteger && 37
(isSignedOp(Op1) | isSignedOp(Op2)) == 319
)
326
37
    // Cannot fold a signed integer setcc with an unsigned integer setcc.
327
5
    return ISD::SETCC_INVALID;
328
32
329
32
  unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
330
32
331
32
  // If the N and U bits get set, then the resultant comparison DOES suddenly
332
32
  // care about orderedness, and it is true when ordered.
333
32
  if (Op > ISD::SETTRUE2)
334
17
    Op &= ~16;     // Clear the U bit if the N bit is set.
335
32
336
32
  // Canonicalize illegal integer setcc's.
337
32
  if (
IsInteger && 32
Op == ISD::SETUNE14
) // e.g. SETUGT | SETULT
338
0
    Op = ISD::SETNE;
339
37
340
37
  return ISD::CondCode(Op);
341
37
}
342
343
ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
344
27
                                        bool IsInteger) {
345
27
  if (
IsInteger && 27
(isSignedOp(Op1) | isSignedOp(Op2)) == 31
)
346
27
    // Cannot fold a signed setcc with an unsigned setcc.
347
0
    return ISD::SETCC_INVALID;
348
27
349
27
  // Combine all of the condition bits.
350
27
  ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
351
27
352
27
  // Canonicalize illegal integer setcc's.
353
27
  if (
IsInteger27
) {
354
1
    switch (Result) {
355
0
    default: break;
356
0
    case ISD::SETUO : Result = ISD::SETFALSE; break;  // SETUGT & SETULT
357
1
    case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
358
1
    case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
359
0
    case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
360
0
    case ISD::SETOGT: Result = ISD::SETUGT  ; break;  // SETUGT & SETNE
361
27
    }
362
27
  }
363
27
364
27
  return Result;
365
27
}
366
367
//===----------------------------------------------------------------------===//
368
//                           SDNode Profile Support
369
//===----------------------------------------------------------------------===//
370
371
/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
372
265M
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
373
265M
  ID.AddInteger(OpC);
374
265M
}
375
376
/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
377
/// solely with their pointer.
378
265M
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
379
265M
  ID.AddPointer(VTList.VTs);
380
265M
}
381
382
/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
383
static void AddNodeIDOperands(FoldingSetNodeID &ID,
384
157M
                              ArrayRef<SDValue> Ops) {
385
191M
  for (auto& Op : Ops) {
386
191M
    ID.AddPointer(Op.getNode());
387
191M
    ID.AddInteger(Op.getResNo());
388
191M
  }
389
157M
}
390
391
/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
392
static void AddNodeIDOperands(FoldingSetNodeID &ID,
393
108M
                              ArrayRef<SDUse> Ops) {
394
144M
  for (auto& Op : Ops) {
395
144M
    ID.AddPointer(Op.getNode());
396
144M
    ID.AddInteger(Op.getResNo());
397
144M
  }
398
108M
}
399
400
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
401
157M
                          SDVTList VTList, ArrayRef<SDValue> OpList) {
402
157M
  AddNodeIDOpcode(ID, OpC);
403
157M
  AddNodeIDValueTypes(ID, VTList);
404
157M
  AddNodeIDOperands(ID, OpList);
405
157M
}
406
407
/// If this is an SDNode with special info, add this info to the NodeID data.
408
109M
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
409
109M
  switch (N->getOpcode()) {
410
0
  case ISD::TargetExternalSymbol:
411
0
  case ISD::ExternalSymbol:
412
0
  case ISD::MCSymbol:
413
0
    llvm_unreachable("Should only be used on nodes with operands");
414
50.7M
  default: break;  // Normal nodes don't need extra info.
415
27.8M
  case ISD::TargetConstant:
416
27.8M
  case ISD::Constant: {
417
27.8M
    const ConstantSDNode *C = cast<ConstantSDNode>(N);
418
27.8M
    ID.AddPointer(C->getConstantIntValue());
419
27.8M
    ID.AddBoolean(C->isOpaque());
420
27.8M
    break;
421
27.8M
  }
422
138k
  case ISD::TargetConstantFP:
423
138k
  case ISD::ConstantFP:
424
138k
    ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
425
138k
    break;
426
2.40M
  case ISD::TargetGlobalAddress:
427
2.40M
  case ISD::GlobalAddress:
428
2.40M
  case ISD::TargetGlobalTLSAddress:
429
2.40M
  case ISD::GlobalTLSAddress: {
430
2.40M
    const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
431
2.40M
    ID.AddPointer(GA->getGlobal());
432
2.40M
    ID.AddInteger(GA->getOffset());
433
2.40M
    ID.AddInteger(GA->getTargetFlags());
434
2.40M
    break;
435
2.40M
  }
436
1.19M
  case ISD::BasicBlock:
437
1.19M
    ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
438
1.19M
    break;
439
15.9M
  case ISD::Register:
440
15.9M
    ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
441
15.9M
    break;
442
1.13M
  case ISD::RegisterMask:
443
1.13M
    ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
444
1.13M
    break;
445
1.02k
  case ISD::SRCVALUE:
446
1.02k
    ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
447
1.02k
    break;
448
969k
  case ISD::FrameIndex:
449
969k
  case ISD::TargetFrameIndex:
450
969k
    ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
451
969k
    break;
452
4.51k
  case ISD::JumpTable:
453
4.51k
  case ISD::TargetJumpTable:
454
4.51k
    ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
455
4.51k
    ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
456
4.51k
    break;
457
122k
  case ISD::ConstantPool:
458
122k
  case ISD::TargetConstantPool: {
459
122k
    const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
460
122k
    ID.AddInteger(CP->getAlignment());
461
122k
    ID.AddInteger(CP->getOffset());
462
122k
    if (CP->isMachineConstantPoolEntry())
463
103
      CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
464
122k
    else
465
122k
      ID.AddPointer(CP->getConstVal());
466
122k
    ID.AddInteger(CP->getTargetFlags());
467
122k
    break;
468
122k
  }
469
0
  case ISD::TargetIndex: {
470
0
    const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
471
0
    ID.AddInteger(TI->getIndex());
472
0
    ID.AddInteger(TI->getOffset());
473
0
    ID.AddInteger(TI->getTargetFlags());
474
0
    break;
475
122k
  }
476
3.08M
  case ISD::LOAD: {
477
3.08M
    const LoadSDNode *LD = cast<LoadSDNode>(N);
478
3.08M
    ID.AddInteger(LD->getMemoryVT().getRawBits());
479
3.08M
    ID.AddInteger(LD->getRawSubclassData());
480
3.08M
    ID.AddInteger(LD->getPointerInfo().getAddrSpace());
481
3.08M
    break;
482
122k
  }
483
5.73M
  case ISD::STORE: {
484
5.73M
    const StoreSDNode *ST = cast<StoreSDNode>(N);
485
5.73M
    ID.AddInteger(ST->getMemoryVT().getRawBits());
486
5.73M
    ID.AddInteger(ST->getRawSubclassData());
487
5.73M
    ID.AddInteger(ST->getPointerInfo().getAddrSpace());
488
5.73M
    break;
489
122k
  }
490
23.2k
  case ISD::ATOMIC_CMP_SWAP:
491
23.2k
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
492
23.2k
  case ISD::ATOMIC_SWAP:
493
23.2k
  case ISD::ATOMIC_LOAD_ADD:
494
23.2k
  case ISD::ATOMIC_LOAD_SUB:
495
23.2k
  case ISD::ATOMIC_LOAD_AND:
496
23.2k
  case ISD::ATOMIC_LOAD_OR:
497
23.2k
  case ISD::ATOMIC_LOAD_XOR:
498
23.2k
  case ISD::ATOMIC_LOAD_NAND:
499
23.2k
  case ISD::ATOMIC_LOAD_MIN:
500
23.2k
  case ISD::ATOMIC_LOAD_MAX:
501
23.2k
  case ISD::ATOMIC_LOAD_UMIN:
502
23.2k
  case ISD::ATOMIC_LOAD_UMAX:
503
23.2k
  case ISD::ATOMIC_LOAD:
504
23.2k
  case ISD::ATOMIC_STORE: {
505
23.2k
    const AtomicSDNode *AT = cast<AtomicSDNode>(N);
506
23.2k
    ID.AddInteger(AT->getMemoryVT().getRawBits());
507
23.2k
    ID.AddInteger(AT->getRawSubclassData());
508
23.2k
    ID.AddInteger(AT->getPointerInfo().getAddrSpace());
509
23.2k
    break;
510
23.2k
  }
511
147
  case ISD::PREFETCH: {
512
147
    const MemSDNode *PF = cast<MemSDNode>(N);
513
147
    ID.AddInteger(PF->getPointerInfo().getAddrSpace());
514
147
    break;
515
23.2k
  }
516
58.7k
  case ISD::VECTOR_SHUFFLE: {
517
58.7k
    const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
518
58.7k
    for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
519
775k
         
i != e775k
;
++i716k
)
520
716k
      ID.AddInteger(SVN->getMaskElt(i));
521
58.7k
    break;
522
23.2k
  }
523
75
  case ISD::TargetBlockAddress:
524
75
  case ISD::BlockAddress: {
525
75
    const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
526
75
    ID.AddPointer(BA->getBlockAddress());
527
75
    ID.AddInteger(BA->getOffset());
528
75
    ID.AddInteger(BA->getTargetFlags());
529
75
    break;
530
109M
  }
531
109M
  } // end switch (N->getOpcode())
532
109M
533
109M
  // Target specific memory nodes could also have address spaces to check.
534
109M
  
if (109M
N->isTargetMemoryOpcode()109M
)
535
9.14k
    ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
536
109M
}
537
538
/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
539
/// data.
540
108M
static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
541
108M
  AddNodeIDOpcode(ID, N->getOpcode());
542
108M
  // Add the return value info.
543
108M
  AddNodeIDValueTypes(ID, N->getVTList());
544
108M
  // Add the operand info.
545
108M
  AddNodeIDOperands(ID, N->ops());
546
108M
547
108M
  // Handle SDNode leafs with special info.
548
108M
  AddNodeIDCustom(ID, N);
549
108M
}
550
551
//===----------------------------------------------------------------------===//
552
//                              SelectionDAG Class
553
//===----------------------------------------------------------------------===//
554
555
/// doNotCSE - Return true if CSE should not be performed for this node.
556
25.9M
static bool doNotCSE(SDNode *N) {
557
25.9M
  if (N->getValueType(0) == MVT::Glue)
558
4.81k
    return true; // Never CSE anything that produces a flag.
559
25.8M
560
25.8M
  switch (N->getOpcode()) {
561
25.7M
  default: break;
562
187k
  case ISD::HANDLENODE:
563
187k
  case ISD::EH_LABEL:
564
187k
    return true;   // Never CSE these nodes.
565
25.7M
  }
566
25.7M
567
25.7M
  // Check that remaining values produced are not flags.
568
32.6M
  
for (unsigned i = 1, e = N->getNumValues(); 25.7M
i != e32.6M
;
++i6.95M
)
569
15.6M
    
if (15.6M
N->getValueType(i) == MVT::Glue15.6M
)
570
8.66M
      return true; // Never CSE anything that produces a flag.
571
25.7M
572
17.0M
  return false;
573
25.9M
}
574
575
/// RemoveDeadNodes - This method deletes all unreachable nodes in the
576
/// SelectionDAG.
577
14.4M
void SelectionDAG::RemoveDeadNodes() {
578
14.4M
  // Create a dummy node (which is not added to allnodes), that adds a reference
579
14.4M
  // to the root node, preventing it from being deleted.
580
14.4M
  HandleSDNode Dummy(getRoot());
581
14.4M
582
14.4M
  SmallVector<SDNode*, 128> DeadNodes;
583
14.4M
584
14.4M
  // Add all obviously-dead nodes to the DeadNodes worklist.
585
14.4M
  for (SDNode &Node : allnodes())
586
310M
    
if (310M
Node.use_empty()310M
)
587
1.62M
      DeadNodes.push_back(&Node);
588
14.4M
589
14.4M
  RemoveDeadNodes(DeadNodes);
590
14.4M
591
14.4M
  // If the root changed (e.g. it was a dead load, update the root).
592
14.4M
  setRoot(Dummy.getValue());
593
14.4M
}
594
595
/// RemoveDeadNodes - This method deletes the unreachable nodes in the
596
/// given list, and any nodes that become unreachable as a result.
597
32.8M
void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
598
32.8M
599
32.8M
  // Process the worklist, deleting the nodes and adding their uses to the
600
32.8M
  // worklist.
601
48.5M
  while (
!DeadNodes.empty()48.5M
) {
602
15.6M
    SDNode *N = DeadNodes.pop_back_val();
603
15.6M
    // Skip to next node if we've already managed to delete the node. This could
604
15.6M
    // happen if replacing a node causes a node previously added to the node to
605
15.6M
    // be deleted.
606
15.6M
    if (N->getOpcode() == ISD::DELETED_NODE)
607
0
      continue;
608
15.6M
609
38.2M
    
for (DAGUpdateListener *DUL = UpdateListeners; 15.6M
DUL38.2M
;
DUL = DUL->Next22.5M
)
610
22.5M
      DUL->NodeDeleted(N, nullptr);
611
15.6M
612
15.6M
    // Take the node out of the appropriate CSE map.
613
15.6M
    RemoveNodeFromCSEMaps(N);
614
15.6M
615
15.6M
    // Next, brutally remove the operand list.  This is safe to do, as there are
616
15.6M
    // no cycles in the graph.
617
30.6M
    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); 
I != E30.6M
; ) {
618
14.9M
      SDUse &Use = *I++;
619
14.9M
      SDNode *Operand = Use.getNode();
620
14.9M
      Use.set(SDValue());
621
14.9M
622
14.9M
      // Now that we removed this operand, see if there are no uses of it left.
623
14.9M
      if (Operand->use_empty())
624
4.79M
        DeadNodes.push_back(Operand);
625
14.9M
    }
626
15.6M
627
15.6M
    DeallocateNode(N);
628
15.6M
  }
629
32.8M
}
630
631
1.55M
void SelectionDAG::RemoveDeadNode(SDNode *N){
632
1.55M
  SmallVector<SDNode*, 16> DeadNodes(1, N);
633
1.55M
634
1.55M
  // Create a dummy node that adds a reference to the root node, preventing
635
1.55M
  // it from being deleted.  (This matters if the root is an operand of the
636
1.55M
  // dead node.)
637
1.55M
  HandleSDNode Dummy(getRoot());
638
1.55M
639
1.55M
  RemoveDeadNodes(DeadNodes);
640
1.55M
}
641
642
30.3M
void SelectionDAG::DeleteNode(SDNode *N) {
643
30.3M
  // First take this out of the appropriate CSE map.
644
30.3M
  RemoveNodeFromCSEMaps(N);
645
30.3M
646
30.3M
  // Finally, remove uses due to operands of this node, remove from the
647
30.3M
  // AllNodes list, and delete the node.
648
30.3M
  DeleteNodeNotInCSEMaps(N);
649
30.3M
}
650
651
30.4M
void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
652
30.4M
  assert(N->getIterator() != AllNodes.begin() &&
653
30.4M
         "Cannot delete the entry node!");
654
30.4M
  assert(N->use_empty() && "Cannot delete a node that is not dead!");
655
30.4M
656
30.4M
  // Drop all of the operands and decrement used node's use counts.
657
30.4M
  N->DropOperands();
658
30.4M
659
30.4M
  DeallocateNode(N);
660
30.4M
}
661
662
115M
void SDDbgInfo::erase(const SDNode *Node) {
663
115M
  DbgValMapType::iterator I = DbgValMap.find(Node);
664
115M
  if (I == DbgValMap.end())
665
115M
    return;
666
319
  for (auto &Val: I->second)
667
349
    Val->setIsInvalidated();
668
115M
  DbgValMap.erase(I);
669
115M
}
670
671
115M
void SelectionDAG::DeallocateNode(SDNode *N) {
672
115M
  // If we have operands, deallocate them.
673
115M
  removeOperands(N);
674
115M
675
115M
  NodeAllocator.Deallocate(AllNodes.remove(N));
676
115M
677
115M
  // Set the opcode to DELETED_NODE to help catch bugs when node
678
115M
  // memory is reallocated.
679
115M
  // FIXME: There are places in SDag that have grown a dependency on the opcode
680
115M
  // value in the released node.
681
115M
  __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
682
115M
  N->NodeType = ISD::DELETED_NODE;
683
115M
684
115M
  // If any of the SDDbgValue nodes refer to this SDNode, invalidate
685
115M
  // them and forget about that node.
686
115M
  DbgInfo->erase(N);
687
115M
}
688
689
#ifndef NDEBUG
690
/// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
691
static void VerifySDNode(SDNode *N) {
692
  switch (N->getOpcode()) {
693
  default:
694
    break;
695
  case ISD::BUILD_PAIR: {
696
    EVT VT = N->getValueType(0);
697
    assert(N->getNumValues() == 1 && "Too many results!");
698
    assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
699
           "Wrong return type!");
700
    assert(N->getNumOperands() == 2 && "Wrong number of operands!");
701
    assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
702
           "Mismatched operand types!");
703
    assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
704
           "Wrong operand type!");
705
    assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
706
           "Wrong return type size");
707
    break;
708
  }
709
  case ISD::BUILD_VECTOR: {
710
    assert(N->getNumValues() == 1 && "Too many results!");
711
    assert(N->getValueType(0).isVector() && "Wrong return type!");
712
    assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
713
           "Wrong number of operands!");
714
    EVT EltVT = N->getValueType(0).getVectorElementType();
715
    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
716
      assert((I->getValueType() == EltVT ||
717
             (EltVT.isInteger() && I->getValueType().isInteger() &&
718
              EltVT.bitsLE(I->getValueType()))) &&
719
            "Wrong operand type!");
720
      assert(I->getValueType() == N->getOperand(0).getValueType() &&
721
             "Operands must all have the same type");
722
    }
723
    break;
724
  }
725
  }
726
}
727
#endif // NDEBUG
728
729
/// \brief Insert a newly allocated node into the DAG.
730
///
731
/// Handles insertion into the all nodes list and CSE map, as well as
732
/// verification and other common operations when a new node is allocated.
733
119M
void SelectionDAG::InsertNode(SDNode *N) {
734
119M
  AllNodes.push_back(N);
735
#ifndef NDEBUG
736
  N->PersistentId = NextPersistentId++;
737
  VerifySDNode(N);
738
#endif
739
}
740
741
/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
742
/// correspond to it.  This is useful when we're about to delete or repurpose
743
/// the node.  We don't want future request for structurally identical nodes
744
/// to return N anymore.
745
92.0M
bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
746
92.0M
  bool Erased = false;
747
92.0M
  switch (N->getOpcode()) {
748
172k
  case ISD::HANDLENODE: return false;  // noop.
749
2.91M
  case ISD::CONDCODE:
750
2.91M
    assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
751
2.91M
           "Cond code doesn't exist!");
752
2.91M
    Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
753
2.91M
    CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
754
2.91M
    break;
755
29.5k
  case ISD::ExternalSymbol:
756
29.5k
    Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
757
29.5k
    break;
758
253
  case ISD::TargetExternalSymbol: {
759
253
    ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
760
253
    Erased = TargetExternalSymbols.erase(
761
253
               std::pair<std::string,unsigned char>(ESN->getSymbol(),
762
253
                                                    ESN->getTargetFlags()));
763
253
    break;
764
92.0M
  }
765
0
  case ISD::MCSymbol: {
766
0
    auto *MCSN = cast<MCSymbolSDNode>(N);
767
0
    Erased = MCSymbols.erase(MCSN->getMCSymbol());
768
0
    break;
769
92.0M
  }
770
876k
  case ISD::VALUETYPE: {
771
876k
    EVT VT = cast<VTSDNode>(N)->getVT();
772
876k
    if (
VT.isExtended()876k
) {
773
1.64k
      Erased = ExtendedValueTypeNodes.erase(VT);
774
876k
    } else {
775
874k
      Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
776
874k
      ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
777
874k
    }
778
876k
    break;
779
92.0M
  }
780
88.0M
  default:
781
88.0M
    // Remove it from the CSE Map.
782
88.0M
    assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
783
88.0M
    assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
784
88.0M
    Erased = CSEMap.RemoveNode(N);
785
88.0M
    break;
786
91.9M
  }
787
#ifndef NDEBUG
788
  // Verify that the node was actually in one of the CSE maps, unless it has a
789
  // flag result (which cannot be CSE'd) or is one of the special cases that are
790
  // not subject to CSE.
791
  if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
792
      !N->isMachineOpcode() && !doNotCSE(N)) {
793
    N->dump(this);
794
    dbgs() << "\n";
795
    llvm_unreachable("Node is not in map!");
796
  }
797
#endif
798
0
  return Erased;
799
91.9M
}
800
801
/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
802
/// maps and modified in place. Add it back to the CSE maps, unless an identical
803
/// node already exists, in which case transfer all its users to the existing
804
/// node. This transfer can potentially trigger recursive merging.
805
void
806
25.1M
SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
807
25.1M
  // For node types that aren't CSE'd, just act as if no identical node
808
25.1M
  // already exists.
809
25.1M
  if (
!doNotCSE(N)25.1M
) {
810
16.2M
    SDNode *Existing = CSEMap.GetOrInsertNode(N);
811
16.2M
    if (
Existing != N16.2M
) {
812
64.3k
      // If there was already an existing matching node, use ReplaceAllUsesWith
813
64.3k
      // to replace the dead one with the existing one.  This can cause
814
64.3k
      // recursive merging of other unrelated nodes down the line.
815
64.3k
      ReplaceAllUsesWith(N, Existing);
816
64.3k
817
64.3k
      // N is now dead. Inform the listeners and delete it.
818
233k
      for (DAGUpdateListener *DUL = UpdateListeners; 
DUL233k
;
DUL = DUL->Next168k
)
819
168k
        DUL->NodeDeleted(N, Existing);
820
64.3k
      DeleteNodeNotInCSEMaps(N);
821
64.3k
      return;
822
64.3k
    }
823
25.0M
  }
824
25.0M
825
25.0M
  // If the node doesn't already exist, we updated it.  Inform listeners.
826
86.7M
  
for (DAGUpdateListener *DUL = UpdateListeners; 25.0M
DUL86.7M
;
DUL = DUL->Next61.7M
)
827
61.7M
    DUL->NodeUpdated(N);
828
25.1M
}
829
830
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
831
/// were replaced with those specified.  If this node is never memoized,
832
/// return null, otherwise return a pointer to the slot it would take.  If a
833
/// node already exists with these operands, the slot will be non-null.
834
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
835
21.0k
                                           void *&InsertPos) {
836
21.0k
  if (doNotCSE(N))
837
0
    return nullptr;
838
21.0k
839
21.0k
  SDValue Ops[] = { Op };
840
21.0k
  FoldingSetNodeID ID;
841
21.0k
  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
842
21.0k
  AddNodeIDCustom(ID, N);
843
21.0k
  SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
844
21.0k
  if (Node)
845
16
    Node->intersectFlagsWith(N->getFlags());
846
21.0k
  return Node;
847
21.0k
}
848
849
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
850
/// were replaced with those specified.  If this node is never memoized,
851
/// return null, otherwise return a pointer to the slot it would take.  If a
852
/// node already exists with these operands, the slot will be non-null.
853
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
854
                                           SDValue Op1, SDValue Op2,
855
32.1k
                                           void *&InsertPos) {
856
32.1k
  if (doNotCSE(N))
857
0
    return nullptr;
858
32.1k
859
32.1k
  SDValue Ops[] = { Op1, Op2 };
860
32.1k
  FoldingSetNodeID ID;
861
32.1k
  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
862
32.1k
  AddNodeIDCustom(ID, N);
863
32.1k
  SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
864
32.1k
  if (Node)
865
853
    Node->intersectFlagsWith(N->getFlags());
866
32.1k
  return Node;
867
32.1k
}
868
869
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
870
/// were replaced with those specified.  If this node is never memoized,
871
/// return null, otherwise return a pointer to the slot it would take.  If a
872
/// node already exists with these operands, the slot will be non-null.
873
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
874
720k
                                           void *&InsertPos) {
875
720k
  if (doNotCSE(N))
876
10.5k
    return nullptr;
877
709k
878
709k
  FoldingSetNodeID ID;
879
709k
  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
880
709k
  AddNodeIDCustom(ID, N);
881
709k
  SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
882
709k
  if (Node)
883
596
    Node->intersectFlagsWith(N->getFlags());
884
720k
  return Node;
885
720k
}
886
887
356k
unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
888
356k
  Type *Ty = VT == MVT::iPTR ?
889
0
                   PointerType::get(Type::getInt8Ty(*getContext()), 0) :
890
356k
                   VT.getTypeForEVT(*getContext());
891
356k
892
356k
  return getDataLayout().getABITypeAlignment(Ty);
893
356k
}
894
895
// EntryNode could meaningfully have debug info if we can find it...
896
SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
897
    : TM(tm), OptLevel(OL),
898
      EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
899
35.1k
      Root(getEntryNode()) {
900
35.1k
  InsertNode(&EntryNode);
901
35.1k
  DbgInfo = new SDDbgInfo();
902
35.1k
}
903
904
void SelectionDAG::init(MachineFunction &NewMF,
905
                        OptimizationRemarkEmitter &NewORE,
906
437k
                        Pass *PassPtr) {
907
437k
  MF = &NewMF;
908
437k
  SDAGISelPass = PassPtr;
909
437k
  ORE = &NewORE;
910
437k
  TLI = getSubtarget().getTargetLowering();
911
437k
  TSI = getSubtarget().getSelectionDAGInfo();
912
437k
  Context = &MF->getFunction()->getContext();
913
437k
}
914
915
35.0k
SelectionDAG::~SelectionDAG() {
916
35.0k
  assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
917
35.0k
  allnodes_clear();
918
35.0k
  OperandRecycler.clear(OperandAllocator);
919
35.0k
  delete DbgInfo;
920
35.0k
}
921
922
3.46M
void SelectionDAG::allnodes_clear() {
923
3.46M
  assert(&*AllNodes.begin() == &EntryNode);
924
3.46M
  AllNodes.remove(AllNodes.begin());
925
73.0M
  while (!AllNodes.empty())
926
69.5M
    DeallocateNode(&AllNodes.front());
927
#ifndef NDEBUG
928
  NextPersistentId = 0;
929
#endif
930
}
931
932
SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
933
31.1M
                                          void *&InsertPos) {
934
31.1M
  SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
935
31.1M
  if (
N31.1M
) {
936
10.6M
    switch (N->getOpcode()) {
937
10.6M
    default: break;
938
0
    case ISD::Constant:
939
0
    case ISD::ConstantFP:
940
0
      llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
941
31.1M
                       "debug location.  Use another overload.");
942
31.1M
    }
943
31.1M
  }
944
31.1M
  return N;
945
31.1M
}
946
947
SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
948
125M
                                          const SDLoc &DL, void *&InsertPos) {
949
125M
  SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
950
125M
  if (
N125M
) {
951
27.5M
    switch (N->getOpcode()) {
952
6.49M
    case ISD::Constant:
953
6.49M
    case ISD::ConstantFP:
954
6.49M
      // Erase debug location from the node if the node is used at several
955
6.49M
      // different places. Do not propagate one location to all uses as it
956
6.49M
      // will cause a worse single stepping debugging experience.
957
6.49M
      if (N->getDebugLoc() != DL.getDebugLoc())
958
84.0k
        N->setDebugLoc(DebugLoc());
959
6.49M
      break;
960
21.0M
    default:
961
21.0M
      // When the node's point of use is located earlier in the instruction
962
21.0M
      // sequence than its prior point of use, update its debug info to the
963
21.0M
      // earlier location.
964
21.0M
      if (
DL.getIROrder() && 21.0M
DL.getIROrder() < N->getIROrder()13.5M
)
965
258k
        N->setDebugLoc(DL.getDebugLoc());
966
6.49M
      break;
967
125M
    }
968
125M
  }
969
125M
  return N;
970
125M
}
971
972
3.42M
void SelectionDAG::clear() {
973
3.42M
  allnodes_clear();
974
3.42M
  OperandRecycler.clear(OperandAllocator);
975
3.42M
  OperandAllocator.Reset();
976
3.42M
  CSEMap.clear();
977
3.42M
978
3.42M
  ExtendedValueTypeNodes.clear();
979
3.42M
  ExternalSymbols.clear();
980
3.42M
  TargetExternalSymbols.clear();
981
3.42M
  MCSymbols.clear();
982
3.42M
  std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
983
3.42M
            static_cast<CondCodeSDNode*>(nullptr));
984
3.42M
  std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
985
3.42M
            static_cast<SDNode*>(nullptr));
986
3.42M
987
3.42M
  EntryNode.UseList = nullptr;
988
3.42M
  InsertNode(&EntryNode);
989
3.42M
  Root = getEntryNode();
990
3.42M
  DbgInfo->clear();
991
3.42M
}
992
993
6
SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
994
6
  return VT.bitsGT(Op.getValueType())
995
0
             ? getNode(ISD::FP_EXTEND, DL, VT, Op)
996
6
             : getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL));
997
6
}
998
999
74.7k
SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1000
74.7k
  return VT.bitsGT(Op.getValueType()) ?
1001
29.8k
    getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1002
44.8k
    getNode(ISD::TRUNCATE, DL, VT, Op);
1003
74.7k
}
1004
1005
840k
SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1006
840k
  return VT.bitsGT(Op.getValueType()) ?
1007
34.4k
    getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1008
806k
    getNode(ISD::TRUNCATE, DL, VT, Op);
1009
840k
}
1010
1011
321k
SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1012
321k
  return VT.bitsGT(Op.getValueType()) ?
1013
14.1k
    getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1014
307k
    getNode(ISD::TRUNCATE, DL, VT, Op);
1015
321k
}
1016
1017
SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1018
9.09k
                                        EVT OpVT) {
1019
9.09k
  if (VT.bitsLE(Op.getValueType()))
1020
8.85k
    return getNode(ISD::TRUNCATE, SL, VT, Op);
1021
233
1022
233
  TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1023
233
  return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1024
233
}
1025
1026
893k
SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1027
893k
  assert(!VT.isVector() &&
1028
893k
         "getZeroExtendInReg should use the vector element type instead of "
1029
893k
         "the vector type!");
1030
893k
  if (
Op.getValueType() == VT893k
)
return Op23
;
1031
893k
  unsigned BitWidth = Op.getScalarValueSizeInBits();
1032
893k
  APInt Imm = APInt::getLowBitsSet(BitWidth,
1033
893k
                                   VT.getSizeInBits());
1034
893k
  return getNode(ISD::AND, DL, Op.getValueType(), Op,
1035
893k
                 getConstant(Imm, DL, Op.getValueType()));
1036
893k
}
1037
1038
SDValue SelectionDAG::getAnyExtendVectorInReg(SDValue Op, const SDLoc &DL,
1039
228
                                              EVT VT) {
1040
228
  assert(VT.isVector() && "This DAG node is restricted to vector types.");
1041
228
  assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
1042
228
         "The sizes of the input and result must match in order to perform the "
1043
228
         "extend in-register.");
1044
228
  assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1045
228
         "The destination vector type must have fewer lanes than the input.");
1046
228
  return getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Op);
1047
228
}
1048
1049
SDValue SelectionDAG::getSignExtendVectorInReg(SDValue Op, const SDLoc &DL,
1050
1.87k
                                               EVT VT) {
1051
1.87k
  assert(VT.isVector() && "This DAG node is restricted to vector types.");
1052
1.87k
  assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
1053
1.87k
         "The sizes of the input and result must match in order to perform the "
1054
1.87k
         "extend in-register.");
1055
1.87k
  assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1056
1.87k
         "The destination vector type must have fewer lanes than the input.");
1057
1.87k
  return getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Op);
1058
1.87k
}
1059
1060
SDValue SelectionDAG::getZeroExtendVectorInReg(SDValue Op, const SDLoc &DL,
1061
1.54k
                                               EVT VT) {
1062
1.54k
  assert(VT.isVector() && "This DAG node is restricted to vector types.");
1063
1.54k
  assert(VT.getSizeInBits() == Op.getValueSizeInBits() &&
1064
1.54k
         "The sizes of the input and result must match in order to perform the "
1065
1.54k
         "extend in-register.");
1066
1.54k
  assert(VT.getVectorNumElements() < Op.getValueType().getVectorNumElements() &&
1067
1.54k
         "The destination vector type must have fewer lanes than the input.");
1068
1.54k
  return getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Op);
1069
1.54k
}
1070
1071
/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1072
9.71k
SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1073
9.71k
  EVT EltVT = VT.getScalarType();
1074
9.71k
  SDValue NegOne =
1075
9.71k
    getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1076
9.71k
  return getNode(ISD::XOR, DL, VT, Val, NegOne);
1077
9.71k
}
1078
1079
12
SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1080
12
  EVT EltVT = VT.getScalarType();
1081
12
  SDValue TrueValue;
1082
12
  switch (TLI->getBooleanContents(VT)) {
1083
0
    case TargetLowering::ZeroOrOneBooleanContent:
1084
0
    case TargetLowering::UndefinedBooleanContent:
1085
0
      TrueValue = getConstant(1, DL, VT);
1086
0
      break;
1087
12
    case TargetLowering::ZeroOrNegativeOneBooleanContent:
1088
12
      TrueValue = getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL,
1089
12
                              VT);
1090
12
      break;
1091
12
  }
1092
12
  return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1093
12
}
1094
1095
SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1096
28.8M
                                  bool isT, bool isO) {
1097
28.8M
  EVT EltVT = VT.getScalarType();
1098
28.8M
  assert((EltVT.getSizeInBits() >= 64 ||
1099
28.8M
         (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1100
28.8M
         "getConstant with a uint64_t value that doesn't fit in the type!");
1101
28.8M
  return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1102
28.8M
}
1103
1104
SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1105
33.8M
                                  bool isT, bool isO) {
1106
33.8M
  return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1107
33.8M
}
1108
1109
SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1110
39.7M
                                  EVT VT, bool isT, bool isO) {
1111
39.7M
  assert(VT.isInteger() && "Cannot create FP integer constant!");
1112
39.7M
1113
39.7M
  EVT EltVT = VT.getScalarType();
1114
39.7M
  const ConstantInt *Elt = &Val;
1115
39.7M
1116
39.7M
  // In some cases the vector type is legal but the element type is illegal and
1117
39.7M
  // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
1118
39.7M
  // inserted value (the type does not need to match the vector element type).
1119
39.7M
  // Any extra bits introduced will be truncated away.
1120
39.7M
  if (
VT.isVector() && 39.7M
TLI->getTypeAction(*getContext(), EltVT) ==
1121
39.7M
      TargetLowering::TypePromoteInteger) {
1122
7.10k
   EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1123
7.10k
   APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1124
7.10k
   Elt = ConstantInt::get(*getContext(), NewVal);
1125
7.10k
  }
1126
39.7M
  // In other cases the element type is illegal and needs to be expanded, for
1127
39.7M
  // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1128
39.7M
  // the value into n parts and use a vector type with n-times the elements.
1129
39.7M
  // Then bitcast to the type requested.
1130
39.7M
  // Legalizing constants too early makes the DAGCombiner's job harder so we
1131
39.7M
  // only legalize if the DAG tells us we must produce legal types.
1132
39.7M
  else 
if (39.7M
NewNodesMustHaveLegalTypes && 39.7M
VT.isVector()17.6M
&&
1133
39.9k
           TLI->getTypeAction(*getContext(), EltVT) ==
1134
39.7M
           TargetLowering::TypeExpandInteger) {
1135
183
    const APInt &NewVal = Elt->getValue();
1136
183
    EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1137
183
    unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1138
183
    unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1139
183
    EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1140
183
1141
183
    // Check the temporary vector is the correct size. If this fails then
1142
183
    // getTypeToTransformTo() probably returned a type whose size (in bits)
1143
183
    // isn't a power-of-2 factor of the requested type size.
1144
183
    assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1145
183
1146
183
    SmallVector<SDValue, 2> EltParts;
1147
549
    for (unsigned i = 0; 
i < ViaVecNumElts / VT.getVectorNumElements()549
;
++i366
) {
1148
366
      EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1149
366
                                           .zextOrTrunc(ViaEltSizeInBits), DL,
1150
366
                                     ViaEltVT, isT, isO));
1151
366
    }
1152
183
1153
183
    // EltParts is currently in little endian order. If we actually want
1154
183
    // big-endian order then reverse it now.
1155
183
    if (getDataLayout().isBigEndian())
1156
56
      std::reverse(EltParts.begin(), EltParts.end());
1157
183
1158
183
    // The elements must be reversed when the element order is different
1159
183
    // to the endianness of the elements (because the BITCAST is itself a
1160
183
    // vector shuffle in this situation). However, we do not need any code to
1161
183
    // perform this reversal because getConstant() is producing a vector
1162
183
    // splat.
1163
183
    // This situation occurs in MIPS MSA.
1164
183
1165
183
    SmallVector<SDValue, 8> Ops;
1166
733
    for (unsigned i = 0, e = VT.getVectorNumElements(); 
i != e733
;
++i550
)
1167
550
      Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1168
39.7M
1169
39.7M
    SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1170
39.7M
    NewSDValueDbgMsg(V, "Creating constant: ", this);
1171
39.7M
    return V;
1172
39.7M
  }
1173
39.7M
1174
39.7M
  assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1175
39.7M
         "APInt size does not match type size!");
1176
39.7M
  unsigned Opc = isT ? 
ISD::TargetConstant21.3M
:
ISD::Constant18.3M
;
1177
39.7M
  FoldingSetNodeID ID;
1178
39.7M
  AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1179
39.7M
  ID.AddPointer(Elt);
1180
39.7M
  ID.AddBoolean(isO);
1181
39.7M
  void *IP = nullptr;
1182
39.7M
  SDNode *N = nullptr;
1183
39.7M
  if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1184
17.8M
    
if (17.8M
!VT.isVector()17.8M
)
1185
17.7M
      return SDValue(N, 0);
1186
22.0M
1187
22.0M
  
if (22.0M
!N22.0M
) {
1188
21.9M
    N = newSDNode<ConstantSDNode>(isT, isO, Elt, DL.getDebugLoc(), EltVT);
1189
21.9M
    CSEMap.InsertNode(N, IP);
1190
21.9M
    InsertNode(N);
1191
21.9M
  }
1192
22.0M
1193
22.0M
  SDValue Result(N, 0);
1194
22.0M
  if (VT.isVector())
1195
61.1k
    Result = getSplatBuildVector(VT, DL, Result);
1196
39.7M
1197
39.7M
  NewSDValueDbgMsg(Result, "Creating constant: ", this);
1198
39.7M
  return Result;
1199
39.7M
}
1200
1201
SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1202
6.62M
                                        bool isTarget) {
1203
6.62M
  return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1204
6.62M
}
1205
1206
SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1207
11.1k
                                    bool isTarget) {
1208
11.1k
  return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1209
11.1k
}
1210
1211
SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1212
162k
                                    EVT VT, bool isTarget) {
1213
162k
  assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1214
162k
1215
162k
  EVT EltVT = VT.getScalarType();
1216
162k
1217
162k
  // Do the map lookup using the actual bit pattern for the floating point
1218
162k
  // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1219
162k
  // we don't have issues with SNANs.
1220
162k
  unsigned Opc = isTarget ? 
ISD::TargetConstantFP34.4k
:
ISD::ConstantFP128k
;
1221
162k
  FoldingSetNodeID ID;
1222
162k
  AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1223
162k
  ID.AddPointer(&V);
1224
162k
  void *IP = nullptr;
1225
162k
  SDNode *N = nullptr;
1226
162k
  if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1227
4.17k
    
if (4.17k
!VT.isVector()4.17k
)
1228
3.00k
      return SDValue(N, 0);
1229
159k
1230
159k
  
if (159k
!N159k
) {
1231
158k
    N = newSDNode<ConstantFPSDNode>(isTarget, &V, DL.getDebugLoc(), EltVT);
1232
158k
    CSEMap.InsertNode(N, IP);
1233
158k
    InsertNode(N);
1234
158k
  }
1235
159k
1236
159k
  SDValue Result(N, 0);
1237
159k
  if (VT.isVector())
1238
2.07k
    Result = getSplatBuildVector(VT, DL, Result);
1239
162k
  NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1240
162k
  return Result;
1241
162k
}
1242
1243
SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1244
5.66k
                                    bool isTarget) {
1245
5.66k
  EVT EltVT = VT.getScalarType();
1246
5.66k
  if (EltVT == MVT::f32)
1247
3.61k
    return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1248
2.05k
  else 
if (2.05k
EltVT == MVT::f642.05k
)
1249
1.74k
    return getConstantFP(APFloat(Val), DL, VT, isTarget);
1250
306
  else 
if (306
EltVT == MVT::f80 || 306
EltVT == MVT::f128168
||
EltVT == MVT::ppcf128161
||
1251
306
           
EltVT == MVT::f16159
) {
1252
306
    bool Ignored;
1253
306
    APFloat APF = APFloat(Val);
1254
306
    APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1255
306
                &Ignored);
1256
306
    return getConstantFP(APF, DL, VT, isTarget);
1257
306
  } else
1258
306
    llvm_unreachable("Unsupported type in getConstantFP");
1259
5.66k
}
1260
1261
SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1262
                                       EVT VT, int64_t Offset, bool isTargetGA,
1263
6.30M
                                       unsigned char TargetFlags) {
1264
6.30M
  assert((TargetFlags == 0 || isTargetGA) &&
1265
6.30M
         "Cannot set target flags on target-independent globals");
1266
6.30M
1267
6.30M
  // Truncate (with sign-extension) the offset value to the pointer size.
1268
6.30M
  unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1269
6.30M
  if (BitWidth < 64)
1270
132k
    Offset = SignExtend64(Offset, BitWidth);
1271
6.30M
1272
6.30M
  unsigned Opc;
1273
6.30M
  if (GV->isThreadLocal())
1274
1.66k
    
Opc = isTargetGA ? 1.66k
ISD::TargetGlobalTLSAddress900
:
ISD::GlobalTLSAddress763
;
1275
6.30M
  else
1276
6.29M
    
Opc = isTargetGA ? 6.29M
ISD::TargetGlobalAddress3.68M
:
ISD::GlobalAddress2.61M
;
1277
6.30M
1278
6.30M
  FoldingSetNodeID ID;
1279
6.30M
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1280
6.30M
  ID.AddPointer(GV);
1281
6.30M
  ID.AddInteger(Offset);
1282
6.30M
  ID.AddInteger(TargetFlags);
1283
6.30M
  void *IP = nullptr;
1284
6.30M
  if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1285
293k
    return SDValue(E, 0);
1286
6.00M
1287
6.00M
  auto *N = newSDNode<GlobalAddressSDNode>(
1288
6.00M
      Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1289
6.00M
  CSEMap.InsertNode(N, IP);
1290
6.00M
    InsertNode(N);
1291
6.00M
  return SDValue(N, 0);
1292
6.00M
}
1293
1294
1.43M
SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1295
1.43M
  unsigned Opc = isTarget ? 
ISD::TargetFrameIndex927k
:
ISD::FrameIndex505k
;
1296
1.43M
  FoldingSetNodeID ID;
1297
1.43M
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1298
1.43M
  ID.AddInteger(FI);
1299
1.43M
  void *IP = nullptr;
1300
1.43M
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1301
431k
    return SDValue(E, 0);
1302
1.00M
1303
1.00M
  auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1304
1.00M
  CSEMap.InsertNode(N, IP);
1305
1.00M
  InsertNode(N);
1306
1.00M
  return SDValue(N, 0);
1307
1.00M
}
1308
1309
SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1310
15.1k
                                   unsigned char TargetFlags) {
1311
15.1k
  assert((TargetFlags == 0 || isTarget) &&
1312
15.1k
         "Cannot set target flags on target-independent jump tables");
1313
15.1k
  unsigned Opc = isTarget ? 
ISD::TargetJumpTable9.96k
:
ISD::JumpTable5.16k
;
1314
15.1k
  FoldingSetNodeID ID;
1315
15.1k
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1316
15.1k
  ID.AddInteger(JTI);
1317
15.1k
  ID.AddInteger(TargetFlags);
1318
15.1k
  void *IP = nullptr;
1319
15.1k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1320
238
    return SDValue(E, 0);
1321
14.8k
1322
14.8k
  auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1323
14.8k
  CSEMap.InsertNode(N, IP);
1324
14.8k
  InsertNode(N);
1325
14.8k
  return SDValue(N, 0);
1326
14.8k
}
1327
1328
SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1329
                                      unsigned Alignment, int Offset,
1330
                                      bool isTarget,
1331
287k
                                      unsigned char TargetFlags) {
1332
287k
  assert((TargetFlags == 0 || isTarget) &&
1333
287k
         "Cannot set target flags on target-independent globals");
1334
287k
  if (Alignment == 0)
1335
98.2k
    Alignment = MF->getFunction()->optForSize()
1336
1.97k
                    ? getDataLayout().getABITypeAlignment(C->getType())
1337
96.2k
                    : getDataLayout().getPrefTypeAlignment(C->getType());
1338
287k
  unsigned Opc = isTarget ? 
ISD::TargetConstantPool191k
:
ISD::ConstantPool96.6k
;
1339
287k
  FoldingSetNodeID ID;
1340
287k
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1341
287k
  ID.AddInteger(Alignment);
1342
287k
  ID.AddInteger(Offset);
1343
287k
  ID.AddPointer(C);
1344
287k
  ID.AddInteger(TargetFlags);
1345
287k
  void *IP = nullptr;
1346
287k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1347
2.97k
    return SDValue(E, 0);
1348
284k
1349
284k
  auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1350
284k
                                          TargetFlags);
1351
284k
  CSEMap.InsertNode(N, IP);
1352
284k
  InsertNode(N);
1353
284k
  return SDValue(N, 0);
1354
284k
}
1355
1356
SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1357
                                      unsigned Alignment, int Offset,
1358
                                      bool isTarget,
1359
356
                                      unsigned char TargetFlags) {
1360
356
  assert((TargetFlags == 0 || isTarget) &&
1361
356
         "Cannot set target flags on target-independent globals");
1362
356
  if (Alignment == 0)
1363
0
    Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1364
356
  unsigned Opc = isTarget ? 
ISD::TargetConstantPool339
:
ISD::ConstantPool17
;
1365
356
  FoldingSetNodeID ID;
1366
356
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1367
356
  ID.AddInteger(Alignment);
1368
356
  ID.AddInteger(Offset);
1369
356
  C->addSelectionDAGCSEId(ID);
1370
356
  ID.AddInteger(TargetFlags);
1371
356
  void *IP = nullptr;
1372
356
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1373
0
    return SDValue(E, 0);
1374
356
1375
356
  auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1376
356
                                          TargetFlags);
1377
356
  CSEMap.InsertNode(N, IP);
1378
356
  InsertNode(N);
1379
356
  return SDValue(N, 0);
1380
356
}
1381
1382
SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1383
0
                                     unsigned char TargetFlags) {
1384
0
  FoldingSetNodeID ID;
1385
0
  AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1386
0
  ID.AddInteger(Index);
1387
0
  ID.AddInteger(Offset);
1388
0
  ID.AddInteger(TargetFlags);
1389
0
  void *IP = nullptr;
1390
0
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1391
0
    return SDValue(E, 0);
1392
0
1393
0
  auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1394
0
  CSEMap.InsertNode(N, IP);
1395
0
  InsertNode(N);
1396
0
  return SDValue(N, 0);
1397
0
}
1398
1399
3.91M
SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1400
3.91M
  FoldingSetNodeID ID;
1401
3.91M
  AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1402
3.91M
  ID.AddPointer(MBB);
1403
3.91M
  void *IP = nullptr;
1404
3.91M
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1405
5
    return SDValue(E, 0);
1406
3.91M
1407
3.91M
  auto *N = newSDNode<BasicBlockSDNode>(MBB);
1408
3.91M
  CSEMap.InsertNode(N, IP);
1409
3.91M
  InsertNode(N);
1410
3.91M
  return SDValue(N, 0);
1411
3.91M
}
1412
1413
1.36M
SDValue SelectionDAG::getValueType(EVT VT) {
1414
1.36M
  if (
VT.isSimple() && 1.36M
(unsigned)VT.getSimpleVT().SimpleTy >=
1415
1.35M
      ValueTypeNodes.size())
1416
24.4k
    ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1417
1.36M
1418
1.36M
  SDNode *&N = VT.isExtended() ?
1419
1.36M
    
ExtendedValueTypeNodes[VT]2.05k
:
ValueTypeNodes[VT.getSimpleVT().SimpleTy]1.35M
;
1420
1.36M
1421
1.36M
  if (
N1.36M
)
return SDValue(N, 0)483k
;
1422
876k
  N = newSDNode<VTSDNode>(VT);
1423
876k
  InsertNode(N);
1424
876k
  return SDValue(N, 0);
1425
876k
}
1426
1427
33.3k
SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1428
33.3k
  SDNode *&N = ExternalSymbols[Sym];
1429
33.3k
  if (
N33.3k
)
return SDValue(N, 0)3.78k
;
1430
29.5k
  N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1431
29.5k
  InsertNode(N);
1432
29.5k
  return SDValue(N, 0);
1433
29.5k
}
1434
1435
65
SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1436
65
  SDNode *&N = MCSymbols[Sym];
1437
65
  if (N)
1438
16
    return SDValue(N, 0);
1439
49
  N = newSDNode<MCSymbolSDNode>(Sym, VT);
1440
49
  InsertNode(N);
1441
49
  return SDValue(N, 0);
1442
49
}
1443
1444
SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1445
47.3k
                                              unsigned char TargetFlags) {
1446
47.3k
  SDNode *&N =
1447
47.3k
    TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1448
47.3k
                                                               TargetFlags)];
1449
47.3k
  if (
N47.3k
)
return SDValue(N, 0)9.23k
;
1450
38.0k
  N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1451
38.0k
  InsertNode(N);
1452
38.0k
  return SDValue(N, 0);
1453
38.0k
}
1454
1455
3.18M
SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1456
3.18M
  if ((unsigned)Cond >= CondCodeNodes.size())
1457
37.8k
    CondCodeNodes.resize(Cond+1);
1458
3.18M
1459
3.18M
  if (
!CondCodeNodes[Cond]3.18M
) {
1460
2.91M
    auto *N = newSDNode<CondCodeSDNode>(Cond);
1461
2.91M
    CondCodeNodes[Cond] = N;
1462
2.91M
    InsertNode(N);
1463
2.91M
  }
1464
3.18M
1465
3.18M
  return SDValue(CondCodeNodes[Cond], 0);
1466
3.18M
}
1467
1468
/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1469
/// point at N1 to point at N2 and indices that point at N2 to point at N1.
1470
915
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1471
915
  std::swap(N1, N2);
1472
915
  ShuffleVectorSDNode::commuteMask(M);
1473
915
}
1474
1475
SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1476
106k
                                       SDValue N2, ArrayRef<int> Mask) {
1477
106k
  assert(VT.getVectorNumElements() == Mask.size() &&
1478
106k
           "Must have the same number of vector elements as mask elements!");
1479
106k
  assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1480
106k
         "Invalid VECTOR_SHUFFLE");
1481
106k
1482
106k
  // Canonicalize shuffle undef, undef -> undef
1483
106k
  if (
N1.isUndef() && 106k
N2.isUndef()352
)
1484
140
    return getUNDEF(VT);
1485
106k
1486
106k
  // Validate that all indices in Mask are within the range of the elements
1487
106k
  // input to the shuffle.
1488
106k
  int NElts = Mask.size();
1489
106k
  assert(llvm::all_of(Mask, [&](int M) { return M < (NElts * 2); }) &&
1490
106k
         "Index out of range");
1491
106k
1492
106k
  // Copy the mask so we can do any needed cleanup.
1493
106k
  SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1494
106k
1495
106k
  // Canonicalize shuffle v, v -> v, undef
1496
106k
  if (
N1 == N2106k
) {
1497
3.52k
    N2 = getUNDEF(VT);
1498
32.6k
    for (int i = 0; 
i != NElts32.6k
;
++i29.1k
)
1499
29.1k
      
if (29.1k
MaskVec[i] >= NElts29.1k
)
MaskVec[i] -= NElts1.54k
;
1500
3.52k
  }
1501
106k
1502
106k
  // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
1503
106k
  if (N1.isUndef())
1504
212
    commuteShuffle(N1, N2, MaskVec);
1505
106k
1506
106k
  // If shuffling a splat, try to blend the splat instead. We do this here so
1507
106k
  // that even when this arises during lowering we don't have to re-handle it.
1508
10.0k
  auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1509
10.0k
    BitVector UndefElements;
1510
10.0k
    SDValue Splat = BV->getSplatValue(&UndefElements);
1511
10.0k
    if (!Splat)
1512
862
      return;
1513
9.20k
1514
122k
    
for (int i = 0; 9.20k
i < NElts122k
;
++i113k
) {
1515
113k
      if (
MaskVec[i] < Offset || 113k
MaskVec[i] >= (Offset + NElts)88.6k
)
1516
36.6k
        continue;
1517
77.0k
1518
77.0k
      // If this input comes from undef, mark it as such.
1519
77.0k
      
if (77.0k
UndefElements[MaskVec[i] - Offset]77.0k
) {
1520
25
        MaskVec[i] = -1;
1521
25
        continue;
1522
25
      }
1523
76.9k
1524
76.9k
      // If we can blend a non-undef lane, use that instead.
1525
76.9k
      
if (76.9k
!UndefElements[i]76.9k
)
1526
75.5k
        MaskVec[i] = i + Offset;
1527
113k
    }
1528
10.0k
  };
1529
106k
  if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1530
4.59k
    BlendSplat(N1BV, 0);
1531
106k
  if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1532
5.47k
    BlendSplat(N2BV, NElts);
1533
106k
1534
106k
  // Canonicalize all index into lhs, -> shuffle lhs, undef
1535
106k
  // Canonicalize all index into rhs, -> shuffle rhs, undef
1536
106k
  bool AllLHS = true, AllRHS = true;
1537
106k
  bool N2Undef = N2.isUndef();
1538
960k
  for (int i = 0; 
i != NElts960k
;
++i854k
) {
1539
854k
    if (
MaskVec[i] >= NElts854k
) {
1540
141k
      if (N2Undef)
1541
2.82k
        MaskVec[i] = -1;
1542
141k
      else
1543
139k
        AllLHS = false;
1544
854k
    } else 
if (712k
MaskVec[i] >= 0712k
) {
1545
526k
      AllRHS = false;
1546
526k
    }
1547
854k
  }
1548
106k
  if (
AllLHS && 106k
AllRHS70.8k
)
1549
45
    return getUNDEF(VT);
1550
106k
  
if (106k
AllLHS && 106k
!N2Undef70.7k
)
1551
2.66k
    N2 = getUNDEF(VT);
1552
106k
  if (
AllRHS106k
) {
1553
703
    N1 = getUNDEF(VT);
1554
703
    commuteShuffle(N1, N2, MaskVec);
1555
703
  }
1556
106k
  // Reset our undef status after accounting for the mask.
1557
106k
  N2Undef = N2.isUndef();
1558
106k
  // Re-check whether both sides ended up undef.
1559
106k
  if (
N1.isUndef() && 106k
N2Undef0
)
1560
0
    return getUNDEF(VT);
1561
106k
1562
106k
  // If Identity shuffle return that node.
1563
106k
  bool Identity = true, AllSame = true;
1564
960k
  for (int i = 0; 
i != NElts960k
;
++i854k
) {
1565
854k
    if (
MaskVec[i] >= 0 && 854k
MaskVec[i] != i665k
)
Identity = false364k
;
1566
854k
    if (
MaskVec[i] != MaskVec[0]854k
)
AllSame = false645k
;
1567
854k
  }
1568
106k
  if (
Identity && 106k
NElts31.8k
)
1569
31.8k
    return N1;
1570
74.2k
1571
74.2k
  // Shuffling a constant splat doesn't change the result.
1572
74.2k
  
if (74.2k
N2Undef74.2k
) {
1573
39.6k
    SDValue V = N1;
1574
39.6k
1575
39.6k
    // Look through any bitcasts. We check that these don't change the number
1576
39.6k
    // (and size) of elements and just changes their types.
1577
50.2k
    while (V.getOpcode() == ISD::BITCAST)
1578
10.5k
      V = V->getOperand(0);
1579
39.6k
1580
39.6k
    // A splat should always show up as a build vector node.
1581
39.6k
    if (auto *
BV39.6k
= dyn_cast<BuildVectorSDNode>(V)) {
1582
835
      BitVector UndefElements;
1583
835
      SDValue Splat = BV->getSplatValue(&UndefElements);
1584
835
      // If this is a splat of an undef, shuffling it is also undef.
1585
835
      if (
Splat && 835
Splat.isUndef()113
)
1586
0
        return getUNDEF(VT);
1587
835
1588
835
      bool SameNumElts =
1589
835
          V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1590
835
1591
835
      // We only have a splat which can skip shuffles if there is a splatted
1592
835
      // value and no undef lanes rearranged by the shuffle.
1593
835
      if (
Splat && 835
UndefElements.none()113
) {
1594
25
        // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1595
25
        // number of elements match or the value splatted is a zero constant.
1596
25
        if (SameNumElts)
1597
4
          return N1;
1598
21
        
if (auto *21
C21
= dyn_cast<ConstantSDNode>(Splat))
1599
20
          
if (20
C->isNullValue()20
)
1600
13
            return N1;
1601
818
      }
1602
818
1603
818
      // If the shuffle itself creates a splat, build the vector directly.
1604
818
      
if (818
AllSame && 818
SameNumElts289
) {
1605
249
        EVT BuildVT = BV->getValueType(0);
1606
249
        const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1607
249
        SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1608
249
1609
249
        // We may have jumped through bitcasts, so the type of the
1610
249
        // BUILD_VECTOR may not match the type of the shuffle.
1611
249
        if (BuildVT != VT)
1612
0
          NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1613
249
        return NewBV;
1614
249
      }
1615
73.9k
    }
1616
39.6k
  }
1617
73.9k
1618
73.9k
  FoldingSetNodeID ID;
1619
73.9k
  SDValue Ops[2] = { N1, N2 };
1620
73.9k
  AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1621
791k
  for (int i = 0; 
i != NElts791k
;
++i717k
)
1622
717k
    ID.AddInteger(MaskVec[i]);
1623
73.9k
1624
73.9k
  void* IP = nullptr;
1625
73.9k
  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1626
3.16k
    return SDValue(E, 0);
1627
70.7k
1628
70.7k
  // Allocate the mask array for the node out of the BumpPtrAllocator, since
1629
70.7k
  // SDNode doesn't have access to it.  This memory will be "leaked" when
1630
70.7k
  // the node is deallocated, but recovered when the NodeAllocator is released.
1631
70.7k
  int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1632
70.7k
  std::copy(MaskVec.begin(), MaskVec.end(), MaskAlloc);
1633
70.7k
1634
70.7k
  auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1635
70.7k
                                           dl.getDebugLoc(), MaskAlloc);
1636
70.7k
  createOperands(N, Ops);
1637
70.7k
1638
70.7k
  CSEMap.InsertNode(N, IP);
1639
70.7k
  InsertNode(N);
1640
70.7k
  return SDValue(N, 0);
1641
70.7k
}
1642
1643
4.32k
SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1644
4.32k
  MVT VT = SV.getSimpleValueType(0);
1645
4.32k
  SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1646
4.32k
  ShuffleVectorSDNode::commuteMask(MaskVec);
1647
4.32k
1648
4.32k
  SDValue Op0 = SV.getOperand(0);
1649
4.32k
  SDValue Op1 = SV.getOperand(1);
1650
4.32k
  return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1651
4.32k
}
1652
1653
23.6M
SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1654
23.6M
  FoldingSetNodeID ID;
1655
23.6M
  AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1656
23.6M
  ID.AddInteger(RegNo);
1657
23.6M
  void *IP = nullptr;
1658
23.6M
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1659
9.48M
    return SDValue(E, 0);
1660
14.1M
1661
14.1M
  auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1662
14.1M
  CSEMap.InsertNode(N, IP);
1663
14.1M
  InsertNode(N);
1664
14.1M
  return SDValue(N, 0);
1665
14.1M
}
1666
1667
1.79M
SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1668
1.79M
  FoldingSetNodeID ID;
1669
1.79M
  AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1670
1.79M
  ID.AddPointer(RegMask);
1671
1.79M
  void *IP = nullptr;
1672
1.79M
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1673
697k
    return SDValue(E, 0);
1674
1.09M
1675
1.09M
  auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
1676
1.09M
  CSEMap.InsertNode(N, IP);
1677
1.09M
  InsertNode(N);
1678
1.09M
  return SDValue(N, 0);
1679
1.09M
}
1680
1681
SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1682
37.0k
                                 MCSymbol *Label) {
1683
37.0k
  return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
1684
37.0k
}
1685
1686
SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1687
37.0k
                                   SDValue Root, MCSymbol *Label) {
1688
37.0k
  FoldingSetNodeID ID;
1689
37.0k
  SDValue Ops[] = { Root };
1690
37.0k
  AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
1691
37.0k
  ID.AddPointer(Label);
1692
37.0k
  void *IP = nullptr;
1693
37.0k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1694
0
    return SDValue(E, 0);
1695
37.0k
1696
37.0k
  auto *N = newSDNode<LabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label);
1697
37.0k
  createOperands(N, Ops);
1698
37.0k
1699
37.0k
  CSEMap.InsertNode(N, IP);
1700
37.0k
  InsertNode(N);
1701
37.0k
  return SDValue(N, 0);
1702
37.0k
}
1703
1704
SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1705
                                      int64_t Offset,
1706
                                      bool isTarget,
1707
246
                                      unsigned char TargetFlags) {
1708
246
  unsigned Opc = isTarget ? 
ISD::TargetBlockAddress136
:
ISD::BlockAddress110
;
1709
246
1710
246
  FoldingSetNodeID ID;
1711
246
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1712
246
  ID.AddPointer(BA);
1713
246
  ID.AddInteger(Offset);
1714
246
  ID.AddInteger(TargetFlags);
1715
246
  void *IP = nullptr;
1716
246
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1717
2
    return SDValue(E, 0);
1718
244
1719
244
  auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
1720
244
  CSEMap.InsertNode(N, IP);
1721
244
  InsertNode(N);
1722
244
  return SDValue(N, 0);
1723
244
}
1724
1725
1.21k
SDValue SelectionDAG::getSrcValue(const Value *V) {
1726
1.21k
  assert((!V || V->getType()->isPointerTy()) &&
1727
1.21k
         "SrcValue is not a pointer?");
1728
1.21k
1729
1.21k
  FoldingSetNodeID ID;
1730
1.21k
  AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
1731
1.21k
  ID.AddPointer(V);
1732
1.21k
1733
1.21k
  void *IP = nullptr;
1734
1.21k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1735
365
    return SDValue(E, 0);
1736
849
1737
849
  auto *N = newSDNode<SrcValueSDNode>(V);
1738
849
  CSEMap.InsertNode(N, IP);
1739
849
  InsertNode(N);
1740
849
  return SDValue(N, 0);
1741
849
}
1742
1743
11.8k
SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1744
11.8k
  FoldingSetNodeID ID;
1745
11.8k
  AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
1746
11.8k
  ID.AddPointer(MD);
1747
11.8k
1748
11.8k
  void *IP = nullptr;
1749
11.8k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1750
0
    return SDValue(E, 0);
1751
11.8k
1752
11.8k
  auto *N = newSDNode<MDNodeSDNode>(MD);
1753
11.8k
  CSEMap.InsertNode(N, IP);
1754
11.8k
  InsertNode(N);
1755
11.8k
  return SDValue(N, 0);
1756
11.8k
}
1757
1758
176k
SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1759
176k
  if (VT == V.getValueType())
1760
60.4k
    return V;
1761
116k
1762
116k
  return getNode(ISD::BITCAST, SDLoc(V), VT, V);
1763
116k
}
1764
1765
SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
1766
199
                                       unsigned SrcAS, unsigned DestAS) {
1767
199
  SDValue Ops[] = {Ptr};
1768
199
  FoldingSetNodeID ID;
1769
199
  AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
1770
199
  ID.AddInteger(SrcAS);
1771
199
  ID.AddInteger(DestAS);
1772
199
1773
199
  void *IP = nullptr;
1774
199
  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1775
0
    return SDValue(E, 0);
1776
199
1777
199
  auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
1778
199
                                           VT, SrcAS, DestAS);
1779
199
  createOperands(N, Ops);
1780
199
1781
199
  CSEMap.InsertNode(N, IP);
1782
199
  InsertNode(N);
1783
199
  return SDValue(N, 0);
1784
199
}
1785
1786
/// getShiftAmountOperand - Return the specified value casted to
1787
/// the target's desired shift amount type.
1788
942k
SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1789
942k
  EVT OpTy = Op.getValueType();
1790
942k
  EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
1791
942k
  if (
OpTy == ShTy || 942k
OpTy.isVector()15.1k
)
return Op927k
;
1792
15.1k
1793
15.1k
  return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
1794
15.1k
}
1795
1796
27
SDValue SelectionDAG::expandVAArg(SDNode *Node) {
1797
27
  SDLoc dl(Node);
1798
27
  const TargetLowering &TLI = getTargetLoweringInfo();
1799
27
  const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
1800
27
  EVT VT = Node->getValueType(0);
1801
27
  SDValue Tmp1 = Node->getOperand(0);
1802
27
  SDValue Tmp2 = Node->getOperand(1);
1803
27
  unsigned Align = Node->getConstantOperandVal(3);
1804
27
1805
27
  SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
1806
27
                               Tmp2, MachinePointerInfo(V));
1807
27
  SDValue VAList = VAListLoad;
1808
27
1809
27
  if (
Align > TLI.getMinStackArgumentAlignment()27
) {
1810
11
    assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
1811
11
1812
11
    VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1813
11
                     getConstant(Align - 1, dl, VAList.getValueType()));
1814
11
1815
11
    VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList,
1816
11
                     getConstant(-(int64_t)Align, dl, VAList.getValueType()));
1817
11
  }
1818
27
1819
27
  // Increment the pointer, VAList, to the next vaarg
1820
27
  Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1821
27
                 getConstant(getDataLayout().getTypeAllocSize(
1822
27
                                               VT.getTypeForEVT(*getContext())),
1823
27
                             dl, VAList.getValueType()));
1824
27
  // Store the incremented VAList to the legalized pointer
1825
27
  Tmp1 =
1826
27
      getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
1827
27
  // Load the actual argument out of the pointer VAList
1828
27
  return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
1829
27
}
1830
1831
35
SDValue SelectionDAG::expandVACopy(SDNode *Node) {
1832
35
  SDLoc dl(Node);
1833
35
  const TargetLowering &TLI = getTargetLoweringInfo();
1834
35
  // This defaults to loading a pointer from the input and storing it to the
1835
35
  // output, returning the chain.
1836
35
  const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
1837
35
  const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
1838
35
  SDValue Tmp1 =
1839
35
      getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
1840
35
              Node->getOperand(2), MachinePointerInfo(VS));
1841
35
  return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
1842
35
                  MachinePointerInfo(VD));
1843
35
}
1844
1845
2.62k
SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1846
2.62k
  MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1847
2.62k
  unsigned ByteSize = VT.getStoreSize();
1848
2.62k
  Type *Ty = VT.getTypeForEVT(*getContext());
1849
2.62k
  unsigned StackAlign =
1850
2.62k
      std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
1851
2.62k
1852
2.62k
  int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
1853
2.62k
  return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1854
2.62k
}
1855
1856
3.92k
SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
1857
3.92k
  unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
1858
3.92k
  Type *Ty1 = VT1.getTypeForEVT(*getContext());
1859
3.92k
  Type *Ty2 = VT2.getTypeForEVT(*getContext());
1860
3.92k
  const DataLayout &DL = getDataLayout();
1861
3.92k
  unsigned Align =
1862
3.92k
      std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
1863
3.92k
1864
3.92k
  MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1865
3.92k
  int FrameIdx = MFI.CreateStackObject(Bytes, Align, false);
1866
3.92k
  return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1867
3.92k
}
1868
1869
SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
1870
3.50M
                                ISD::CondCode Cond, const SDLoc &dl) {
1871
3.50M
  // These setcc operations always fold.
1872
3.50M
  switch (Cond) {
1873
3.44M
  default: break;
1874
37
  case ISD::SETFALSE:
1875
37
  case ISD::SETFALSE2: return getConstant(0, dl, VT);
1876
42
  case ISD::SETTRUE:
1877
42
  case ISD::SETTRUE2: {
1878
42
    TargetLowering::BooleanContent Cnt =
1879
42
        TLI->getBooleanContents(N1->getValueType(0));
1880
42
    return getConstant(
1881
42
        Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? 
-1ULL21
:
121
, dl,
1882
42
        VT);
1883
42
  }
1884
42
1885
54.5k
  case ISD::SETOEQ:
1886
54.5k
  case ISD::SETOGT:
1887
54.5k
  case ISD::SETOGE:
1888
54.5k
  case ISD::SETOLT:
1889
54.5k
  case ISD::SETOLE:
1890
54.5k
  case ISD::SETONE:
1891
54.5k
  case ISD::SETO:
1892
54.5k
  case ISD::SETUO:
1893
54.5k
  case ISD::SETUEQ:
1894
54.5k
  case ISD::SETUNE:
1895
54.5k
    assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!");
1896
54.5k
    break;
1897
3.50M
  }
1898
3.50M
1899
3.50M
  
if (ConstantSDNode *3.50M
N2C3.50M
= dyn_cast<ConstantSDNode>(N2)) {
1900
2.71M
    const APInt &C2 = N2C->getAPIntValue();
1901
2.71M
    if (ConstantSDNode *
N1C2.71M
= dyn_cast<ConstantSDNode>(N1)) {
1902
1.81k
      const APInt &C1 = N1C->getAPIntValue();
1903
1.81k
1904
1.81k
      switch (Cond) {
1905
0
      
default: 0
llvm_unreachable0
("Unknown integer setcc!");
1906
218
      case ISD::SETEQ:  return getConstant(C1 == C2, dl, VT);
1907
120
      case ISD::SETNE:  return getConstant(C1 != C2, dl, VT);
1908
198
      case ISD::SETULT: return getConstant(C1.ult(C2), dl, VT);
1909
222
      case ISD::SETUGT: return getConstant(C1.ugt(C2), dl, VT);
1910
194
      case ISD::SETULE: return getConstant(C1.ule(C2), dl, VT);
1911
176
      case ISD::SETUGE: return getConstant(C1.uge(C2), dl, VT);
1912
196
      case ISD::SETLT:  return getConstant(C1.slt(C2), dl, VT);
1913
194
      case ISD::SETGT:  return getConstant(C1.sgt(C2), dl, VT);
1914
143
      case ISD::SETLE:  return getConstant(C1.sle(C2), dl, VT);
1915
153
      case ISD::SETGE:  return getConstant(C1.sge(C2), dl, VT);
1916
3.49M
      }
1917
3.49M
    }
1918
2.71M
  }
1919
3.49M
  
if (ConstantFPSDNode *3.49M
N1C3.49M
= dyn_cast<ConstantFPSDNode>(N1)) {
1920
449
    if (ConstantFPSDNode *
N2C449
= dyn_cast<ConstantFPSDNode>(N2)) {
1921
230
      APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
1922
230
      switch (Cond) {
1923
0
      default: break;
1924
4
      
case ISD::SETEQ: if (4
R==APFloat::cmpUnordered4
)
1925
0
                          return getUNDEF(VT);
1926
4
                        
LLVM_FALLTHROUGH4
;
1927
91
      case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, dl, VT);
1928
4
      
case ISD::SETNE: if (4
R==APFloat::cmpUnordered4
)
1929
0
                          return getUNDEF(VT);
1930
4
                        
LLVM_FALLTHROUGH4
;
1931
4
      case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan ||
1932
0
                                           R==APFloat::cmpLessThan, dl, VT);
1933
0
      
case ISD::SETLT: if (0
R==APFloat::cmpUnordered0
)
1934
0
                          return getUNDEF(VT);
1935
0
                        
LLVM_FALLTHROUGH0
;
1936
22
      case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, dl, VT);
1937
0
      
case ISD::SETGT: if (0
R==APFloat::cmpUnordered0
)
1938
0
                          return getUNDEF(VT);
1939
0
                        
LLVM_FALLTHROUGH0
;
1940
2
      case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, dl, VT);
1941
0
      
case ISD::SETLE: if (0
R==APFloat::cmpUnordered0
)
1942
0
                          return getUNDEF(VT);
1943
0
                        
LLVM_FALLTHROUGH0
;
1944
4
      case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan ||
1945
4
                                           R==APFloat::cmpEqual, dl, VT);
1946
0
      
case ISD::SETGE: if (0
R==APFloat::cmpUnordered0
)
1947
0
                          return getUNDEF(VT);
1948
0
                        
LLVM_FALLTHROUGH0
;
1949
5
      case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan ||
1950
5
                                           R==APFloat::cmpEqual, dl, VT);
1951
8
      case ISD::SETO:   return getConstant(R!=APFloat::cmpUnordered, dl, VT);
1952
31
      case ISD::SETUO:  return getConstant(R==APFloat::cmpUnordered, dl, VT);
1953
0
      case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered ||
1954
0
                                           R==APFloat::cmpEqual, dl, VT);
1955
18
      case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, dl, VT);
1956
6
      case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered ||
1957
6
                                           R==APFloat::cmpLessThan, dl, VT);
1958
36
      case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan ||
1959
36
                                           R==APFloat::cmpUnordered, dl, VT);
1960
1
      case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, dl, VT);
1961
2
      case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, dl, VT);
1962
449
      }
1963
219
    } else {
1964
219
      // Ensure that the constant occurs on the RHS.
1965
219
      ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
1966
219
      MVT CompVT = N1.getValueType().getSimpleVT();
1967
219
      if (!TLI->isCondCodeLegal(SwappedCond, CompVT))
1968
103
        return SDValue();
1969
116
1970
116
      return getSetCC(dl, VT, N2, N1, SwappedCond);
1971
116
    }
1972
449
  }
1973
3.49M
1974
3.49M
  // Could not fold it.
1975
3.49M
  return SDValue();
1976
3.49M
}
1977
1978
/// See if the specified operand can be simplified with the knowledge that only
1979
/// the bits specified by Mask are used.
1980
1.86M
SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) {
1981
1.86M
  switch (V.getOpcode()) {
1982
1.35M
  default:
1983
1.35M
    break;
1984
344k
  case ISD::Constant: {
1985
344k
    const ConstantSDNode *CV = cast<ConstantSDNode>(V.getNode());
1986
344k
    assert(CV && "Const value should be ConstSDNode.");
1987
344k
    const APInt &CVal = CV->getAPIntValue();
1988
344k
    APInt NewVal = CVal & Mask;
1989
344k
    if (NewVal != CVal)
1990
786
      return getConstant(NewVal, SDLoc(V), V.getValueType());
1991
343k
    break;
1992
343k
  }
1993
23.1k
  case ISD::OR:
1994
23.1k
  case ISD::XOR:
1995
23.1k
    // If the LHS or RHS don't contribute bits to the or, drop them.
1996
23.1k
    if (MaskedValueIsZero(V.getOperand(0), Mask))
1997
712
      return V.getOperand(1);
1998
22.4k
    
if (22.4k
MaskedValueIsZero(V.getOperand(1), Mask)22.4k
)
1999
942
      return V.getOperand(0);
2000
21.4k
    break;
2001
122k
  case ISD::SRL:
2002
122k
    // Only look at single-use SRLs.
2003
122k
    if (!V.getNode()->hasOneUse())
2004
13.1k
      break;
2005
109k
    
if (ConstantSDNode *109k
RHSC109k
= dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2006
108k
      // See if we can recursively simplify the LHS.
2007
108k
      unsigned Amt = RHSC->getZExtValue();
2008
108k
2009
108k
      // Watch out for shift count overflow though.
2010
108k
      if (Amt >= Mask.getBitWidth())
2011
1
        break;
2012
108k
      APInt NewMask = Mask << Amt;
2013
108k
      if (SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask))
2014
1.56k
        return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2015
1.56k
                       V.getOperand(1));
2016
107k
    }
2017
107k
    break;
2018
12.8k
  case ISD::AND: {
2019
12.8k
    // X & -1 -> X (ignoring bits which aren't demanded).
2020
12.8k
    ConstantSDNode *AndVal = isConstOrConstSplat(V.getOperand(1));
2021
12.8k
    if (
AndVal && 12.8k
Mask.isSubsetOf(AndVal->getAPIntValue())12.0k
)
2022
1.79k
      return V.getOperand(0);
2023
11.0k
    break;
2024
11.0k
  }
2025
12.8k
  case ISD::ANY_EXTEND: {
2026
12.8k
    SDValue Src = V.getOperand(0);
2027
12.8k
    unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
2028
12.8k
    // Being conservative here - only peek through if we only demand bits in the
2029
12.8k
    // non-extended source (even though the extended bits are technically undef).
2030
12.8k
    if (Mask.getActiveBits() > SrcBitWidth)
2031
30
      break;
2032
12.8k
    APInt SrcMask = Mask.trunc(SrcBitWidth);
2033
12.8k
    if (SDValue DemandedSrc = GetDemandedBits(Src, SrcMask))
2034
151
      return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc);
2035
12.6k
    break;
2036
12.6k
  }
2037
1.86M
  }
2038
1.86M
  return SDValue();
2039
1.86M
}
2040
2041
/// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
2042
/// use this predicate to simplify operations downstream.
2043
507k
bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2044
507k
  unsigned BitWidth = Op.getScalarValueSizeInBits();
2045
507k
  return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2046
507k
}
2047
2048
/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
2049
/// this predicate to simplify operations downstream.  Mask is known to be zero
2050
/// for bits that V cannot have.
2051
bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask,
2052
3.71M
                                     unsigned Depth) const {
2053
3.71M
  KnownBits Known;
2054
3.71M
  computeKnownBits(Op, Known, Depth);
2055
3.71M
  return Mask.isSubsetOf(Known.Zero);
2056
3.71M
}
2057
2058
/// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
2059
/// is less than the element bit-width of the shift node, return it.
2060
4.95M
static const APInt *getValidShiftAmountConstant(SDValue V) {
2061
4.95M
  if (ConstantSDNode *
SA4.95M
= isConstOrConstSplat(V.getOperand(1))) {
2062
4.76M
    // Shifting more than the bitwidth is not valid.
2063
4.76M
    const APInt &ShAmt = SA->getAPIntValue();
2064
4.76M
    if (ShAmt.ult(V.getScalarValueSizeInBits()))
2065
4.76M
      return &ShAmt;
2066
193k
  }
2067
193k
  return nullptr;
2068
193k
}
2069
2070
/// Determine which bits of Op are known to be either zero or one and return
2071
/// them in Known. For vectors, the known bits are those that are shared by
2072
/// every vector element.
2073
void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known,
2074
54.7M
                                    unsigned Depth) const {
2075
54.7M
  EVT VT = Op.getValueType();
2076
54.7M
  APInt DemandedElts = VT.isVector()
2077
751k
                           ? APInt::getAllOnesValue(VT.getVectorNumElements())
2078
54.0M
                           : APInt(1, 1);
2079
54.7M
  computeKnownBits(Op, Known, DemandedElts, Depth);
2080
54.7M
}
2081
2082
/// Determine which bits of Op are known to be either zero or one and return
2083
/// them in Known. The DemandedElts argument allows us to only collect the known
2084
/// bits that are shared by the requested vector elements.
2085
void SelectionDAG::computeKnownBits(SDValue Op, KnownBits &Known,
2086
                                    const APInt &DemandedElts,
2087
107M
                                    unsigned Depth) const {
2088
107M
  unsigned BitWidth = Op.getScalarValueSizeInBits();
2089
107M
2090
107M
  Known = KnownBits(BitWidth);   // Don't know anything.
2091
107M
  if (Depth == 6)
2092
2.73M
    return;  // Limit search depth.
2093
104M
2094
104M
  KnownBits Known2;
2095
104M
  unsigned NumElts = DemandedElts.getBitWidth();
2096
104M
2097
104M
  if (!DemandedElts)
2098
0
    return;  // No demanded elts, better to assume we don't know anything.
2099
104M
2100
104M
  unsigned Opcode = Op.getOpcode();
2101
104M
  switch (Opcode) {
2102
23.0M
  case ISD::Constant:
2103
23.0M
    // We know all of the bits for a constant!
2104
23.0M
    Known.One = cast<ConstantSDNode>(Op)->getAPIntValue();
2105
23.0M
    Known.Zero = ~Known.One;
2106
23.0M
    break;
2107
377k
  case ISD::BUILD_VECTOR:
2108
377k
    // Collect the known bits that are shared by every demanded vector element.
2109
377k
    assert(NumElts == Op.getValueType().getVectorNumElements() &&
2110
377k
           "Unexpected vector size");
2111
377k
    Known.Zero.setAllBits(); Known.One.setAllBits();
2112
2.22M
    for (unsigned i = 0, e = Op.getNumOperands(); 
i != e2.22M
;
++i1.84M
) {
2113
2.00M
      if (!DemandedElts[i])
2114
1.15M
        continue;
2115
853k
2116
853k
      SDValue SrcOp = Op.getOperand(i);
2117
853k
      computeKnownBits(SrcOp, Known2, Depth + 1);
2118
853k
2119
853k
      // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2120
853k
      if (
SrcOp.getValueSizeInBits() != BitWidth853k
) {
2121
51.0k
        assert(SrcOp.getValueSizeInBits() > BitWidth &&
2122
51.0k
               "Expected BUILD_VECTOR implicit truncation");
2123
51.0k
        Known2 = Known2.trunc(BitWidth);
2124
51.0k
      }
2125
853k
2126
853k
      // Known bits are the values that are shared by every demanded element.
2127
853k
      Known.One &= Known2.One;
2128
853k
      Known.Zero &= Known2.Zero;
2129
853k
2130
853k
      // If we don't know any bits, early out.
2131
853k
      if (
!Known.One && 853k
!Known.Zero311k
)
2132
164k
        break;
2133
2.00M
    }
2134
377k
    break;
2135
32.9k
  case ISD::VECTOR_SHUFFLE: {
2136
32.9k
    // Collect the known bits that are shared by every vector element referenced
2137
32.9k
    // by the shuffle.
2138
32.9k
    APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2139
32.9k
    Known.Zero.setAllBits(); Known.One.setAllBits();
2140
32.9k
    const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2141
32.9k
    assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
2142
405k
    for (unsigned i = 0; 
i != NumElts405k
;
++i372k
) {
2143
377k
      if (!DemandedElts[i])
2144
205k
        continue;
2145
172k
2146
172k
      int M = SVN->getMaskElt(i);
2147
172k
      if (
M < 0172k
) {
2148
4.95k
        // For UNDEF elements, we don't know anything about the common state of
2149
4.95k
        // the shuffle result.
2150
4.95k
        Known.resetAll();
2151
4.95k
        DemandedLHS.clearAllBits();
2152
4.95k
        DemandedRHS.clearAllBits();
2153
4.95k
        break;
2154
4.95k
      }
2155
167k
2156
167k
      
if (167k
(unsigned)M < NumElts167k
)
2157
125k
        DemandedLHS.setBit((unsigned)M % NumElts);
2158
167k
      else
2159
41.3k
        DemandedRHS.setBit((unsigned)M % NumElts);
2160
377k
    }
2161
32.9k
    // Known bits are the values that are shared by every demanded element.
2162
32.9k
    if (
!!DemandedLHS32.9k
) {
2163
24.9k
      SDValue LHS = Op.getOperand(0);
2164
24.9k
      computeKnownBits(LHS, Known2, DemandedLHS, Depth + 1);
2165
24.9k
      Known.One &= Known2.One;
2166
24.9k
      Known.Zero &= Known2.Zero;
2167
24.9k
    }
2168
32.9k
    // If we don't know any bits, early out.
2169
32.9k
    if (
!Known.One && 32.9k
!Known.Zero29.8k
)
2170
28.7k
      break;
2171
4.18k
    
if (4.18k
!!DemandedRHS4.18k
) {
2172
3.18k
      SDValue RHS = Op.getOperand(1);
2173
3.18k
      computeKnownBits(RHS, Known2, DemandedRHS, Depth + 1);
2174
3.18k
      Known.One &= Known2.One;
2175
3.18k
      Known.Zero &= Known2.Zero;
2176
3.18k
    }
2177
4.18k
    break;
2178
4.18k
  }
2179
21.9k
  case ISD::CONCAT_VECTORS: {
2180
21.9k
    // Split DemandedElts and test each of the demanded subvectors.
2181
21.9k
    Known.Zero.setAllBits(); Known.One.setAllBits();
2182
21.9k
    EVT SubVectorVT = Op.getOperand(0).getValueType();
2183
21.9k
    unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2184
21.9k
    unsigned NumSubVectors = Op.getNumOperands();
2185
29.8k
    for (unsigned i = 0; 
i != NumSubVectors29.8k
;
++i7.91k
) {
2186
27.9k
      APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
2187
27.9k
      DemandedSub = DemandedSub.trunc(NumSubVectorElts);
2188
27.9k
      if (
!!DemandedSub27.9k
) {
2189
24.5k
        SDValue Sub = Op.getOperand(i);
2190
24.5k
        computeKnownBits(Sub, Known2, DemandedSub, Depth + 1);
2191
24.5k
        Known.One &= Known2.One;
2192
24.5k
        Known.Zero &= Known2.Zero;
2193
24.5k
      }
2194
27.9k
      // If we don't know any bits, early out.
2195
27.9k
      if (
!Known.One && 27.9k
!Known.Zero24.6k
)
2196
20.0k
        break;
2197
27.9k
    }
2198
21.9k
    break;
2199
4.18k
  }
2200
142k
  case ISD::EXTRACT_SUBVECTOR: {
2201
142k
    // If we know the element index, just demand that subvector elements,
2202
142k
    // otherwise demand them all.
2203
142k
    SDValue Src = Op.getOperand(0);
2204
142k
    ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2205
142k
    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2206
142k
    if (
SubIdx && 142k
SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)142k
) {
2207
142k
      // Offset the demanded elts by the subvector index.
2208
142k
      uint64_t Idx = SubIdx->getZExtValue();
2209
142k
      APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx);
2210
142k
      computeKnownBits(Src, Known, DemandedSrc, Depth + 1);
2211
142k
    } else {
2212
0
      computeKnownBits(Src, Known, Depth + 1);
2213
0
    }
2214
142k
    break;
2215
4.18k
  }
2216
485k
  case ISD::BITCAST: {
2217
485k
    SDValue N0 = Op.getOperand(0);
2218
485k
    unsigned SubBitWidth = N0.getScalarValueSizeInBits();
2219
485k
2220
485k
    // Ignore bitcasts from floating point.
2221
485k
    if (!N0.getValueType().isInteger())
2222
61.8k
      break;
2223
423k
2224
423k
    // Fast handling of 'identity' bitcasts.
2225
423k
    
if (423k
BitWidth == SubBitWidth423k
) {
2226
143
      computeKnownBits(N0, Known, DemandedElts, Depth + 1);
2227
143
      break;
2228
143
    }
2229
423k
2230
423k
    // Support big-endian targets when it becomes useful.
2231
423k
    bool IsLE = getDataLayout().isLittleEndian();
2232
423k
    if (!IsLE)
2233
3.98k
      break;
2234
419k
2235
419k
    // Bitcast 'small element' vector to 'large element' scalar/vector.
2236
419k
    
if (419k
(BitWidth % SubBitWidth) == 0419k
) {
2237
233k
      assert(N0.getValueType().isVector() && "Expected bitcast from vector");
2238
233k
2239
233k
      // Collect known bits for the (larger) output by collecting the known
2240
233k
      // bits from each set of sub elements and shift these into place.
2241
233k
      // We need to separately call computeKnownBits for each set of
2242
233k
      // sub elements as the knownbits for each is likely to be different.
2243
233k
      unsigned SubScale = BitWidth / SubBitWidth;
2244
233k
      APInt SubDemandedElts(NumElts * SubScale, 0);
2245
705k
      for (unsigned i = 0; 
i != NumElts705k
;
++i472k
)
2246
472k
        
if (472k
DemandedElts[i]472k
)
2247
434k
          SubDemandedElts.setBit(i * SubScale);
2248
233k
2249
1.09M
      for (unsigned i = 0; 
i != SubScale1.09M
;
++i862k
) {
2250
862k
        computeKnownBits(N0, Known2, SubDemandedElts.shl(i),
2251
862k
                         Depth + 1);
2252
862k
        Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * i);
2253
862k
        Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * i);
2254
862k
      }
2255
233k
    }
2256
419k
2257
419k
    // Bitcast 'large element' scalar/vector to 'small element' vector.
2258
419k
    if (
(SubBitWidth % BitWidth) == 0419k
) {
2259
185k
      assert(Op.getValueType().isVector() && "Expected bitcast to vector");
2260
185k
2261
185k
      // Collect known bits for the (smaller) output by collecting the known
2262
185k
      // bits from the overlapping larger input elements and extracting the
2263
185k
      // sub sections we actually care about.
2264
185k
      unsigned SubScale = SubBitWidth / BitWidth;
2265
185k
      APInt SubDemandedElts(NumElts / SubScale, 0);
2266
5.36M
      for (unsigned i = 0; 
i != NumElts5.36M
;
++i5.18M
)
2267
5.18M
        
if (5.18M
DemandedElts[i]5.18M
)
2268
584k
          SubDemandedElts.setBit(i / SubScale);
2269
185k
2270
185k
      computeKnownBits(N0, Known2, SubDemandedElts, Depth + 1);
2271
185k
2272
185k
      Known.Zero.setAllBits(); Known.One.setAllBits();
2273
2.34M
      for (unsigned i = 0; 
i != NumElts2.34M
;
++i2.15M
)
2274
2.33M
        
if (2.33M
DemandedElts[i]2.33M
) {
2275
227k
          unsigned Offset = (i % SubScale) * BitWidth;
2276
227k
          Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
2277
227k
          Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
2278
227k
          // If we don't know any bits, early out.
2279
227k
          if (
!Known.One && 227k
!Known.Zero221k
)
2280
179k
            break;
2281
2.33M
        }
2282
185k
    }
2283
419k
    break;
2284
419k
  }
2285
2.22M
  case ISD::AND:
2286
2.22M
    // If either the LHS or the RHS are Zero, the result is zero.
2287
2.22M
    computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
2288
2.22M
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2289
2.22M
2290
2.22M
    // Output known-1 bits are only known if set in both the LHS & RHS.
2291
2.22M
    Known.One &= Known2.One;
2292
2.22M
    // Output known-0 are known to be clear if zero in either the LHS | RHS.
2293
2.22M
    Known.Zero |= Known2.Zero;
2294
2.22M
    break;
2295
611k
  case ISD::OR:
2296
611k
    computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
2297
611k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2298
611k
2299
611k
    // Output known-0 bits are only known if clear in both the LHS & RHS.
2300
611k
    Known.Zero &= Known2.Zero;
2301
611k
    // Output known-1 are known to be set if set in either the LHS | RHS.
2302
611k
    Known.One |= Known2.One;
2303
611k
    break;
2304
518k
  case ISD::XOR: {
2305
518k
    computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
2306
518k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2307
518k
2308
518k
    // Output known-0 bits are known if clear or set in both the LHS & RHS.
2309
518k
    APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
2310
518k
    // Output known-1 are known to be set if set in only one of the LHS, RHS.
2311
518k
    Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
2312
518k
    Known.Zero = KnownZeroOut;
2313
518k
    break;
2314
419k
  }
2315
1.99M
  case ISD::MUL: {
2316
1.99M
    computeKnownBits(Op.getOperand(1), Known, DemandedElts, Depth + 1);
2317
1.99M
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2318
1.99M
2319
1.99M
    // If low bits are zero in either operand, output low known-0 bits.
2320
1.99M
    // Also compute a conservative estimate for high known-0 bits.
2321
1.99M
    // More trickiness is possible, but this is sufficient for the
2322
1.99M
    // interesting case of alignment computation.
2323
1.99M
    unsigned TrailZ = Known.countMinTrailingZeros() +
2324
1.99M
                      Known2.countMinTrailingZeros();
2325
1.99M
    unsigned LeadZ =  std::max(Known.countMinLeadingZeros() +
2326
1.99M
                               Known2.countMinLeadingZeros(),
2327
1.99M
                               BitWidth) - BitWidth;
2328
1.99M
2329
1.99M
    Known.resetAll();
2330
1.99M
    Known.Zero.setLowBits(std::min(TrailZ, BitWidth));
2331
1.99M
    Known.Zero.setHighBits(std::min(LeadZ, BitWidth));
2332
1.99M
    break;
2333
419k
  }
2334
59.2k
  case ISD::UDIV: {
2335
59.2k
    // For the purposes of computing leading zeros we can conservatively
2336
59.2k
    // treat a udiv as a logical right shift by the power of 2 known to
2337
59.2k
    // be less than the denominator.
2338
59.2k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2339
59.2k
    unsigned LeadZ = Known2.countMinLeadingZeros();
2340
59.2k
2341
59.2k
    computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
2342
59.2k
    unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros();
2343
59.2k
    if (RHSMaxLeadingZeros != BitWidth)
2344
38.2k
      LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1);
2345
59.2k
2346
59.2k
    Known.Zero.setHighBits(LeadZ);
2347
59.2k
    break;
2348
419k
  }
2349
308k
  case ISD::SELECT:
2350
308k
    computeKnownBits(Op.getOperand(2), Known, Depth+1);
2351
308k
    // If we don't know any bits, early out.
2352
308k
    if (
!Known.One && 308k
!Known.Zero173k
)
2353
152k
      break;
2354
156k
    computeKnownBits(Op.getOperand(1), Known2, Depth+1);
2355
156k
2356
156k
    // Only known if known in both the LHS and RHS.
2357
156k
    Known.One &= Known2.One;
2358
156k
    Known.Zero &= Known2.Zero;
2359
156k
    break;
2360
221k
  case ISD::SELECT_CC:
2361
221k
    computeKnownBits(Op.getOperand(3), Known, Depth+1);
2362
221k
    // If we don't know any bits, early out.
2363
221k
    if (
!Known.One && 221k
!Known.Zero121k
)
2364
108k
      break;
2365
113k
    computeKnownBits(Op.getOperand(2), Known2, Depth+1);
2366
113k
2367
113k
    // Only known if known in both the LHS and RHS.
2368
113k
    Known.One &= Known2.One;
2369
113k
    Known.Zero &= Known2.Zero;
2370
113k
    break;
2371
2.39k
  case ISD::SMULO:
2372
2.39k
  case ISD::UMULO:
2373
2.39k
    if (Op.getResNo() != 1)
2374
60
      break;
2375
2.33k
    // The boolean result conforms to getBooleanContents.
2376
2.33k
    // If we know the result of a setcc has the top bits zero, use this info.
2377
2.33k
    // We know that we have an integer-based boolean since these operations
2378
2.33k
    // are only available for integer.
2379
2.33k
    
if (2.33k
TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2380
2.33k
            TargetLowering::ZeroOrOneBooleanContent &&
2381
2.33k
        BitWidth > 1)
2382
2.31k
      Known.Zero.setBitsFrom(1);
2383
2.33k
    break;
2384
779k
  case ISD::SETCC:
2385
779k
    // If we know the result of a setcc has the top bits zero, use this info.
2386
779k
    if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2387
779k
            TargetLowering::ZeroOrOneBooleanContent &&
2388
682k
        BitWidth > 1)
2389
643k
      Known.Zero.setBitsFrom(1);
2390
779k
    break;
2391
3.23M
  case ISD::SHL:
2392
3.23M
    if (const APInt *
ShAmt3.23M
= getValidShiftAmountConstant(Op)) {
2393
3.11M
      computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2394
3.11M
      Known.Zero <<= *ShAmt;
2395
3.11M
      Known.One <<= *ShAmt;
2396
3.11M
      // Low bits are known zero.
2397
3.11M
      Known.Zero.setLowBits(ShAmt->getZExtValue());
2398
3.11M
    }
2399
3.23M
    break;
2400
1.25M
  case ISD::SRL:
2401
1.25M
    if (const APInt *
ShAmt1.25M
= getValidShiftAmountConstant(Op)) {
2402
1.20M
      computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2403
1.20M
      Known.Zero.lshrInPlace(*ShAmt);
2404
1.20M
      Known.One.lshrInPlace(*ShAmt);
2405
1.20M
      // High bits are known zero.
2406
1.20M
      Known.Zero.setHighBits(ShAmt->getZExtValue());
2407
1.20M
    }
2408
1.25M
    break;
2409
465k
  case ISD::SRA:
2410
465k
    if (const APInt *
ShAmt465k
= getValidShiftAmountConstant(Op)) {
2411
439k
      computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2412
439k
      Known.Zero.lshrInPlace(*ShAmt);
2413
439k
      Known.One.lshrInPlace(*ShAmt);
2414
439k
      // If we know the value of the sign bit, then we know it is copied across
2415
439k
      // the high bits by the shift amount.
2416
439k
      APInt SignMask = APInt::getSignMask(BitWidth);
2417
439k
      SignMask.lshrInPlace(*ShAmt);  // Adjust to where it is now in the mask.
2418
439k
      if (
Known.Zero.intersects(SignMask)439k
) {
2419
10
        Known.Zero.setHighBits(ShAmt->getZExtValue());// New bits are known zero.
2420
439k
      } else 
if (439k
Known.One.intersects(SignMask)439k
) {
2421
1
        Known.One.setHighBits(ShAmt->getZExtValue()); // New bits are known one.
2422
1
      }
2423
439k
    }
2424
465k
    break;
2425
164k
  case ISD::SIGN_EXTEND_INREG: {
2426
164k
    EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2427
164k
    unsigned EBits = EVT.getScalarSizeInBits();
2428
164k
2429
164k
    // Sign extension.  Compute the demanded bits in the result that are not
2430
164k
    // present in the input.
2431
164k
    APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2432
164k
2433
164k
    APInt InSignMask = APInt::getSignMask(EBits);
2434
164k
    APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2435
164k
2436
164k
    // If the sign extended bits are demanded, we know that the sign
2437
164k
    // bit is demanded.
2438
164k
    InSignMask = InSignMask.zext(BitWidth);
2439
164k
    if (NewBits.getBoolValue())
2440
164k
      InputDemandedBits |= InSignMask;
2441
164k
2442
164k
    computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2443
164k
    Known.One &= InputDemandedBits;
2444
164k
    Known.Zero &= InputDemandedBits;
2445
164k
2446
164k
    // If the sign bit of the input is known set or clear, then we know the
2447
164k
    // top bits of the result.
2448
164k
    if (
Known.Zero.intersects(InSignMask)164k
) { // Input sign bit known clear
2449
8
      Known.Zero |= NewBits;
2450
8
      Known.One  &= ~NewBits;
2451
164k
    } else 
if (164k
Known.One.intersects(InSignMask)164k
) { // Input sign bit known set
2452
2
      Known.One  |= NewBits;
2453
2
      Known.Zero &= ~NewBits;
2454
164k
    } else {                              // Input sign bit unknown
2455
164k
      Known.Zero &= ~NewBits;
2456
164k
      Known.One  &= ~NewBits;
2457
164k
    }
2458
164k
    break;
2459
2.33k
  }
2460
4.34k
  case ISD::CTTZ:
2461
4.34k
  case ISD::CTTZ_ZERO_UNDEF: {
2462
4.34k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2463
4.34k
    // If we have a known 1, its position is our upper bound.
2464
4.34k
    unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2465
4.34k
    unsigned LowBits = Log2_32(PossibleTZ) + 1;
2466
4.34k
    Known.Zero.setBitsFrom(LowBits);
2467
4.34k
    break;
2468
4.34k
  }
2469
56.9k
  case ISD::CTLZ:
2470
56.9k
  case ISD::CTLZ_ZERO_UNDEF: {
2471
56.9k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2472
56.9k
    // If we have a known 1, its position is our upper bound.
2473
56.9k
    unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2474
56.9k
    unsigned LowBits = Log2_32(PossibleLZ) + 1;
2475
56.9k
    Known.Zero.setBitsFrom(LowBits);
2476
56.9k
    break;
2477
56.9k
  }
2478
1.04k
  case ISD::CTPOP: {
2479
1.04k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2480
1.04k
    // If we know some of the bits are zero, they can't be one.
2481
1.04k
    unsigned PossibleOnes = Known2.countMaxPopulation();
2482
1.04k
    Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
2483
1.04k
    break;
2484
56.9k
  }
2485
9.47M
  case ISD::LOAD: {
2486
9.47M
    LoadSDNode *LD = cast<LoadSDNode>(Op);
2487
9.47M
    // If this is a ZEXTLoad and we are looking at the loaded value.
2488
9.47M
    if (
ISD::isZEXTLoad(Op.getNode()) && 9.47M
Op.getResNo() == 01.15M
) {
2489
1.14M
      EVT VT = LD->getMemoryVT();
2490
1.14M
      unsigned MemBits = VT.getScalarSizeInBits();
2491
1.14M
      Known.Zero.setBitsFrom(MemBits);
2492
9.47M
    } else 
if (const MDNode *8.33M
Ranges8.33M
= LD->getRanges()) {
2493
65.6k
      if (LD->getExtensionType() == ISD::NON_EXTLOAD)
2494
1.25k
        computeKnownBitsFromRangeMetadata(*Ranges, Known);
2495
8.33M
    }
2496
9.47M
    break;
2497
56.9k
  }
2498
5.93k
  case ISD::ZERO_EXTEND_VECTOR_INREG: {
2499
5.93k
    EVT InVT = Op.getOperand(0).getValueType();
2500
5.93k
    unsigned InBits = InVT.getScalarSizeInBits();
2501
5.93k
    Known = Known.trunc(InBits);
2502
5.93k
    computeKnownBits(Op.getOperand(0), Known,
2503
5.93k
                     DemandedElts.zext(InVT.getVectorNumElements()),
2504
5.93k
                     Depth + 1);
2505
5.93k
    Known = Known.zext(BitWidth);
2506
5.93k
    Known.Zero.setBitsFrom(InBits);
2507
5.93k
    break;
2508
56.9k
  }
2509
816k
  case ISD::ZERO_EXTEND: {
2510
816k
    EVT InVT = Op.getOperand(0).getValueType();
2511
816k
    unsigned InBits = InVT.getScalarSizeInBits();
2512
816k
    Known = Known.trunc(InBits);
2513
816k
    computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2514
816k
    Known = Known.zext(BitWidth);
2515
816k
    Known.Zero.setBitsFrom(InBits);
2516
816k
    break;
2517
56.9k
  }
2518
56.9k
  // TODO ISD::SIGN_EXTEND_VECTOR_INREG
2519
1.15M
  case ISD::SIGN_EXTEND: {
2520
1.15M
    EVT InVT = Op.getOperand(0).getValueType();
2521
1.15M
    unsigned InBits = InVT.getScalarSizeInBits();
2522
1.15M
2523
1.15M
    Known = Known.trunc(InBits);
2524
1.15M
    computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2525
1.15M
2526
1.15M
    // If the sign bit is known to be zero or one, then sext will extend
2527
1.15M
    // it to the top bits, else it will just zext.
2528
1.15M
    Known = Known.sext(BitWidth);
2529
1.15M
    break;
2530
56.9k
  }
2531
216k
  case ISD::ANY_EXTEND: {
2532
216k
    EVT InVT = Op.getOperand(0).getValueType();
2533
216k
    unsigned InBits = InVT.getScalarSizeInBits();
2534
216k
    Known = Known.trunc(InBits);
2535
216k
    computeKnownBits(Op.getOperand(0), Known, Depth+1);
2536
216k
    Known = Known.zext(BitWidth);
2537
216k
    break;
2538
56.9k
  }
2539
1.63M
  case ISD::TRUNCATE: {
2540
1.63M
    EVT InVT = Op.getOperand(0).getValueType();
2541
1.63M
    unsigned InBits = InVT.getScalarSizeInBits();
2542
1.63M
    Known = Known.zext(InBits);
2543
1.63M
    computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2544
1.63M
    Known = Known.trunc(BitWidth);
2545
1.63M
    break;
2546
56.9k
  }
2547
970k
  case ISD::AssertZext: {
2548
970k
    EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2549
970k
    APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
2550
970k
    computeKnownBits(Op.getOperand(0), Known, Depth+1);
2551
970k
    Known.Zero |= (~InMask);
2552
970k
    Known.One  &= (~Known.Zero);
2553
970k
    break;
2554
56.9k
  }
2555
26
  case ISD::FGETSIGN:
2556
26
    // All bits are zero except the low bit.
2557
26
    Known.Zero.setBitsFrom(1);
2558
26
    break;
2559
6.02k
  case ISD::USUBO:
2560
6.02k
  case ISD::SSUBO:
2561
6.02k
    if (
Op.getResNo() == 16.02k
) {
2562
4.39k
      // If we know the result of a setcc has the top bits zero, use this info.
2563
4.39k
      if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2564
4.39k
              TargetLowering::ZeroOrOneBooleanContent &&
2565
154
          BitWidth > 1)
2566
123
        Known.Zero.setBitsFrom(1);
2567
4.39k
      break;
2568
4.39k
    }
2569
1.62k
    
LLVM_FALLTHROUGH1.62k
;
2570
854k
  case ISD::SUB:
2571
854k
  case ISD::SUBC: {
2572
854k
    if (ConstantSDNode *
CLHS854k
= isConstOrConstSplat(Op.getOperand(0))) {
2573
205k
      // We know that the top bits of C-X are clear if X contains less bits
2574
205k
      // than C (i.e. no wrap-around can happen).  For example, 20-X is
2575
205k
      // positive if we can prove that X is >= 0 and < 16.
2576
205k
      if (
CLHS->getAPIntValue().isNonNegative()205k
) {
2577
195k
        unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros();
2578
195k
        // NLZ can't be BitWidth with no sign bit
2579
195k
        APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1);
2580
195k
        computeKnownBits(Op.getOperand(1), Known2, DemandedElts,
2581
195k
                         Depth + 1);
2582
195k
2583
195k
        // If all of the MaskV bits are known to be zero, then we know the
2584
195k
        // output top bits are zero, because we now know that the output is
2585
195k
        // from [0-C].
2586
195k
        if (
(Known2.Zero & MaskV) == MaskV195k
) {
2587
2.48k
          unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros();
2588
2.48k
          // Top bits known zero.
2589
2.48k
          Known.Zero.setHighBits(NLZ2);
2590
2.48k
        }
2591
195k
      }
2592
205k
    }
2593
854k
2594
854k
    // If low bits are know to be zero in both operands, then we know they are
2595
854k
    // going to be 0 in the result. Both addition and complement operations
2596
854k
    // preserve the low zero bits.
2597
854k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2598
854k
    unsigned KnownZeroLow = Known2.countMinTrailingZeros();
2599
854k
    if (KnownZeroLow == 0)
2600
671k
      break;
2601
182k
2602
182k
    computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
2603
182k
    KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros());
2604
182k
    Known.Zero.setLowBits(KnownZeroLow);
2605
182k
    break;
2606
182k
  }
2607
13.3k
  case ISD::UADDO:
2608
13.3k
  case ISD::SADDO:
2609
13.3k
  case ISD::ADDCARRY:
2610
13.3k
    if (
Op.getResNo() == 113.3k
) {
2611
1.28k
      // If we know the result of a setcc has the top bits zero, use this info.
2612
1.28k
      if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2613
1.28k
              TargetLowering::ZeroOrOneBooleanContent &&
2614
1.14k
          BitWidth > 1)
2615
1.10k
        Known.Zero.setBitsFrom(1);
2616
1.28k
      break;
2617
1.28k
    }
2618
12.0k
    
LLVM_FALLTHROUGH12.0k
;
2619
14.0M
  case ISD::ADD:
2620
14.0M
  case ISD::ADDC:
2621
14.0M
  case ISD::ADDE: {
2622
14.0M
    // Output known-0 bits are known if clear or set in both the low clear bits
2623
14.0M
    // common to both LHS & RHS.  For example, 8+(X<<3) is known to have the
2624
14.0M
    // low 3 bits clear.
2625
14.0M
    // Output known-0 bits are also known if the top bits of each input are
2626
14.0M
    // known to be clear. For example, if one input has the top 10 bits clear
2627
14.0M
    // and the other has the top 8 bits clear, we know the top 7 bits of the
2628
14.0M
    // output must be clear.
2629
14.0M
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2630
14.0M
    unsigned KnownZeroHigh = Known2.countMinLeadingZeros();
2631
14.0M
    unsigned KnownZeroLow = Known2.countMinTrailingZeros();
2632
14.0M
2633
14.0M
    computeKnownBits(Op.getOperand(1), Known2, DemandedElts,
2634
14.0M
                     Depth + 1);
2635
14.0M
    KnownZeroHigh = std::min(KnownZeroHigh, Known2.countMinLeadingZeros());
2636
14.0M
    KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros());
2637
14.0M
2638
14.0M
    if (
Opcode == ISD::ADDE || 14.0M
Opcode == ISD::ADDCARRY14.0M
) {
2639
5.17k
      // With ADDE and ADDCARRY, a carry bit may be added in, so we can only
2640
5.17k
      // use this information if we know (at least) that the low two bits are
2641
5.17k
      // clear. We then return to the caller that the low bit is unknown but
2642
5.17k
      // that other bits are known zero.
2643
5.17k
      if (KnownZeroLow >= 2)
2644
641
        Known.Zero.setBits(1, KnownZeroLow);
2645
5.17k
      break;
2646
5.17k
    }
2647
14.0M
2648
14.0M
    Known.Zero.setLowBits(KnownZeroLow);
2649
14.0M
    if (KnownZeroHigh > 1)
2650
249k
      Known.Zero.setHighBits(KnownZeroHigh - 1);
2651
14.0M
    break;
2652
14.0M
  }
2653
6.20k
  case ISD::SREM:
2654
6.20k
    if (ConstantSDNode *
Rem6.20k
= isConstOrConstSplat(Op.getOperand(1))) {
2655
3.11k
      const APInt &RA = Rem->getAPIntValue().abs();
2656
3.11k
      if (
RA.isPowerOf2()3.11k
) {
2657
604
        APInt LowBits = RA - 1;
2658
604
        computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2659
604
2660
604
        // The low bits of the first operand are unchanged by the srem.
2661
604
        Known.Zero = Known2.Zero & LowBits;
2662
604
        Known.One = Known2.One & LowBits;
2663
604
2664
604
        // If the first operand is non-negative or has all low bits zero, then
2665
604
        // the upper bits are all zero.
2666
604
        if (
Known2.Zero[BitWidth-1] || 604
((Known2.Zero & LowBits) == LowBits)604
)
2667
2
          Known.Zero |= ~LowBits;
2668
604
2669
604
        // If the first operand is negative and not all low bits are zero, then
2670
604
        // the upper bits are all one.
2671
604
        if (
Known2.One[BitWidth-1] && 604
((Known2.One & LowBits) != 0)0
)
2672
0
          Known.One |= ~LowBits;
2673
604
        assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?");
2674
604
      }
2675
3.11k
    }
2676
6.20k
    break;
2677
10.3k
  case ISD::UREM: {
2678
10.3k
    if (ConstantSDNode *
Rem10.3k
= isConstOrConstSplat(Op.getOperand(1))) {
2679
7.31k
      const APInt &RA = Rem->getAPIntValue();
2680
7.31k
      if (
RA.isPowerOf2()7.31k
) {
2681
8
        APInt LowBits = (RA - 1);
2682
8
        computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2683
8
2684
8
        // The upper bits are all zero, the lower ones are unchanged.
2685
8
        Known.Zero = Known2.Zero | ~LowBits;
2686
8
        Known.One = Known2.One & LowBits;
2687
8
        break;
2688
8
      }
2689
10.2k
    }
2690
10.2k
2691
10.2k
    // Since the result is less than or equal to either operand, any leading
2692
10.2k
    // zero bits in either operand must also exist in the result.
2693
10.2k
    computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2694
10.2k
    computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
2695
10.2k
2696
10.2k
    uint32_t Leaders =
2697
10.2k
        std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
2698
10.2k
    Known.resetAll();
2699
10.2k
    Known.Zero.setHighBits(Leaders);
2700
10.2k
    break;
2701
10.2k
  }
2702
50
  case ISD::EXTRACT_ELEMENT: {
2703
50
    computeKnownBits(Op.getOperand(0), Known, Depth+1);
2704
50
    const unsigned Index = Op.getConstantOperandVal(1);
2705
50
    const unsigned BitWidth = Op.getValueSizeInBits();
2706
50
2707
50
    // Remove low part of known bits mask
2708
50
    Known.Zero = Known.Zero.getHiBits(Known.Zero.getBitWidth() - Index * BitWidth);
2709
50
    Known.One = Known.One.getHiBits(Known.One.getBitWidth() - Index * BitWidth);
2710
50
2711
50
    // Remove high part of known bit mask
2712
50
    Known = Known.trunc(BitWidth);
2713
50
    break;
2714
10.2k
  }
2715
393k
  case ISD::EXTRACT_VECTOR_ELT: {
2716
393k
    SDValue InVec = Op.getOperand(0);
2717
393k
    SDValue EltNo = Op.getOperand(1);
2718
393k
    EVT VecVT = InVec.getValueType();
2719
393k
    const unsigned BitWidth = Op.getValueSizeInBits();
2720
393k
    const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
2721
393k
    const unsigned NumSrcElts = VecVT.getVectorNumElements();
2722
393k
    // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2723
393k
    // anything about the extended bits.
2724
393k
    if (BitWidth > EltBitWidth)
2725
63.7k
      Known = Known.trunc(EltBitWidth);
2726
393k
    ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
2727
393k
    if (
ConstEltNo && 393k
ConstEltNo->getAPIntValue().ult(NumSrcElts)392k
) {
2728
392k
      // If we know the element index, just demand that vector element.
2729
392k
      unsigned Idx = ConstEltNo->getZExtValue();
2730
392k
      APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
2731
392k
      computeKnownBits(InVec, Known, DemandedElt, Depth + 1);
2732
393k
    } else {
2733
535
      // Unknown element index, so ignore DemandedElts and demand them all.
2734
535
      computeKnownBits(InVec, Known, Depth + 1);
2735
535
    }
2736
393k
    if (BitWidth > EltBitWidth)
2737
63.7k
      Known = Known.zext(BitWidth);
2738
393k
    break;
2739
10.2k
  }
2740
8.18k
  case ISD::INSERT_VECTOR_ELT: {
2741
8.18k
    SDValue InVec = Op.getOperand(0);
2742
8.18k
    SDValue InVal = Op.getOperand(1);
2743
8.18k
    SDValue EltNo = Op.getOperand(2);
2744
8.18k
2745
8.18k
    ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
2746
8.18k
    if (
CEltNo && 8.18k
CEltNo->getAPIntValue().ult(NumElts)8.17k
) {
2747
8.17k
      // If we know the element index, split the demand between the
2748
8.17k
      // source vector and the inserted element.
2749
8.17k
      Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth);
2750
8.17k
      unsigned EltIdx = CEltNo->getZExtValue();
2751
8.17k
2752
8.17k
      // If we demand the inserted element then add its common known bits.
2753
8.17k
      if (
DemandedElts[EltIdx]8.17k
) {
2754
7.10k
        computeKnownBits(InVal, Known2, Depth + 1);
2755
7.10k
        Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
2756
7.10k
        Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
2757
7.10k
      }
2758
8.17k
2759
8.17k
      // If we demand the source vector then add its common known bits, ensuring
2760
8.17k
      // that we don't demand the inserted element.
2761
8.17k
      APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx));
2762
8.17k
      if (
!!VectorElts8.17k
) {
2763
4.61k
        computeKnownBits(InVec, Known2, VectorElts, Depth + 1);
2764
4.61k
        Known.One &= Known2.One;
2765
4.61k
        Known.Zero &= Known2.Zero;
2766
4.61k
      }
2767
8.18k
    } else {
2768
4
      // Unknown element index, so ignore DemandedElts and demand them all.
2769
4
      computeKnownBits(InVec, Known, Depth + 1);
2770
4
      computeKnownBits(InVal, Known2, Depth + 1);
2771
4
      Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
2772
4
      Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
2773
4
    }
2774
8.18k
    break;
2775
10.2k
  }
2776
499
  case ISD::BITREVERSE: {
2777
499
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2778
499
    Known.Zero = Known2.Zero.reverseBits();
2779
499
    Known.One = Known2.One.reverseBits();
2780
499
    break;
2781
10.2k
  }
2782
6.35k
  case ISD::BSWAP: {
2783
6.35k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2784
6.35k
    Known.Zero = Known2.Zero.byteSwap();
2785
6.35k
    Known.One = Known2.One.byteSwap();
2786
6.35k
    break;
2787
10.2k
  }
2788
2.11k
  case ISD::ABS: {
2789
2.11k
    computeKnownBits(Op.getOperand(0), Known2, DemandedElts, Depth + 1);
2790
2.11k
2791
2.11k
    // If the source's MSB is zero then we know the rest of the bits already.
2792
2.11k
    if (
Known2.isNonNegative()2.11k
) {
2793
0
      Known.Zero = Known2.Zero;
2794
0
      Known.One = Known2.One;
2795
0
      break;
2796
0
    }
2797
2.11k
2798
2.11k
    // We only know that the absolute values's MSB will be zero iff there is
2799
2.11k
    // a set bit that isn't the sign bit (otherwise it could be INT_MIN).
2800
2.11k
    Known2.One.clearSignBit();
2801
2.11k
    if (
Known2.One.getBoolValue()2.11k
) {
2802
2
      Known.Zero = APInt::getSignMask(BitWidth);
2803
2
      break;
2804
2
    }
2805
2.11k
    break;
2806
2.11k
  }
2807
626
  case ISD::UMIN: {
2808
626
    computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
2809
626
    computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
2810
626
2811
626
    // UMIN - we know that the result will have the maximum of the
2812
626
    // known zero leading bits of the inputs.
2813
626
    unsigned LeadZero = Known.countMinLeadingZeros();
2814
626
    LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros());
2815
626
2816
626
    Known.Zero &= Known2.Zero;
2817
626
    Known.One &= Known2.One;
2818
626
    Known.Zero.setHighBits(LeadZero);
2819
626
    break;
2820
2.11k
  }
2821
303
  case ISD::UMAX: {
2822
303
    computeKnownBits(Op.getOperand(0), Known, DemandedElts,
2823
303
                     Depth + 1);
2824
303
    computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
2825
303
2826
303
    // UMAX - we know that the result will have the maximum of the
2827
303
    // known one leading bits of the inputs.
2828
303
    unsigned LeadOne = Known.countMinLeadingOnes();
2829
303
    LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes());
2830
303
2831
303
    Known.Zero &= Known2.Zero;
2832
303
    Known.One &= Known2.One;
2833
303
    Known.One.setHighBits(LeadOne);
2834
303
    break;
2835
2.11k
  }
2836
6.09k
  case ISD::SMIN:
2837
6.09k
  case ISD::SMAX: {
2838
6.09k
    computeKnownBits(Op.getOperand(0), Known, DemandedElts,
2839
6.09k
                     Depth + 1);
2840
6.09k
    // If we don't know any bits, early out.
2841
6.09k
    if (
!Known.One && 6.09k
!Known.Zero6.09k
)
2842
6.09k
      break;
2843
0
    computeKnownBits(Op.getOperand(1), Known2, DemandedElts, Depth + 1);
2844
0
    Known.Zero &= Known2.Zero;
2845
0
    Known.One &= Known2.One;
2846
0
    break;
2847
0
  }
2848
4.09M
  case ISD::FrameIndex:
2849
4.09M
  case ISD::TargetFrameIndex:
2850
4.09M
    if (unsigned 
Align4.09M
= InferPtrAlignment(Op)) {
2851
4.09M
      // The low bits are known zero if the pointer is aligned.
2852
4.09M
      Known.Zero.setLowBits(Log2_32(Align));
2853
4.09M
      break;
2854
4.09M
    }
2855
0
    break;
2856
0
2857
34.7M
  default:
2858
34.7M
    if (Opcode < ISD::BUILTIN_OP_END)
2859
32.5M
      break;
2860
2.19M
    
LLVM_FALLTHROUGH2.19M
;
2861
2.55M
  case ISD::INTRINSIC_WO_CHAIN:
2862
2.55M
  case ISD::INTRINSIC_W_CHAIN:
2863
2.55M
  case ISD::INTRINSIC_VOID:
2864
2.55M
    // Allow the target to implement this method for its nodes.
2865
2.55M
    TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
2866
2.55M
    break;
2867
104M
  }
2868
104M
2869
104M
  assert((Known.Zero & Known.One) == 0 && "Bits known to be one AND zero?");
2870
104M
}
2871
2872
SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
2873
4.43k
                                                             SDValue N1) const {
2874
4.43k
  // X + 0 never overflow
2875
4.43k
  if (isNullConstant(N1))
2876
0
    return OFK_Never;
2877
4.43k
2878
4.43k
  KnownBits N1Known;
2879
4.43k
  computeKnownBits(N1, N1Known);
2880
4.43k
  if (
N1Known.Zero.getBoolValue()4.43k
) {
2881
1.56k
    KnownBits N0Known;
2882
1.56k
    computeKnownBits(N0, N0Known);
2883
1.56k
2884
1.56k
    bool overflow;
2885
1.56k
    (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow);
2886
1.56k
    if (!overflow)
2887
124
      return OFK_Never;
2888
4.31k
  }
2889
4.31k
2890
4.31k
  // mulhi + 1 never overflow
2891
4.31k
  
if (4.31k
N0.getOpcode() == ISD::UMUL_LOHI && 4.31k
N0.getResNo() == 1593
&&
2892
95
      (~N1Known.Zero & 0x01) == ~N1Known.Zero)
2893
42
    return OFK_Never;
2894
4.27k
2895
4.27k
  
if (4.27k
N1.getOpcode() == ISD::UMUL_LOHI && 4.27k
N1.getResNo() == 1346
) {
2896
113
    KnownBits N0Known;
2897
113
    computeKnownBits(N0, N0Known);
2898
113
2899
113
    if ((~N0Known.Zero & 0x01) == ~N0Known.Zero)
2900
0
      return OFK_Never;
2901
4.27k
  }
2902
4.27k
2903
4.27k
  return OFK_Sometime;
2904
4.27k
}
2905
2906
250k
bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
2907
250k
  EVT OpVT = Val.getValueType();
2908
250k
  unsigned BitWidth = OpVT.getScalarSizeInBits();
2909
250k
2910
250k
  // Is the constant a known power of 2?
2911
250k
  if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
2912
242k
    return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
2913
8.33k
2914
8.33k
  // A left-shift of a constant one will have exactly one bit set because
2915
8.33k
  // shifting the bit off the end is undefined.
2916
8.33k
  
if (8.33k
Val.getOpcode() == ISD::SHL8.33k
) {
2917
179
    auto *C = isConstOrConstSplat(Val.getOperand(0));
2918
179
    if (
C && 179
C->getAPIntValue() == 144
)
2919
41
      return true;
2920
8.29k
  }
2921
8.29k
2922
8.29k
  // Similarly, a logical right-shift of a constant sign-bit will have exactly
2923
8.29k
  // one bit set.
2924
8.29k
  
if (8.29k
Val.getOpcode() == ISD::SRL8.29k
) {
2925
23
    auto *C = isConstOrConstSplat(Val.getOperand(0));
2926
23
    if (
C && 23
C->getAPIntValue().isSignMask()5
)
2927
5
      return true;
2928
8.29k
  }
2929
8.29k
2930
8.29k
  // Are all operands of a build vector constant powers of two?
2931
8.29k
  
if (8.29k
Val.getOpcode() == ISD::BUILD_VECTOR8.29k
)
2932
2.79k
    
if (2.79k
llvm::all_of(Val->ops(), [BitWidth](SDValue E) 2.79k
{
2933
5.74k
          if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
2934
5.72k
            return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
2935
17
          return false;
2936
17
        }))
2937
315
      return true;
2938
7.97k
2939
7.97k
  // More could be done here, though the above checks are enough
2940
7.97k
  // to handle some common cases.
2941
7.97k
2942
7.97k
  // Fall back to computeKnownBits to catch other known cases.
2943
7.97k
  KnownBits Known;
2944
7.97k
  computeKnownBits(Val, Known);
2945
87
  return (Known.countMaxPopulation() == 1) && (Known.countMinPopulation() == 1);
2946
250k
}
2947
2948
4.33M
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
2949
4.33M
  EVT VT = Op.getValueType();
2950
4.33M
  APInt DemandedElts = VT.isVector()
2951
43.4k
                           ? APInt::getAllOnesValue(VT.getVectorNumElements())
2952
4.28M
                           : APInt(1, 1);
2953
4.33M
  return ComputeNumSignBits(Op, DemandedElts, Depth);
2954
4.33M
}
2955
2956
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
2957
4.37M
                                          unsigned Depth) const {
2958
4.37M
  EVT VT = Op.getValueType();
2959
4.37M
  assert(VT.isInteger() && "Invalid VT!");
2960
4.37M
  unsigned VTBits = VT.getScalarSizeInBits();
2961
4.37M
  unsigned NumElts = DemandedElts.getBitWidth();
2962
4.37M
  unsigned Tmp, Tmp2;
2963
4.37M
  unsigned FirstAnswer = 1;
2964
4.37M
2965
4.37M
  if (Depth == 6)
2966
2.21k
    return 1;  // Limit search depth.
2967
4.36M
2968
4.36M
  
if (4.36M
!DemandedElts4.36M
)
2969
0
    return 1;  // No demanded elts, better to assume we don't know anything.
2970
4.36M
2971
4.36M
  switch (Op.getOpcode()) {
2972
2.30M
  default: break;
2973
55.1k
  case ISD::AssertSext:
2974
55.1k
    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2975
55.1k
    return VTBits-Tmp+1;
2976
73.2k
  case ISD::AssertZext:
2977
73.2k
    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
2978
73.2k
    return VTBits-Tmp;
2979
4.36M
2980
821k
  case ISD::Constant: {
2981
821k
    const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue();
2982
821k
    return Val.getNumSignBits();
2983
4.36M
  }
2984
4.36M
2985
11.7k
  case ISD::BUILD_VECTOR:
2986
11.7k
    Tmp = VTBits;
2987
31.8k
    for (unsigned i = 0, e = Op.getNumOperands(); 
(i < e) && 31.8k
(Tmp > 1)29.7k
;
++i20.1k
) {
2988
20.1k
      if (!DemandedElts[i])
2989
3.54k
        continue;
2990
16.5k
2991
16.5k
      SDValue SrcOp = Op.getOperand(i);
2992
16.5k
      Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1);
2993
16.5k
2994
16.5k
      // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2995
16.5k
      if (
SrcOp.getValueSizeInBits() != VTBits16.5k
) {
2996
242
        assert(SrcOp.getValueSizeInBits() > VTBits &&
2997
242
               "Expected BUILD_VECTOR implicit truncation");
2998
242
        unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
2999
242
        Tmp2 = (Tmp2 > ExtraBits ? 
Tmp2 - ExtraBits116
:
1126
);
3000
242
      }
3001
20.1k
      Tmp = std::min(Tmp, Tmp2);
3002
20.1k
    }
3003
11.7k
    return Tmp;
3004
4.36M
3005
355
  case ISD::VECTOR_SHUFFLE: {
3006
355
    // Collect the minimum number of sign bits that are shared by every vector
3007
355
    // element referenced by the shuffle.
3008
355
    APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3009
355
    const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3010
355
    assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3011
2.58k
    for (unsigned i = 0; 
i != NumElts2.58k
;
++i2.23k
) {
3012
2.27k
      int M = SVN->getMaskElt(i);
3013
2.27k
      if (!DemandedElts[i])
3014
383
        continue;
3015
1.89k
      // For UNDEF elements, we don't know anything about the common state of
3016
1.89k
      // the shuffle result.
3017
1.89k
      
if (1.89k
M < 01.89k
)
3018
42
        return 1;
3019
1.85k
      
if (1.85k
(unsigned)M < NumElts1.85k
)
3020
1.28k
        DemandedLHS.setBit((unsigned)M % NumElts);
3021
1.85k
      else
3022
564
        DemandedRHS.setBit((unsigned)M % NumElts);
3023
2.27k
    }
3024
313
    Tmp = std::numeric_limits<unsigned>::max();
3025
313
    if (!!DemandedLHS)
3026
308
      Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3027
313
    if (
!!DemandedRHS313
) {
3028
151
      Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3029
151
      Tmp = std::min(Tmp, Tmp2);
3030
151
    }
3031
313
    // If we don't know anything, early out and try computeKnownBits fall-back.
3032
313
    if (Tmp == 1)
3033
194
      break;
3034
313
    assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3035
119
    return Tmp;
3036
119
  }
3037
119
3038
16.0k
  case ISD::BITCAST: {
3039
16.0k
    SDValue N0 = Op.getOperand(0);
3040
16.0k
    unsigned SrcBits = N0.getScalarValueSizeInBits();
3041
16.0k
3042
16.0k
    // Ignore bitcasts from floating point.
3043
16.0k
    if (!N0.getValueType().isInteger())
3044
2.30k
      break;
3045
13.7k
3046
13.7k
    // Fast handling of 'identity' bitcasts.
3047
13.7k
    
if (13.7k
VTBits == SrcBits13.7k
)
3048
1
      return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3049
13.7k
3050
13.7k
    // Bitcast 'large element' scalar/vector to 'small element' vector.
3051
13.7k
    // TODO: Handle cases other than 'sign splat' when we have a use case.
3052
13.7k
    // Requires handling of DemandedElts and Endianness.
3053
13.7k
    
if (13.7k
(SrcBits % VTBits) == 013.7k
) {
3054
6.25k
      assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3055
6.25k
      Tmp = ComputeNumSignBits(N0, Depth + 1);
3056
6.25k
      if (Tmp == SrcBits)
3057
2.03k
        return VTBits;
3058
11.7k
    }
3059
11.7k
    break;
3060
11.7k
  }
3061
11.7k
3062
51.9k
  case ISD::SIGN_EXTEND:
3063
51.9k
  case ISD::SIGN_EXTEND_VECTOR_INREG:
3064
51.9k
    Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3065
51.9k
    return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp;
3066
51.9k
3067
14.3k
  case ISD::SIGN_EXTEND_INREG:
3068
14.3k
    // Max of the input and what this extends.
3069
14.3k
    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3070
14.3k
    Tmp = VTBits-Tmp+1;
3071
14.3k
3072
14.3k
    Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3073
14.3k
    return std::max(Tmp, Tmp2);
3074
51.9k
3075
13.0k
  case ISD::SRA:
3076
13.0k
    Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3077
13.0k
    // SRA X, C   -> adds C sign bits.
3078
13.0k
    if (ConstantSDNode *
C13.0k
= isConstOrConstSplat(Op.getOperand(1))) {
3079
10.9k
      APInt ShiftVal = C->getAPIntValue();
3080
10.9k
      ShiftVal += Tmp;
3081
10.9k
      Tmp = ShiftVal.uge(VTBits) ? 
VTBits2.52k
:
ShiftVal.getZExtValue()8.47k
;
3082
10.9k
    }
3083
13.0k
    return Tmp;
3084
87.0k
  case ISD::SHL:
3085
87.0k
    if (ConstantSDNode *
C87.0k
= isConstOrConstSplat(Op.getOperand(1))) {
3086
76.1k
      // shl destroys sign bits.
3087
76.1k
      Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3088
76.1k
      if (C->getAPIntValue().uge(VTBits) ||      // Bad shift.
3089
76.1k
          
C->getAPIntValue().uge(Tmp)76.1k
)
break38.2k
; // Shifted all sign bits out.
3090
37.9k
      return Tmp - C->getZExtValue();
3091
37.9k
    }
3092
10.9k
    break;
3093
115k
  case ISD::AND:
3094
115k
  case ISD::OR:
3095
115k
  case ISD::XOR:    // NOT is handled here.
3096
115k
    // Logical binary ops preserve the number of sign bits at the worst.
3097
115k
    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3098
115k
    if (
Tmp != 1115k
) {
3099
46.7k
      Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3100
46.7k
      FirstAnswer = std::min(Tmp, Tmp2);
3101
46.7k
      // We computed what we know about the sign bits as our first
3102
46.7k
      // answer. Now proceed to the generic code that uses
3103
46.7k
      // computeKnownBits, and pick whichever answer is better.
3104
46.7k
    }
3105
115k
    break;
3106
115k
3107
2.57k
  case ISD::SELECT:
3108
2.57k
    Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3109
2.57k
    if (
Tmp == 12.57k
)
return 1917
; // Early out.
3110
1.65k
    Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1);
3111
1.65k
    return std::min(Tmp, Tmp2);
3112
957
  case ISD::SELECT_CC:
3113
957
    Tmp = ComputeNumSignBits(Op.getOperand(2), Depth+1);
3114
957
    if (
Tmp == 1957
)
return 1274
; // Early out.
3115
683
    Tmp2 = ComputeNumSignBits(Op.getOperand(3), Depth+1);
3116
683
    return std::min(Tmp, Tmp2);
3117
110
  case ISD::SMIN:
3118
110
  case ISD::SMAX:
3119
110
  case ISD::UMIN:
3120
110
  case ISD::UMAX:
3121
110
    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3122
110
    if (Tmp == 1)
3123
10
      return 1;  // Early out.
3124
100
    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3125
100
    return std::min(Tmp, Tmp2);
3126
126
  case ISD::SADDO:
3127
126
  case ISD::UADDO:
3128
126
  case ISD::SSUBO:
3129
126
  case ISD::USUBO:
3130
126
  case ISD::SMULO:
3131
126
  case ISD::UMULO:
3132
126
    if (Op.getResNo() != 1)
3133
52
      break;
3134
74
    // The boolean result conforms to getBooleanContents.  Fall through.
3135
74
    // If setcc returns 0/-1, all bits are sign bits.
3136
74
    // We know that we have an integer-based boolean since these operations
3137
74
    // are only available for integer.
3138
74
    
if (74
TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3139
74
        TargetLowering::ZeroOrNegativeOneBooleanContent)
3140
1
      return VTBits;
3141
73
    break;
3142
4.83k
  case ISD::SETCC:
3143
4.83k
    // If setcc returns 0/-1, all bits are sign bits.
3144
4.83k
    if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3145
4.83k
        TargetLowering::ZeroOrNegativeOneBooleanContent)
3146
3.97k
      return VTBits;
3147
853
    break;
3148
262
  case ISD::ROTL:
3149
262
  case ISD::ROTR:
3150
262
    if (ConstantSDNode *
C262
= dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
3151
248
      unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3152
248
3153
248
      // Handle rotate right by N like a rotate left by 32-N.
3154
248
      if (Op.getOpcode() == ISD::ROTR)
3155
230
        RotAmt = (VTBits - RotAmt) % VTBits;
3156
248
3157
248
      // If we aren't rotating out all of the known-in sign bits, return the
3158
248
      // number that are left.  This handles rotl(sext(x), 1) for example.
3159
248
      Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3160
248
      if (
Tmp > (RotAmt + 1)248
)
return (Tmp - RotAmt)0
;
3161
262
    }
3162
262
    break;
3163
608k
  case ISD::ADD:
3164
608k
  case ISD::ADDC:
3165
608k
    // Add can have at most one carry bit.  Thus we know that the output
3166
608k
    // is, at worst, one more bit than the inputs.
3167
608k
    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3168
608k
    if (
Tmp == 1608k
)
return 1567k
; // Early out.
3169
41.2k
3170
41.2k
    // Special case decrementing a value (ADD X, -1):
3171
41.2k
    
if (ConstantSDNode *41.2k
CRHS41.2k
= dyn_cast<ConstantSDNode>(Op.getOperand(1)))
3172
29.2k
      
if (29.2k
CRHS->isAllOnesValue()29.2k
) {
3173
2.24k
        KnownBits Known;
3174
2.24k
        computeKnownBits(Op.getOperand(0), Known, Depth+1);
3175
2.24k
3176
2.24k
        // If the input is known to be 0 or 1, the output is 0/-1, which is all
3177
2.24k
        // sign bits set.
3178
2.24k
        if ((Known.Zero | 1).isAllOnesValue())
3179
26
          return VTBits;
3180
2.22k
3181
2.22k
        // If we are subtracting one from a positive number, there is no carry
3182
2.22k
        // out of the result.
3183
2.22k
        
if (2.22k
Known.isNonNegative()2.22k
)
3184
1.26k
          return Tmp;
3185
39.9k
      }
3186
39.9k
3187
39.9k
    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3188
39.9k
    if (
Tmp2 == 139.9k
)
return 13.49k
;
3189
36.4k
    return std::min(Tmp, Tmp2)-1;
3190
36.4k
3191
62.0k
  case ISD::SUB:
3192
62.0k
    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3193
62.0k
    if (
Tmp2 == 162.0k
)
return 147.7k
;
3194
14.3k
3195
14.3k
    // Handle NEG.
3196
14.3k
    
if (ConstantSDNode *14.3k
CLHS14.3k
= isConstOrConstSplat(Op.getOperand(0)))
3197
1.98k
      
if (1.98k
CLHS->isNullValue()1.98k
) {
3198
970
        KnownBits Known;
3199
970
        computeKnownBits(Op.getOperand(1), Known, Depth+1);
3200
970
        // If the input is known to be 0 or 1, the output is 0/-1, which is all
3201
970
        // sign bits set.
3202
970
        if ((Known.Zero | 1).isAllOnesValue())
3203
64
          return VTBits;
3204
906
3205
906
        // If the input is known to be positive (the sign bit is known clear),
3206
906
        // the output of the NEG has the same number of sign bits as the input.
3207
906
        
if (906
Known.isNonNegative()906
)
3208
390
          return Tmp2;
3209
13.9k
3210
13.9k
        // Otherwise, we treat this like a SUB.
3211
13.9k
      }
3212
13.9k
3213
13.9k
    // Sub can have at most one carry bit.  Thus we know that the output
3214
13.9k
    // is, at worst, one more bit than the inputs.
3215
13.9k
    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3216
13.9k
    if (
Tmp == 113.9k
)
return 16.91k
; // Early out.
3217
7.00k
    return std::min(Tmp, Tmp2)-1;
3218
95.6k
  case ISD::TRUNCATE: {
3219
95.6k
    // Check if the sign bits of source go down as far as the truncated value.
3220
95.6k
    unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
3221
95.6k
    unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3222
95.6k
    if (NumSrcSignBits > (NumSrcBits - VTBits))
3223
36.0k
      return NumSrcSignBits - (NumSrcBits - VTBits);
3224
59.6k
    break;
3225
59.6k
  }
3226
24
  case ISD::EXTRACT_ELEMENT: {
3227
24
    const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3228
24
    const int BitWidth = Op.getValueSizeInBits();
3229
24
    const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
3230
24
3231
24
    // Get reverse index (starting from 1), Op1 value indexes elements from
3232
24
    // little end. Sign starts at big end.
3233
24
    const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
3234
24
3235
24
    // If the sign portion ends in our element the subtraction gives correct
3236
24
    // result. Otherwise it gives either negative or > bitwidth result
3237
24
    return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
3238
59.6k
  }
3239
224
  case ISD::INSERT_VECTOR_ELT: {
3240
224
    SDValue InVec = Op.getOperand(0);
3241
224
    SDValue InVal = Op.getOperand(1);
3242
224
    SDValue EltNo = Op.getOperand(2);
3243
224
    unsigned NumElts = InVec.getValueType().getVectorNumElements();
3244
224
3245
224
    ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3246
224
    if (
CEltNo && 224
CEltNo->getAPIntValue().ult(NumElts)220
) {
3247
220
      // If we know the element index, split the demand between the
3248
220
      // source vector and the inserted element.
3249
220
      unsigned EltIdx = CEltNo->getZExtValue();
3250
220
3251
220
      // If we demand the inserted element then get its sign bits.
3252
220
      Tmp = std::numeric_limits<unsigned>::max();
3253
220
      if (
DemandedElts[EltIdx]220
) {
3254
218
        // TODO - handle implicit truncation of inserted elements.
3255
218
        if (InVal.getScalarValueSizeInBits() != VTBits)
3256
0
          break;
3257
218
        Tmp = ComputeNumSignBits(InVal, Depth + 1);
3258
218
      }
3259
220
3260
220
      // If we demand the source vector then get its sign bits, and determine
3261
220
      // the minimum.
3262
220
      APInt VectorElts = DemandedElts;
3263
220
      VectorElts.clearBit(EltIdx);
3264
220
      if (
!!VectorElts220
) {
3265
177
        Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1);
3266
177
        Tmp = std::min(Tmp, Tmp2);
3267
177
      }
3268
224
    } else {
3269
4
      // Unknown element index, so ignore DemandedElts and demand them all.
3270
4
      Tmp = ComputeNumSignBits(InVec, Depth + 1);
3271
4
      Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
3272
4
      Tmp = std::min(Tmp, Tmp2);
3273
4
    }
3274
224
    assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3275
224
    return Tmp;
3276
224
  }
3277
29.8k
  case ISD::EXTRACT_VECTOR_ELT: {
3278
29.8k
    SDValue InVec = Op.getOperand(0);
3279
29.8k
    SDValue EltNo = Op.getOperand(1);
3280
29.8k
    EVT VecVT = InVec.getValueType();
3281
29.8k
    const unsigned BitWidth = Op.getValueSizeInBits();
3282
29.8k
    const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
3283
29.8k
    const unsigned NumSrcElts = VecVT.getVectorNumElements();
3284
29.8k
3285
29.8k
    // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
3286
29.8k
    // anything about sign bits. But if the sizes match we can derive knowledge
3287
29.8k
    // about sign bits from the vector operand.
3288
29.8k
    if (BitWidth != EltBitWidth)
3289
8.85k
      break;
3290
21.0k
3291
21.0k
    // If we know the element index, just demand that vector element, else for
3292
21.0k
    // an unknown element index, ignore DemandedElts and demand them all.
3293
21.0k
    APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3294
21.0k
    ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3295
21.0k
    if (
ConstEltNo && 21.0k
ConstEltNo->getAPIntValue().ult(NumSrcElts)21.0k
)
3296
21.0k
      DemandedSrcElts =
3297
21.0k
          APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3298
21.0k
3299
21.0k
    return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
3300
21.0k
  }
3301
1.66k
  case ISD::EXTRACT_SUBVECTOR: {
3302
1.66k
    // If we know the element index, just demand that subvector elements,
3303
1.66k
    // otherwise demand them all.
3304
1.66k
    SDValue Src = Op.getOperand(0);
3305
1.66k
    ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
3306
1.66k
    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3307
1.66k
    if (
SubIdx && 1.66k
SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)1.66k
) {
3308
1.66k
      // Offset the demanded elts by the subvector index.
3309
1.66k
      uint64_t Idx = SubIdx->getZExtValue();
3310
1.66k
      APInt DemandedSrc = DemandedElts.zext(NumSrcElts).shl(Idx);
3311
1.66k
      return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
3312
1.66k
    }
3313
0
    return ComputeNumSignBits(Src, Depth + 1);
3314
0
  }
3315
959
  case ISD::CONCAT_VECTORS:
3316
959
    // Determine the minimum number of sign bits across all demanded
3317
959
    // elts of the input vectors. Early out if the result is already 1.
3318
959
    Tmp = std::numeric_limits<unsigned>::max();
3319
959
    EVT SubVectorVT = Op.getOperand(0).getValueType();
3320
959
    unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3321
959
    unsigned NumSubVectors = Op.getNumOperands();
3322
2.15k
    for (unsigned i = 0; 
(i < NumSubVectors) && 2.15k
(Tmp > 1)1.91k
;
++i1.19k
) {
3323
1.19k
      APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
3324
1.19k
      DemandedSub = DemandedSub.trunc(NumSubVectorElts);
3325
1.19k
      if (!DemandedSub)
3326
139
        continue;
3327
1.05k
      Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
3328
1.05k
      Tmp = std::min(Tmp, Tmp2);
3329
1.05k
    }
3330
355
    assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3331
355
    return Tmp;
3332
2.54M
  }
3333
2.54M
3334
2.54M
  // If we are looking at the loaded value of the SDNode.
3335
2.54M
  
if (2.54M
Op.getResNo() == 02.54M
) {
3336
2.48M
    // Handle LOADX separately here. EXTLOAD case will fallthrough.
3337
2.48M
    if (LoadSDNode *
LD2.48M
= dyn_cast<LoadSDNode>(Op)) {
3338
856k
      unsigned ExtType = LD->getExtensionType();
3339
856k
      switch (ExtType) {
3340
723k
        default: break;
3341
54.7k
        case ISD::SEXTLOAD:    // '17' bits known
3342
54.7k
          Tmp = LD->getMemoryVT().getScalarSizeInBits();
3343
54.7k
          return VTBits-Tmp+1;
3344
77.7k
        case ISD::ZEXTLOAD:    // '16' bits known
3345
77.7k
          Tmp = LD->getMemoryVT().getScalarSizeInBits();
3346
77.7k
          return VTBits-Tmp;
3347
2.41M
      }
3348
2.41M
    }
3349
2.48M
  }
3350
2.41M
3351
2.41M
  // Allow the target to implement this method for its nodes.
3352
2.41M
  
if (2.41M
Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3353
2.27M
      Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3354
2.27M
      Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3355
2.41M
      
Op.getOpcode() == ISD::INTRINSIC_VOID2.24M
) {
3356
175k
    unsigned NumBits =
3357
175k
        TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
3358
175k
    if (NumBits > 1)
3359
3.88k
      FirstAnswer = std::max(FirstAnswer, NumBits);
3360
175k
  }
3361
2.41M
3362
2.41M
  // Finally, if we can prove that the top bits of the result are 0's or 1's,
3363
2.41M
  // use this information.
3364
2.41M
  KnownBits Known;
3365
2.41M
  computeKnownBits(Op, Known, DemandedElts, Depth);
3366
2.41M
3367
2.41M
  APInt Mask;
3368
2.41M
  if (
Known.isNonNegative()2.41M
) { // sign bit is 0
3369
205k
    Mask = Known.Zero;
3370
2.41M
  } else 
if (2.21M
Known.isNegative()2.21M
) { // sign bit is 1;
3371
587
    Mask = Known.One;
3372
2.21M
  } else {
3373
2.21M
    // Nothing known.
3374
2.21M
    return FirstAnswer;
3375
2.21M
  }
3376
205k
3377
205k
  // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
3378
205k
  // the number of identical bits in the top of the input value.
3379
205k
  Mask = ~Mask;
3380
205k
  Mask <<= Mask.getBitWidth()-VTBits;
3381
205k
  // Return # leading zeros.  We use 'min' here in case Val was zero before
3382
205k
  // shifting.  We don't want to return '64' as for an i32 "0".
3383
205k
  return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
3384
205k
}
3385
3386
17.6M
bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
3387
17.6M
  if (
(Op.getOpcode() != ISD::ADD && 17.6M
Op.getOpcode() != ISD::OR3.67M
) ||
3388
14.1M
      !isa<ConstantSDNode>(Op.getOperand(1)))
3389
4.24M
    return false;
3390
13.3M
3391
13.3M
  
if (13.3M
Op.getOpcode() == ISD::OR &&
3392
190k
      !MaskedValueIsZero(Op.getOperand(0),
3393
190k
                     cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue()))
3394
176
    return false;
3395
13.3M
3396
13.3M
  return true;
3397
13.3M
}
3398
3399
593
bool SelectionDAG::isKnownNeverNaN(SDValue Op) const {
3400
593
  // If we're told that NaNs won't happen, assume they won't.
3401
593
  if (getTarget().Options.NoNaNsFPMath)
3402
194
    return true;
3403
399
3404
399
  
if (399
Op->getFlags().hasNoNaNs()399
)
3405
26
    return true;
3406
373
3407
373
  // If the value is a constant, we can obviously see if it is a NaN or not.
3408
373
  
if (const ConstantFPSDNode *373
C373
= dyn_cast<ConstantFPSDNode>(Op))
3409
43
    return !C->getValueAPF().isNaN();
3410
330
3411
330
  // TODO: Recognize more cases here.
3412
330
3413
330
  return false;
3414
330
}
3415
3416
172
bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
3417
172
  // If the value is a constant, we can obviously see if it is a zero or not.
3418
172
  if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
3419
26
    return !C->isZero();
3420
146
3421
146
  // TODO: Recognize more cases here.
3422
146
  switch (Op.getOpcode()) {
3423
146
  default: break;
3424
0
  case ISD::OR:
3425
0
    if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
3426
0
      return !C->isNullValue();
3427
0
    break;
3428
146
  }
3429
146
3430
146
  return false;
3431
146
}
3432
3433
14.4k
bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
3434
14.4k
  // Check the obvious case.
3435
14.4k
  if (
A == B14.4k
)
return true3.86k
;
3436
10.5k
3437
10.5k
  // For for negative and positive zero.
3438
10.5k
  
if (const ConstantFPSDNode *10.5k
CA10.5k
= dyn_cast<ConstantFPSDNode>(A))
3439
1.25k
    
if (const ConstantFPSDNode *1.25k
CB1.25k
= dyn_cast<ConstantFPSDNode>(B))
3440
114
      
if (114
CA->isZero() && 114
CB->isZero()4
)
return true4
;
3441
10.5k
3442
10.5k
  // Otherwise they may not be equal.
3443
10.5k
  return false;
3444
10.5k
}
3445
3446
8.64M
bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
3447
8.64M
  assert(A.getValueType() == B.getValueType() &&
3448
8.64M
         "Values must have the same type");
3449
8.64M
  KnownBits AKnown, BKnown;
3450
8.64M
  computeKnownBits(A, AKnown);
3451
8.64M
  computeKnownBits(B, BKnown);
3452
8.64M
  return (AKnown.Zero | BKnown.Zero).isAllOnesValue();
3453
8.64M
}
3454
3455
static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
3456
                                  ArrayRef<SDValue> Ops,
3457
50.1k
                                  SelectionDAG &DAG) {
3458
50.1k
  assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
3459
50.1k
  assert(llvm::all_of(Ops,
3460
50.1k
                      [Ops](SDValue Op) {
3461
50.1k
                        return Ops[0].getValueType() == Op.getValueType();
3462
50.1k
                      }) &&
3463
50.1k
         "Concatenation of vectors with inconsistent value types!");
3464
50.1k
  assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) ==
3465
50.1k
             VT.getVectorNumElements() &&
3466
50.1k
         "Incorrect element count in vector concatenation!");
3467
50.1k
3468
50.1k
  if (Ops.size() == 1)
3469
0
    return Ops[0];
3470
50.1k
3471
50.1k
  // Concat of UNDEFs is UNDEF.
3472
50.6k
  
if (50.1k
llvm::all_of(Ops, [](SDValue Op) 50.1k
{ return Op.isUndef(); }50.6k
))
3473
150
    return DAG.getUNDEF(VT);
3474
49.9k
3475
49.9k
  // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
3476
49.9k
  // simplified to one big BUILD_VECTOR.
3477
49.9k
  // FIXME: Add support for SCALAR_TO_VECTOR as well.
3478
49.9k
  EVT SVT = VT.getScalarType();
3479
49.9k
  SmallVector<SDValue, 16> Elts;
3480
52.0k
  for (SDValue Op : Ops) {
3481
52.0k
    EVT OpVT = Op.getValueType();
3482
52.0k
    if (Op.isUndef())
3483
529
      Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
3484
51.5k
    else 
if (51.5k
Op.getOpcode() == ISD::BUILD_VECTOR51.5k
)
3485
2.10k
      Elts.append(Op->op_begin(), Op->op_end());
3486
51.5k
    else
3487
49.3k
      return SDValue();
3488
567
  }
3489
567
3490
567
  // BUILD_VECTOR requires all inputs to be of the same type, find the
3491
567
  // maximum type and extend them all.
3492
567
  for (SDValue Op : Elts)
3493
10.4k
    
SVT = (SVT.bitsLT(Op.getValueType()) ? 10.4k
Op.getValueType()145
:
SVT10.2k
);
3494
567
3495
567
  if (SVT.bitsGT(VT.getScalarType()))
3496
145
    for (SDValue &Op : Elts)
3497
4.44k
      Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
3498
0
               ? DAG.getZExtOrTrunc(Op, DL, SVT)
3499
4.44k
               : DAG.getSExtOrTrunc(Op, DL, SVT);
3500
50.1k
3501
50.1k
  SDValue V = DAG.getBuildVector(VT, DL, Elts);
3502
50.1k
  NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
3503
50.1k
  return V;
3504
50.1k
}
3505
3506
/// Gets or creates the specified node.
3507
7.11M
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
3508
7.11M
  FoldingSetNodeID ID;
3509
7.11M
  AddNodeIDNode(ID, Opcode, getVTList(VT), None);
3510
7.11M
  void *IP = nullptr;
3511
7.11M
  if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
3512
5.47M
    return SDValue(E, 0);
3513
1.64M
3514
1.64M
  auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
3515
1.64M
                              getVTList(VT));
3516
1.64M
  CSEMap.InsertNode(N, IP);
3517
1.64M
3518
1.64M
  InsertNode(N);
3519
1.64M
  SDValue V = SDValue(N, 0);
3520
1.64M
  NewSDValueDbgMsg(V, "Creating new node: ", this);
3521
1.64M
  return V;
3522
1.64M
}
3523
3524
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
3525
20.1M
                              SDValue Operand, const SDNodeFlags Flags) {
3526
20.1M
  // Constant fold unary operations with an integer constant operand. Even
3527
20.1M
  // opaque constant will be folded, because the folding of unary operations
3528
20.1M
  // doesn't create new constants with different values. Nevertheless, the
3529
20.1M
  // opaque flag is preserved during folding to prevent future folding with
3530
20.1M
  // other constants.
3531
20.1M
  if (ConstantSDNode *
C20.1M
= dyn_cast<ConstantSDNode>(Operand)) {
3532
920k
    const APInt &Val = C->getAPIntValue();
3533
920k
    switch (Opcode) {
3534
48.3k
    default: break;
3535
432k
    case ISD::SIGN_EXTEND:
3536
432k
      return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
3537
432k
                         C->isTargetOpcode(), C->isOpaque());
3538
400k
    case ISD::ANY_EXTEND:
3539
400k
    case ISD::ZERO_EXTEND:
3540
400k
    case ISD::TRUNCATE:
3541
400k
      return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
3542
400k
                         C->isTargetOpcode(), C->isOpaque());
3543
411
    case ISD::UINT_TO_FP:
3544
411
    case ISD::SINT_TO_FP: {
3545
411
      APFloat apf(EVTToAPFloatSemantics(VT),
3546
411
                  APInt::getNullValue(VT.getSizeInBits()));
3547
411
      (void)apf.convertFromAPInt(Val,
3548
411
                                 Opcode==ISD::SINT_TO_FP,
3549
411
                                 APFloat::rmNearestTiesToEven);
3550
411
      return getConstantFP(apf, DL, VT);
3551
411
    }
3552
10.2k
    case ISD::BITCAST:
3553
10.2k
      if (
VT == MVT::f16 && 10.2k
C->getValueType(0) == MVT::i169
)
3554
9
        return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
3555
10.2k
      
if (10.2k
VT == MVT::f32 && 10.2k
C->getValueType(0) == MVT::i32319
)
3556
319
        return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
3557
9.96k
      
if (9.96k
VT == MVT::f64 && 9.96k
C->getValueType(0) == MVT::i64107
)
3558
107
        return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
3559
9.85k
      
if (9.85k
VT == MVT::f128 && 9.85k
C->getValueType(0) == MVT::i1280
)
3560
0
        return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
3561
9.85k
      break;
3562
190
    case ISD::ABS:
3563
190
      return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
3564
190
                         C->isOpaque());
3565
403
    case ISD::BITREVERSE:
3566
403
      return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
3567
403
                         C->isOpaque());
3568
170
    case ISD::BSWAP:
3569
170
      return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
3570
170
                         C->isOpaque());
3571
390
    case ISD::CTPOP:
3572
390
      return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
3573
390
                         C->isOpaque());
3574
25.5k
    case ISD::CTLZ:
3575
25.5k
    case ISD::CTLZ_ZERO_UNDEF:
3576
25.5k
      return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
3577
25.5k
                         C->isOpaque());
3578
1.32k
    case ISD::CTTZ:
3579
1.32k
    case ISD::CTTZ_ZERO_UNDEF:
3580
1.32k
      return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
3581
1.32k
                         C->isOpaque());
3582
281
    case ISD::FP16_TO_FP: {
3583
281
      bool Ignored;
3584
281
      APFloat FPV(APFloat::IEEEhalf(),
3585
281
                  (Val.getBitWidth() == 16) ? 
Val281
:
Val.trunc(16)0
);
3586
281
3587
281
      // This can return overflow, underflow, or inexact; we don't care.
3588
281
      // FIXME need to be more flexible about rounding mode.
3589
281
      (void)FPV.convert(EVTToAPFloatSemantics(VT),
3590
281
                        APFloat::rmNearestTiesToEven, &Ignored);
3591
281
      return getConstantFP(FPV, DL, VT);
3592
19.2M
    }
3593
920k
    }
3594
920k
  }
3595
19.2M
3596
19.2M
  // Constant fold unary operations with a floating point constant operand.
3597
19.2M
  
if (ConstantFPSDNode *19.2M
C19.2M
= dyn_cast<ConstantFPSDNode>(Operand)) {
3598
6.30k
    APFloat V = C->getValueAPF();    // make copy
3599
6.30k
    switch (Opcode) {
3600
317
    case ISD::FNEG:
3601
317
      V.changeSign();
3602
317
      return getConstantFP(V, DL, VT);
3603
120
    case ISD::FABS:
3604
120
      V.clearSign();
3605
120
      return getConstantFP(V, DL, VT);
3606
16
    case ISD::FCEIL: {
3607
16
      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
3608
16
      if (
fs == APFloat::opOK || 16
fs == APFloat::opInexact0
)
3609
16
        return getConstantFP(V, DL, VT);
3610
0
      break;
3611
0
    }
3612
13
    case ISD::FTRUNC: {
3613
13
      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
3614
13
      if (
fs == APFloat::opOK || 13
fs == APFloat::opInexact0
)
3615
13
        return getConstantFP(V, DL, VT);
3616
0
      break;
3617
0
    }
3618
13
    case ISD::FFLOOR: {
3619
13
      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
3620
13
      if (
fs == APFloat::opOK || 13
fs == APFloat::opInexact0
)
3621
13
        return getConstantFP(V, DL, VT);
3622
0
      break;
3623
0
    }
3624
104
    case ISD::FP_EXTEND: {
3625
104
      bool ignored;
3626
104
      // This can return overflow, underflow, or inexact; we don't care.
3627
104
      // FIXME need to be more flexible about rounding mode.
3628
104
      (void)V.convert(EVTToAPFloatSemantics(VT),
3629
104
                      APFloat::rmNearestTiesToEven, &ignored);
3630
104
      return getConstantFP(V, DL, VT);
3631
0
    }
3632
625
    case ISD::FP_TO_SINT:
3633
625
    case ISD::FP_TO_UINT: {
3634
625
      bool ignored;
3635
625
      APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
3636
625
      // FIXME need to be more flexible about rounding mode.
3637
625
      APFloat::opStatus s =
3638
625
          V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
3639
625
      if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
3640
28
        break;
3641
597
      return getConstant(IntVal, DL, VT);
3642
597
    }
3643
3.73k
    case ISD::BITCAST:
3644
3.73k
      if (
VT == MVT::i16 && 3.73k
C->getValueType(0) == MVT::f16193
)
3645
193
        return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
3646
3.54k
      else 
if (3.54k
VT == MVT::i32 && 3.54k
C->getValueType(0) == MVT::f322.55k
)
3647
2.55k
        return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
3648
990
      else 
if (990
VT == MVT::i64 && 990
C->getValueType(0) == MVT::f64798
)
3649
798
        return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
3650
192
      break;
3651
18
    case ISD::FP_TO_FP16: {
3652
18
      bool Ignored;
3653
18
      // This can return overflow, underflow, or inexact; we don't care.
3654
18
      // FIXME need to be more flexible about rounding mode.
3655
18
      (void)V.convert(APFloat::IEEEhalf(),
3656
18
                      APFloat::rmNearestTiesToEven, &Ignored);
3657
18
      return getConstant(V.bitcastToAPInt(), DL, VT);
3658
19.2M
    }
3659
6.30k
    }
3660
6.30k
  }
3661
19.2M
3662
19.2M
  // Constant fold unary operations with a vector integer or float operand.
3663
19.2M
  
if (BuildVectorSDNode *19.2M
BV19.2M
= dyn_cast<BuildVectorSDNode>(Operand)) {
3664
191k
    if (
BV->isConstant()191k
) {
3665
33.2k
      switch (Opcode) {
3666
31.3k
      default:
3667
31.3k
        // FIXME: Entirely reasonable to perform folding of other unary
3668
31.3k
        // operations here as the need arises.
3669
31.3k
        break;
3670
1.85k
      case ISD::FNEG:
3671
1.85k
      case ISD::FABS:
3672
1.85k
      case ISD::FCEIL:
3673
1.85k
      case ISD::FTRUNC:
3674
1.85k
      case ISD::FFLOOR:
3675
1.85k
      case ISD::FP_EXTEND:
3676
1.85k
      case ISD::FP_TO_SINT:
3677
1.85k
      case ISD::FP_TO_UINT:
3678
1.85k
      case ISD::TRUNCATE:
3679
1.85k
      case ISD::UINT_TO_FP:
3680
1.85k
      case ISD::SINT_TO_FP:
3681
1.85k
      case ISD::ABS:
3682
1.85k
      case ISD::BITREVERSE:
3683
1.85k
      case ISD::BSWAP:
3684
1.85k
      case ISD::CTLZ:
3685
1.85k
      case ISD::CTLZ_ZERO_UNDEF:
3686
1.85k
      case ISD::CTTZ:
3687
1.85k
      case ISD::CTTZ_ZERO_UNDEF:
3688
1.85k
      case ISD::CTPOP: {
3689
1.85k
        SDValue Ops = { Operand };
3690
1.85k
        if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
3691
1.79k
          return Fold;
3692
19.2M
      }
3693
33.2k
      }
3694
33.2k
    }
3695
191k
  }
3696
19.2M
3697
19.2M
  unsigned OpOpcode = Operand.getNode()->getOpcode();
3698
19.2M
  switch (Opcode) {
3699
12.7M
  case ISD::TokenFactor:
3700
12.7M
  case ISD::MERGE_VALUES:
3701
12.7M
  case ISD::CONCAT_VECTORS:
3702
12.7M
    return Operand;         // Factor, merge or concat of one node?  No need.
3703
0
  
case ISD::FP_ROUND: 0
llvm_unreachable0
("Invalid method to make FP_ROUND node");
3704
33.4k
  case ISD::FP_EXTEND:
3705
33.4k
    assert(VT.isFloatingPoint() &&
3706
33.4k
           Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
3707
33.4k
    if (
Operand.getValueType() == VT33.4k
)
return Operand2.29k
; // noop conversion.
3708
33.4k
    assert((!VT.isVector() ||
3709
31.1k
            VT.getVectorNumElements() ==
3710
31.1k
            Operand.getValueType().getVectorNumElements()) &&
3711
31.1k
           "Vector element count mismatch!");
3712
31.1k
    assert(Operand.getValueType().bitsLT(VT) &&
3713
31.1k
           "Invalid fpext node, dst < src!");
3714
31.1k
    if (Operand.isUndef())
3715
53
      return getUNDEF(VT);
3716
31.1k
    break;
3717
340k
  case ISD::SIGN_EXTEND:
3718
340k
    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3719
340k
           "Invalid SIGN_EXTEND!");
3720
340k
    if (
Operand.getValueType() == VT340k
)
return Operand3.49k
; // noop extension
3721
340k
    assert((!VT.isVector() ||
3722
336k
            VT.getVectorNumElements() ==
3723
336k
            Operand.getValueType().getVectorNumElements()) &&
3724
336k
           "Vector element count mismatch!");
3725
336k
    assert(Operand.getValueType().bitsLT(VT) &&
3726
336k
           "Invalid sext node, dst < src!");
3727
336k
    if (
OpOpcode == ISD::SIGN_EXTEND || 336k
OpOpcode == ISD::ZERO_EXTEND336k
)
3728
394
      return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
3729
336k
    else 
if (336k
OpOpcode == ISD::UNDEF336k
)
3730
336k
      // sext(undef) = 0, because the top bits will all be the same.
3731
123
      return getConstant(0, DL, VT);
3732
336k
    break;
3733
684k
  case ISD::ZERO_EXTEND:
3734
684k
    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3735
684k
           "Invalid ZERO_EXTEND!");
3736
684k
    if (
Operand.getValueType() == VT684k
)
return Operand15.3k
; // noop extension
3737
684k
    assert((!VT.isVector() ||
3738
669k
            VT.getVectorNumElements() ==
3739
669k
            Operand.getValueType().getVectorNumElements()) &&
3740
669k
           "Vector element count mismatch!");
3741
669k
    assert(Operand.getValueType().bitsLT(VT) &&
3742
669k
           "Invalid zext node, dst < src!");
3743
669k
    if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
3744
4.05k
      return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
3745
665k
    else 
if (665k
OpOpcode == ISD::UNDEF665k
)
3746
665k
      // zext(undef) = 0, because the top bits will be zero.
3747
882
      return getConstant(0, DL, VT);
3748
664k
    break;
3749
734k
  case ISD::ANY_EXTEND:
3750
734k
    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3751
734k
           "Invalid ANY_EXTEND!");
3752
734k
    if (
Operand.getValueType() == VT734k
)
return Operand452k
; // noop extension
3753
734k
    assert((!VT.isVector() ||
3754
282k
            VT.getVectorNumElements() ==
3755
282k
            Operand.getValueType().getVectorNumElements()) &&
3756
282k
           "Vector element count mismatch!");
3757
282k
    assert(Operand.getValueType().bitsLT(VT) &&
3758
282k
           "Invalid anyext node, dst < src!");
3759
282k
3760
282k
    if (
OpOpcode == ISD::ZERO_EXTEND || 282k
OpOpcode == ISD::SIGN_EXTEND280k
||
3761
279k
        OpOpcode == ISD::ANY_EXTEND)
3762
282k
      // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
3763
24.2k
      return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
3764
257k
    else 
if (257k
OpOpcode == ISD::UNDEF257k
)
3765
967
      return getUNDEF(VT);
3766
257k
3767
257k
    // (ext (trunx x)) -> x
3768
257k
    
if (257k
OpOpcode == ISD::TRUNCATE257k
) {
3769
21.5k
      SDValue OpOp = Operand.getOperand(0);
3770
21.5k
      if (OpOp.getValueType() == VT)
3771
16.3k
        return OpOp;
3772
240k
    }
3773
240k
    break;
3774
2.26M
  case ISD::TRUNCATE:
3775
2.26M
    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
3776
2.26M
           "Invalid TRUNCATE!");
3777
2.26M
    if (
Operand.getValueType() == VT2.26M
)
return Operand1.22M
; // noop truncate
3778
2.26M
    assert((!VT.isVector() ||
3779
1.04M
            VT.getVectorNumElements() ==
3780
1.04M
            Operand.getValueType().getVectorNumElements()) &&
3781
1.04M
           "Vector element count mismatch!");
3782
1.04M
    assert(Operand.getValueType().bitsGT(VT) &&
3783
1.04M
           "Invalid truncate node, src < dst!");
3784
1.04M
    if (OpOpcode == ISD::TRUNCATE)
3785
37.0k
      return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
3786
1.00M
    
if (1.00M
OpOpcode == ISD::ZERO_EXTEND || 1.00M
OpOpcode == ISD::SIGN_EXTEND1.00M
||
3787
1.00M
        
OpOpcode == ISD::ANY_EXTEND997k
) {
3788
19.9k
      // If the source is smaller than the dest, we still need an extend.
3789
19.9k
      if (Operand.getOperand(0).getValueType().getScalarType()
3790
19.9k
            .bitsLT(VT.getScalarType()))
3791
4.70k
        return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
3792
15.2k
      
if (15.2k
Operand.getOperand(0).getValueType().bitsGT(VT)15.2k
)
3793
785
        return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
3794
14.4k
      return Operand.getOperand(0);
3795
14.4k
    }
3796
984k
    
if (984k
OpOpcode == ISD::UNDEF984k
)
3797
12.0k
      return getUNDEF(VT);
3798
972k
    break;
3799
569
  case ISD::ABS:
3800
569
    assert(VT.isInteger() && VT == Operand.getValueType() &&
3801
569
           "Invalid ABS!");
3802
569
    if (OpOpcode == ISD::UNDEF)
3803
1
      return getUNDEF(VT);
3804
568
    break;
3805
1.52k
  case ISD::BSWAP:
3806
1.52k
    assert(VT.isInteger() && VT == Operand.getValueType() &&
3807
1.52k
           "Invalid BSWAP!");
3808
1.52k
    assert((VT.getScalarSizeInBits() % 16 == 0) &&
3809
1.52k
           "BSWAP types must be a multiple of 16 bits!");
3810
1.52k
    if (OpOpcode == ISD::UNDEF)
3811
4
      return getUNDEF(VT);
3812
1.52k
    break;
3813
714
  case ISD::BITREVERSE:
3814
714
    assert(VT.isInteger() && VT == Operand.getValueType() &&
3815
714
           "Invalid BITREVERSE!");
3816
714
    if (OpOpcode == ISD::UNDEF)
3817
4
      return getUNDEF(VT);
3818
710
    break;
3819
757k
  case ISD::BITCAST:
3820
757k
    // Basic sanity checking.
3821
757k
    assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&
3822
757k
           "Cannot BITCAST between types of different sizes!");
3823
757k
    if (
VT == Operand.getValueType()757k
)
return Operand200k
; // noop conversion.
3824
556k
    
if (556k
OpOpcode == ISD::BITCAST556k
) // bitconv(bitconv(x)) -> bitconv(x)
3825
32.1k
      return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
3826
524k
    
if (524k
OpOpcode == ISD::UNDEF524k
)
3827
5.41k
      return getUNDEF(VT);
3828
518k
    break;
3829
40.8k
  case ISD::SCALAR_TO_VECTOR:
3830
40.8k
    assert(VT.isVector() && !Operand.getValueType().isVector() &&
3831
40.8k
           (VT.getVectorElementType() == Operand.getValueType() ||
3832
40.8k
            (VT.getVectorElementType().isInteger() &&
3833
40.8k
             Operand.getValueType().isInteger() &&
3834
40.8k
             VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
3835
40.8k
           "Illegal SCALAR_TO_VECTOR node!");
3836
40.8k
    if (OpOpcode == ISD::UNDEF)
3837
1
      return getUNDEF(VT);
3838
40.8k
    // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
3839
40.8k
    
if (40.8k
OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
3840
16.5k
        isa<ConstantSDNode>(Operand.getOperand(1)) &&
3841
16.3k
        Operand.getConstantOperandVal(1) == 0 &&
3842
5.87k
        Operand.getOperand(0).getValueType() == VT)
3843
644
      return Operand.getOperand(0);
3844
40.2k
    break;
3845
8.39k
  case ISD::FNEG:
3846
8.39k
    // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
3847
8.39k
    if (
getTarget().Options.UnsafeFPMath && 8.39k
OpOpcode == ISD::FSUB646
)
3848
8.39k
      // FIXME: FNEG has no fast-math-flags to propagate; use the FSUB's flags?
3849
59
      return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1),
3850
59
                     Operand.getOperand(0), Operand.getNode()->getFlags());
3851
8.33k
    
if (8.33k
OpOpcode == ISD::FNEG8.33k
) // --X -> X
3852
0
      return Operand.getOperand(0);
3853
8.33k
    break;
3854
4.17k
  case ISD::FABS:
3855
4.17k
    if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
3856
0
      return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
3857
4.17k
    break;
3858
4.51M
  }
3859
4.51M
3860
4.51M
  SDNode *N;
3861
4.51M
  SDVTList VTs = getVTList(VT);
3862
4.51M
  SDValue Ops[] = {Operand};
3863
4.51M
  if (
VT != MVT::Glue4.51M
) { // Don't CSE flag producing nodes
3864
4.51M
    FoldingSetNodeID ID;
3865
4.51M
    AddNodeIDNode(ID, Opcode, VTs, Ops);
3866
4.51M
    void *IP = nullptr;
3867
4.51M
    if (SDNode *
E4.51M
= FindNodeOrInsertPos(ID, DL, IP)) {
3868
282k
      E->intersectFlagsWith(Flags);
3869
282k
      return SDValue(E, 0);
3870
282k
    }
3871
4.23M
3872
4.23M
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
3873
4.23M
    N->setFlags(Flags);
3874
4.23M
    createOperands(N, Ops);
3875
4.23M
    CSEMap.InsertNode(N, IP);
3876
4.51M
  } else {
3877
1.26k
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
3878
1.26k
    createOperands(N, Ops);
3879
1.26k
  }
3880
4.51M
3881
4.23M
  InsertNode(N);
3882
4.23M
  SDValue V = SDValue(N, 0);
3883
4.23M
  NewSDValueDbgMsg(V, "Creating new node: ", this);
3884
4.23M
  return V;
3885
20.1M
}
3886
3887
static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
3888
1.07M
                                        const APInt &C2) {
3889
1.07M
  switch (Opcode) {
3890
592k
  case ISD::ADD:  return std::make_pair(C1 + C2, true);
3891
25.2k
  case ISD::SUB:  return std::make_pair(C1 - C2, true);
3892
4.97k
  case ISD::MUL:  return std::make_pair(C1 * C2, true);
3893
266k
  case ISD::AND:  return std::make_pair(C1 & C2, true);
3894
1.56k
  case ISD::OR:   return std::make_pair(C1 | C2, true);
3895
2.50k
  case ISD::XOR:  return std::make_pair(C1 ^ C2, true);
3896
26.3k
  case ISD::SHL:  return std::make_pair(C1 << C2, true);
3897
13.6k
  case ISD::SRL:  return std::make_pair(C1.lshr(C2), true);
3898
927
  case ISD::SRA:  return std::make_pair(C1.ashr(C2), true);
3899
16
  case ISD::ROTL: return std::make_pair(C1.rotl(C2), true);
3900
0
  case ISD::ROTR: return std::make_pair(C1.rotr(C2), true);
3901
840
  
case ISD::SMIN: return std::make_pair(C1.sle(C2) ? 840
C1446
:
C2394
, true);
3902
844
  
case ISD::SMAX: return std::make_pair(C1.sge(C2) ? 844
C1496
:
C2348
, true);
3903
824
  
case ISD::UMIN: return std::make_pair(C1.ule(C2) ? 824
C1443
:
C2381
, true);
3904
825
  
case ISD::UMAX: return std::make_pair(C1.uge(C2) ? 825
C1481
:
C2344
, true);
3905
28
  case ISD::UDIV:
3906
28
    if (!C2.getBoolValue())
3907
0
      break;
3908
28
    return std::make_pair(C1.udiv(C2), true);
3909
32
  case ISD::UREM:
3910
32
    if (!C2.getBoolValue())
3911
0
      break;
3912
32
    return std::make_pair(C1.urem(C2), true);
3913
99
  case ISD::SDIV:
3914
99
    if (!C2.getBoolValue())
3915
0
      break;
3916
99
    return std::make_pair(C1.sdiv(C2), true);
3917
57
  case ISD::SREM:
3918
57
    if (!C2.getBoolValue())
3919
0
      break;
3920
57
    return std::make_pair(C1.srem(C2), true);
3921
136k
  }
3922
136k
  return std::make_pair(APInt(1, 0), false);
3923
136k
}
3924
3925
SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
3926
                                             EVT VT, const ConstantSDNode *Cst1,
3927
1.07M
                                             const ConstantSDNode *Cst2) {
3928
1.07M
  if (
Cst1->isOpaque() || 1.07M
Cst2->isOpaque()1.07M
)
3929
1.40k
    return SDValue();
3930
1.07M
3931
1.07M
  std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
3932
1.07M
                                            Cst2->getAPIntValue());
3933
1.07M
  if (!Folded.second)
3934
136k
    return SDValue();
3935
937k
  return getConstant(Folded.first, DL, VT);
3936
937k
}
3937
3938
SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
3939
                                       const GlobalAddressSDNode *GA,
3940
298k
                                       const SDNode *N2) {
3941
298k
  if (GA->getOpcode() != ISD::GlobalAddress)
3942
11
    return SDValue();
3943
298k
  
if (298k
!TLI->isOffsetFoldingLegal(GA)298k
)
3944
295k
    return SDValue();
3945
2.64k
  const ConstantSDNode *Cst2 = dyn_cast<ConstantSDNode>(N2);
3946
2.64k
  if (!Cst2)
3947
1.38k
    return SDValue();
3948
1.26k
  int64_t Offset = Cst2->getSExtValue();
3949
1.26k
  switch (Opcode) {
3950
1.25k
  case ISD::ADD: break;
3951
1
  case ISD::SUB: Offset = -uint64_t(Offset); break;
3952
5
  default: return SDValue();
3953
1.25k
  }
3954
1.25k
  return getGlobalAddress(GA->getGlobal(), SDLoc(Cst2), VT,
3955
1.25k
                          GA->getOffset() + uint64_t(Offset));
3956
1.25k
}
3957
3958
20.2M
bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
3959
20.2M
  switch (Opcode) {
3960
87.4k
  case ISD::SDIV:
3961
87.4k
  case ISD::UDIV:
3962
87.4k
  case ISD::SREM:
3963
87.4k
  case ISD::UREM: {
3964
87.4k
    // If a divisor is zero/undef or any element of a divisor vector is
3965
87.4k
    // zero/undef, the whole op is undef.
3966
87.4k
    assert(Ops.size() == 2 && "Div/rem should have 2 operands");
3967
87.4k
    SDValue Divisor = Ops[1];
3968
87.4k
    if (
Divisor.isUndef() || 87.4k
isNullConstant(Divisor)87.4k
)
3969
37
      return true;
3970
87.4k
3971
87.4k
    return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
3972
1.34k
           llvm::any_of(Divisor->op_values(),
3973
13.0k
                        [](SDValue V) { return V.isUndef() ||
3974
13.0k
                                        isNullConstant(V); });
3975
87.4k
    // TODO: Handle signed overflow.
3976
87.4k
  }
3977
87.4k
  // TODO: Handle oversized shifts.
3978
20.1M
  default:
3979
20.1M
    return false;
3980
0
  }
3981
0
}
3982
3983
SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
3984
                                             EVT VT, SDNode *Cst1,
3985
17.4M
                                             SDNode *Cst2) {
3986
17.4M
  // If the opcode is a target-specific ISD node, there's nothing we can
3987
17.4M
  // do here and the operand rules may not line up with the below, so
3988
17.4M
  // bail early.
3989
17.4M
  if (Opcode >= ISD::BUILTIN_OP_END)
3990
1.10M
    return SDValue();
3991
16.3M
3992
16.3M
  
if (16.3M
isUndef(Opcode, {SDValue(Cst1, 0), SDValue(Cst2, 0)})16.3M
)
3993
76
    return getUNDEF(VT);
3994
16.3M
3995
16.3M
  // Handle the case of two scalars.
3996
16.3M
  
if (const ConstantSDNode *16.3M
Scalar116.3M
= dyn_cast<ConstantSDNode>(Cst1)) {
3997
1.15M
    if (const ConstantSDNode *
Scalar21.15M
= dyn_cast<ConstantSDNode>(Cst2)) {
3998
1.06M
      SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2);
3999
1.06M
      assert((!Folded || !VT.isVector()) &&
4000
1.06M
             "Can't fold vectors ops with scalar operands");
4001
1.06M
      return Folded;
4002
1.06M
    }
4003
15.2M
  }
4004
15.2M
4005
15.2M
  // fold (add Sym, c) -> Sym+c
4006
15.2M
  
if (GlobalAddressSDNode *15.2M
GA15.2M
= dyn_cast<GlobalAddressSDNode>(Cst1))
4007
297k
    return FoldSymbolOffset(Opcode, VT, GA, Cst2);
4008
14.9M
  
if (14.9M
TLI->isCommutativeBinOp(Opcode)14.9M
)
4009
7.28M
    
if (GlobalAddressSDNode *7.28M
GA7.28M
= dyn_cast<GlobalAddressSDNode>(Cst2))
4010
1.11k
      return FoldSymbolOffset(Opcode, VT, GA, Cst1);
4011
14.9M
4012
14.9M
  // For vectors extract each constant element into Inputs so we can constant
4013
14.9M
  // fold them individually.
4014
14.9M
  BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
4015
14.9M
  BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
4016
14.9M
  if (
!BV1 || 14.9M
!BV254.1k
)
4017
14.9M
    return SDValue();
4018
19.6k
4019
14.9M
  assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!");
4020
19.6k
4021
19.6k
  EVT SVT = VT.getScalarType();
4022
19.6k
  EVT LegalSVT = SVT;
4023
19.6k
  if (
NewNodesMustHaveLegalTypes && 19.6k
LegalSVT.isInteger()8.61k
) {
4024
8.61k
    LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4025
8.61k
    if (LegalSVT.bitsLT(SVT))
4026
0
      return SDValue();
4027
19.6k
  }
4028
19.6k
  SmallVector<SDValue, 4> Outputs;
4029
31.8k
  for (unsigned I = 0, E = BV1->getNumOperands(); 
I != E31.8k
;
++I12.2k
) {
4030
29.6k
    SDValue V1 = BV1->getOperand(I);
4031
29.6k
    SDValue V2 = BV2->getOperand(I);
4032
29.6k
4033
29.6k
    if (
SVT.isInteger()29.6k
) {
4034
28.5k
        if (V1->getValueType(0).bitsGT(SVT))
4035
1.20k
          V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
4036
28.5k
        if (V2->getValueType(0).bitsGT(SVT))
4037
1.60k
          V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
4038
28.5k
    }
4039
29.6k
4040
29.6k
    if (
V1->getValueType(0) != SVT || 29.6k
V2->getValueType(0) != SVT29.6k
)
4041
0
      return SDValue();
4042
29.6k
4043
29.6k
    // Fold one vector element.
4044
29.6k
    SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
4045
29.6k
    if (LegalSVT != SVT)
4046
939
      ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4047
29.6k
4048
29.6k
    // Scalar folding only succeeded if the result is a constant or UNDEF.
4049
29.6k
    if (
!ScalarResult.isUndef() && 29.6k
ScalarResult.getOpcode() != ISD::Constant29.6k
&&
4050
17.9k
        ScalarResult.getOpcode() != ISD::ConstantFP)
4051
17.3k
      return SDValue();
4052
12.2k
    Outputs.push_back(ScalarResult);
4053
12.2k
  }
4054
19.6k
4055
2.21k
  assert(VT.getVectorNumElements() == Outputs.size() &&
4056
2.21k
         "Vector size mismatch!");
4057
2.21k
4058
2.21k
  // We may have a vector type but a scalar result. Create a splat.
4059
2.21k
  Outputs.resize(VT.getVectorNumElements(), Outputs.back());
4060
2.21k
4061
2.21k
  // Build a big vector out of the scalar elements we generated.
4062
2.21k
  return getBuildVector(VT, SDLoc(), Outputs);
4063
17.4M
}
4064
4065
// TODO: Merge with FoldConstantArithmetic
4066
SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
4067
                                                   const SDLoc &DL, EVT VT,
4068
                                                   ArrayRef<SDValue> Ops,
4069
3.83M
                                                   const SDNodeFlags Flags) {
4070
3.83M
  // If the opcode is a target-specific ISD node, there's nothing we can
4071
3.83M
  // do here and the operand rules may not line up with the below, so
4072
3.83M
  // bail early.
4073
3.83M
  if (Opcode >= ISD::BUILTIN_OP_END)
4074
0
    return SDValue();
4075
3.83M
4076
3.83M
  
if (3.83M
isUndef(Opcode, Ops)3.83M
)
4077
1
    return getUNDEF(VT);
4078
3.83M
4079
3.83M
  // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
4080
3.83M
  
if (3.83M
!VT.isVector()3.83M
)
4081
3.46M
    return SDValue();
4082
375k
4083
375k
  unsigned NumElts = VT.getVectorNumElements();
4084
375k
4085
3.67k
  auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
4086
3.67k
    return !Op.getValueType().isVector() ||
4087
3.28k
           Op.getValueType().getVectorNumElements() == NumElts;
4088
3.67k
  };
4089
375k
4090
395k
  auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
4091
395k
    BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
4092
395k
    return (Op.isUndef()) || (Op.getOpcode() == ISD::CONDCODE) ||
4093
395k
           
(BV && 395k
BV->isConstant()43.6k
);
4094
395k
  };
4095
375k
4096
375k
  // All operands must be vector types with the same number of elements as
4097
375k
  // the result type and must be either UNDEF or a build vector of constant
4098
375k
  // or UNDEF scalars.
4099
375k
  if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
4100
2.56k
      !llvm::all_of(Ops, IsScalarOrSameVectorSize))
4101
372k
    return SDValue();
4102
2.56k
4103
2.56k
  // If we are comparing vectors, then the result needs to be a i1 boolean
4104
2.56k
  // that is then sign-extended back to the legal result type.
4105
2.56k
  
EVT SVT = (Opcode == ISD::SETCC ? 2.56k
MVT::i1397
:
VT.getScalarType()2.17k
);
4106
2.56k
4107
2.56k
  // Find legal integer scalar type for constant promotion and
4108
2.56k
  // ensure that its scalar size is at least as large as source.
4109
2.56k
  EVT LegalSVT = VT.getScalarType();
4110
2.56k
  if (
NewNodesMustHaveLegalTypes && 2.56k
LegalSVT.isInteger()571
) {
4111
543
    LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4112
543
    if (LegalSVT.bitsLT(VT.getScalarType()))
4113
0
      return SDValue();
4114
2.56k
  }
4115
2.56k
4116
2.56k
  // Constant fold each scalar lane separately.
4117
2.56k
  SmallVector<SDValue, 4> ScalarResults;
4118
22.3k
  for (unsigned i = 0; 
i != NumElts22.3k
;
i++19.8k
) {
4119
19.9k
    SmallVector<SDValue, 4> ScalarOps;
4120
25.8k
    for (SDValue Op : Ops) {
4121
25.8k
      EVT InSVT = Op.getValueType().getScalarType();
4122
25.8k
      BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
4123
25.8k
      if (
!InBV25.8k
) {
4124
1.62k
        // We've checked that this is UNDEF or a constant of some kind.
4125
1.62k
        if (Op.isUndef())
4126
120
          ScalarOps.push_back(getUNDEF(InSVT));
4127
1.62k
        else
4128
1.50k
          ScalarOps.push_back(Op);
4129
1.62k
        continue;
4130
1.62k
      }
4131
24.2k
4132
24.2k
      SDValue ScalarOp = InBV->getOperand(i);
4133
24.2k
      EVT ScalarVT = ScalarOp.getValueType();
4134
24.2k
4135
24.2k
      // Build vector (integer) scalar operands may need implicit
4136
24.2k
      // truncation - do this before constant folding.
4137
24.2k
      if (
ScalarVT.isInteger() && 24.2k
ScalarVT.bitsGT(InSVT)23.1k
)
4138
424
        ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
4139
25.8k
4140
25.8k
      ScalarOps.push_back(ScalarOp);
4141
25.8k
    }
4142
19.9k
4143
19.9k
    // Constant fold the scalar operands.
4144
19.9k
    SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
4145
19.9k
4146
19.9k
    // Legalize the (integer) scalar constant if necessary.
4147
19.9k
    if (LegalSVT != SVT)
4148
6.05k
      ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4149
19.9k
4150
19.9k
    // Scalar folding only succeeded if the result is a constant or UNDEF.
4151
19.9k
    if (
!ScalarResult.isUndef() && 19.9k
ScalarResult.getOpcode() != ISD::Constant19.7k
&&
4152
593
        ScalarResult.getOpcode() != ISD::ConstantFP)
4153
153
      return SDValue();
4154
19.8k
    ScalarResults.push_back(ScalarResult);
4155
19.8k
  }
4156
2.56k
4157
2.41k
  SDValue V = getBuildVector(VT, DL, ScalarResults);
4158
2.41k
  NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
4159
2.41k
  return V;
4160
3.83M
}
4161
4162
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4163
21.6M
                              SDValue N1, SDValue N2, const SDNodeFlags Flags) {
4164
21.6M
  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4165
21.6M
  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
4166
21.6M
  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
4167
21.6M
  ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
4168
21.6M
4169
21.6M
  // Canonicalize constant to RHS if commutative.
4170
21.6M
  if (
TLI->isCommutativeBinOp(Opcode)21.6M
) {
4171
11.5M
    if (
N1C && 11.5M
!N2C419k
) {
4172
16.8k
      std::swap(N1C, N2C);
4173
16.8k
      std::swap(N1, N2);
4174
11.5M
    } else 
if (11.5M
N1CFP && 11.5M
!N2CFP1.37k
) {
4175
747
      std::swap(N1CFP, N2CFP);
4176
747
      std::swap(N1, N2);
4177
747
    }
4178
11.5M
  }
4179
21.6M
4180
21.6M
  switch (Opcode) {
4181
3.67M
  default: break;
4182
2.95M
  case ISD::TokenFactor:
4183
2.95M
    assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
4184
2.95M
           N2.getValueType() == MVT::Other && "Invalid token factor!");
4185
2.95M
    // Fold trivial token factors.
4186
2.95M
    if (
N1.getOpcode() == ISD::EntryToken2.95M
)
return N210.8k
;
4187
2.94M
    
if (2.94M
N2.getOpcode() == ISD::EntryToken2.94M
)
return N11.08k
;
4188
2.94M
    
if (2.94M
N1 == N22.94M
)
return N1107k
;
4189
2.83M
    break;
4190
43.3k
  case ISD::CONCAT_VECTORS: {
4191
43.3k
    // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
4192
43.3k
    SDValue Ops[] = {N1, N2};
4193
43.3k
    if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
4194
482
      return V;
4195
42.8k
    break;
4196
42.8k
  }
4197
1.24M
  case ISD::AND:
4198
1.24M
    assert(VT.isInteger() && "This operator does not apply to FP types!");
4199
1.24M
    assert(N1.getValueType() == N2.getValueType() &&
4200
1.24M
           N1.getValueType() == VT && "Binary operator types must match!");
4201
1.24M
    // (X & 0) -> 0.  This commonly occurs when legalizing i64 values, so it's
4202
1.24M
    // worth handling here.
4203
1.24M
    if (
N2C && 1.24M
N2C->isNullValue()1.08M
)
4204
3.79k
      return N2;
4205
1.24M
    
if (1.24M
N2C && 1.24M
N2C->isAllOnesValue()1.08M
) // X & -1 -> X
4206
3.17k
      return N1;
4207
1.24M
    break;
4208
10.1M
  case ISD::OR:
4209
10.1M
  case ISD::XOR:
4210
10.1M
  case ISD::ADD:
4211
10.1M
  case ISD::SUB:
4212
10.1M
    assert(VT.isInteger() && "This operator does not apply to FP types!");
4213
10.1M
    assert(N1.getValueType() == N2.getValueType() &&
4214
10.1M
           N1.getValueType() == VT && "Binary operator types must match!");
4215
10.1M
    // (X ^|+- 0) -> X.  This commonly occurs when legalizing i64 values, so
4216
10.1M
    // it's worth handling here.
4217
10.1M
    if (
N2C && 10.1M
N2C->isNullValue()8.92M
)
4218
3.58M
      return N1;
4219
6.53M
    break;
4220
318k
  case ISD::UDIV:
4221
318k
  case ISD::UREM:
4222
318k
  case ISD::MULHU:
4223
318k
  case ISD::MULHS:
4224
318k
  case ISD::MUL:
4225
318k
  case ISD::SDIV:
4226
318k
  case ISD::SREM:
4227
318k
  case ISD::SMIN:
4228
318k
  case ISD::SMAX:
4229
318k
  case ISD::UMIN:
4230
318k
  case ISD::UMAX:
4231
318k
    assert(VT.isInteger() && "This operator does not apply to FP types!");
4232
318k
    assert(N1.getValueType() == N2.getValueType() &&
4233
318k
           N1.getValueType() == VT && "Binary operator types must match!");
4234
318k
    break;
4235
192k
  case ISD::FADD:
4236
192k
  case ISD::FSUB:
4237
192k
  case ISD::FMUL:
4238
192k
  case ISD::FDIV:
4239
192k
  case ISD::FREM:
4240
192k
    if (
getTarget().Options.UnsafeFPMath192k
) {
4241
5.51k
      if (
Opcode == ISD::FADD5.51k
) {
4242
1.57k
        // x+0 --> x
4243
1.57k
        if (
N2CFP && 1.57k
N2CFP->getValueAPF().isZero()263
)
4244
16
          return N1;
4245
3.93k
      } else 
if (3.93k
Opcode == ISD::FSUB3.93k
) {
4246
674
        // x-0 --> x
4247
674
        if (
N2CFP && 674
N2CFP->getValueAPF().isZero()10
)
4248
0
          return N1;
4249
3.26k
      } else 
if (3.26k
Opcode == ISD::FMUL3.26k
) {
4250
2.77k
        // x*0 --> 0
4251
2.77k
        if (
N2CFP && 2.77k
N2CFP->isZero()498
)
4252
6
          return N2;
4253
2.76k
        // x*1 --> x
4254
2.76k
        
if (2.76k
N2CFP && 2.76k
N2CFP->isExactlyValue(1.0)492
)
4255
61
          return N1;
4256
192k
      }
4257
5.51k
    }
4258
192k
    assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
4259
192k
    assert(N1.getValueType() == N2.getValueType() &&
4260
192k
           N1.getValueType() == VT && "Binary operator types must match!");
4261
192k
    break;
4262
2.13k
  case ISD::FCOPYSIGN:   // N1 and result must match.  N1/N2 need not match.
4263
2.13k
    assert(N1.getValueType() == VT &&
4264
2.13k
           N1.getValueType().isFloatingPoint() &&
4265
2.13k
           N2.getValueType().isFloatingPoint() &&
4266
2.13k
           "Invalid FCOPYSIGN!");
4267
2.13k
    break;
4268
786k
  case ISD::SHL:
4269
786k
  case ISD::SRA:
4270
786k
  case ISD::SRL:
4271
786k
  case ISD::ROTL:
4272
786k
  case ISD::ROTR:
4273
786k
    assert(VT == N1.getValueType() &&
4274
786k
           "Shift operators return type must be the same as their first arg");
4275
786k
    assert(VT.isInteger() && N2.getValueType().isInteger() &&
4276
786k
           "Shifts only work on integers");
4277
786k
    assert((!VT.isVector() || VT == N2.getValueType()) &&
4278
786k
           "Vector shift amounts must be in the same as their first arg");
4279
786k
    // Verify that the shift amount VT is bit enough to hold valid shift
4280
786k
    // amounts.  This catches things like trying to shift an i1024 value by an
4281
786k
    // i8, which is easy to fall into in generic code that uses
4282
786k
    // TLI.getShiftAmount().
4283
786k
    assert(N2.getValueSizeInBits() >= Log2_32_Ceil(N1.getValueSizeInBits()) &&
4284
786k
           "Invalid use of small shift amount with oversized value!");
4285
786k
4286
786k
    // Always fold shifts of i1 values so the code generator doesn't need to
4287
786k
    // handle them.  Since we know the size of the shift has to be less than the
4288
786k
    // size of the value, the shift/rotate count is guaranteed to be zero.
4289
786k
    if (VT == MVT::i1)
4290
45
      return N1;
4291
786k
    
if (786k
N2C && 786k
N2C->isNullValue()711k
)
4292
16.1k
      return N1;
4293
770k
    break;
4294
8
  case ISD::FP_ROUND_INREG: {
4295
8
    EVT EVT = cast<VTSDNode>(N2)->getVT();
4296
8
    assert(VT == N1.getValueType() && "Not an inreg round!");
4297
8
    assert(VT.isFloatingPoint() && EVT.isFloatingPoint() &&
4298
8
           "Cannot FP_ROUND_INREG integer types");
4299
8
    assert(EVT.isVector() == VT.isVector() &&
4300
8
           "FP_ROUND_INREG type should be vector iff the operand "
4301
8
           "type is vector!");
4302
8
    assert((!EVT.isVector() ||
4303
8
            EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
4304
8
           "Vector element counts must match in FP_ROUND_INREG");
4305
8
    assert(EVT.bitsLE(VT) && "Not rounding down!");
4306
8
    (void)EVT;
4307
8
    if (
cast<VTSDNode>(N2)->getVT() == VT8
)
return N10
; // Not actually rounding.
4308
8
    break;
4309
8
  }
4310
13.1k
  case ISD::FP_ROUND:
4311
13.1k
    assert(VT.isFloatingPoint() &&
4312
13.1k
           N1.getValueType().isFloatingPoint() &&
4313
13.1k
           VT.bitsLE(N1.getValueType()) &&
4314
13.1k
           N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
4315
13.1k
           "Invalid FP_ROUND!");
4316
13.1k
    if (
N1.getValueType() == VT13.1k
)
return N177
; // noop conversion.
4317
13.0k
    break;
4318
1.29M
  case ISD::AssertSext:
4319
1.29M
  case ISD::AssertZext: {
4320
1.29M
    EVT EVT = cast<VTSDNode>(N2)->getVT();
4321
1.29M
    assert(VT == N1.getValueType() && "Not an inreg extend!");
4322
1.29M
    assert(VT.isInteger() && EVT.isInteger() &&
4323
1.29M
           "Cannot *_EXTEND_INREG FP types");
4324
1.29M
    assert(!EVT.isVector() &&
4325
1.29M
           "AssertSExt/AssertZExt type should be the vector element type "
4326
1.29M
           "rather than the vector type!");
4327
1.29M
    assert(EVT.bitsLE(VT) && "Not extending!");
4328
1.29M
    if (
VT == EVT1.29M
)
return N1703k
; // noop assertion.
4329
590k
    break;
4330
590k
  }
4331
65.1k
  case ISD::SIGN_EXTEND_INREG: {
4332
65.1k
    EVT EVT = cast<VTSDNode>(N2)->getVT();
4333
65.1k
    assert(VT == N1.getValueType() && "Not an inreg extend!");
4334
65.1k
    assert(VT.isInteger() && EVT.isInteger() &&
4335
65.1k
           "Cannot *_EXTEND_INREG FP types");
4336
65.1k
    assert(EVT.isVector() == VT.isVector() &&
4337
65.1k
           "SIGN_EXTEND_INREG type should be vector iff the operand "
4338
65.1k
           "type is vector!");
4339
65.1k
    assert((!EVT.isVector() ||
4340
65.1k
            EVT.getVectorNumElements() == VT.getVectorNumElements()) &&
4341
65.1k
           "Vector element counts must match in SIGN_EXTEND_INREG");
4342
65.1k
    assert(EVT.bitsLE(VT) && "Not extending!");
4343
65.1k
    if (
EVT == VT65.1k
)
return N190
; // Not actually extending
4344
65.0k
4345
65.0k
    
auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) 65.0k
{
4346
11.7k
      unsigned FromBits = EVT.getScalarSizeInBits();
4347
11.7k
      Val <<= Val.getBitWidth() - FromBits;
4348
11.7k
      Val.ashrInPlace(Val.getBitWidth() - FromBits);
4349
11.7k
      return getConstant(Val, DL, ConstantVT);
4350
11.7k
    };
4351
65.0k
4352
65.0k
    if (
N1C65.0k
) {
4353
10.2k
      const APInt &Val = N1C->getAPIntValue();
4354
10.2k
      return SignExtendInReg(Val, VT);
4355
10.2k
    }
4356
54.7k
    
if (54.7k
ISD::isBuildVectorOfConstantSDNodes(N1.getNode())54.7k
) {
4357
347
      SmallVector<SDValue, 8> Ops;
4358
347
      llvm::EVT OpVT = N1.getOperand(0).getValueType();
4359
1.92k
      for (int i = 0, e = VT.getVectorNumElements(); 
i != e1.92k
;
++i1.58k
) {
4360
1.58k
        SDValue Op = N1.getOperand(i);
4361
1.58k
        if (
Op.isUndef()1.58k
) {
4362
42
          Ops.push_back(getUNDEF(OpVT));
4363
42
          continue;
4364
42
        }
4365
1.54k
        ConstantSDNode *C = cast<ConstantSDNode>(Op);
4366
1.54k
        APInt Val = C->getAPIntValue();
4367
1.54k
        Ops.push_back(SignExtendInReg(Val, OpVT));
4368
1.54k
      }
4369
347
      return getBuildVector(VT, DL, Ops);
4370
347
    }
4371
54.4k
    break;
4372
54.4k
  }
4373
516k
  case ISD::EXTRACT_VECTOR_ELT:
4374
516k
    // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF.
4375
516k
    if (N1.isUndef())
4376
14.3k
      return getUNDEF(VT);
4377
501k
4378
501k
    // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF
4379
501k
    
if (501k
N2C && 501k
N2C->getZExtValue() >= N1.getValueType().getVectorNumElements()500k
)
4380
84
      return getUNDEF(VT);
4381
501k
4382
501k
    // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
4383
501k
    // expanding copies of large vectors from registers.
4384
501k
    
if (501k
N2C &&
4385
500k
        N1.getOpcode() == ISD::CONCAT_VECTORS &&
4386
501k
        
N1.getNumOperands() > 03.79k
) {
4387
3.79k
      unsigned Factor =
4388
3.79k
        N1.getOperand(0).getValueType().getVectorNumElements();
4389
3.79k
      return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
4390
3.79k
                     N1.getOperand(N2C->getZExtValue() / Factor),
4391
3.79k
                     getConstant(N2C->getZExtValue() % Factor, DL,
4392
3.79k
                                 N2.getValueType()));
4393
3.79k
    }
4394
497k
4395
497k
    // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is
4396
497k
    // expanding large vector constants.
4397
497k
    
if (497k
N2C && 497k
N1.getOpcode() == ISD::BUILD_VECTOR496k
) {
4398
125k
      SDValue Elt = N1.getOperand(N2C->getZExtValue());
4399
125k
4400
125k
      if (VT != Elt.getValueType())
4401
125k
        // If the vector element type is not legal, the BUILD_VECTOR operands
4402
125k
        // are promoted and implicitly truncated, and the result implicitly
4403
125k
        // extended. Make that explicit here.
4404
6.36k
        Elt = getAnyExtOrTrunc(Elt, DL, VT);
4405
125k
4406
125k
      return Elt;
4407
125k
    }
4408
372k
4409
372k
    // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
4410
372k
    // operations are lowered to scalars.
4411
372k
    
if (372k
N1.getOpcode() == ISD::INSERT_VECTOR_ELT372k
) {
4412
4.13k
      // If the indices are the same, return the inserted element else
4413
4.13k
      // if the indices are known different, extract the element from
4414
4.13k
      // the original vector.
4415
4.13k
      SDValue N1Op2 = N1.getOperand(2);
4416
4.13k
      ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
4417
4.13k
4418
4.13k
      if (
N1Op2C && 4.13k
N2C3.02k
) {
4419
3.01k
        if (
N1Op2C->getZExtValue() == N2C->getZExtValue()3.01k
) {
4420
504
          if (VT == N1.getOperand(1).getValueType())
4421
487
            return N1.getOperand(1);
4422
504
          else
4423
17
            return getSExtOrTrunc(N1.getOperand(1), DL, VT);
4424
2.51k
        }
4425
2.51k
4426
2.51k
        return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
4427
2.51k
      }
4428
4.13k
    }
4429
369k
4430
369k
    // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
4431
369k
    // when vector types are scalarized and v1iX is legal.
4432
369k
    // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx)
4433
369k
    
if (369k
N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
4434
369k
        
N1.getValueType().getVectorNumElements() == 1106k
) {
4435
26
      return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
4436
26
                     N1.getOperand(1));
4437
26
    }
4438
369k
    break;
4439
61.3k
  case ISD::EXTRACT_ELEMENT:
4440
61.3k
    assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
4441
61.3k
    assert(!N1.getValueType().isVector() && !VT.isVector() &&
4442
61.3k
           (N1.getValueType().isInteger() == VT.isInteger()) &&
4443
61.3k
           N1.getValueType() != VT &&
4444
61.3k
           "Wrong types for EXTRACT_ELEMENT!");
4445
61.3k
4446
61.3k
    // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
4447
61.3k
    // 64-bit integers into 32-bit parts.  Instead of building the extract of
4448
61.3k
    // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
4449
61.3k
    if (N1.getOpcode() == ISD::BUILD_PAIR)
4450
12.3k
      return N1.getOperand(N2C->getZExtValue());
4451
48.9k
4452
48.9k
    // EXTRACT_ELEMENT of a constant int is also very common.
4453
48.9k
    
if (48.9k
N1C48.9k
) {
4454
11.0k
      unsigned ElementSize = VT.getSizeInBits();
4455
11.0k
      unsigned Shift = ElementSize * N2C->getZExtValue();
4456
11.0k
      APInt ShiftedVal = N1C->getAPIntValue().lshr(Shift);
4457
11.0k
      return getConstant(ShiftedVal.trunc(ElementSize), DL, VT);
4458
11.0k
    }
4459
37.9k
    break;
4460
328k
  case ISD::EXTRACT_SUBVECTOR:
4461
328k
    if (
VT.isSimple() && 328k
N1.getValueType().isSimple()327k
) {
4462
291k
      assert(VT.isVector() && N1.getValueType().isVector() &&
4463
291k
             "Extract subvector VTs must be a vectors!");
4464
291k
      assert(VT.getVectorElementType() ==
4465
291k
             N1.getValueType().getVectorElementType() &&
4466
291k
             "Extract subvector VTs must have the same element type!");
4467
291k
      assert(VT.getSimpleVT() <= N1.getSimpleValueType() &&
4468
291k
             "Extract subvector must be from larger vector to smaller vector!");
4469
291k
4470
291k
      if (
N2C291k
) {
4471
291k
        assert((VT.getVectorNumElements() + N2C->getZExtValue()
4472
291k
                <= N1.getValueType().getVectorNumElements())
4473
291k
               && "Extract subvector overflow!");
4474
291k
      }
4475
291k
4476
291k
      // Trivial extraction.
4477
291k
      if (VT.getSimpleVT() == N1.getSimpleValueType())
4478
37.6k
        return N1;
4479
253k
4480
253k
      // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
4481
253k
      
if (253k
N1.isUndef()253k
)
4482
1.07k
        return getUNDEF(VT);
4483
252k
4484
252k
      // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
4485
252k
      // the concat have the same type as the extract.
4486
252k
      
if (252k
N2C && 252k
N1.getOpcode() == ISD::CONCAT_VECTORS252k
&&
4487
11.2k
          N1.getNumOperands() > 0 &&
4488
252k
          
VT == N1.getOperand(0).getValueType()11.2k
) {
4489
9.45k
        unsigned Factor = VT.getVectorNumElements();
4490
9.45k
        return N1.getOperand(N2C->getZExtValue() / Factor);
4491
9.45k
      }
4492
242k
4493
242k
      // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
4494
242k
      // during shuffle legalization.
4495
242k
      
if (242k
N1.getOpcode() == ISD::INSERT_SUBVECTOR && 242k
N2 == N1.getOperand(2)240
&&
4496
186
          VT == N1.getOperand(1).getValueType())
4497
186
        return N1.getOperand(1);
4498
280k
    }
4499
280k
    break;
4500
16.9M
  }
4501
16.9M
4502
16.9M
  // Perform trivial constant folding.
4503
16.9M
  
if (SDValue 16.9M
SV16.9M
=
4504
16.9M
          FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode()))
4505
434k
    return SV;
4506
16.5M
4507
16.5M
  // Constant fold FP operations.
4508
16.5M
  bool HasFPExceptions = TLI->hasFloatingPointExceptions();
4509
16.5M
  if (
N1CFP16.5M
) {
4510
30.5k
    if (
N2CFP30.5k
) {
4511
10.7k
      APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
4512
10.7k
      APFloat::opStatus s;
4513
10.7k
      switch (Opcode) {
4514
305
      case ISD::FADD:
4515
305
        s = V1.add(V2, APFloat::rmNearestTiesToEven);
4516
305
        if (
!HasFPExceptions || 305
s != APFloat::opInvalidOp305
)
4517
305
          return getConstantFP(V1, DL, VT);
4518
0
        break;
4519
27
      case ISD::FSUB:
4520
27
        s = V1.subtract(V2, APFloat::rmNearestTiesToEven);
4521
27
        if (
!HasFPExceptions || 27
s!=APFloat::opInvalidOp27
)
4522
27
          return getConstantFP(V1, DL, VT);
4523
0
        break;
4524
263
      case ISD::FMUL:
4525
263
        s = V1.multiply(V2, APFloat::rmNearestTiesToEven);
4526
263
        if (
!HasFPExceptions || 263
s!=APFloat::opInvalidOp252
)
4527
263
          return getConstantFP(V1, DL, VT);
4528
0
        break;
4529
39
      case ISD::FDIV:
4530
39
        s = V1.divide(V2, APFloat::rmNearestTiesToEven);
4531
39
        if (
!HasFPExceptions || 39
(s!=APFloat::opInvalidOp &&
4532
39
                                 
s!=APFloat::opDivByZero23
)) {
4533
18
          return getConstantFP(V1, DL, VT);
4534
18
        }
4535
21
        break;
4536
25
      case ISD::FREM :
4537
25
        s = V1.mod(V2);
4538
25
        if (
!HasFPExceptions || 25
(s!=APFloat::opInvalidOp &&
4539
25
                                 
s!=APFloat::opDivByZero11
)) {
4540
11
          return getConstantFP(V1, DL, VT);
4541
11
        }
4542
14
        break;
4543
4
      case ISD::FCOPYSIGN:
4544
4
        V1.copySign(V2);
4545
4
        return getConstantFP(V1, DL, VT);
4546
10.0k
      default: break;
4547
29.9k
      }
4548
29.9k
    }
4549
29.9k
4550
29.9k
    
if (29.9k
Opcode == ISD::FP_ROUND29.9k
) {
4551
4
      APFloat V = N1CFP->getValueAPF();    // make copy
4552
4
      bool ignored;
4553
4
      // This can return overflow, underflow, or inexact; we don't care.
4554
4
      // FIXME need to be more flexible about rounding mode.
4555
4
      (void)V.convert(EVTToAPFloatSemantics(VT),
4556
4
                      APFloat::rmNearestTiesToEven, &ignored);
4557
4
      return getConstantFP(V, DL, VT);
4558
4
    }
4559
16.5M
  }
4560
16.5M
4561
16.5M
  // Canonicalize an UNDEF to the RHS, even over a constant.
4562
16.5M
  
if (16.5M
N1.isUndef()16.5M
) {
4563
5.10k
    if (
TLI->isCommutativeBinOp(Opcode)5.10k
) {
4564
2.85k
      std::swap(N1, N2);
4565
5.10k
    } else {
4566
2.25k
      switch (Opcode) {
4567
129
      case ISD::FP_ROUND_INREG:
4568
129
      case ISD::SIGN_EXTEND_INREG:
4569
129
      case ISD::SUB:
4570
129
      case ISD::FSUB:
4571
129
      case ISD::FDIV:
4572
129
      case ISD::FREM:
4573
129
      case ISD::SRA:
4574
129
        return N1;     // fold op(undef, arg2) -> undef
4575
353
      case ISD::UDIV:
4576
353
      case ISD::SDIV:
4577
353
      case ISD::UREM:
4578
353
      case ISD::SREM:
4579
353
      case ISD::SRL:
4580
353
      case ISD::SHL:
4581
353
        if (!VT.isVector())
4582
270
          return getConstant(0, DL, VT);    // fold op(undef, arg2) -> 0
4583
83
        // For vectors, we can't easily build an all zero vector, just return
4584
83
        // the LHS.
4585
83
        return N2;
4586
2.25k
      }
4587
2.25k
    }
4588
5.10k
  }
4589
16.5M
4590
16.5M
  // Fold a bunch of operators when the RHS is undef.
4591
16.5M
  
if (16.5M
N2.isUndef()16.5M
) {
4592
20.9k
    switch (Opcode) {
4593
480
    case ISD::XOR:
4594
480
      if (N1.isUndef())
4595
480
        // Handle undef ^ undef -> 0 special case. This is a common
4596
480
        // idiom (misuse).
4597
6
        return getConstant(0, DL, VT);
4598
474
      
LLVM_FALLTHROUGH474
;
4599
1.95k
    case ISD::ADD:
4600
1.95k
    case ISD::ADDC:
4601
1.95k
    case ISD::ADDE:
4602
1.95k
    case ISD::SUB:
4603
1.95k
    case ISD::UDIV:
4604
1.95k
    case ISD::SDIV:
4605
1.95k
    case ISD::UREM:
4606
1.95k
    case ISD::SREM:
4607
1.95k
      return N2;       // fold op(arg1, undef) -> undef
4608
1.31k
    case ISD::FADD:
4609
1.31k
    case ISD::FSUB:
4610
1.31k
    case ISD::FMUL:
4611
1.31k
    case ISD::FDIV:
4612
1.31k
    case ISD::FREM:
4613
1.31k
      if (getTarget().Options.UnsafeFPMath)
4614
5
        return N2;
4615
1.31k
      break;
4616
387
    case ISD::MUL:
4617
387
    case ISD::AND:
4618
387
    case ISD::SRL:
4619
387
    case ISD::SHL:
4620
387
      if (!VT.isVector())
4621
258
        return getConstant(0, DL, VT);  // fold op(arg1, undef) -> 0
4622
129
      // For vectors, we can't easily build an all zero vector, just return
4623
129
      // the LHS.
4624
129
      return N1;
4625
81
    case ISD::OR:
4626
81
      if (!VT.isVector())
4627
41
        return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), DL, VT);
4628
40
      // For vectors, we can't easily build an all one vector, just return
4629
40
      // the LHS.
4630
40
      return N1;
4631
6
    case ISD::SRA:
4632
6
      return N1;
4633
16.5M
    }
4634
16.5M
  }
4635
16.5M
4636
16.5M
  // Memoize this node if possible.
4637
16.5M
  SDNode *N;
4638
16.5M
  SDVTList VTs = getVTList(VT);
4639
16.5M
  SDValue Ops[] = {N1, N2};
4640
16.5M
  if (
VT != MVT::Glue16.5M
) {
4641
16.4M
    FoldingSetNodeID ID;
4642
16.4M
    AddNodeIDNode(ID, Opcode, VTs, Ops);
4643
16.4M
    void *IP = nullptr;
4644
16.4M
    if (SDNode *
E16.4M
= FindNodeOrInsertPos(ID, DL, IP)) {
4645
1.29M
      E->intersectFlagsWith(Flags);
4646
1.29M
      return SDValue(E, 0);
4647
1.29M
    }
4648
15.1M
4649
15.1M
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4650
15.1M
    N->setFlags(Flags);
4651
15.1M
    createOperands(N, Ops);
4652
15.1M
    CSEMap.InsertNode(N, IP);
4653
16.5M
  } else {
4654
27.3k
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4655
27.3k
    createOperands(N, Ops);
4656
27.3k
  }
4657
16.5M
4658
15.2M
  InsertNode(N);
4659
15.2M
  SDValue V = SDValue(N, 0);
4660
15.2M
  NewSDValueDbgMsg(V, "Creating new node: ", this);
4661
15.2M
  return V;
4662
21.6M
}
4663
4664
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4665
9.97M
                              SDValue N1, SDValue N2, SDValue N3) {
4666
9.97M
  // Perform various simplifications.
4667
9.97M
  switch (Opcode) {
4668
3.55k
  case ISD::FMA: {
4669
3.55k
    ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
4670
3.55k
    ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
4671
3.55k
    ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
4672
3.55k
    if (
N1CFP && 3.55k
N2CFP55
&&
N3CFP15
) {
4673
9
      APFloat  V1 = N1CFP->getValueAPF();
4674
9
      const APFloat &V2 = N2CFP->getValueAPF();
4675
9
      const APFloat &V3 = N3CFP->getValueAPF();
4676
9
      APFloat::opStatus s =
4677
9
        V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
4678
9
      if (
!TLI->hasFloatingPointExceptions() || 9
s != APFloat::opInvalidOp9
)
4679
9
        return getConstantFP(V1, DL, VT);
4680
3.54k
    }
4681
3.54k
    break;
4682
3.54k
  }
4683
22
  case ISD::CONCAT_VECTORS: {
4684
22
    // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
4685
22
    SDValue Ops[] = {N1, N2, N3};
4686
22
    if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
4687
2
      return V;
4688
20
    break;
4689
20
  }
4690
3.50M
  case ISD::SETCC: {
4691
3.50M
    // Use FoldSetCC to simplify SETCC's.
4692
3.50M
    if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
4693
2.23k
      return V;
4694
3.49M
    // Vector constant folding.
4695
3.49M
    SDValue Ops[] = {N1, N2, N3};
4696
3.49M
    if (SDValue 
V3.49M
= FoldConstantVectorArithmetic(Opcode, DL, VT, Ops)) {
4697
340
      NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
4698
340
      return V;
4699
340
    }
4700
3.49M
    break;
4701
3.49M
  }
4702
164k
  case ISD::SELECT:
4703
164k
    if (ConstantSDNode *
N1C164k
= dyn_cast<ConstantSDNode>(N1)) {
4704
400
     if (N1C->getZExtValue())
4705
138
       return N2;             // select true, X, Y -> X
4706
262
     return N3;             // select false, X, Y -> Y
4707
262
    }
4708
163k
4709
163k
    
if (163k
N2 == N3163k
)
return N2135
; // select C, X, X -> X
4710
163k
    break;
4711
0
  case ISD::VECTOR_SHUFFLE:
4712
0
    llvm_unreachable("should use getVectorShuffle constructor!");
4713
78.4k
  case ISD::INSERT_VECTOR_ELT: {
4714
78.4k
    ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
4715
78.4k
    // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF
4716
78.4k
    if (
N3C && 78.4k
N3C->getZExtValue() >= N1.getValueType().getVectorNumElements()78.1k
)
4717
12
      return getUNDEF(VT);
4718
78.4k
    break;
4719
78.4k
  }
4720
66.3k
  case ISD::INSERT_SUBVECTOR: {
4721
66.3k
    SDValue Index = N3;
4722
66.3k
    if (
VT.isSimple() && 66.3k
N1.getValueType().isSimple()66.3k
4723
66.3k
        && 
N2.getValueType().isSimple()66.3k
) {
4724
66.3k
      assert(VT.isVector() && N1.getValueType().isVector() &&
4725
66.3k
             N2.getValueType().isVector() &&
4726
66.3k
             "Insert subvector VTs must be a vectors");
4727
66.3k
      assert(VT == N1.getValueType() &&
4728
66.3k
             "Dest and insert subvector source types must match!");
4729
66.3k
      assert(N2.getSimpleValueType() <= N1.getSimpleValueType() &&
4730
66.3k
             "Insert subvector must be from smaller vector to larger vector!");
4731
66.3k
      if (
isa<ConstantSDNode>(Index)66.3k
) {
4732
66.3k
        assert((N2.getValueType().getVectorNumElements() +
4733
66.3k
                cast<ConstantSDNode>(Index)->getZExtValue()
4734
66.3k
                <= VT.getVectorNumElements())
4735
66.3k
               && "Insert subvector overflow!");
4736
66.3k
      }
4737
66.3k
4738
66.3k
      // Trivial insertion.
4739
66.3k
      if (VT.getSimpleVT() == N2.getSimpleValueType())
4740
101
        return N2;
4741
66.2k
    }
4742
66.2k
    break;
4743
66.2k
  }
4744
0
  case ISD::BITCAST:
4745
0
    // Fold bit_convert nodes from a type to themselves.
4746
0
    if (N1.getValueType() == VT)
4747
0
      return N1;
4748
0
    break;
4749
9.97M
  }
4750
9.97M
4751
9.97M
  // Memoize node if it doesn't produce a flag.
4752
9.97M
  SDNode *N;
4753
9.97M
  SDVTList VTs = getVTList(VT);
4754
9.97M
  SDValue Ops[] = {N1, N2, N3};
4755
9.97M
  if (
VT != MVT::Glue9.97M
) {
4756
9.97M
    FoldingSetNodeID ID;
4757
9.97M
    AddNodeIDNode(ID, Opcode, VTs, Ops);
4758
9.97M
    void *IP = nullptr;
4759
9.97M
    if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
4760
330k
      return SDValue(E, 0);
4761
9.64M
4762
9.64M
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4763
9.64M
    createOperands(N, Ops);
4764
9.64M
    CSEMap.InsertNode(N, IP);
4765
9.97M
  } else {
4766
2.80k
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4767
2.80k
    createOperands(N, Ops);
4768
2.80k
  }
4769
9.97M
4770
9.64M
  InsertNode(N);
4771
9.64M
  SDValue V = SDValue(N, 0);
4772
9.64M
  NewSDValueDbgMsg(V, "Creating new node: ", this);
4773
9.64M
  return V;
4774
9.97M
}
4775
4776
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4777
1.03M
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
4778
1.03M
  SDValue Ops[] = { N1, N2, N3, N4 };
4779
1.03M
  return getNode(Opcode, DL, VT, Ops);
4780
1.03M
}
4781
4782
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4783
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
4784
2.08M
                              SDValue N5) {
4785
2.08M
  SDValue Ops[] = { N1, N2, N3, N4, N5 };
4786
2.08M
  return getNode(Opcode, DL, VT, Ops);
4787
2.08M
}
4788
4789
/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
4790
/// the incoming stack arguments to be loaded from the stack.
4791
154
SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
4792
154
  SmallVector<SDValue, 8> ArgChains;
4793
154
4794
154
  // Include the original chain at the beginning of the list. When this is
4795
154
  // used by target LowerCall hooks, this helps legalize find the
4796
154
  // CALLSEQ_BEGIN node.
4797
154
  ArgChains.push_back(Chain);
4798
154
4799
154
  // Add a chain value for each stack argument.
4800
154
  for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(),
4801
738
       UE = getEntryNode().getNode()->use_end(); 
U != UE738
;
++U584
)
4802
584
    
if (LoadSDNode *584
L584
= dyn_cast<LoadSDNode>(*U))
4803
153
      
if (FrameIndexSDNode *153
FI153
= dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
4804
102
        
if (102
FI->getIndex() < 0102
)
4805
102
          ArgChains.push_back(SDValue(L, 1));
4806
154
4807
154
  // Build a tokenfactor for all the chains.
4808
154
  return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
4809
154
}
4810
4811
/// getMemsetValue - Vectorized representation of the memset value
4812
/// operand.
4813
static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
4814
16.5k
                              const SDLoc &dl) {
4815
16.5k
  assert(!Value.isUndef());
4816
16.5k
4817
16.5k
  unsigned NumBits = VT.getScalarSizeInBits();
4818
16.5k
  if (ConstantSDNode *
C16.5k
= dyn_cast<ConstantSDNode>(Value)) {
4819
16.4k
    assert(C->getAPIntValue().getBitWidth() == 8);
4820
16.4k
    APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
4821
16.4k
    if (VT.isInteger())
4822
16.4k
      return DAG.getConstant(Val, dl, VT);
4823
31
    return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
4824
31
                             VT);
4825
31
  }
4826
110
4827
16.5k
  assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
4828
110
  EVT IntVT = VT.getScalarType();
4829
110
  if (!IntVT.isInteger())
4830
1
    IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
4831
110
4832
110
  Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
4833
110
  if (
NumBits > 8110
) {
4834
94
    // Use a multiplication with 0x010101... to extend the input to the
4835
94
    // required length.
4836
94
    APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
4837
94
    Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
4838
94
                        DAG.getConstant(Magic, dl, IntVT));
4839
94
  }
4840
110
4841
110
  if (
VT != Value.getValueType() && 110
!VT.isInteger()18
)
4842
1
    Value = DAG.getBitcast(VT.getScalarType(), Value);
4843
110
  if (VT != Value.getValueType())
4844
18
    Value = DAG.getSplatBuildVector(VT, dl, Value);
4845
16.5k
4846
16.5k
  return Value;
4847
16.5k
}
4848
4849
/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
4850
/// used when a memcpy is turned into a memset when the source is a constant
4851
/// string ptr.
4852
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
4853
                                  const TargetLowering &TLI,
4854
753
                                  const ConstantDataArraySlice &Slice) {
4855
753
  // Handle vector with all elements zero.
4856
753
  if (
Slice.Array == nullptr753
) {
4857
70
    if (VT.isInteger())
4858
64
      return DAG.getConstant(0, dl, VT);
4859
6
    else 
if (6
VT == MVT::f32 || 6
VT == MVT::f646
||
VT == MVT::f1283
)
4860
6
      return DAG.getConstantFP(0.0, dl, VT);
4861
0
    else 
if (0
VT.isVector()0
) {
4862
0
      unsigned NumElts = VT.getVectorNumElements();
4863
0
      MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? 
MVT::i320
:
MVT::i640
;
4864
0
      return DAG.getNode(ISD::BITCAST, dl, VT,
4865
0
                         DAG.getConstant(0, dl,
4866
0
                                         EVT::getVectorVT(*DAG.getContext(),
4867
0
                                                          EltVT, NumElts)));
4868
0
    } else
4869
0
      llvm_unreachable("Expected type!");
4870
70
  }
4871
753
4872
683
  assert(!VT.isVector() && "Can't handle vector type here!");
4873
683
  unsigned NumVTBits = VT.getSizeInBits();
4874
683
  unsigned NumVTBytes = NumVTBits / 8;
4875
683
  unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
4876
683
4877
683
  APInt Val(NumVTBits, 0);
4878
683
  if (
DAG.getDataLayout().isLittleEndian()683
) {
4879
3.15k
    for (unsigned i = 0; 
i != NumBytes3.15k
;
++i2.47k
)
4880
2.47k
      Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
4881
0
  } else {
4882
0
    for (unsigned i = 0; 
i != NumBytes0
;
++i0
)
4883
0
      Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
4884
0
  }
4885
683
4886
683
  // If the "cost" of materializing the integer immediate is less than the cost
4887
683
  // of a load, then it is cost effective to turn the load into the immediate.
4888
683
  Type *Ty = VT.getTypeForEVT(*DAG.getContext());
4889
683
  if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
4890
562
    return DAG.getConstant(Val, dl, VT);
4891
121
  return SDValue(nullptr, 0);
4892
121
}
4893
4894
SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, unsigned Offset,
4895
85.6k
                                           const SDLoc &DL) {
4896
85.6k
  EVT VT = Base.getValueType();
4897
85.6k
  return getNode(ISD::ADD, DL, VT, Base, getConstant(Offset, DL, VT));
4898
85.6k
}
4899
4900
/// Returns true if memcpy source is constant data.
4901
13.5k
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
4902
13.5k
  uint64_t SrcDelta = 0;
4903
13.5k
  GlobalAddressSDNode *G = nullptr;
4904
13.5k
  if (Src.getOpcode() == ISD::GlobalAddress)
4905
976
    G = cast<GlobalAddressSDNode>(Src);
4906
12.6k
  else 
if (12.6k
Src.getOpcode() == ISD::ADD &&
4907
1.69k
           Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
4908
12.6k
           
Src.getOperand(1).getOpcode() == ISD::Constant201
) {
4909
187
    G = cast<GlobalAddressSDNode>(Src.getOperand(0));
4910
187
    SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue();
4911
187
  }
4912
13.5k
  if (!G)
4913
12.4k
    return false;
4914
1.16k
4915
1.16k
  return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
4916
1.16k
                                  SrcDelta + G->getOffset());
4917
1.16k
}
4918
4919
/// Determines the optimal series of memory ops to replace the memset / memcpy.
4920
/// Return true if the number of memory ops is below the threshold (Limit).
4921
/// It returns the types of the sequence of memory ops to perform
4922
/// memset / memcpy by reference.
4923
static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
4924
                                     unsigned Limit, uint64_t Size,
4925
                                     unsigned DstAlign, unsigned SrcAlign,
4926
                                     bool IsMemset,
4927
                                     bool ZeroMemset,
4928
                                     bool MemcpyStrSrc,
4929
                                     bool AllowOverlap,
4930
                                     unsigned DstAS, unsigned SrcAS,
4931
                                     SelectionDAG &DAG,
4932
35.3k
                                     const TargetLowering &TLI) {
4933
35.3k
  assert((SrcAlign == 0 || SrcAlign >= DstAlign) &&
4934
35.3k
         "Expecting memcpy / memset source to meet alignment requirement!");
4935
35.3k
  // If 'SrcAlign' is zero, that means the memory operation does not need to
4936
35.3k
  // load the value, i.e. memset or memcpy from constant string. Otherwise,
4937
35.3k
  // it's the inferred alignment of the source. 'DstAlign', on the other hand,
4938
35.3k
  // is the specified alignment of the memory operation. If it is zero, that
4939
35.3k
  // means it's possible to change the alignment of the destination.
4940
35.3k
  // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does
4941
35.3k
  // not need to be loaded.
4942
35.3k
  EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign,
4943
35.3k
                                   IsMemset, ZeroMemset, MemcpyStrSrc,
4944
35.3k
                                   DAG.getMachineFunction());
4945
35.3k
4946
35.3k
  if (
VT == MVT::Other35.3k
) {
4947
637
    // Use the largest integer type whose alignment constraints are satisfied.
4948
637
    // We only need to check DstAlign here as SrcAlign is always greater or
4949
637
    // equal to DstAlign (or zero).
4950
637
    VT = MVT::i64;
4951
1.02k
    while (
DstAlign && 1.02k
DstAlign < VT.getSizeInBits() / 81.00k
&&
4952
803
           !TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign))
4953
392
      VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
4954
637
    assert(VT.isInteger());
4955
637
4956
637
    // Find the largest legal integer type.
4957
637
    MVT LVT = MVT::i64;
4958
1.03k
    while (!TLI.isTypeLegal(LVT))
4959
398
      LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
4960
637
    assert(LVT.isInteger());
4961
637
4962
637
    // If the type we've chosen is larger than the largest legal integer type
4963
637
    // then use that instead.
4964
637
    if (VT.bitsGT(LVT))
4965
63
      VT = LVT;
4966
637
  }
4967
35.3k
4968
35.3k
  unsigned NumMemOps = 0;
4969
176k
  while (
Size != 0176k
) {
4970
153k
    unsigned VTSize = VT.getSizeInBits() / 8;
4971
165k
    while (
VTSize > Size165k
) {
4972
11.8k
      // For now, only use non-vector load / store's for the left-over pieces.
4973
11.8k
      EVT NewVT = VT;
4974
11.8k
      unsigned NewVTSize;
4975
11.8k
4976
11.8k
      bool Found = false;
4977
11.8k
      if (
VT.isVector() || 11.8k
VT.isFloatingPoint()11.7k
) {
4978
6.33k
        NewVT = (VT.getSizeInBits() > 64) ? 
MVT::i646.29k
:
MVT::i3244
;
4979
6.33k
        if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) &&
4980
6.26k
            TLI.isSafeMemOpType(NewVT.getSimpleVT()))
4981
6.26k
          Found = true;
4982
71
        else 
if (71
NewVT == MVT::i64 &&
4983
71
                 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
4984
71
                 
TLI.isSafeMemOpType(MVT::f64)13
) {
4985
12
          // i64 is usually not legal on 32-bit targets, but f64 may be.
4986
12
          NewVT = MVT::f64;
4987
12
          Found = true;
4988
12
        }
4989
6.33k
      }
4990
11.8k
4991
11.8k
      if (
!Found11.8k
) {
4992
5.56k
        do {
4993
5.56k
          NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
4994
5.56k
          if (NewVT == MVT::i8)
4995
399
            break;
4996
5.16k
        } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT()));
4997
5.56k
      }
4998
11.4k
      NewVTSize = NewVT.getSizeInBits() / 8;
4999
11.4k
5000
11.4k
      // If the new VT cannot cover all of the remaining bits, then consider
5001
11.4k
      // issuing a (or a pair of) unaligned and overlapping load / store.
5002
11.4k
      // FIXME: Only does this for 64-bit or more since we don't have proper
5003
11.4k
      // cost model for unaligned load / store.
5004
11.4k
      bool Fast;
5005
11.4k
      if (
NumMemOps && 11.4k
AllowOverlap11.4k
&&
5006
11.4k
          
VTSize >= 811.3k
&&
NewVTSize < Size10.4k
&&
5007
11.4k
          
TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, &Fast)225
&&
Fast221
)
5008
207
        VTSize = Size;
5009
11.2k
      else {
5010
11.2k
        VT = NewVT;
5011
11.2k
        VTSize = NewVTSize;
5012
11.2k
      }
5013
11.8k
    }
5014
153k
5015
153k
    
if (153k
++NumMemOps > Limit153k
)
5016
11.9k
      return false;
5017
141k
5018
141k
    MemOps.push_back(VT);
5019
141k
    Size -= VTSize;
5020
141k
  }
5021
35.3k
5022
22.9k
  return true;
5023
35.3k
}
5024
5025
35.3k
static bool shouldLowerMemFuncForSize(const MachineFunction &MF) {
5026
35.3k
  // On Darwin, -Os means optimize for size without hurting performance, so
5027
35.3k
  // only really optimize for size when -Oz (MinSize) is used.
5028
35.3k
  if (MF.getTarget().getTargetTriple().isOSDarwin())
5029
34.3k
    return MF.getFunction()->optForMinSize();
5030
973
  return MF.getFunction()->optForSize();
5031
973
}
5032
5033
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
5034
                                       SDValue Chain, SDValue Dst, SDValue Src,
5035
                                       uint64_t Size, unsigned Align,
5036
                                       bool isVol, bool AlwaysInline,
5037
                                       MachinePointerInfo DstPtrInfo,
5038
13.5k
                                       MachinePointerInfo SrcPtrInfo) {
5039
13.5k
  // Turn a memcpy of undef to nop.
5040
13.5k
  if (Src.isUndef())
5041
3
    return Chain;
5042
13.5k
5043
13.5k
  // Expand memcpy to a series of load and store ops if the size operand falls
5044
13.5k
  // below a certain threshold.
5045
13.5k
  // TODO: In the AlwaysInline case, if the size is big then generate a loop
5046
13.5k
  // rather than maybe a humongous number of loads and stores.
5047
13.5k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5048
13.5k
  const DataLayout &DL = DAG.getDataLayout();
5049
13.5k
  LLVMContext &C = *DAG.getContext();
5050
13.5k
  std::vector<EVT> MemOps;
5051
13.5k
  bool DstAlignCanChange = false;
5052
13.5k
  MachineFunction &MF = DAG.getMachineFunction();
5053
13.5k
  MachineFrameInfo &MFI = MF.getFrameInfo();
5054
13.5k
  bool OptSize = shouldLowerMemFuncForSize(MF);
5055
13.5k
  FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
5056
13.5k
  if (
FI && 13.5k
!MFI.isFixedObjectIndex(FI->getIndex())1.26k
)
5057
1.26k
    DstAlignCanChange = true;
5058
13.5k
  unsigned SrcAlign = DAG.InferPtrAlignment(Src);
5059
13.5k
  if (Align > SrcAlign)
5060
8.86k
    SrcAlign = Align;
5061
13.5k
  ConstantDataArraySlice Slice;
5062
13.5k
  bool CopyFromConstant = isMemSrcFromConstant(Src, Slice);
5063
462
  bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
5064
13.5k
  unsigned Limit = AlwaysInline ? 
~0U3
:
TLI.getMaxStoresPerMemcpy(OptSize)13.5k
;
5065
13.5k
5066
13.5k
  if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
5067
13.5k
                                (DstAlignCanChange ? 
01.26k
:
Align12.3k
),
5068
13.5k
                                (isZeroConstant ? 
024
:
SrcAlign13.5k
),
5069
13.5k
                                false, false, CopyFromConstant, true,
5070
13.5k
                                DstPtrInfo.getAddrSpace(),
5071
13.5k
                                SrcPtrInfo.getAddrSpace(),
5072
13.5k
                                DAG, TLI))
5073
6.84k
    return SDValue();
5074
6.74k
5075
6.74k
  
if (6.74k
DstAlignCanChange6.74k
) {
5076
860
    Type *Ty = MemOps[0].getTypeForEVT(C);
5077
860
    unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
5078
860
5079
860
    // Don't promote to an alignment that would require dynamic stack
5080
860
    // realignment.
5081
860
    const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
5082
860
    if (!TRI->needsStackRealignment(MF))
5083
837
      
while (837
NewAlign > Align &&
5084
734
             DL.exceedsNaturalStackAlignment(NewAlign))
5085
9
          NewAlign /= 2;
5086
860
5087
860
    if (
NewAlign > Align860
) {
5088
730
      // Give the stack frame object a larger alignment if needed.
5089
730
      if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
5090
560
        MFI.setObjectAlignment(FI->getIndex(), NewAlign);
5091
730
      Align = NewAlign;
5092
730
    }
5093
860
  }
5094
6.74k
5095
6.74k
  MachineMemOperand::Flags MMOFlags =
5096
6.74k
      isVol ? 
MachineMemOperand::MOVolatile1
:
MachineMemOperand::MONone6.74k
;
5097
6.74k
  SmallVector<SDValue, 8> OutChains;
5098
6.74k
  unsigned NumMemOps = MemOps.size();
5099
6.74k
  uint64_t SrcOff = 0, DstOff = 0;
5100
17.3k
  for (unsigned i = 0; 
i != NumMemOps17.3k
;
++i10.5k
) {
5101
10.5k
    EVT VT = MemOps[i];
5102
10.5k
    unsigned VTSize = VT.getSizeInBits() / 8;
5103
10.5k
    SDValue Value, Store;
5104
10.5k
5105
10.5k
    if (
VTSize > Size10.5k
) {
5106
174
      // Issuing an unaligned load / store pair  that overlaps with the previous
5107
174
      // pair. Adjust the offset accordingly.
5108
174
      assert(i == NumMemOps-1 && i != 0);
5109
174
      SrcOff -= VTSize - Size;
5110
174
      DstOff -= VTSize - Size;
5111
174
    }
5112
10.5k
5113
10.5k
    if (CopyFromConstant &&
5114
10.5k
        
(isZeroConstant || 918
(VT.isInteger() && 890
!VT.isVector()740
))) {
5115
753
      // It's unlikely a store of a vector immediate can be done in a single
5116
753
      // instruction. It would require a load from a constantpool first.
5117
753
      // We only handle zero vectors here.
5118
753
      // FIXME: Handle other cases where store of vector immediate is done in
5119
753
      // a single instruction.
5120
753
      ConstantDataArraySlice SubSlice;
5121
753
      if (
SrcOff < Slice.Length753
) {
5122
709
        SubSlice = Slice;
5123
709
        SubSlice.move(SrcOff);
5124
753
      } else {
5125
44
        // This is an out-of-bounds access and hence UB. Pretend we read zero.
5126
44
        SubSlice.Array = nullptr;
5127
44
        SubSlice.Offset = 0;
5128
44
        SubSlice.Length = VTSize;
5129
44
      }
5130
753
      Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
5131
753
      if (Value.getNode())
5132
632
        Store = DAG.getStore(Chain, dl, Value,
5133
632
                             DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5134
632
                             DstPtrInfo.getWithOffset(DstOff), Align,
5135
632
                             MMOFlags);
5136
753
    }
5137
10.5k
5138
10.5k
    if (
!Store.getNode()10.5k
) {
5139
9.94k
      // The type might not be legal for the target.  This should only happen
5140
9.94k
      // if the type is smaller than a legal type, as on PPC, so the right
5141
9.94k
      // thing to do is generate a LoadExt/StoreTrunc pair.  These simplify
5142
9.94k
      // to Load/Store if NVT==VT.
5143
9.94k
      // FIXME does the case above also need this?
5144
9.94k
      EVT NVT = TLI.getTypeToTransformTo(C, VT);
5145
9.94k
      assert(NVT.bitsGE(VT));
5146
9.94k
5147
9.94k
      bool isDereferenceable =
5148
9.94k
        SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
5149
9.94k
      MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
5150
9.94k
      if (isDereferenceable)
5151
5.85k
        SrcMMOFlags |= MachineMemOperand::MODereferenceable;
5152
9.94k
5153
9.94k
      Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain,
5154
9.94k
                             DAG.getMemBasePlusOffset(Src, SrcOff, dl),
5155
9.94k
                             SrcPtrInfo.getWithOffset(SrcOff), VT,
5156
9.94k
                             MinAlign(SrcAlign, SrcOff), SrcMMOFlags);
5157
9.94k
      OutChains.push_back(Value.getValue(1));
5158
9.94k
      Store = DAG.getTruncStore(
5159
9.94k
          Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5160
9.94k
          DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags);
5161
9.94k
    }
5162
10.5k
    OutChains.push_back(Store);
5163
10.5k
    SrcOff += VTSize;
5164
10.5k
    DstOff += VTSize;
5165
10.5k
    Size -= VTSize;
5166
10.5k
  }
5167
13.5k
5168
13.5k
  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
5169
13.5k
}
5170
5171
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
5172
                                        SDValue Chain, SDValue Dst, SDValue Src,
5173
                                        uint64_t Size, unsigned Align,
5174
                                        bool isVol, bool AlwaysInline,
5175
                                        MachinePointerInfo DstPtrInfo,
5176
147
                                        MachinePointerInfo SrcPtrInfo) {
5177
147
  // Turn a memmove of undef to nop.
5178
147
  if (Src.isUndef())
5179
0
    return Chain;
5180
147
5181
147
  // Expand memmove to a series of load and store ops if the size operand falls
5182
147
  // below a certain threshold.
5183
147
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5184
147
  const DataLayout &DL = DAG.getDataLayout();
5185
147
  LLVMContext &C = *DAG.getContext();
5186
147
  std::vector<EVT> MemOps;
5187
147
  bool DstAlignCanChange = false;
5188
147
  MachineFunction &MF = DAG.getMachineFunction();
5189
147
  MachineFrameInfo &MFI = MF.getFrameInfo();
5190
147
  bool OptSize = shouldLowerMemFuncForSize(MF);
5191
147
  FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
5192
147
  if (
FI && 147
!MFI.isFixedObjectIndex(FI->getIndex())1
)
5193
1
    DstAlignCanChange = true;
5194
147
  unsigned SrcAlign = DAG.InferPtrAlignment(Src);
5195
147
  if (Align > SrcAlign)
5196
134
    SrcAlign = Align;
5197
147
  unsigned Limit = AlwaysInline ? 
~0U0
:
TLI.getMaxStoresPerMemmove(OptSize)147
;
5198
147
5199
147
  if (!FindOptimalMemOpLowering(MemOps, Limit, Size,
5200
147
                                (DstAlignCanChange ? 
01
:
Align146
), SrcAlign,
5201
147
                                false, false, false, false,
5202
147
                                DstPtrInfo.getAddrSpace(),
5203
147
                                SrcPtrInfo.getAddrSpace(),
5204
147
                                DAG, TLI))
5205
57
    return SDValue();
5206
90
5207
90
  
if (90
DstAlignCanChange90
) {
5208
1
    Type *Ty = MemOps[0].getTypeForEVT(C);
5209
1
    unsigned NewAlign = (unsigned)DL.getABITypeAlignment(Ty);
5210
1
    if (
NewAlign > Align1
) {
5211
1
      // Give the stack frame object a larger alignment if needed.
5212
1
      if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
5213
0
        MFI.setObjectAlignment(FI->getIndex(), NewAlign);
5214
1
      Align = NewAlign;
5215
1
    }
5216
1
  }
5217
90
5218
90
  MachineMemOperand::Flags MMOFlags =
5219
90
      isVol ? 
MachineMemOperand::MOVolatile0
:
MachineMemOperand::MONone90
;
5220
90
  uint64_t SrcOff = 0, DstOff = 0;
5221
90
  SmallVector<SDValue, 8> LoadValues;
5222
90
  SmallVector<SDValue, 8> LoadChains;
5223
90
  SmallVector<SDValue, 8> OutChains;
5224
90
  unsigned NumMemOps = MemOps.size();
5225
226
  for (unsigned i = 0; 
i < NumMemOps226
;
i++136
) {
5226
136
    EVT VT = MemOps[i];
5227
136
    unsigned VTSize = VT.getSizeInBits() / 8;
5228
136
    SDValue Value;
5229
136
5230
136
    bool isDereferenceable =
5231
136
      SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
5232
136
    MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
5233
136
    if (isDereferenceable)
5234
31
      SrcMMOFlags |= MachineMemOperand::MODereferenceable;
5235
136
5236
136
    Value =
5237
136
        DAG.getLoad(VT, dl, Chain, DAG.getMemBasePlusOffset(Src, SrcOff, dl),
5238
136
                    SrcPtrInfo.getWithOffset(SrcOff), SrcAlign, SrcMMOFlags);
5239
136
    LoadValues.push_back(Value);
5240
136
    LoadChains.push_back(Value.getValue(1));
5241
136
    SrcOff += VTSize;
5242
136
  }
5243
90
  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
5244
90
  OutChains.clear();
5245
226
  for (unsigned i = 0; 
i < NumMemOps226
;
i++136
) {
5246
136
    EVT VT = MemOps[i];
5247
136
    unsigned VTSize = VT.getSizeInBits() / 8;
5248
136
    SDValue Store;
5249
136
5250
136
    Store = DAG.getStore(Chain, dl, LoadValues[i],
5251
136
                         DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5252
136
                         DstPtrInfo.getWithOffset(DstOff), Align, MMOFlags);
5253
136
    OutChains.push_back(Store);
5254
136
    DstOff += VTSize;
5255
136
  }
5256
147
5257
147
  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
5258
147
}
5259
5260
/// \brief Lower the call to 'memset' intrinsic function into a series of store
5261
/// operations.
5262
///
5263
/// \param DAG Selection DAG where lowered code is placed.
5264
/// \param dl Link to corresponding IR location.
5265
/// \param Chain Control flow dependency.
5266
/// \param Dst Pointer to destination memory location.
5267
/// \param Src Value of byte to write into the memory.
5268
/// \param Size Number of bytes to write.
5269
/// \param Align Alignment of the destination in bytes.
5270
/// \param isVol True if destination is volatile.
5271
/// \param DstPtrInfo IR information on the memory pointer.
5272
/// \returns New head in the control flow, if lowering was successful, empty
5273
/// SDValue otherwise.
5274
///
5275
/// The function tries to replace 'llvm.memset' intrinsic with several store
5276
/// operations and value calculation code. This is usually profitable for small
5277
/// memory size.
5278
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
5279
                               SDValue Chain, SDValue Dst, SDValue Src,
5280
                               uint64_t Size, unsigned Align, bool isVol,
5281
21.5k
                               MachinePointerInfo DstPtrInfo) {
5282
21.5k
  // Turn a memset of undef to nop.
5283
21.5k
  if (Src.isUndef())
5284
1
    return Chain;
5285
21.5k
5286
21.5k
  // Expand memset to a series of load/store ops if the size operand
5287
21.5k
  // falls below a certain threshold.
5288
21.5k
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5289
21.5k
  std::vector<EVT> MemOps;
5290
21.5k
  bool DstAlignCanChange = false;
5291
21.5k
  MachineFunction &MF = DAG.getMachineFunction();
5292
21.5k
  MachineFrameInfo &MFI = MF.getFrameInfo();
5293
21.5k
  bool OptSize = shouldLowerMemFuncForSize(MF);
5294
21.5k
  FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
5295
21.5k
  if (
FI && 21.5k
!MFI.isFixedObjectIndex(FI->getIndex())3.62k
)
5296
3.62k
    DstAlignCanChange = true;
5297
21.5k
  bool IsZeroVal =
5298
21.4k
    isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue();
5299
21.5k
  if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize),
5300
21.5k
                                Size, (DstAlignCanChange ? 
03.62k
:
Align17.9k
), 0,
5301
21.5k
                                true, IsZeroVal, false, true,
5302
21.5k
                                DstPtrInfo.getAddrSpace(), ~0u,
5303
21.5k
                                DAG, TLI))
5304
5.03k
    return SDValue();
5305
16.5k
5306
16.5k
  
if (16.5k
DstAlignCanChange16.5k
) {
5307
3.36k
    Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
5308
3.36k
    unsigned NewAlign = (unsigned)DAG.getDataLayout().getABITypeAlignment(Ty);
5309
3.36k
    if (
NewAlign > Align3.36k
) {
5310
2.90k
      // Give the stack frame object a larger alignment if needed.
5311
2.90k
      if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
5312
2.87k
        MFI.setObjectAlignment(FI->getIndex(), NewAlign);
5313
2.90k
      Align = NewAlign;
5314
2.90k
    }
5315
3.36k
  }
5316
16.5k
5317
16.5k
  SmallVector<SDValue, 8> OutChains;
5318
16.5k
  uint64_t DstOff = 0;
5319
16.5k
  unsigned NumMemOps = MemOps.size();
5320
16.5k
5321
16.5k
  // Find the largest store and generate the bit pattern for it.
5322
16.5k
  EVT LargestVT = MemOps[0];
5323
63.3k
  for (unsigned i = 1; 
i < NumMemOps63.3k
;
i++46.8k
)
5324
46.8k
    
if (46.8k
MemOps[i].bitsGT(LargestVT)46.8k
)
5325
0
      LargestVT = MemOps[i];
5326
16.5k
  SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
5327
16.5k
5328
79.9k
  for (unsigned i = 0; 
i < NumMemOps79.9k
;
i++63.3k
) {
5329
63.3k
    EVT VT = MemOps[i];
5330
63.3k
    unsigned VTSize = VT.getSizeInBits() / 8;
5331
63.3k
    if (
VTSize > Size63.3k
) {
5332
33
      // Issuing an unaligned load / store pair  that overlaps with the previous
5333
33
      // pair. Adjust the offset accordingly.
5334
33
      assert(i == NumMemOps-1 && i != 0);
5335
33
      DstOff -= VTSize - Size;
5336
33
    }
5337
63.3k
5338
63.3k
    // If this store is smaller than the largest store see whether we can get
5339
63.3k
    // the smaller value for free with a truncate.
5340
63.3k
    SDValue Value = MemSetValue;
5341
63.3k
    if (
VT.bitsLT(LargestVT)63.3k
) {
5342
3.85k
      if (
!LargestVT.isVector() && 3.85k
!VT.isVector()3.84k
&&
5343
3.84k
          TLI.isTruncateFree(LargestVT, VT))
5344
3.83k
        Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
5345
3.85k
      else
5346
18
        Value = getMemsetValue(Src, VT, DAG, dl);
5347
3.85k
    }
5348
63.3k
    assert(Value.getValueType() == VT && "Value with wrong type.");
5349
63.3k
    SDValue Store = DAG.getStore(
5350
63.3k
        Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
5351
63.3k
        DstPtrInfo.getWithOffset(DstOff), Align,
5352
63.3k
        isVol ? 
MachineMemOperand::MOVolatile0
:
MachineMemOperand::MONone63.3k
);
5353
63.3k
    OutChains.push_back(Store);
5354
63.3k
    DstOff += VT.getSizeInBits() / 8;
5355
63.3k
    Size -= VTSize;
5356
63.3k
  }
5357
21.5k
5358
21.5k
  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
5359
21.5k
}
5360
5361
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
5362
18.8k
                                            unsigned AS) {
5363
18.8k
  // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
5364
18.8k
  // pointer operands can be losslessly bitcasted to pointers of address space 0
5365
18.8k
  if (
AS != 0 && 18.8k
!TLI->isNoopAddrSpaceCast(AS, 0)0
) {
5366
0
    report_fatal_error("cannot lower memory intrinsic in address space " +
5367
0
                       Twine(AS));
5368
0
  }
5369
18.8k
}
5370
5371
SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
5372
                                SDValue Src, SDValue Size, unsigned Align,
5373
                                bool isVol, bool AlwaysInline, bool isTailCall,
5374
                                MachinePointerInfo DstPtrInfo,
5375
15.4k
                                MachinePointerInfo SrcPtrInfo) {
5376
15.4k
  assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
5377
15.4k
5378
15.4k
  // Check to see if we should lower the memcpy to loads and stores first.
5379
15.4k
  // For cases within the target-specified limits, this is the best choice.
5380
15.4k
  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
5381
15.4k
  if (
ConstantSize15.4k
) {
5382
13.6k
    // Memcpy with size zero? Just return the original chain.
5383
13.6k
    if (ConstantSize->isNullValue())
5384
5
      return Chain;
5385
13.5k
5386
13.5k
    SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
5387
13.5k
                                             ConstantSize->getZExtValue(),Align,
5388
13.5k
                                isVol, false, DstPtrInfo, SrcPtrInfo);
5389
13.5k
    if (Result.getNode())
5390
6.74k
      return Result;
5391
8.65k
  }
5392
8.65k
5393
8.65k
  // Then check to see if we should lower the memcpy with target-specific
5394
8.65k
  // code. If the target chooses to do this, this is the next best.
5395
8.65k
  
if (8.65k
TSI8.65k
) {
5396
8.65k
    SDValue Result = TSI->EmitTargetCodeForMemcpy(
5397
8.65k
        *this, dl, Chain, Dst, Src, Size, Align, isVol, AlwaysInline,
5398
8.65k
        DstPtrInfo, SrcPtrInfo);
5399
8.65k
    if (Result.getNode())
5400
182
      return Result;
5401
8.47k
  }
5402
8.47k
5403
8.47k
  // If we really need inline code and the target declined to provide it,
5404
8.47k
  // use a (potentially long) sequence of loads and stores.
5405
8.47k
  
if (8.47k
AlwaysInline8.47k
) {
5406
3
    assert(ConstantSize && "AlwaysInline requires a constant size!");
5407
3
    return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src,
5408
3
                                   ConstantSize->getZExtValue(), Align, isVol,
5409
3
                                   true, DstPtrInfo, SrcPtrInfo);
5410
3
  }
5411
8.46k
5412
8.46k
  checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
5413
8.46k
  checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
5414
8.46k
5415
8.46k
  // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
5416
8.46k
  // memcpy is not guaranteed to be safe. libc memcpys aren't required to
5417
8.46k
  // respect volatile, so they may do things like read or write memory
5418
8.46k
  // beyond the given memory regions. But fixing this isn't easy, and most
5419
8.46k
  // people don't care.
5420
8.46k
5421
8.46k
  // Emit a library call.
5422
8.46k
  TargetLowering::ArgListTy Args;
5423
8.46k
  TargetLowering::ArgListEntry Entry;
5424
8.46k
  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
5425
8.46k
  Entry.Node = Dst; Args.push_back(Entry);
5426
8.46k
  Entry.Node = Src; Args.push_back(Entry);
5427
8.46k
  Entry.Node = Size; Args.push_back(Entry);
5428
8.46k
  // FIXME: pass in SDLoc
5429
8.46k
  TargetLowering::CallLoweringInfo CLI(*this);
5430
8.46k
  CLI.setDebugLoc(dl)
5431
8.46k
      .setChain(Chain)
5432
8.46k
      .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
5433
8.46k
                    Dst.getValueType().getTypeForEVT(*getContext()),
5434
8.46k
                    getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
5435
8.46k
                                      TLI->getPointerTy(getDataLayout())),
5436
8.46k
                    std::move(Args))
5437
8.46k
      .setDiscardResult()
5438
8.46k
      .setTailCall(isTailCall);
5439
8.46k
5440
8.46k
  std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
5441
8.46k
  return CallResult.second;
5442
8.46k
}
5443
5444
SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
5445
                                 SDValue Src, SDValue Size, unsigned Align,
5446
                                 bool isVol, bool isTailCall,
5447
                                 MachinePointerInfo DstPtrInfo,
5448
442
                                 MachinePointerInfo SrcPtrInfo) {
5449
442
  assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
5450
442
5451
442
  // Check to see if we should lower the memmove to loads and stores first.
5452
442
  // For cases within the target-specified limits, this is the best choice.
5453
442
  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
5454
442
  if (
ConstantSize442
) {
5455
147
    // Memmove with size zero? Just return the original chain.
5456
147
    if (ConstantSize->isNullValue())
5457
0
      return Chain;
5458
147
5459
147
    SDValue Result =
5460
147
      getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src,
5461
147
                               ConstantSize->getZExtValue(), Align, isVol,
5462
147
                               false, DstPtrInfo, SrcPtrInfo);
5463
147
    if (Result.getNode())
5464
90
      return Result;
5465
352
  }
5466
352
5467
352
  // Then check to see if we should lower the memmove with target-specific
5468
352
  // code. If the target chooses to do this, this is the next best.
5469
352
  
if (352
TSI352
) {
5470
352
    SDValue Result = TSI->EmitTargetCodeForMemmove(
5471
352
        *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo, SrcPtrInfo);
5472
352
    if (Result.getNode())
5473
48
      return Result;
5474
304
  }
5475
304
5476
304
  checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
5477
304
  checkAddrSpaceIsValidForLibcall(TLI, SrcPtrInfo.getAddrSpace());
5478
304
5479
304
  // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
5480
304
  // not be safe.  See memcpy above for more details.
5481
304
5482
304
  // Emit a library call.
5483
304
  TargetLowering::ArgListTy Args;
5484
304
  TargetLowering::ArgListEntry Entry;
5485
304
  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
5486
304
  Entry.Node = Dst; Args.push_back(Entry);
5487
304
  Entry.Node = Src; Args.push_back(Entry);
5488
304
  Entry.Node = Size; Args.push_back(Entry);
5489
304
  // FIXME:  pass in SDLoc
5490
304
  TargetLowering::CallLoweringInfo CLI(*this);
5491
304
  CLI.setDebugLoc(dl)
5492
304
      .setChain(Chain)
5493
304
      .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
5494
304
                    Dst.getValueType().getTypeForEVT(*getContext()),
5495
304
                    getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
5496
304
                                      TLI->getPointerTy(getDataLayout())),
5497
304
                    std::move(Args))
5498
304
      .setDiscardResult()
5499
304
      .setTailCall(isTailCall);
5500
304
5501
304
  std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
5502
304
  return CallResult.second;
5503
304
}
5504
5505
SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
5506
                                SDValue Src, SDValue Size, unsigned Align,
5507
                                bool isVol, bool isTailCall,
5508
29.5k
                                MachinePointerInfo DstPtrInfo) {
5509
29.5k
  assert(Align && "The SDAG layer expects explicit alignment and reserves 0");
5510
29.5k
5511
29.5k
  // Check to see if we should lower the memset to stores first.
5512
29.5k
  // For cases within the target-specified limits, this is the best choice.
5513
29.5k
  ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
5514
29.5k
  if (
ConstantSize29.5k
) {
5515
21.5k
    // Memset with size zero? Just return the original chain.
5516
21.5k
    if (ConstantSize->isNullValue())
5517
8
      return Chain;
5518
21.5k
5519
21.5k
    SDValue Result =
5520
21.5k
      getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(),
5521
21.5k
                      Align, isVol, DstPtrInfo);
5522
21.5k
5523
21.5k
    if (Result.getNode())
5524
16.5k
      return Result;
5525
12.9k
  }
5526
12.9k
5527
12.9k
  // Then check to see if we should lower the memset with target-specific
5528
12.9k
  // code. If the target chooses to do this, this is the next best.
5529
12.9k
  
if (12.9k
TSI12.9k
) {
5530
12.9k
    SDValue Result = TSI->EmitTargetCodeForMemset(
5531
12.9k
        *this, dl, Chain, Dst, Src, Size, Align, isVol, DstPtrInfo);
5532
12.9k
    if (Result.getNode())
5533
11.6k
      return Result;
5534
1.34k
  }
5535
1.34k
5536
1.34k
  checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace());
5537
1.34k
5538
1.34k
  // Emit a library call.
5539
1.34k
  Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
5540
1.34k
  TargetLowering::ArgListTy Args;
5541
1.34k
  TargetLowering::ArgListEntry Entry;
5542
1.34k
  Entry.Node = Dst; Entry.Ty = IntPtrTy;
5543
1.34k
  Args.push_back(Entry);
5544
1.34k
  Entry.Node = Src;
5545
1.34k
  Entry.Ty = Src.getValueType().getTypeForEVT(*getContext());
5546
1.34k
  Args.push_back(Entry);
5547
1.34k
  Entry.Node = Size;
5548
1.34k
  Entry.Ty = IntPtrTy;
5549
1.34k
  Args.push_back(Entry);
5550
1.34k
5551
1.34k
  // FIXME: pass in SDLoc
5552
1.34k
  TargetLowering::CallLoweringInfo CLI(*this);
5553
1.34k
  CLI.setDebugLoc(dl)
5554
1.34k
      .setChain(Chain)
5555
1.34k
      .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
5556
1.34k
                    Dst.getValueType().getTypeForEVT(*getContext()),
5557
1.34k
                    getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
5558
1.34k
                                      TLI->getPointerTy(getDataLayout())),
5559
1.34k
                    std::move(Args))
5560
1.34k
      .setDiscardResult()
5561
1.34k
      .setTailCall(isTailCall);
5562
1.34k
5563
1.34k
  std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
5564
1.34k
  return CallResult.second;
5565
1.34k
}
5566
5567
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
5568
                                SDVTList VTList, ArrayRef<SDValue> Ops,
5569
26.5k
                                MachineMemOperand *MMO) {
5570
26.5k
  FoldingSetNodeID ID;
5571
26.5k
  ID.AddInteger(MemVT.getRawBits());
5572
26.5k
  AddNodeIDNode(ID, Opcode, VTList, Ops);
5573
26.5k
  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5574
26.5k
  void* IP = nullptr;
5575
26.5k
  if (SDNode *
E26.5k
= FindNodeOrInsertPos(ID, dl, IP)) {
5576
0
    cast<AtomicSDNode>(E)->refineAlignment(MMO);
5577
0
    return SDValue(E, 0);
5578
0
  }
5579
26.5k
5580
26.5k
  auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
5581
26.5k
                                    VTList, MemVT, MMO);
5582
26.5k
  createOperands(N, Ops);
5583
26.5k
5584
26.5k
  CSEMap.InsertNode(N, IP);
5585
26.5k
  InsertNode(N);
5586
26.5k
  return SDValue(N, 0);
5587
26.5k
}
5588
5589
SDValue SelectionDAG::getAtomicCmpSwap(
5590
    unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain,
5591
    SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo,
5592
    unsigned Alignment, AtomicOrdering SuccessOrdering,
5593
2.47k
    AtomicOrdering FailureOrdering, SyncScope::ID SSID) {
5594
2.47k
  assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
5595
2.47k
         Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
5596
2.47k
  assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
5597
2.47k
5598
2.47k
  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
5599
2.47k
    Alignment = getEVTAlignment(MemVT);
5600
2.47k
5601
2.47k
  MachineFunction &MF = getMachineFunction();
5602
2.47k
5603
2.47k
  // FIXME: Volatile isn't really correct; we should keep track of atomic
5604
2.47k
  // orderings in the memoperand.
5605
2.47k
  auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad |
5606
2.47k
               MachineMemOperand::MOStore;
5607
2.47k
  MachineMemOperand *MMO =
5608
2.47k
    MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment,
5609
2.47k
                            AAMDNodes(), nullptr, SSID, SuccessOrdering,
5610
2.47k
                            FailureOrdering);
5611
2.47k
5612
2.47k
  return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO);
5613
2.47k
}
5614
5615
SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
5616
                                       EVT MemVT, SDVTList VTs, SDValue Chain,
5617
                                       SDValue Ptr, SDValue Cmp, SDValue Swp,
5618
6.31k
                                       MachineMemOperand *MMO) {
5619
6.31k
  assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
5620
6.31k
         Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
5621
6.31k
  assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
5622
6.31k
5623
6.31k
  SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
5624
6.31k
  return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
5625
6.31k
}
5626
5627
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
5628
                                SDValue Chain, SDValue Ptr, SDValue Val,
5629
                                const Value *PtrVal, unsigned Alignment,
5630
                                AtomicOrdering Ordering,
5631
17.2k
                                SyncScope::ID SSID) {
5632
17.2k
  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
5633
5.10k
    Alignment = getEVTAlignment(MemVT);
5634
17.2k
5635
17.2k
  MachineFunction &MF = getMachineFunction();
5636
17.2k
  // An atomic store does not load. An atomic load does not store.
5637
17.2k
  // (An atomicrmw obviously both loads and stores.)
5638
17.2k
  // For now, atomics are considered to be volatile always, and they are
5639
17.2k
  // chained as such.
5640
17.2k
  // FIXME: Volatile isn't really correct; we should keep track of atomic
5641
17.2k
  // orderings in the memoperand.
5642
17.2k
  auto Flags = MachineMemOperand::MOVolatile;
5643
17.2k
  if (Opcode != ISD::ATOMIC_STORE)
5644
5.10k
    Flags |= MachineMemOperand::MOLoad;
5645
17.2k
  if (Opcode != ISD::ATOMIC_LOAD)
5646
17.2k
    Flags |= MachineMemOperand::MOStore;
5647
17.2k
5648
17.2k
  MachineMemOperand *MMO =
5649
17.2k
    MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
5650
17.2k
                            MemVT.getStoreSize(), Alignment, AAMDNodes(),
5651
17.2k
                            nullptr, SSID, Ordering);
5652
17.2k
5653
17.2k
  return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
5654
17.2k
}
5655
5656
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
5657
                                SDValue Chain, SDValue Ptr, SDValue Val,
5658
18.8k
                                MachineMemOperand *MMO) {
5659
18.8k
  assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
5660
18.8k
          Opcode == ISD::ATOMIC_LOAD_SUB ||
5661
18.8k
          Opcode == ISD::ATOMIC_LOAD_AND ||
5662
18.8k
          Opcode == ISD::ATOMIC_LOAD_OR ||
5663
18.8k
          Opcode == ISD::ATOMIC_LOAD_XOR ||
5664
18.8k
          Opcode == ISD::ATOMIC_LOAD_NAND ||
5665
18.8k
          Opcode == ISD::ATOMIC_LOAD_MIN ||
5666
18.8k
          Opcode == ISD::ATOMIC_LOAD_MAX ||
5667
18.8k
          Opcode == ISD::ATOMIC_LOAD_UMIN ||
5668
18.8k
          Opcode == ISD::ATOMIC_LOAD_UMAX ||
5669
18.8k
          Opcode == ISD::ATOMIC_SWAP ||
5670
18.8k
          Opcode == ISD::ATOMIC_STORE) &&
5671
18.8k
         "Invalid Atomic Op");
5672
18.8k
5673
18.8k
  EVT VT = Val.getValueType();
5674
18.8k
5675
12.3k
  SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
5676
6.49k
                                               getVTList(VT, MVT::Other);
5677
18.8k
  SDValue Ops[] = {Chain, Ptr, Val};
5678
18.8k
  return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
5679
18.8k
}
5680
5681
SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
5682
                                EVT VT, SDValue Chain, SDValue Ptr,
5683
1.35k
                                MachineMemOperand *MMO) {
5684
1.35k
  assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
5685
1.35k
5686
1.35k
  SDVTList VTs = getVTList(VT, MVT::Other);
5687
1.35k
  SDValue Ops[] = {Chain, Ptr};
5688
1.35k
  return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
5689
1.35k
}
5690
5691
/// getMergeValues - Create a MERGE_VALUES node from the given operands.
5692
959k
SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
5693
959k
  if (Ops.size() == 1)
5694
908k
    return Ops[0];
5695
51.6k
5696
51.6k
  SmallVector<EVT, 4> VTs;
5697
51.6k
  VTs.reserve(Ops.size());
5698
157k
  for (unsigned i = 0; 
i < Ops.size()157k
;
++i105k
)
5699
105k
    VTs.push_back(Ops[i].getValueType());
5700
959k
  return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
5701
959k
}
5702
5703
SDValue SelectionDAG::getMemIntrinsicNode(
5704
    unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
5705
    EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, bool Vol,
5706
126k
    bool ReadMem, bool WriteMem, unsigned Size) {
5707
126k
  if (Align == 0)  // Ensure that codegen never sees alignment 0
5708
29.5k
    Align = getEVTAlignment(MemVT);
5709
126k
5710
126k
  MachineFunction &MF = getMachineFunction();
5711
126k
  auto Flags = MachineMemOperand::MONone;
5712
126k
  if (WriteMem)
5713
74.5k
    Flags |= MachineMemOperand::MOStore;
5714
126k
  if (ReadMem)
5715
52.1k
    Flags |= MachineMemOperand::MOLoad;
5716
126k
  if (Vol)
5717
94.3k
    Flags |= MachineMemOperand::MOVolatile;
5718
126k
  if (!Size)
5719
126k
    Size = MemVT.getStoreSize();
5720
126k
  MachineMemOperand *MMO =
5721
126k
    MF.getMachineMemOperand(PtrInfo, Flags, Size, Align);
5722
126k
5723
126k
  return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
5724
126k
}
5725
5726
SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
5727
                                          SDVTList VTList,
5728
                                          ArrayRef<SDValue> Ops, EVT MemVT,
5729
134k
                                          MachineMemOperand *MMO) {
5730
134k
  assert((Opcode == ISD::INTRINSIC_VOID ||
5731
134k
          Opcode == ISD::INTRINSIC_W_CHAIN ||
5732
134k
          Opcode == ISD::PREFETCH ||
5733
134k
          Opcode == ISD::LIFETIME_START ||
5734
134k
          Opcode == ISD::LIFETIME_END ||
5735
134k
          ((int)Opcode <= std::numeric_limits<int>::max() &&
5736
134k
           (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
5737
134k
         "Opcode is not a memory-accessing opcode!");
5738
134k
5739
134k
  // Memoize the node unless it returns a flag.
5740
134k
  MemIntrinsicSDNode *N;
5741
134k
  if (
VTList.VTs[VTList.NumVTs-1] != MVT::Glue134k
) {
5742
132k
    FoldingSetNodeID ID;
5743
132k
    AddNodeIDNode(ID, Opcode, VTList, Ops);
5744
132k
    ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5745
132k
    void *IP = nullptr;
5746
132k
    if (SDNode *
E132k
= FindNodeOrInsertPos(ID, dl, IP)) {
5747
5
      cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
5748
5
      return SDValue(E, 0);
5749
5
    }
5750
132k
5751
132k
    N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
5752
132k
                                      VTList, MemVT, MMO);
5753
132k
    createOperands(N, Ops);
5754
132k
5755
132k
  CSEMap.InsertNode(N, IP);
5756
134k
  } else {
5757
2.37k
    N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
5758
2.37k
                                      VTList, MemVT, MMO);
5759
2.37k
    createOperands(N, Ops);
5760
2.37k
  }
5761
134k
  InsertNode(N);
5762
134k
  return SDValue(N, 0);
5763
134k
}
5764
5765
/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
5766
/// MachinePointerInfo record from it.  This is particularly useful because the
5767
/// code generator has many cases where it doesn't bother passing in a
5768
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
5769
static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr,
5770
38.3k
                                           int64_t Offset = 0) {
5771
38.3k
  // If this is FI+Offset, we can model it.
5772
38.3k
  if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
5773
12.4k
    return MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
5774
12.4k
                                             FI->getIndex(), Offset);
5775
25.8k
5776
25.8k
  // If this is (FI+Offset1)+Offset2, we can model it.
5777
25.8k
  
if (25.8k
Ptr.getOpcode() != ISD::ADD ||
5778
14.5k
      !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
5779
13.5k
      !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
5780
23.1k
    return MachinePointerInfo();
5781
2.76k
5782
2.76k
  int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
5783
2.76k
  return MachinePointerInfo::getFixedStack(
5784
2.76k
      DAG.getMachineFunction(), FI,
5785
2.76k
      Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
5786
2.76k
}
5787
5788
/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
5789
/// MachinePointerInfo record from it.  This is particularly useful because the
5790
/// code generator has many cases where it doesn't bother passing in a
5791
/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
5792
static MachinePointerInfo InferPointerInfo(SelectionDAG &DAG, SDValue Ptr,
5793
17.6k
                                           SDValue OffsetOp) {
5794
17.6k
  // If the 'Offset' value isn't a constant, we can't handle this.
5795
17.6k
  if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
5796
15
    return InferPointerInfo(DAG, Ptr, OffsetNode->getSExtValue());
5797
17.6k
  
if (17.6k
OffsetOp.isUndef()17.6k
)
5798
17.6k
    return InferPointerInfo(DAG, Ptr);
5799
0
  return MachinePointerInfo();
5800
0
}
5801
5802
SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
5803
                              EVT VT, const SDLoc &dl, SDValue Chain,
5804
                              SDValue Ptr, SDValue Offset,
5805
                              MachinePointerInfo PtrInfo, EVT MemVT,
5806
                              unsigned Alignment,
5807
                              MachineMemOperand::Flags MMOFlags,
5808
2.39M
                              const AAMDNodes &AAInfo, const MDNode *Ranges) {
5809
2.39M
  assert(Chain.getValueType() == MVT::Other &&
5810
2.39M
        "Invalid chain type");
5811
2.39M
  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
5812
130k
    Alignment = getEVTAlignment(MemVT);
5813
2.39M
5814
2.39M
  MMOFlags |= MachineMemOperand::MOLoad;
5815
2.39M
  assert((MMOFlags & MachineMemOperand::MOStore) == 0);
5816
2.39M
  // If we don't have a PtrInfo, infer the trivial frame index case to simplify
5817
2.39M
  // clients.
5818
2.39M
  if (PtrInfo.V.isNull())
5819
17.6k
    PtrInfo = InferPointerInfo(*this, Ptr, Offset);
5820
2.39M
5821
2.39M
  MachineFunction &MF = getMachineFunction();
5822
2.39M
  MachineMemOperand *MMO = MF.getMachineMemOperand(
5823
2.39M
      PtrInfo, MMOFlags, MemVT.getStoreSize(), Alignment, AAInfo, Ranges);
5824
2.39M
  return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
5825
2.39M
}
5826
5827
SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
5828
                              EVT VT, const SDLoc &dl, SDValue Chain,
5829
                              SDValue Ptr, SDValue Offset, EVT MemVT,
5830
3.38M
                              MachineMemOperand *MMO) {
5831
3.38M
  if (
VT == MemVT3.38M
) {
5832
2.64M
    ExtType = ISD::NON_EXTLOAD;
5833
3.38M
  } else 
if (746k
ExtType == ISD::NON_EXTLOAD746k
) {
5834
0
    assert(VT == MemVT && "Non-extending load from different memory type!");
5835
746k
  } else {
5836
746k
    // Extending load.
5837
746k
    assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
5838
746k
           "Should only be an extending load, not truncating!");
5839
746k
    assert(VT.isInteger() == MemVT.isInteger() &&
5840
746k
           "Cannot convert from FP to Int or Int -> FP!");
5841
746k
    assert(VT.isVector() == MemVT.isVector() &&
5842
746k
           "Cannot use an ext load to convert to or from a vector!");
5843
746k
    assert((!VT.isVector() ||
5844
746k
            VT.getVectorNumElements() == MemVT.getVectorNumElements()) &&
5845
746k
           "Cannot use an ext load to change the number of vector elements!");
5846
746k
  }
5847
3.38M
5848
3.38M
  bool Indexed = AM != ISD::UNINDEXED;
5849
3.38M
  assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
5850
3.38M
5851
3.38M
  SDVTList VTs = Indexed ?
5852
3.38M
    
getVTList(VT, Ptr.getValueType(), MVT::Other)64.8k
:
getVTList(VT, MVT::Other)3.32M
;
5853
3.38M
  SDValue Ops[] = { Chain, Ptr, Offset };
5854
3.38M
  FoldingSetNodeID ID;
5855
3.38M
  AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
5856
3.38M
  ID.AddInteger(MemVT.getRawBits());
5857
3.38M
  ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
5858
3.38M
      dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
5859
3.38M
  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5860
3.38M
  void *IP = nullptr;
5861
3.38M
  if (SDNode *
E3.38M
= FindNodeOrInsertPos(ID, dl, IP)) {
5862
54.2k
    cast<LoadSDNode>(E)->refineAlignment(MMO);
5863
54.2k
    return SDValue(E, 0);
5864
54.2k
  }
5865
3.33M
  auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
5866
3.33M
                                  ExtType, MemVT, MMO);
5867
3.33M
  createOperands(N, Ops);
5868
3.33M
5869
3.33M
  CSEMap.InsertNode(N, IP);
5870
3.33M
  InsertNode(N);
5871
3.33M
  return SDValue(N, 0);
5872
3.33M
}
5873
5874
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
5875
                              SDValue Ptr, MachinePointerInfo PtrInfo,
5876
                              unsigned Alignment,
5877
                              MachineMemOperand::Flags MMOFlags,
5878
2.18M
                              const AAMDNodes &AAInfo, const MDNode *Ranges) {
5879
2.18M
  SDValue Undef = getUNDEF(Ptr.getValueType());
5880
2.18M
  return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
5881
2.18M
                 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
5882
2.18M
}
5883
5884
SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
5885
295k
                              SDValue Ptr, MachineMemOperand *MMO) {
5886
295k
  SDValue Undef = getUNDEF(Ptr.getValueType());
5887
295k
  return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
5888
295k
                 VT, MMO);
5889
295k
}
5890
5891
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
5892
                                 EVT VT, SDValue Chain, SDValue Ptr,
5893
                                 MachinePointerInfo PtrInfo, EVT MemVT,
5894
                                 unsigned Alignment,
5895
                                 MachineMemOperand::Flags MMOFlags,
5896
122k
                                 const AAMDNodes &AAInfo) {
5897
122k
  SDValue Undef = getUNDEF(Ptr.getValueType());
5898
122k
  return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
5899
122k
                 MemVT, Alignment, MMOFlags, AAInfo);
5900
122k
}
5901
5902
SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
5903
                                 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
5904
518k
                                 MachineMemOperand *MMO) {
5905
518k
  SDValue Undef = getUNDEF(Ptr.getValueType());
5906
518k
  return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
5907
518k
                 MemVT, MMO);
5908
518k
}
5909
5910
SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
5911
                                     SDValue Base, SDValue Offset,
5912
64.8k
                                     ISD::MemIndexedMode AM) {
5913
64.8k
  LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
5914
64.8k
  assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
5915
64.8k
  // Don't propagate the invariant or dereferenceable flags.
5916
64.8k
  auto MMOFlags =
5917
64.8k
      LD->getMemOperand()->getFlags() &
5918
64.8k
      ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
5919
64.8k
  return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
5920
64.8k
                 LD->getChain(), Base, Offset, LD->getPointerInfo(),
5921
64.8k
                 LD->getMemoryVT(), LD->getAlignment(), MMOFlags,
5922
64.8k
                 LD->getAAInfo());
5923
64.8k
}
5924
5925
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
5926
                               SDValue Ptr, MachinePointerInfo PtrInfo,
5927
                               unsigned Alignment,
5928
                               MachineMemOperand::Flags MMOFlags,
5929
2.11M
                               const AAMDNodes &AAInfo) {
5930
2.11M
  assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
5931
2.11M
  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
5932
178k
    Alignment = getEVTAlignment(Val.getValueType());
5933
2.11M
5934
2.11M
  MMOFlags |= MachineMemOperand::MOStore;
5935
2.11M
  assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
5936
2.11M
5937
2.11M
  if (PtrInfo.V.isNull())
5938
16.4k
    PtrInfo = InferPointerInfo(*this, Ptr);
5939
2.11M
5940
2.11M
  MachineFunction &MF = getMachineFunction();
5941
2.11M
  MachineMemOperand *MMO = MF.getMachineMemOperand(
5942
2.11M
      PtrInfo, MMOFlags, Val.getValueType().getStoreSize(), Alignment, AAInfo);
5943
2.11M
  return getStore(Chain, dl, Val, Ptr, MMO);
5944
2.11M
}
5945
5946
SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
5947
3.12M
                               SDValue Ptr, MachineMemOperand *MMO) {
5948
3.12M
  assert(Chain.getValueType() == MVT::Other &&
5949
3.12M
        "Invalid chain type");
5950
3.12M
  EVT VT = Val.getValueType();
5951
3.12M
  SDVTList VTs = getVTList(MVT::Other);
5952
3.12M
  SDValue Undef = getUNDEF(Ptr.getValueType());
5953
3.12M
  SDValue Ops[] = { Chain, Val, Ptr, Undef };
5954
3.12M
  FoldingSetNodeID ID;
5955
3.12M
  AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
5956
3.12M
  ID.AddInteger(VT.getRawBits());
5957
3.12M
  ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
5958
3.12M
      dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
5959
3.12M
  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
5960
3.12M
  void *IP = nullptr;
5961
3.12M
  if (SDNode *
E3.12M
= FindNodeOrInsertPos(ID, dl, IP)) {
5962
28.8k
    cast<StoreSDNode>(E)->refineAlignment(MMO);
5963
28.8k
    return SDValue(E, 0);
5964
28.8k
  }
5965
3.10M
  auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
5966
3.10M
                                   ISD::UNINDEXED, false, VT, MMO);
5967
3.10M
  createOperands(N, Ops);
5968
3.10M
5969
3.10M
  CSEMap.InsertNode(N, IP);
5970
3.10M
  InsertNode(N);
5971
3.10M
  return SDValue(N, 0);
5972
3.10M
}
5973
5974
SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
5975
                                    SDValue Ptr, MachinePointerInfo PtrInfo,
5976
                                    EVT SVT, unsigned Alignment,
5977
                                    MachineMemOperand::Flags MMOFlags,
5978
104k
                                    const AAMDNodes &AAInfo) {
5979
104k
  assert(Chain.getValueType() == MVT::Other &&
5980
104k
        "Invalid chain type");
5981
104k
  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
5982
1.13k
    Alignment = getEVTAlignment(SVT);
5983
104k
5984
104k
  MMOFlags |= MachineMemOperand::MOStore;
5985
104k
  assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
5986
104k
5987
104k
  if (PtrInfo.V.isNull())
5988
4.23k
    PtrInfo = InferPointerInfo(*this, Ptr);
5989
104k
5990
104k
  MachineFunction &MF = getMachineFunction();
5991
104k
  MachineMemOperand *MMO = MF.getMachineMemOperand(
5992
104k
      PtrInfo, MMOFlags, SVT.getStoreSize(), Alignment, AAInfo);
5993
104k
  return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
5994
104k
}
5995
5996
SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
5997
                                    SDValue Ptr, EVT SVT,
5998
450k
                                    MachineMemOperand *MMO) {
5999
450k
  EVT VT = Val.getValueType();
6000
450k
6001
450k
  assert(Chain.getValueType() == MVT::Other &&
6002
450k
        "Invalid chain type");
6003
450k
  if (VT == SVT)
6004
59.0k
    return getStore(Chain, dl, Val, Ptr, MMO);
6005
391k
6006
450k
  assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
6007
391k
         "Should only be a truncating store, not extending!");
6008
391k
  assert(VT.isInteger() == SVT.isInteger() &&
6009
391k
         "Can't do FP-INT conversion!");
6010
391k
  assert(VT.isVector() == SVT.isVector() &&
6011
391k
         "Cannot use trunc store to convert to or from a vector!");
6012
391k
  assert((!VT.isVector() ||
6013
391k
          VT.getVectorNumElements() == SVT.getVectorNumElements()) &&
6014
391k
         "Cannot use trunc store to change the number of vector elements!");
6015
391k
6016
391k
  SDVTList VTs = getVTList(MVT::Other);
6017
391k
  SDValue Undef = getUNDEF(Ptr.getValueType());
6018
391k
  SDValue Ops[] = { Chain, Val, Ptr, Undef };
6019
391k
  FoldingSetNodeID ID;
6020
391k
  AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
6021
391k
  ID.AddInteger(SVT.getRawBits());
6022
391k
  ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
6023
391k
      dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
6024
391k
  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6025
391k
  void *IP = nullptr;
6026
391k
  if (SDNode *
E391k
= FindNodeOrInsertPos(ID, dl, IP)) {
6027
1.15k
    cast<StoreSDNode>(E)->refineAlignment(MMO);
6028
1.15k
    return SDValue(E, 0);
6029
1.15k
  }
6030
390k
  auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
6031
390k
                                   ISD::UNINDEXED, true, SVT, MMO);
6032
390k
  createOperands(N, Ops);
6033
390k
6034
390k
  CSEMap.InsertNode(N, IP);
6035
390k
  InsertNode(N);
6036
390k
  return SDValue(N, 0);
6037
390k
}
6038
6039
SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
6040
                                      SDValue Base, SDValue Offset,
6041
23.5k
                                      ISD::MemIndexedMode AM) {
6042
23.5k
  StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
6043
23.5k
  assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
6044
23.5k
  SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
6045
23.5k
  SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
6046
23.5k
  FoldingSetNodeID ID;
6047
23.5k
  AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
6048
23.5k
  ID.AddInteger(ST->getMemoryVT().getRawBits());
6049
23.5k
  ID.AddInteger(ST->getRawSubclassData());
6050
23.5k
  ID.AddInteger(ST->getPointerInfo().getAddrSpace());
6051
23.5k
  void *IP = nullptr;
6052
23.5k
  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
6053
0
    return SDValue(E, 0);
6054
23.5k
6055
23.5k
  auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
6056
23.5k
                                   ST->isTruncatingStore(), ST->getMemoryVT(),
6057
23.5k
                                   ST->getMemOperand());
6058
23.5k
  createOperands(N, Ops);
6059
23.5k
6060
23.5k
  CSEMap.InsertNode(N, IP);
6061
23.5k
  InsertNode(N);
6062
23.5k
  return SDValue(N, 0);
6063
23.5k
}
6064
6065
SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
6066
                                    SDValue Ptr, SDValue Mask, SDValue Src0,
6067
                                    EVT MemVT, MachineMemOperand *MMO,
6068
316
                                    ISD::LoadExtType ExtTy, bool isExpanding) {
6069
316
  SDVTList VTs = getVTList(VT, MVT::Other);
6070
316
  SDValue Ops[] = { Chain, Ptr, Mask, Src0 };
6071
316
  FoldingSetNodeID ID;
6072
316
  AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
6073
316
  ID.AddInteger(VT.getRawBits());
6074
316
  ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
6075
316
      dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO));
6076
316
  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6077
316
  void *IP = nullptr;
6078
316
  if (SDNode *
E316
= FindNodeOrInsertPos(ID, dl, IP)) {
6079
0
    cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
6080
0
    return SDValue(E, 0);
6081
0
  }
6082
316
  auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
6083
316
                                        ExtTy, isExpanding, MemVT, MMO);
6084
316
  createOperands(N, Ops);
6085
316
6086
316
  CSEMap.InsertNode(N, IP);
6087
316
  InsertNode(N);
6088
316
  return SDValue(N, 0);
6089
316
}
6090
6091
SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
6092
                                     SDValue Val, SDValue Ptr, SDValue Mask,
6093
                                     EVT MemVT, MachineMemOperand *MMO,
6094
199
                                     bool IsTruncating, bool IsCompressing) {
6095
199
  assert(Chain.getValueType() == MVT::Other &&
6096
199
        "Invalid chain type");
6097
199
  EVT VT = Val.getValueType();
6098
199
  SDVTList VTs = getVTList(MVT::Other);
6099
199
  SDValue Ops[] = { Chain, Ptr, Mask, Val };
6100
199
  FoldingSetNodeID ID;
6101
199
  AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
6102
199
  ID.AddInteger(VT.getRawBits());
6103
199
  ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
6104
199
      dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO));
6105
199
  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6106
199
  void *IP = nullptr;
6107
199
  if (SDNode *
E199
= FindNodeOrInsertPos(ID, dl, IP)) {
6108
0
    cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
6109
0
    return SDValue(E, 0);
6110
0
  }
6111
199
  auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
6112
199
                                         IsTruncating, IsCompressing, MemVT, MMO);
6113
199
  createOperands(N, Ops);
6114
199
6115
199
  CSEMap.InsertNode(N, IP);
6116
199
  InsertNode(N);
6117
199
  return SDValue(N, 0);
6118
199
}
6119
6120
SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
6121
                                      ArrayRef<SDValue> Ops,
6122
290
                                      MachineMemOperand *MMO) {
6123
290
  assert(Ops.size() == 5 && "Incompatible number of operands");
6124
290
6125
290
  FoldingSetNodeID ID;
6126
290
  AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
6127
290
  ID.AddInteger(VT.getRawBits());
6128
290
  ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
6129
290
      dl.getIROrder(), VTs, VT, MMO));
6130
290
  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6131
290
  void *IP = nullptr;
6132
290
  if (SDNode *
E290
= FindNodeOrInsertPos(ID, dl, IP)) {
6133
0
    cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
6134
0
    return SDValue(E, 0);
6135
0
  }
6136
290
6137
290
  auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
6138
290
                                          VTs, VT, MMO);
6139
290
  createOperands(N, Ops);
6140
290
6141
290
  assert(N->getValue().getValueType() == N->getValueType(0) &&
6142
290
         "Incompatible type of the PassThru value in MaskedGatherSDNode");
6143
290
  assert(N->getMask().getValueType().getVectorNumElements() ==
6144
290
             N->getValueType(0).getVectorNumElements() &&
6145
290
         "Vector width mismatch between mask and data");
6146
290
  assert(N->getIndex().getValueType().getVectorNumElements() ==
6147
290
             N->getValueType(0).getVectorNumElements() &&
6148
290
         "Vector width mismatch between index and data");
6149
290
6150
290
  CSEMap.InsertNode(N, IP);
6151
290
  InsertNode(N);
6152
290
  return SDValue(N, 0);
6153
290
}
6154
6155
SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
6156
                                       ArrayRef<SDValue> Ops,
6157
178
                                       MachineMemOperand *MMO) {
6158
178
  assert(Ops.size() == 5 && "Incompatible number of operands");
6159
178
6160
178
  FoldingSetNodeID ID;
6161
178
  AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
6162
178
  ID.AddInteger(VT.getRawBits());
6163
178
  ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
6164
178
      dl.getIROrder(), VTs, VT, MMO));
6165
178
  ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
6166
178
  void *IP = nullptr;
6167
178
  if (SDNode *
E178
= FindNodeOrInsertPos(ID, dl, IP)) {
6168
0
    cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
6169
0
    return SDValue(E, 0);
6170
0
  }
6171
178
  auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
6172
178
                                           VTs, VT, MMO);
6173
178
  createOperands(N, Ops);
6174
178
6175
178
  assert(N->getMask().getValueType().getVectorNumElements() ==
6176
178
             N->getValue().getValueType().getVectorNumElements() &&
6177
178
         "Vector width mismatch between mask and data");
6178
178
  assert(N->getIndex().getValueType().getVectorNumElements() ==
6179
178
             N->getValue().getValueType().getVectorNumElements() &&
6180
178
         "Vector width mismatch between index and data");
6181
178
6182
178
  CSEMap.InsertNode(N, IP);
6183
178
  InsertNode(N);
6184
178
  return SDValue(N, 0);
6185
178
}
6186
6187
SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
6188
421
                               SDValue Ptr, SDValue SV, unsigned Align) {
6189
421
  SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
6190
421
  return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
6191
421
}
6192
6193
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6194
6.77k
                              ArrayRef<SDUse> Ops) {
6195
6.77k
  switch (Ops.size()) {
6196
0
  case 0: return getNode(Opcode, DL, VT);
6197
142
  case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
6198
3.23k
  case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
6199
45
  case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
6200
3.34k
  default: break;
6201
3.34k
  }
6202
3.34k
6203
3.34k
  // Copy from an SDUse array into an SDValue array for use with
6204
3.34k
  // the regular getNode logic.
6205
3.34k
  SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
6206
3.34k
  return getNode(Opcode, DL, VT, NewOps);
6207
3.34k
}
6208
6209
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6210
20.3M
                              ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
6211
20.3M
  unsigned NumOps = Ops.size();
6212
20.3M
  switch (NumOps) {
6213
0
  case 0: return getNode(Opcode, DL, VT);
6214
12.8M
  case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
6215
2.15M
  case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
6216
1.00M
  case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
6217
4.39M
  default: break;
6218
4.39M
  }
6219
4.39M
6220
4.39M
  switch (Opcode) {
6221
2.33M
  default: break;
6222
6.78k
  case ISD::CONCAT_VECTORS:
6223
6.78k
    // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF.
6224
6.78k
    if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this))
6225
233
      return V;
6226
6.55k
    break;
6227
144k
  case ISD::SELECT_CC:
6228
144k
    assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
6229
144k
    assert(Ops[0].getValueType() == Ops[1].getValueType() &&
6230
144k
           "LHS and RHS of condition must have same type!");
6231
144k
    assert(Ops[2].getValueType() == Ops[3].getValueType() &&
6232
144k
           "True and False arms of SelectCC must have same type!");
6233
144k
    assert(Ops[2].getValueType() == VT &&
6234
144k
           "select_cc node must be of same type as true and false value!");
6235
144k
    break;
6236
1.90M
  case ISD::BR_CC:
6237
1.90M
    assert(NumOps == 5 && "BR_CC takes 5 operands!");
6238
1.90M
    assert(Ops[2].getValueType() == Ops[3].getValueType() &&
6239
1.90M
           "LHS/RHS of comparison should match types!");
6240
1.90M
    break;
6241
4.38M
  }
6242
4.38M
6243
4.38M
  // Memoize nodes.
6244
4.38M
  SDNode *N;
6245
4.38M
  SDVTList VTs = getVTList(VT);
6246
4.38M
6247
4.38M
  if (
VT != MVT::Glue4.38M
) {
6248
4.38M
    FoldingSetNodeID ID;
6249
4.38M
    AddNodeIDNode(ID, Opcode, VTs, Ops);
6250
4.38M
    void *IP = nullptr;
6251
4.38M
6252
4.38M
    if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6253
326k
      return SDValue(E, 0);
6254
4.06M
6255
4.06M
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6256
4.06M
    createOperands(N, Ops);
6257
4.06M
6258
4.06M
    CSEMap.InsertNode(N, IP);
6259
4.38M
  } else {
6260
1
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6261
1
    createOperands(N, Ops);
6262
1
  }
6263
4.38M
6264
4.06M
  InsertNode(N);
6265
4.06M
  return SDValue(N, 0);
6266
20.3M
}
6267
6268
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
6269
12.9k
                              ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
6270
12.9k
  return getNode(Opcode, DL, getVTList(ResultTys), Ops);
6271
12.9k
}
6272
6273
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6274
30.3M
                              ArrayRef<SDValue> Ops) {
6275
30.3M
  if (VTList.NumVTs == 1)
6276
8.84M
    return getNode(Opcode, DL, VTList.VTs[0], Ops);
6277
21.4M
6278
#if 0
6279
  switch (Opcode) {
6280
  // FIXME: figure out how to safely handle things like
6281
  // int foo(int x) { return 1 << (x & 255); }
6282
  // int bar() { return foo(256); }
6283
  case ISD::SRA_PARTS:
6284
  case ISD::SRL_PARTS:
6285
  case ISD::SHL_PARTS:
6286
    if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
6287
        cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
6288
      return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
6289
    else if (N3.getOpcode() == ISD::AND)
6290
      if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
6291
        // If the and is only masking out bits that cannot effect the shift,
6292
        // eliminate the and.
6293
        unsigned NumBits = VT.getScalarSizeInBits()*2;
6294
        if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
6295
          return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
6296
      }
6297
    break;
6298
  }
6299
#endif
6300
6301
21.4M
  // Memoize the node unless it returns a flag.
6302
21.4M
  SDNode *N;
6303
21.4M
  if (
VTList.VTs[VTList.NumVTs-1] != MVT::Glue21.4M
) {
6304
10.2M
    FoldingSetNodeID ID;
6305
10.2M
    AddNodeIDNode(ID, Opcode, VTList, Ops);
6306
10.2M
    void *IP = nullptr;
6307
10.2M
    if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6308
1.36M
      return SDValue(E, 0);
6309
8.85M
6310
8.85M
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
6311
8.85M
    createOperands(N, Ops);
6312
8.85M
    CSEMap.InsertNode(N, IP);
6313
21.4M
  } else {
6314
11.2M
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
6315
11.2M
    createOperands(N, Ops);
6316
11.2M
  }
6317
20.1M
  InsertNode(N);
6318
20.1M
  return SDValue(N, 0);
6319
30.3M
}
6320
6321
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
6322
0
                              SDVTList VTList) {
6323
0
  return getNode(Opcode, DL, VTList, None);
6324
0
}
6325
6326
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6327
5.14k
                              SDValue N1) {
6328
5.14k
  SDValue Ops[] = { N1 };
6329
5.14k
  return getNode(Opcode, DL, VTList, Ops);
6330
5.14k
}
6331
6332
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6333
946k
                              SDValue N1, SDValue N2) {
6334
946k
  SDValue Ops[] = { N1, N2 };
6335
946k
  return getNode(Opcode, DL, VTList, Ops);
6336
946k
}
6337
6338
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6339
13.4k
                              SDValue N1, SDValue N2, SDValue N3) {
6340
13.4k
  SDValue Ops[] = { N1, N2, N3 };
6341
13.4k
  return getNode(Opcode, DL, VTList, Ops);
6342
13.4k
}
6343
6344
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6345
2.50k
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
6346
2.50k
  SDValue Ops[] = { N1, N2, N3, N4 };
6347
2.50k
  return getNode(Opcode, DL, VTList, Ops);
6348
2.50k
}
6349
6350
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
6351
                              SDValue N1, SDValue N2, SDValue N3, SDValue N4,
6352
335
                              SDValue N5) {
6353
335
  SDValue Ops[] = { N1, N2, N3, N4, N5 };
6354
335
  return getNode(Opcode, DL, VTList, Ops);
6355
335
}
6356
6357
138M
SDVTList SelectionDAG::getVTList(EVT VT) {
6358
138M
  return makeVTList(SDNode::getValueTypeList(VT), 1);
6359
138M
}
6360
6361
27.4M
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
6362
27.4M
  FoldingSetNodeID ID;
6363
27.4M
  ID.AddInteger(2U);
6364
27.4M
  ID.AddInteger(VT1.getRawBits());
6365
27.4M
  ID.AddInteger(VT2.getRawBits());
6366
27.4M
6367
27.4M
  void *IP = nullptr;
6368
27.4M
  SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
6369
27.4M
  if (
!Result27.4M
) {
6370
179k
    EVT *Array = Allocator.Allocate<EVT>(2);
6371
179k
    Array[0] = VT1;
6372
179k
    Array[1] = VT2;
6373
179k
    Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
6374
179k
    VTListMap.InsertNode(Result, IP);
6375
179k
  }
6376
27.4M
  return Result->getSDVTList();
6377
27.4M
}
6378
6379
1.19M
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
6380
1.19M
  FoldingSetNodeID ID;
6381
1.19M
  ID.AddInteger(3U);
6382
1.19M
  ID.AddInteger(VT1.getRawBits());
6383
1.19M
  ID.AddInteger(VT2.getRawBits());
6384
1.19M
  ID.AddInteger(VT3.getRawBits());
6385
1.19M
6386
1.19M
  void *IP = nullptr;
6387
1.19M
  SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
6388
1.19M
  if (
!Result1.19M
) {
6389
36.0k
    EVT *Array = Allocator.Allocate<EVT>(3);
6390
36.0k
    Array[0] = VT1;
6391
36.0k
    Array[1] = VT2;
6392
36.0k
    Array[2] = VT3;
6393
36.0k
    Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
6394
36.0k
    VTListMap.InsertNode(Result, IP);
6395
36.0k
  }
6396
1.19M
  return Result->getSDVTList();
6397
1.19M
}
6398
6399
64
SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
6400
64
  FoldingSetNodeID ID;
6401
64
  ID.AddInteger(4U);
6402
64
  ID.AddInteger(VT1.getRawBits());
6403
64
  ID.AddInteger(VT2.getRawBits());
6404
64
  ID.AddInteger(VT3.getRawBits());
6405
64
  ID.AddInteger(VT4.getRawBits());
6406
64
6407
64
  void *IP = nullptr;
6408
64
  SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
6409
64
  if (
!Result64
) {
6410
20
    EVT *Array = Allocator.Allocate<EVT>(4);
6411
20
    Array[0] = VT1;
6412
20
    Array[1] = VT2;
6413
20
    Array[2] = VT3;
6414
20
    Array[3] = VT4;
6415
20
    Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
6416
20
    VTListMap.InsertNode(Result, IP);
6417
20
  }
6418
64
  return Result->getSDVTList();
6419
64
}
6420
6421
13.7M
SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
6422
13.7M
  unsigned NumVTs = VTs.size();
6423
13.7M
  FoldingSetNodeID ID;
6424
13.7M
  ID.AddInteger(NumVTs);
6425
37.2M
  for (unsigned index = 0; 
index < NumVTs37.2M
;
index++23.4M
) {
6426
23.4M
    ID.AddInteger(VTs[index].getRawBits());
6427
23.4M
  }
6428
13.7M
6429
13.7M
  void *IP = nullptr;
6430
13.7M
  SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
6431
13.7M
  if (
!Result13.7M
) {
6432
104k
    EVT *Array = Allocator.Allocate<EVT>(NumVTs);
6433
104k
    std::copy(VTs.begin(), VTs.end(), Array);
6434
104k
    Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
6435
104k
    VTListMap.InsertNode(Result, IP);
6436
104k
  }
6437
13.7M
  return Result->getSDVTList();
6438
13.7M
}
6439
6440
6441
/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
6442
/// specified operands.  If the resultant node already exists in the DAG,
6443
/// this does not modify the specified node, instead it returns the node that
6444
/// already exists.  If the resultant node does not exist in the DAG, the
6445
/// input node is returned.  As a degenerate case, if you specify the same
6446
/// input operands as the node already has, the input node is returned.
6447
21.0k
SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
6448
21.0k
  assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
6449
21.0k
6450
21.0k
  // Check to see if there is no change.
6451
21.0k
  if (
Op == N->getOperand(0)21.0k
)
return N0
;
6452
21.0k
6453
21.0k
  // See if the modified node already exists.
6454
21.0k
  void *InsertPos = nullptr;
6455
21.0k
  if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
6456
16
    return Existing;
6457
21.0k
6458
21.0k
  // Nope it doesn't.  Remove the node from its current place in the maps.
6459
21.0k
  
if (21.0k
InsertPos21.0k
)
6460
21.0k
    
if (21.0k
!RemoveNodeFromCSEMaps(N)21.0k
)
6461
0
      InsertPos = nullptr;
6462
21.0k
6463
21.0k
  // Now we update the operands.
6464
21.0k
  N->OperandList[0].set(Op);
6465
21.0k
6466
21.0k
  // If this gets put into a CSE map, add it.
6467
21.0k
  if (
InsertPos21.0k
)
CSEMap.InsertNode(N, InsertPos)21.0k
;
6468
21.0k
  return N;
6469
21.0k
}
6470
6471
32.2k
SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
6472
32.2k
  assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
6473
32.2k
6474
32.2k
  // Check to see if there is no change.
6475
32.2k
  if (
Op1 == N->getOperand(0) && 32.2k
Op2 == N->getOperand(1)17.1k
)
6476
68
    return N;   // No operands changed, just return the input node.
6477
32.1k
6478
32.1k
  // See if the modified node already exists.
6479
32.1k
  void *InsertPos = nullptr;
6480
32.1k
  if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
6481
853
    return Existing;
6482
31.3k
6483
31.3k
  // Nope it doesn't.  Remove the node from its current place in the maps.
6484
31.3k
  
if (31.3k
InsertPos31.3k
)
6485
31.3k
    
if (31.3k
!RemoveNodeFromCSEMaps(N)31.3k
)
6486
0
      InsertPos = nullptr;
6487
31.3k
6488
31.3k
  // Now we update the operands.
6489
31.3k
  if (N->OperandList[0] != Op1)
6490
14.3k
    N->OperandList[0].set(Op1);
6491
31.3k
  if (N->OperandList[1] != Op2)
6492
23.7k
    N->OperandList[1].set(Op2);
6493
31.3k
6494
31.3k
  // If this gets put into a CSE map, add it.
6495
31.3k
  if (
InsertPos31.3k
)
CSEMap.InsertNode(N, InsertPos)31.3k
;
6496
32.2k
  return N;
6497
32.2k
}
6498
6499
SDNode *SelectionDAG::
6500
558k
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
6501
558k
  SDValue Ops[] = { Op1, Op2, Op3 };
6502
558k
  return UpdateNodeOperands(N, Ops);
6503
558k
}
6504
6505
SDNode *SelectionDAG::
6506
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
6507
315
                   SDValue Op3, SDValue Op4) {
6508
315
  SDValue Ops[] = { Op1, Op2, Op3, Op4 };
6509
315
  return UpdateNodeOperands(N, Ops);
6510
315
}
6511
6512
SDNode *SelectionDAG::
6513
UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
6514
26.5k
                   SDValue Op3, SDValue Op4, SDValue Op5) {
6515
26.5k
  SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
6516
26.5k
  return UpdateNodeOperands(N, Ops);
6517
26.5k
}
6518
6519
SDNode *SelectionDAG::
6520
5.81M
UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
6521
5.81M
  unsigned NumOps = Ops.size();
6522
5.81M
  assert(N->getNumOperands() == NumOps &&
6523
5.81M
         "Update with wrong number of operands");
6524
5.81M
6525
5.81M
  // If no operands changed just return the input node.
6526
5.81M
  if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
6527
5.09M
    return N;
6528
720k
6529
720k
  // See if the modified node already exists.
6530
720k
  void *InsertPos = nullptr;
6531
720k
  if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
6532
596
    return Existing;
6533
719k
6534
719k
  // Nope it doesn't.  Remove the node from its current place in the maps.
6535
719k
  
if (719k
InsertPos719k
)
6536
708k
    
if (708k
!RemoveNodeFromCSEMaps(N)708k
)
6537
0
      InsertPos = nullptr;
6538
719k
6539
719k
  // Now we update the operands.
6540
2.93M
  for (unsigned i = 0; 
i != NumOps2.93M
;
++i2.21M
)
6541
2.21M
    
if (2.21M
N->OperandList[i] != Ops[i]2.21M
)
6542
1.12M
      N->OperandList[i].set(Ops[i]);
6543
719k
6544
719k
  // If this gets put into a CSE map, add it.
6545
719k
  if (
InsertPos719k
)
CSEMap.InsertNode(N, InsertPos)708k
;
6546
5.81M
  return N;
6547
5.81M
}
6548
6549
/// DropOperands - Release the operands and set this node to have
6550
/// zero operands.
6551
60.8M
void SDNode::DropOperands() {
6552
60.8M
  // Unlike the code in MorphNodeTo that does this, we don't need to
6553
60.8M
  // watch for dead nodes here.
6554
146M
  for (op_iterator I = op_begin(), E = op_end(); 
I != E146M
; ) {
6555
85.2M
    SDUse &Use = *I++;
6556
85.2M
    Use.set(SDValue());
6557
85.2M
  }
6558
60.8M
}
6559
6560
/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
6561
/// machine opcode.
6562
///
6563
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6564
7.32k
                                   EVT VT) {
6565
7.32k
  SDVTList VTs = getVTList(VT);
6566
7.32k
  return SelectNodeTo(N, MachineOpc, VTs, None);
6567
7.32k
}
6568
6569
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6570
205
                                   EVT VT, SDValue Op1) {
6571
205
  SDVTList VTs = getVTList(VT);
6572
205
  SDValue Ops[] = { Op1 };
6573
205
  return SelectNodeTo(N, MachineOpc, VTs, Ops);
6574
205
}
6575
6576
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6577
                                   EVT VT, SDValue Op1,
6578
1.74k
                                   SDValue Op2) {
6579
1.74k
  SDVTList VTs = getVTList(VT);
6580
1.74k
  SDValue Ops[] = { Op1, Op2 };
6581
1.74k
  return SelectNodeTo(N, MachineOpc, VTs, Ops);
6582
1.74k
}
6583
6584
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6585
                                   EVT VT, SDValue Op1,
6586
66
                                   SDValue Op2, SDValue Op3) {
6587
66
  SDVTList VTs = getVTList(VT);
6588
66
  SDValue Ops[] = { Op1, Op2, Op3 };
6589
66
  return SelectNodeTo(N, MachineOpc, VTs, Ops);
6590
66
}
6591
6592
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6593
184k
                                   EVT VT, ArrayRef<SDValue> Ops) {
6594
184k
  SDVTList VTs = getVTList(VT);
6595
184k
  return SelectNodeTo(N, MachineOpc, VTs, Ops);
6596
184k
}
6597
6598
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6599
2
                                   EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
6600
2
  SDVTList VTs = getVTList(VT1, VT2);
6601
2
  return SelectNodeTo(N, MachineOpc, VTs, Ops);
6602
2
}
6603
6604
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6605
4
                                   EVT VT1, EVT VT2) {
6606
4
  SDVTList VTs = getVTList(VT1, VT2);
6607
4
  return SelectNodeTo(N, MachineOpc, VTs, None);
6608
4
}
6609
6610
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6611
                                   EVT VT1, EVT VT2, EVT VT3,
6612
5
                                   ArrayRef<SDValue> Ops) {
6613
5
  SDVTList VTs = getVTList(VT1, VT2, VT3);
6614
5
  return SelectNodeTo(N, MachineOpc, VTs, Ops);
6615
5
}
6616
6617
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6618
                                   EVT VT1, EVT VT2,
6619
0
                                   SDValue Op1, SDValue Op2) {
6620
0
  SDVTList VTs = getVTList(VT1, VT2);
6621
0
  SDValue Ops[] = { Op1, Op2 };
6622
0
  return SelectNodeTo(N, MachineOpc, VTs, Ops);
6623
0
}
6624
6625
SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
6626
212k
                                   SDVTList VTs,ArrayRef<SDValue> Ops) {
6627
212k
  SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
6628
212k
  // Reset the NodeID to -1.
6629
212k
  New->setNodeId(-1);
6630
212k
  if (
New != N212k
) {
6631
86
    ReplaceAllUsesWith(N, New);
6632
86
    RemoveDeadNode(N);
6633
86
  }
6634
212k
  return New;
6635
212k
}
6636
6637
/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
6638
/// the line number information on the merged node since it is not possible to
6639
/// preserve the information that operation is associated with multiple lines.
6640
/// This will make the debugger working better at -O0, were there is a higher
6641
/// probability having other instructions associated with that line.
6642
///
6643
/// For IROrder, we keep the smaller of the two
6644
218k
SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
6645
218k
  DebugLoc NLoc = N->getDebugLoc();
6646
218k
  if (
NLoc && 218k
OptLevel == CodeGenOpt::None1.75k
&&
OLoc.getDebugLoc() != NLoc14
) {
6647
0
    N->setDebugLoc(DebugLoc());
6648
0
  }
6649
218k
  unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
6650
218k
  N->setIROrder(Order);
6651
218k
  return N;
6652
218k
}
6653
6654
/// MorphNodeTo - This *mutates* the specified node to have the specified
6655
/// return type, opcode, and operands.
6656
///
6657
/// Note that MorphNodeTo returns the resultant node.  If there is already a
6658
/// node of the specified opcode and operands, it returns that node instead of
6659
/// the current one.  Note that the SDLoc need not be the same.
6660
///
6661
/// Using MorphNodeTo is faster than creating a new node and swapping it in
6662
/// with ReplaceAllUsesWith both because it often avoids allocating a new
6663
/// node, and because it doesn't require CSE recalculation for any of
6664
/// the node's users.
6665
///
6666
/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
6667
/// As a consequence it isn't appropriate to use from within the DAG combiner or
6668
/// the legalizer which maintain worklists that would need to be updated when
6669
/// deleting things.
6670
SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
6671
20.1M
                                  SDVTList VTs, ArrayRef<SDValue> Ops) {
6672
20.1M
  // If an identical node already exists, use it.
6673
20.1M
  void *IP = nullptr;
6674
20.1M
  if (
VTs.VTs[VTs.NumVTs-1] != MVT::Glue20.1M
) {
6675
15.3M
    FoldingSetNodeID ID;
6676
15.3M
    AddNodeIDNode(ID, Opc, VTs, Ops);
6677
15.3M
    if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
6678
25.2k
      return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
6679
20.1M
  }
6680
20.1M
6681
20.1M
  
if (20.1M
!RemoveNodeFromCSEMaps(N)20.1M
)
6682
4.96M
    IP = nullptr;
6683
20.1M
6684
20.1M
  // Start the morphing.
6685
20.1M
  N->NodeType = Opc;
6686
20.1M
  N->ValueList = VTs.VTs;
6687
20.1M
  N->NumValues = VTs.NumVTs;
6688
20.1M
6689
20.1M
  // Clear the operands list, updating used nodes to remove this from their
6690
20.1M
  // use list.  Keep track of any operands that become dead as a result.
6691
20.1M
  SmallPtrSet<SDNode*, 16> DeadNodeSet;
6692
78.0M
  for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); 
I != E78.0M
; ) {
6693
57.9M
    SDUse &Use = *I++;
6694
57.9M
    SDNode *Used = Use.getNode();
6695
57.9M
    Use.set(SDValue());
6696
57.9M
    if (Used->use_empty())
6697
28.1M
      DeadNodeSet.insert(Used);
6698
57.9M
  }
6699
20.1M
6700
20.1M
  // For MachineNode, initialize the memory references information.
6701
20.1M
  if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
6702
20.1M
    MN->setMemRefs(nullptr, nullptr);
6703
20.1M
6704
20.1M
  // Swap for an appropriately sized array from the recycler.
6705
20.1M
  removeOperands(N);
6706
20.1M
  createOperands(N, Ops);
6707
20.1M
6708
20.1M
  // Delete any nodes that are still dead after adding the uses for the
6709
20.1M
  // new operands.
6710
20.1M
  if (
!DeadNodeSet.empty()20.1M
) {
6711
16.8M
    SmallVector<SDNode *, 16> DeadNodes;
6712
16.8M
    for (SDNode *N : DeadNodeSet)
6713
28.1M
      
if (28.1M
N->use_empty()28.1M
)
6714
7.71M
        DeadNodes.push_back(N);
6715
16.8M
    RemoveDeadNodes(DeadNodes);
6716
16.8M
  }
6717
20.1M
6718
20.1M
  if (IP)
6719
15.1M
    CSEMap.InsertNode(N, IP);   // Memoize the new node.
6720
20.1M
  return N;
6721
20.1M
}
6722
6723
40
SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
6724
40
  unsigned OrigOpc = Node->getOpcode();
6725
40
  unsigned NewOpc;
6726
40
  bool IsUnary = false;
6727
40
  bool IsTernary = false;
6728
40
  switch (OrigOpc) {
6729
0
  default:
6730
0
    llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
6731
2
  case ISD::STRICT_FADD: NewOpc = ISD::FADD; break;
6732
6
  case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break;
6733
2
  case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break;
6734
2
  case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break;
6735
0
  case ISD::STRICT_FREM: NewOpc = ISD::FREM; break;
6736
4
  case ISD::STRICT_FMA: NewOpc = ISD::FMA; IsTernary = true; break;
6737
2
  case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; IsUnary = true; break;
6738
2
  case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break;
6739
2
  case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break;
6740
2
  case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; IsUnary = true; break;
6741
2
  case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; IsUnary = true; break;
6742
2
  case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; IsUnary = true; break;
6743
2
  case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; IsUnary = true; break;
6744
2
  case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; IsUnary = true; break;
6745
2
  case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; IsUnary = true; break;
6746
2
  case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; IsUnary = true; break;
6747
2
  case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; IsUnary = true; break;
6748
2
  case ISD::STRICT_FNEARBYINT:
6749
2
    NewOpc = ISD::FNEARBYINT;
6750
2
    IsUnary = true;
6751
2
    break;
6752
40
  }
6753
40
6754
40
  // We're taking this node out of the chain, so we need to re-link things.
6755
40
  SDValue InputChain = Node->getOperand(0);
6756
40
  SDValue OutputChain = SDValue(Node, 1);
6757
40
  ReplaceAllUsesOfValueWith(OutputChain, InputChain);
6758
40
6759
40
  SDVTList VTs = getVTList(Node->getOperand(1).getValueType());
6760
40
  SDNode *Res = nullptr;
6761
40
  if (IsUnary)
6762
20
    Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1) });
6763
20
  else 
if (20
IsTernary20
)
6764
4
    Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1),
6765
4
                                           Node->getOperand(2),
6766
4
                                           Node->getOperand(3)});
6767
20
  else
6768
16
    Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1),
6769
16
                                           Node->getOperand(2) });
6770
40
6771
40
  // MorphNodeTo can operate in two ways: if an existing node with the
6772
40
  // specified operands exists, it can just return it.  Otherwise, it
6773
40
  // updates the node in place to have the requested operands.
6774
40
  if (
Res == Node40
) {
6775
40
    // If we updated the node in place, reset the node ID.  To the isel,
6776
40
    // this should be just like a newly allocated machine node.
6777
40
    Res->setNodeId(-1);
6778
40
  } else {
6779
0
    ReplaceAllUsesWith(Node, Res);
6780
0
    RemoveDeadNode(Node);
6781
0
  }
6782
40
6783
40
  return Res;
6784
40
}
6785
6786
/// getMachineNode - These are used for target selectors to create a new node
6787
/// with specified return type(s), MachineInstr opcode, and operands.
6788
///
6789
/// Note that getMachineNode returns the resultant node.  If there is already a
6790
/// node of the specified opcode and operands, it returns that node instead of
6791
/// the current one.
6792
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6793
2.25k
                                            EVT VT) {
6794
2.25k
  SDVTList VTs = getVTList(VT);
6795
2.25k
  return getMachineNode(Opcode, dl, VTs, None);
6796
2.25k
}
6797
6798
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6799
59.9k
                                            EVT VT, SDValue Op1) {
6800
59.9k
  SDVTList VTs = getVTList(VT);
6801
59.9k
  SDValue Ops[] = { Op1 };
6802
59.9k
  return getMachineNode(Opcode, dl, VTs, Ops);
6803
59.9k
}
6804
6805
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6806
120k
                                            EVT VT, SDValue Op1, SDValue Op2) {
6807
120k
  SDVTList VTs = getVTList(VT);
6808
120k
  SDValue Ops[] = { Op1, Op2 };
6809
120k
  return getMachineNode(Opcode, dl, VTs, Ops);
6810
120k
}
6811
6812
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6813
                                            EVT VT, SDValue Op1, SDValue Op2,
6814
5.42k
                                            SDValue Op3) {
6815
5.42k
  SDVTList VTs = getVTList(VT);
6816
5.42k
  SDValue Ops[] = { Op1, Op2, Op3 };
6817
5.42k
  return getMachineNode(Opcode, dl, VTs, Ops);
6818
5.42k
}
6819
6820
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6821
101k
                                            EVT VT, ArrayRef<SDValue> Ops) {
6822
101k
  SDVTList VTs = getVTList(VT);
6823
101k
  return getMachineNode(Opcode, dl, VTs, Ops);
6824
101k
}
6825
6826
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6827
                                            EVT VT1, EVT VT2, SDValue Op1,
6828
7.62k
                                            SDValue Op2) {
6829
7.62k
  SDVTList VTs = getVTList(VT1, VT2);
6830
7.62k
  SDValue Ops[] = { Op1, Op2 };
6831
7.62k
  return getMachineNode(Opcode, dl, VTs, Ops);
6832
7.62k
}
6833
6834
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6835
                                            EVT VT1, EVT VT2, SDValue Op1,
6836
16
                                            SDValue Op2, SDValue Op3) {
6837
16
  SDVTList VTs = getVTList(VT1, VT2);
6838
16
  SDValue Ops[] = { Op1, Op2, Op3 };
6839
16
  return getMachineNode(Opcode, dl, VTs, Ops);
6840
16
}
6841
6842
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6843
                                            EVT VT1, EVT VT2,
6844
27.5k
                                            ArrayRef<SDValue> Ops) {
6845
27.5k
  SDVTList VTs = getVTList(VT1, VT2);
6846
27.5k
  return getMachineNode(Opcode, dl, VTs, Ops);
6847
27.5k
}
6848
6849
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6850
                                            EVT VT1, EVT VT2, EVT VT3,
6851
16
                                            SDValue Op1, SDValue Op2) {
6852
16
  SDVTList VTs = getVTList(VT1, VT2, VT3);
6853
16
  SDValue Ops[] = { Op1, Op2 };
6854
16
  return getMachineNode(Opcode, dl, VTs, Ops);
6855
16
}
6856
6857
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6858
                                            EVT VT1, EVT VT2, EVT VT3,
6859
                                            SDValue Op1, SDValue Op2,
6860
53
                                            SDValue Op3) {
6861
53
  SDVTList VTs = getVTList(VT1, VT2, VT3);
6862
53
  SDValue Ops[] = { Op1, Op2, Op3 };
6863
53
  return getMachineNode(Opcode, dl, VTs, Ops);
6864
53
}
6865
6866
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6867
                                            EVT VT1, EVT VT2, EVT VT3,
6868
64.8k
                                            ArrayRef<SDValue> Ops) {
6869
64.8k
  SDVTList VTs = getVTList(VT1, VT2, VT3);
6870
64.8k
  return getMachineNode(Opcode, dl, VTs, Ops);
6871
64.8k
}
6872
6873
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
6874
                                            ArrayRef<EVT> ResultTys,
6875
4.23k
                                            ArrayRef<SDValue> Ops) {
6876
4.23k
  SDVTList VTs = getVTList(ResultTys);
6877
4.23k
  return getMachineNode(Opcode, dl, VTs, Ops);
6878
4.23k
}
6879
6880
MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
6881
                                            SDVTList VTs,
6882
1.37M
                                            ArrayRef<SDValue> Ops) {
6883
1.37M
  bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
6884
1.37M
  MachineSDNode *N;
6885
1.37M
  void *IP = nullptr;
6886
1.37M
6887
1.37M
  if (
DoCSE1.37M
) {
6888
1.32M
    FoldingSetNodeID ID;
6889
1.32M
    AddNodeIDNode(ID, ~Opcode, VTs, Ops);
6890
1.32M
    IP = nullptr;
6891
1.32M
    if (SDNode *
E1.32M
= FindNodeOrInsertPos(ID, DL, IP)) {
6892
192k
      return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
6893
192k
    }
6894
1.18M
  }
6895
1.18M
6896
1.18M
  // Allocate a new MachineSDNode.
6897
1.18M
  N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6898
1.18M
  createOperands(N, Ops);
6899
1.18M
6900
1.18M
  if (DoCSE)
6901
1.13M
    CSEMap.InsertNode(N, IP);
6902
1.37M
6903
1.37M
  InsertNode(N);
6904
1.37M
  return N;
6905
1.37M
}
6906
6907
/// getTargetExtractSubreg - A convenience function for creating
6908
/// TargetOpcode::EXTRACT_SUBREG nodes.
6909
SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
6910
69.9k
                                             SDValue Operand) {
6911
69.9k
  SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
6912
69.9k
  SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
6913
69.9k
                                  VT, Operand, SRIdxVal);
6914
69.9k
  return SDValue(Subreg, 0);
6915
69.9k
}
6916
6917
/// getTargetInsertSubreg - A convenience function for creating
6918
/// TargetOpcode::INSERT_SUBREG nodes.
6919
SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
6920
1.26k
                                            SDValue Operand, SDValue Subreg) {
6921
1.26k
  SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
6922
1.26k
  SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
6923
1.26k
                                  VT, Operand, Subreg, SRIdxVal);
6924
1.26k
  return SDValue(Result, 0);
6925
1.26k
}
6926
6927
/// getNodeIfExists - Get the specified node if it's already available, or
6928
/// else return NULL.
6929
SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
6930
                                      ArrayRef<SDValue> Ops,
6931
2.37M
                                      const SDNodeFlags Flags) {
6932
2.37M
  if (
VTList.VTs[VTList.NumVTs - 1] != MVT::Glue2.37M
) {
6933
2.37M
    FoldingSetNodeID ID;
6934
2.37M
    AddNodeIDNode(ID, Opcode, VTList, Ops);
6935
2.37M
    void *IP = nullptr;
6936
2.37M
    if (SDNode *
E2.37M
= FindNodeOrInsertPos(ID, SDLoc(), IP)) {
6937
378
      E->intersectFlagsWith(Flags);
6938
378
      return E;
6939
378
    }
6940
2.37M
  }
6941
2.37M
  return nullptr;
6942
2.37M
}
6943
6944
/// getDbgValue - Creates a SDDbgValue node.
6945
///
6946
/// SDNode
6947
SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
6948
                                      SDNode *N, unsigned R, bool IsIndirect,
6949
304
                                      const DebugLoc &DL, unsigned O) {
6950
304
  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6951
304
         "Expected inlined-at fields to agree");
6952
304
  return new (DbgInfo->getAlloc())
6953
304
      SDDbgValue(Var, Expr, N, R, IsIndirect, DL, O);
6954
304
}
6955
6956
/// Constant
6957
SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
6958
                                              DIExpression *Expr,
6959
                                              const Value *C,
6960
95
                                              const DebugLoc &DL, unsigned O) {
6961
95
  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6962
95
         "Expected inlined-at fields to agree");
6963
95
  return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, C, DL, O);
6964
95
}
6965
6966
/// FrameIndex
6967
SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
6968
                                                DIExpression *Expr, unsigned FI,
6969
                                                const DebugLoc &DL,
6970
46
                                                unsigned O) {
6971
46
  assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
6972
46
         "Expected inlined-at fields to agree");
6973
46
  return new (DbgInfo->getAlloc()) SDDbgValue(Var, Expr, FI, DL, O);
6974
46
}
6975
6976
namespace {
6977
6978
/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
6979
/// pointed to by a use iterator is deleted, increment the use iterator
6980
/// so that it doesn't dangle.
6981
///
6982
class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
6983
  SDNode::use_iterator &UI;
6984
  SDNode::use_iterator &UE;
6985
6986
74.2k
  void NodeDeleted(SDNode *N, SDNode *E) override {
6987
74.2k
    // Increment the iterator as needed.
6988
74.2k
    while (
UI != UE && 74.2k
N == *UI1.08k
)
6989
0
      ++UI;
6990
74.2k
  }
6991
6992
public:
6993
  RAUWUpdateListener(SelectionDAG &d,
6994
                     SDNode::use_iterator &ui,
6995
                     SDNode::use_iterator &ue)
6996
24.8M
    : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
6997
};
6998
6999
} // end anonymous namespace
7000
7001
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
7002
/// This can cause recursive merging of nodes in the DAG.
7003
///
7004
/// This version assumes From has a single result value.
7005
///
7006
8.01M
void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
7007
8.01M
  SDNode *From = FromN.getNode();
7008
8.01M
  assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
7009
8.01M
         "Cannot replace with this method!");
7010
8.01M
  assert(From != To.getNode() && "Cannot replace uses of with self");
7011
8.01M
7012
8.01M
  // Preserve Debug Values
7013
8.01M
  TransferDbgValues(FromN, To);
7014
8.01M
7015
8.01M
  // Iterate over all the existing uses of From. New uses will be added
7016
8.01M
  // to the beginning of the use list, which we avoid visiting.
7017
8.01M
  // This specifically avoids visiting uses of From that arise while the
7018
8.01M
  // replacement is happening, because any such uses would be the result
7019
8.01M
  // of CSE: If an existing node looks like From after one of its operands
7020
8.01M
  // is replaced by To, we don't want to replace of all its users with To
7021
8.01M
  // too. See PR3018 for more info.
7022
8.01M
  SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
7023
8.01M
  RAUWUpdateListener Listener(*this, UI, UE);
7024
16.9M
  while (
UI != UE16.9M
) {
7025
8.90M
    SDNode *User = *UI;
7026
8.90M
7027
8.90M
    // This node is about to morph, remove its old self from the CSE maps.
7028
8.90M
    RemoveNodeFromCSEMaps(User);
7029
8.90M
7030
8.90M
    // A user can appear in a use list multiple times, and when this
7031
8.90M
    // happens the uses are usually next to each other in the list.
7032
8.90M
    // To help reduce the number of CSE recomputations, process all
7033
8.90M
    // the uses of this user that we can find this way.
7034
8.92M
    do {
7035
8.92M
      SDUse &Use = UI.getUse();
7036
8.92M
      ++UI;
7037
8.92M
      Use.set(To);
7038
8.92M
    } while (
UI != UE && 8.92M
*UI == User955k
);
7039
8.90M
7040
8.90M
    // Now that we have modified User, add it back to the CSE maps.  If it
7041
8.90M
    // already exists there, recursively merge the results together.
7042
8.90M
    AddModifiedNodeToCSEMaps(User);
7043
8.90M
  }
7044
8.01M
7045
8.01M
  // If we just RAUW'd the root, take note.
7046
8.01M
  if (FromN == getRoot())
7047
35.4k
    setRoot(To);
7048
8.01M
}
7049
7050
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
7051
/// This can cause recursive merging of nodes in the DAG.
7052
///
7053
/// This version assumes that for each value of From, there is a
7054
/// corresponding value in To in the same position with the same type.
7055
///
7056
5.79M
void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
7057
#ifndef NDEBUG
7058
  for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
7059
    assert((!From->hasAnyUseOfValue(i) ||
7060
            From->getValueType(i) == To->getValueType(i)) &&
7061
           "Cannot use this version of ReplaceAllUsesWith!");
7062
#endif
7063
7064
5.79M
  // Handle the trivial case.
7065
5.79M
  if (From == To)
7066
0
    return;
7067
5.79M
7068
5.79M
  // Preserve Debug Info. Only do this if there's a use.
7069
11.6M
  
for (unsigned i = 0, e = From->getNumValues(); 5.79M
i != e11.6M
;
++i5.82M
)
7070
5.82M
    
if (5.82M
From->hasAnyUseOfValue(i)5.82M
) {
7071
5.80M
      assert((i < To->getNumValues()) && "Invalid To location");
7072
5.80M
      TransferDbgValues(SDValue(From, i), SDValue(To, i));
7073
5.80M
    }
7074
5.79M
7075
5.79M
  // Iterate over just the existing users of From. See the comments in
7076
5.79M
  // the ReplaceAllUsesWith above.
7077
5.79M
  SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
7078
5.79M
  RAUWUpdateListener Listener(*this, UI, UE);
7079
12.1M
  while (
UI != UE12.1M
) {
7080
6.38M
    SDNode *User = *UI;
7081
6.38M
7082
6.38M
    // This node is about to morph, remove its old self from the CSE maps.
7083
6.38M
    RemoveNodeFromCSEMaps(User);
7084
6.38M
7085
6.38M
    // A user can appear in a use list multiple times, and when this
7086
6.38M
    // happens the uses are usually next to each other in the list.
7087
6.38M
    // To help reduce the number of CSE recomputations, process all
7088
6.38M
    // the uses of this user that we can find this way.
7089
6.41M
    do {
7090
6.41M
      SDUse &Use = UI.getUse();
7091
6.41M
      ++UI;
7092
6.41M
      Use.setNode(To);
7093
6.41M
    } while (
UI != UE && 6.41M
*UI == User626k
);
7094
6.38M
7095
6.38M
    // Now that we have modified User, add it back to the CSE maps.  If it
7096
6.38M
    // already exists there, recursively merge the results together.
7097
6.38M
    AddModifiedNodeToCSEMaps(User);
7098
6.38M
  }
7099
5.79M
7100
5.79M
  // If we just RAUW'd the root, take note.
7101
5.79M
  if (From == getRoot().getNode())
7102
49.0k
    setRoot(SDValue(To, getRoot().getResNo()));
7103
5.79M
}
7104
7105
/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
7106
/// This can cause recursive merging of nodes in the DAG.
7107
///
7108
/// This version can replace From with any result values.  To must match the
7109
/// number and types of values returned by From.
7110
2.48M
void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
7111
2.48M
  if (From->getNumValues() == 1)  // Handle the simple case efficiently.
7112
1.91M
    return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
7113
570k
7114
570k
  // Preserve Debug Info.
7115
1.71M
  
for (unsigned i = 0, e = From->getNumValues(); 570k
i != e1.71M
;
++i1.14M
)
7116
1.14M
    TransferDbgValues(SDValue(From, i), *To);
7117
570k
7118
570k
  // Iterate over just the existing users of From. See the comments in
7119
570k
  // the ReplaceAllUsesWith above.
7120
570k
  SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
7121
570k
  RAUWUpdateListener Listener(*this, UI, UE);
7122
1.67M
  while (
UI != UE1.67M
) {
7123
1.10M
    SDNode *User = *UI;
7124
1.10M
7125
1.10M
    // This node is about to morph, remove its old self from the CSE maps.
7126
1.10M
    RemoveNodeFromCSEMaps(User);
7127
1.10M
7128
1.10M
    // A user can appear in a use list multiple times, and when this
7129
1.10M
    // happens the uses are usually next to each other in the list.
7130
1.10M
    // To help reduce the number of CSE recomputations, process all
7131
1.10M
    // the uses of this user that we can find this way.
7132
1.20M
    do {
7133
1.20M
      SDUse &Use = UI.getUse();
7134
1.20M
      const SDValue &ToOp = To[Use.getResNo()];
7135
1.20M
      ++UI;
7136
1.20M
      Use.set(ToOp);
7137
1.20M
    } while (
UI != UE && 1.20M
*UI == User637k
);
7138
1.10M
7139
1.10M
    // Now that we have modified User, add it back to the CSE maps.  If it
7140
1.10M
    // already exists there, recursively merge the results together.
7141
1.10M
    AddModifiedNodeToCSEMaps(User);
7142
1.10M
  }
7143
570k
7144
570k
  // If we just RAUW'd the root, take note.
7145
570k
  if (From == getRoot().getNode())
7146
13
    setRoot(SDValue(To[getRoot().getResNo()]));
7147
2.48M
}
7148
7149
/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
7150
/// uses of other values produced by From.getNode() alone.  The Deleted
7151
/// vector is handled the same way as for ReplaceAllUsesWith.
7152
12.8M
void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
7153
12.8M
  // Handle the really simple, really trivial case efficiently.
7154
12.8M
  if (
From == To12.8M
)
return6.09k
;
7155
12.8M
7156
12.8M
  // Handle the simple, trivial, case efficiently.
7157
12.8M
  
if (12.8M
From.getNode()->getNumValues() == 112.8M
) {
7158
2.40M
    ReplaceAllUsesWith(From, To);
7159
2.40M
    return;
7160
2.40M
  }
7161
10.4M
7162
10.4M
  // Preserve Debug Info.
7163
10.4M
  TransferDbgValues(From, To);
7164
10.4M
7165
10.4M
  // Iterate over just the existing users of From. See the comments in
7166
10.4M
  // the ReplaceAllUsesWith above.
7167
10.4M
  SDNode::use_iterator UI = From.getNode()->use_begin(),
7168
10.4M
                       UE = From.getNode()->use_end();
7169
10.4M
  RAUWUpdateListener Listener(*this, UI, UE);
7170
22.7M
  while (
UI != UE22.7M
) {
7171
12.2M
    SDNode *User = *UI;
7172
12.2M
    bool UserRemovedFromCSEMaps = false;
7173
12.2M
7174
12.2M
    // A user can appear in a use list multiple times, and when this
7175
12.2M
    // happens the uses are usually next to each other in the list.
7176
12.2M
    // To help reduce the number of CSE recomputations, process all
7177
12.2M
    // the uses of this user that we can find this way.
7178
17.3M
    do {
7179
17.3M
      SDUse &Use = UI.getUse();
7180
17.3M
7181
17.3M
      // Skip uses of different values from the same node.
7182
17.3M
      if (
Use.getResNo() != From.getResNo()17.3M
) {
7183
8.57M
        ++UI;
7184
8.57M
        continue;
7185
8.57M
      }
7186
8.73M
7187
8.73M
      // If this node hasn't been modified yet, it's still in the CSE maps,
7188
8.73M
      // so remove its old self from the CSE maps.
7189
8.73M
      
if (8.73M
!UserRemovedFromCSEMaps8.73M
) {
7190
8.73M
        RemoveNodeFromCSEMaps(User);
7191
8.73M
        UserRemovedFromCSEMaps = true;
7192
8.73M
      }
7193
17.3M
7194
17.3M
      ++UI;
7195
17.3M
      Use.set(To);
7196
17.3M
    } while (
UI != UE && 17.3M
*UI == User7.04M
);
7197
12.2M
7198
12.2M
    // We are iterating over all uses of the From node, so if a use
7199
12.2M
    // doesn't use the specific value, no changes are made.
7200
12.2M
    if (!UserRemovedFromCSEMaps)
7201
3.49M
      continue;
7202
8.73M
7203
8.73M
    // Now that we have modified User, add it back to the CSE maps.  If it
7204
8.73M
    // already exists there, recursively merge the results together.
7205
8.73M
    AddModifiedNodeToCSEMaps(User);
7206
8.73M
  }
7207
10.4M
7208
10.4M
  // If we just RAUW'd the root, take note.
7209
10.4M
  if (From == getRoot())
7210
97.3k
    setRoot(To);
7211
12.8M
}
7212
7213
namespace {
7214
7215
  /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
7216
  /// to record information about a use.
7217
  struct UseMemo {
7218
    SDNode *User;
7219
    unsigned Index;
7220
    SDUse *Use;
7221
  };
7222
7223
  /// operator< - Sort Memos by User.
7224
643
  bool operator<(const UseMemo &L, const UseMemo &R) {
7225
643
    return (intptr_t)L.User < (intptr_t)R.User;
7226
643
  }
7227
7228
} // end anonymous namespace
7229
7230
/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
7231
/// uses of other values produced by From.getNode() alone.  The same value
7232
/// may appear in both the From and To list.  The Deleted vector is
7233
/// handled the same way as for ReplaceAllUsesWith.
7234
void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
7235
                                              const SDValue *To,
7236
605
                                              unsigned Num){
7237
605
  // Handle the simple, trivial case efficiently.
7238
605
  if (Num == 1)
7239
0
    return ReplaceAllUsesOfValueWith(*From, *To);
7240
605
7241
605
  TransferDbgValues(*From, *To);
7242
605
7243
605
  // Read up all the uses and make records of them. This helps
7244
605
  // processing new uses that are introduced during the
7245
605
  // replacement process.
7246
605
  SmallVector<UseMemo, 4> Uses;
7247
1.93k
  for (unsigned i = 0; 
i != Num1.93k
;
++i1.33k
) {
7248
1.33k
    unsigned FromResNo = From[i].getResNo();
7249
1.33k
    SDNode *FromNode = From[i].getNode();
7250
1.33k
    for (SDNode::use_iterator UI = FromNode->use_begin(),
7251
3.47k
         E = FromNode->use_end(); 
UI != E3.47k
;
++UI2.14k
) {
7252
2.14k
      SDUse &Use = UI.getUse();
7253
2.14k
      if (
Use.getResNo() == FromResNo2.14k
) {
7254
1.19k
        UseMemo Memo = { *UI, i, &Use };
7255
1.19k
        Uses.push_back(Memo);
7256
1.19k
      }
7257
2.14k
    }
7258
1.33k
  }
7259
605
7260
605
  // Sort the uses, so that all the uses from a given User are together.
7261
605
  std::sort(Uses.begin(), Uses.end());
7262
605
7263
605
  for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
7264
1.72k
       
UseIndex != UseIndexEnd1.72k
; ) {
7265
1.12k
    // We know that this user uses some value of From.  If it is the right
7266
1.12k
    // value, update it.
7267
1.12k
    SDNode *User = Uses[UseIndex].User;
7268
1.12k
7269
1.12k
    // This node is about to morph, remove its old self from the CSE maps.
7270
1.12k
    RemoveNodeFromCSEMaps(User);
7271
1.12k
7272
1.12k
    // The Uses array is sorted, so all the uses for a given User
7273
1.12k
    // are next to each other in the list.
7274
1.12k
    // To help reduce the number of CSE recomputations, process all
7275
1.12k
    // the uses of this user that we can find this way.
7276
1.19k
    do {
7277
1.19k
      unsigned i = Uses[UseIndex].Index;
7278
1.19k
      SDUse &Use = *Uses[UseIndex].Use;
7279
1.19k
      ++UseIndex;
7280
1.19k
7281
1.19k
      Use.set(To[i]);
7282
1.19k
    } while (
UseIndex != UseIndexEnd && 1.19k
Uses[UseIndex].User == User585
);
7283
1.12k
7284
1.12k
    // Now that we have modified User, add it back to the CSE maps.  If it
7285
1.12k
    // already exists there, recursively merge the results together.
7286
1.12k
    AddModifiedNodeToCSEMaps(User);
7287
1.12k
  }
7288
605
}
7289
7290
/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
7291
/// based on their topological order. It returns the maximum id and a vector
7292
/// of the SDNodes* in assigned order by reference.
7293
6.98M
unsigned SelectionDAG::AssignTopologicalOrder() {
7294
6.98M
  unsigned DAGSize = 0;
7295
6.98M
7296
6.98M
  // SortedPos tracks the progress of the algorithm. Nodes before it are
7297
6.98M
  // sorted, nodes after it are unsorted. When the algorithm completes
7298
6.98M
  // it is at the end of the list.
7299
6.98M
  allnodes_iterator SortedPos = allnodes_begin();
7300
6.98M
7301
6.98M
  // Visit all the nodes. Move nodes with no operands to the front of
7302
6.98M
  // the list immediately. Annotate nodes that do have operands with their
7303
6.98M
  // operand count. Before we do this, the Node Id fields of the nodes
7304
6.98M
  // may contain arbitrary values. After, the Node Id fields for nodes
7305
6.98M
  // before SortedPos will contain the topological sort index, and the
7306
6.98M
  // Node Id fields for nodes At SortedPos and after will contain the
7307
6.98M
  // count of outstanding operands.
7308
152M
  for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); 
I != E152M
; ) {
7309
145M
    SDNode *N = &*I++;
7310
145M
    checkForCycles(N, this);
7311
145M
    unsigned Degree = N->getNumOperands();
7312
145M
    if (
Degree == 0145M
) {
7313
69.3M
      // A node with no uses, add it to the result array immediately.
7314
69.3M
      N->setNodeId(DAGSize++);
7315
69.3M
      allnodes_iterator Q(N);
7316
69.3M
      if (Q != SortedPos)
7317
29.6M
        SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
7318
69.3M
      assert(SortedPos != AllNodes.end() && "Overran node list");
7319
69.3M
      ++SortedPos;
7320
145M
    } else {
7321
75.6M
      // Temporarily use the Node Id as scratch space for the degree count.
7322
75.6M
      N->setNodeId(Degree);
7323
75.6M
    }
7324
145M
  }
7325
6.98M
7326
6.98M
  // Visit all the nodes. As we iterate, move nodes into sorted order,
7327
6.98M
  // such that by the time the end is reached all nodes will be sorted.
7328
145M
  for (SDNode &Node : allnodes()) {
7329
145M
    SDNode *N = &Node;
7330
145M
    checkForCycles(N, this);
7331
145M
    // N is in sorted position, so all its uses have one less operand
7332
145M
    // that needs to be sorted.
7333
145M
    for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
7334
365M
         
UI != UE365M
;
++UI220M
) {
7335
220M
      SDNode *P = *UI;
7336
220M
      unsigned Degree = P->getNodeId();
7337
220M
      assert(Degree != 0 && "Invalid node degree");
7338
220M
      --Degree;
7339
220M
      if (
Degree == 0220M
) {
7340
75.6M
        // All of P's operands are sorted, so P may sorted now.
7341
75.6M
        P->setNodeId(DAGSize++);
7342
75.6M
        if (P->getIterator() != SortedPos)
7343
18.3M
          SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
7344
75.6M
        assert(SortedPos != AllNodes.end() && "Overran node list");
7345
75.6M
        ++SortedPos;
7346
220M
      } else {
7347
144M
        // Update P's outstanding operand count.
7348
144M
        P->setNodeId(Degree);
7349
144M
      }
7350
220M
    }
7351
145M
    if (
Node.getIterator() == SortedPos145M
) {
7352
#ifndef NDEBUG
7353
      allnodes_iterator I(N);
7354
      SDNode *S = &*++I;
7355
      dbgs() << "Overran sorted position:\n";
7356
      S->dumprFull(this); dbgs() << "\n";
7357
      dbgs() << "Checking if this is due to cycles\n";
7358
      checkForCycles(this, true);
7359
#endif
7360
0
      llvm_unreachable(nullptr);
7361
0
    }
7362
6.98M
  }
7363
6.98M
7364
6.98M
  assert(SortedPos == AllNodes.end() &&
7365
6.98M
         "Topological sort incomplete!");
7366
6.98M
  assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
7367
6.98M
         "First node in topological sort is not the entry token!");
7368
6.98M
  assert(AllNodes.front().getNodeId() == 0 &&
7369
6.98M
         "First node in topological sort has non-zero id!");
7370
6.98M
  assert(AllNodes.front().getNumOperands() == 0 &&
7371
6.98M
         "First node in topological sort has operands!");
7372
6.98M
  assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
7373
6.98M
         "Last node in topologic sort has unexpected id!");
7374
6.98M
  assert(AllNodes.back().use_empty() &&
7375
6.98M
         "Last node in topologic sort has users!");
7376
6.98M
  assert(DAGSize == allnodes_size() && "Node count mismatch!");
7377
6.98M
  return DAGSize;
7378
6.98M
}
7379
7380
/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
7381
/// value is produced by SD.
7382
445
void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) {
7383
445
  if (
SD445
) {
7384
350
    assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
7385
350
    SD->setHasDebugValue(true);
7386
350
  }
7387
445
  DbgInfo->add(DB, SD, isParameter);
7388
445
}
7389
7390
/// TransferDbgValues - Transfer SDDbgValues. Called in replace nodes.
7391
25.4M
void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) {
7392
25.4M
  if (
From == To || 25.4M
!From.getNode()->getHasDebugValue()25.4M
)
7393
25.4M
    return;
7394
76
  SDNode *FromNode = From.getNode();
7395
76
  SDNode *ToNode = To.getNode();
7396
76
  ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode);
7397
76
  SmallVector<SDDbgValue *, 2> ClonedDVs;
7398
76
  for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end();
7399
152
       
I != E152
;
++I76
) {
7400
76
    SDDbgValue *Dbg = *I;
7401
76
    // Only add Dbgvalues attached to same ResNo.
7402
76
    if (Dbg->getKind() == SDDbgValue::SDNODE &&
7403
76
        Dbg->getSDNode() == From.getNode() &&
7404
76
        
Dbg->getResNo() == From.getResNo()76
&&
!Dbg->isInvalidated()47
) {
7405
47
      assert(FromNode != ToNode &&
7406
47
             "Should not transfer Debug Values intranode");
7407
47
      SDDbgValue *Clone = getDbgValue(Dbg->getVariable(), Dbg->getExpression(),
7408
47
                                      ToNode, To.getResNo(), Dbg->isIndirect(),
7409
47
                                      Dbg->getDebugLoc(), Dbg->getOrder());
7410
47
      ClonedDVs.push_back(Clone);
7411
47
      Dbg->setIsInvalidated();
7412
47
    }
7413
76
  }
7414
76
  for (SDDbgValue *I : ClonedDVs)
7415
47
    AddDbgValue(I, ToNode, false);
7416
25.4M
}
7417
7418
SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
7419
1.38k
                                                   SDValue NewMemOp) {
7420
1.38k
  assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
7421
1.38k
  // The new memory operation must have the same position as the old load in
7422
1.38k
  // terms of memory dependency. Create a TokenFactor for the old load and new
7423
1.38k
  // memory operation and update uses of the old load's output chain to use that
7424
1.38k
  // TokenFactor.
7425
1.38k
  SDValue OldChain = SDValue(OldLoad, 1);
7426
1.38k
  SDValue NewChain = SDValue(NewMemOp.getNode(), 1);
7427
1.38k
  if (!OldLoad->hasAnyUseOfValue(1))
7428
1.17k
    return NewChain;
7429
212
7430
212
  SDValue TokenFactor =
7431
212
      getNode(ISD::TokenFactor, SDLoc(OldLoad), MVT::Other, OldChain, NewChain);
7432
212
  ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
7433
212
  UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewChain);
7434
212
  return TokenFactor;
7435
212
}
7436
7437
//===----------------------------------------------------------------------===//
7438
//                              SDNode Class
7439
//===----------------------------------------------------------------------===//
7440
7441
18.7M
bool llvm::isNullConstant(SDValue V) {
7442
18.7M
  ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
7443
10.9M
  return Const != nullptr && Const->isNullValue();
7444
18.7M
}
7445
7446
5.61M
bool llvm::isNullFPConstant(SDValue V) {
7447
5.61M
  ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
7448
5.61M
  return Const != nullptr && 
Const->isZero()40.3k
&&
!Const->isNegative()20.8k
;
7449
5.61M
}
7450
7451
1.54M
bool llvm::isAllOnesConstant(SDValue V) {
7452
1.54M
  ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
7453
1.10M
  return Const != nullptr && Const->isAllOnesValue();
7454
1.54M
}
7455
7456
334k
bool llvm::isOneConstant(SDValue V) {
7457
334k
  ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
7458
145k
  return Const != nullptr && Const->isOne();
7459
334k
}
7460
7461
522k
bool llvm::isBitwiseNot(SDValue V) {
7462
7.64k
  return V.getOpcode() == ISD::XOR && isAllOnesConstant(V.getOperand(1));
7463
522k
}
7464
7465
13.6M
ConstantSDNode *llvm::isConstOrConstSplat(SDValue N) {
7466
13.6M
  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
7467
10.3M
    return CN;
7468
3.29M
7469
3.29M
  
if (BuildVectorSDNode *3.29M
BV3.29M
= dyn_cast<BuildVectorSDNode>(N)) {
7470
139k
    BitVector UndefElements;
7471
139k
    ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
7472
139k
7473
139k
    // BuildVectors can truncate their operands. Ignore that case here.
7474
139k
    // FIXME: We blindly ignore splats which include undef which is overly
7475
139k
    // pessimistic.
7476
139k
    if (
CN && 139k
UndefElements.none()131k
&&
7477
130k
        CN->getValueType(0) == N.getValueType().getScalarType())
7478
116k
      return CN;
7479
3.18M
  }
7480
3.18M
7481
3.18M
  return nullptr;
7482
3.18M
}
7483
7484
781k
ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N) {
7485
781k
  if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
7486
51.9k
    return CN;
7487
729k
7488
729k
  
if (BuildVectorSDNode *729k
BV729k
= dyn_cast<BuildVectorSDNode>(N)) {
7489
12.1k
    BitVector UndefElements;
7490
12.1k
    ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
7491
12.1k
7492
12.1k
    if (
CN && 12.1k
UndefElements.none()5.20k
)
7493
5.18k
      return CN;
7494
724k
  }
7495
724k
7496
724k
  return nullptr;
7497
724k
}
7498
7499
30.4M
HandleSDNode::~HandleSDNode() {
7500
30.4M
  DropOperands();
7501
30.4M
}
7502
7503
GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
7504
                                         const DebugLoc &DL,
7505
                                         const GlobalValue *GA, EVT VT,
7506
                                         int64_t o, unsigned char TF)
7507
6.00M
    : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
7508
6.00M
  TheGlobal = GA;
7509
6.00M
}
7510
7511
AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl,
7512
                                         EVT VT, unsigned SrcAS,
7513
                                         unsigned DestAS)
7514
    : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
7515
199
      SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
7516
7517
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
7518
                     SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
7519
13.9M
    : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
7520
13.9M
  MemSDNodeBits.IsVolatile = MMO->isVolatile();
7521
13.9M
  MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
7522
13.9M
  MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
7523
13.9M
  MemSDNodeBits.IsInvariant = MMO->isInvariant();
7524
13.9M
7525
13.9M
  // We check here that the size of the memory operand fits within the size of
7526
13.9M
  // the MMO. This is because the MMO might indicate only a possible address
7527
13.9M
  // range instead of specifying the affected memory addresses precisely.
7528
13.9M
  assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
7529
13.9M
}
7530
7531
/// Profile - Gather unique data for the node.
7532
///
7533
108M
void SDNode::Profile(FoldingSetNodeID &ID) const {
7534
108M
  AddNodeIDNode(ID, this);
7535
108M
}
7536
7537
namespace {
7538
7539
  struct EVTArray {
7540
    std::vector<EVT> VTs;
7541
7542
33.2k
    EVTArray() {
7543
33.2k
      VTs.reserve(MVT::LAST_VALUETYPE);
7544
3.76M
      for (unsigned i = 0; 
i < MVT::LAST_VALUETYPE3.76M
;
++i3.72M
)
7545
3.72M
        VTs.push_back(MVT((MVT::SimpleValueType)i));
7546
33.2k
    }
7547
  };
7548
7549
} // end anonymous namespace
7550
7551
static ManagedStatic<std::set<EVT, EVT::compareRawBits>> EVTs;
7552
static ManagedStatic<EVTArray> SimpleVTArray;
7553
static ManagedStatic<sys::SmartMutex<true>> VTMutex;
7554
7555
/// getValueTypeList - Return a pointer to the specified value type.
7556
///
7557
221M
const EVT *SDNode::getValueTypeList(EVT VT) {
7558
221M
  if (
VT.isExtended()221M
) {
7559
88.4k
    sys::SmartScopedLock<true> Lock(*VTMutex);
7560
88.4k
    return &(*EVTs->insert(VT).first);
7561
0
  } else {
7562
221M
    assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE &&
7563
221M
           "Value type out of range!");
7564
221M
    return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy];
7565
221M
  }
7566
0
}
7567
7568
/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
7569
/// indicated value.  This method ignores uses of other values defined by this
7570
/// operation.
7571
37.6M
bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
7572
37.6M
  assert(Value < getNumValues() && "Bad value!");
7573
37.6M
7574
37.6M
  // TODO: Only iterate over uses of a given value of the node
7575
80.3M
  for (SDNode::use_iterator UI = use_begin(), E = use_end(); 
UI != E80.3M
;
++UI42.7M
) {
7576
54.9M
    if (
UI.getUse().getResNo() == Value54.9M
) {
7577
49.6M
      if (NUses == 0)
7578
12.2M
        return false;
7579
37.3M
      --NUses;
7580
37.3M
    }
7581
54.9M
  }
7582
37.6M
7583
37.6M
  // Found exactly the right number of uses?
7584
25.3M
  return NUses == 0;
7585
37.6M
}
7586
7587
/// hasAnyUseOfValue - Return true if there are any use of the indicated
7588
/// value. This method ignores uses of other values defined by this operation.
7589
44.0M
bool SDNode::hasAnyUseOfValue(unsigned Value) const {
7590
44.0M
  assert(Value < getNumValues() && "Bad value!");
7591
44.0M
7592
65.5M
  for (SDNode::use_iterator UI = use_begin(), E = use_end(); 
UI != E65.5M
;
++UI21.4M
)
7593
52.7M
    
if (52.7M
UI.getUse().getResNo() == Value52.7M
)
7594
31.2M
      return true;
7595
44.0M
7596
12.8M
  return false;
7597
44.0M
}
7598
7599
/// isOnlyUserOf - Return true if this node is the only use of N.
7600
81.8k
bool SDNode::isOnlyUserOf(const SDNode *N) const {
7601
81.8k
  bool Seen = false;
7602
173k
  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); 
I != E173k
;
++I91.8k
) {
7603
107k
    SDNode *User = *I;
7604
107k
    if (User == this)
7605
91.8k
      Seen = true;
7606
107k
    else
7607
15.9k
      return false;
7608
107k
  }
7609
81.8k
7610
65.8k
  return Seen;
7611
81.8k
}
7612
7613
/// Return true if the only users of N are contained in Nodes.
7614
76.2k
bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
7615
76.2k
  bool Seen = false;
7616
162k
  for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); 
I != E162k
;
++I86.0k
) {
7617
129k
    SDNode *User = *I;
7618
129k
    if (llvm::any_of(Nodes,
7619
665k
                     [&User](const SDNode *Node) { return User == Node; }))
7620
86.0k
      Seen = true;
7621
129k
    else
7622
43.7k
      return false;
7623
129k
  }
7624
76.2k
7625
32.5k
  return Seen;
7626
76.2k
}
7627
7628
/// isOperand - Return true if this node is an operand of N.
7629
4.63M
bool SDValue::isOperandOf(const SDNode *N) const {
7630
4.63M
  for (const SDValue &Op : N->op_values())
7631
14.0M
    
if (14.0M
*this == Op14.0M
)
7632
163k
      return true;
7633
4.46M
  return false;
7634
4.46M
}
7635
7636
4
bool SDNode::isOperandOf(const SDNode *N) const {
7637
4
  for (const SDValue &Op : N->op_values())
7638
11
    
if (11
this == Op.getNode()11
)
7639
3
      return true;
7640
1
  return false;
7641
1
}
7642
7643
/// reachesChainWithoutSideEffects - Return true if this operand (which must
7644
/// be a chain) reaches the specified operand without crossing any
7645
/// side-effecting instructions on any chain path.  In practice, this looks
7646
/// through token factors and non-volatile loads.  In order to remain efficient,
7647
/// this only looks a couple of nodes in, it does not do an exhaustive search.
7648
///
7649
/// Note that we only need to examine chains when we're searching for
7650
/// side-effects; SelectionDAG requires that all side-effects are represented
7651
/// by chains, even if another operand would force a specific ordering. This
7652
/// constraint is necessary to allow transformations like splitting loads.
7653
bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
7654
3.52k
                                             unsigned Depth) const {
7655
3.52k
  if (
*this == Dest3.52k
)
return true1.76k
;
7656
1.76k
7657
1.76k
  // Don't search too deeply, we just want to be able to see through
7658
1.76k
  // TokenFactor's etc.
7659
1.76k
  
if (1.76k
Depth == 01.76k
)
return false161
;
7660
1.60k
7661
1.60k
  // If this is a token factor, all inputs to the TF happen in parallel.
7662
1.60k
  
if (1.60k
getOpcode() == ISD::TokenFactor1.60k
) {
7663
328
    // First, try a shallow search.
7664
328
    if (
is_contained((*this)->ops(), Dest)328
) {
7665
241
      // We found the chain we want as an operand of this TokenFactor.
7666
241
      // Essentially, we reach the chain without side-effects if we could
7667
241
      // serialize the TokenFactor into a simple chain of operations with
7668
241
      // Dest as the last operation. This is automatically true if the
7669
241
      // chain has one use: there are no other ordering constraints.
7670
241
      // If the chain has more than one use, we give up: some other
7671
241
      // use of Dest might force a side-effect between Dest and the current
7672
241
      // node.
7673
241
      if (Dest.hasOneUse())
7674
139
        return true;
7675
189
    }
7676
189
    // Next, try a deep search: check whether every operand of the TokenFactor
7677
189
    // reaches Dest.
7678
189
    
return llvm::all_of((*this)->ops(), [=](SDValue Op) 189
{
7679
201
      return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
7680
201
    });
7681
328
  }
7682
1.28k
7683
1.28k
  // Loads don't have side effects, look through them.
7684
1.28k
  
if (LoadSDNode *1.28k
Ld1.28k
= dyn_cast<LoadSDNode>(*this)) {
7685
226
    if (!Ld->isVolatile())
7686
226
      return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
7687
1.05k
  }
7688
1.05k
  return false;
7689
1.05k
}
7690
7691
259k
bool SDNode::hasPredecessor(const SDNode *N) const {
7692
259k
  SmallPtrSet<const SDNode *, 32> Visited;
7693
259k
  SmallVector<const SDNode *, 16> Worklist;
7694
259k
  Worklist.push_back(this);
7695
259k
  return hasPredecessorHelper(N, Visited, Worklist);
7696
259k
}
7697
7698
1.58M
void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
7699
1.58M
  this->Flags.intersectWith(Flags);
7700
1.58M
}
7701
7702
6.35k
SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
7703
6.35k
  assert(N->getNumValues() == 1 &&
7704
6.35k
         "Can't unroll a vector with multiple results!");
7705
6.35k
7706
6.35k
  EVT VT = N->getValueType(0);
7707
6.35k
  unsigned NE = VT.getVectorNumElements();
7708
6.35k
  EVT EltVT = VT.getVectorElementType();
7709
6.35k
  SDLoc dl(N);
7710
6.35k
7711
6.35k
  SmallVector<SDValue, 8> Scalars;
7712
6.35k
  SmallVector<SDValue, 4> Operands(N->getNumOperands());
7713
6.35k
7714
6.35k
  // If ResNE is 0, fully unroll the vector op.
7715
6.35k
  if (ResNE == 0)
7716
6.34k
    ResNE = NE;
7717
15
  else 
if (15
NE > ResNE15
)
7718
0
    NE = ResNE;
7719
6.35k
7720
6.35k
  unsigned i;
7721
22.5k
  for (i= 0; 
i != NE22.5k
;
++i16.1k
) {
7722
44.4k
    for (unsigned j = 0, e = N->getNumOperands(); 
j != e44.4k
;
++j28.2k
) {
7723
28.2k
      SDValue Operand = N->getOperand(j);
7724
28.2k
      EVT OperandVT = Operand.getValueType();
7725
28.2k
      if (
OperandVT.isVector()28.2k
) {
7726
27.8k
        // A vector operand; extract a single element.
7727
27.8k
        EVT OperandEltVT = OperandVT.getVectorElementType();
7728
27.8k
        Operands[j] =
7729
27.8k
            getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand,
7730
27.8k
                    getConstant(i, dl, TLI->getVectorIdxTy(getDataLayout())));
7731
28.2k
      } else {
7732
440
        // A scalar operand; just use it as is.
7733
440
        Operands[j] = Operand;
7734
440
      }
7735
28.2k
    }
7736
16.1k
7737
16.1k
    switch (N->getOpcode()) {
7738
15.6k
    default: {
7739
15.6k
      Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
7740
15.6k
                                N->getFlags()));
7741
15.6k
      break;
7742
16.1k
    }
7743
46
    case ISD::VSELECT:
7744
46
      Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
7745
46
      break;
7746
400
    case ISD::SHL:
7747
400
    case ISD::SRA:
7748
400
    case ISD::SRL:
7749
400
    case ISD::ROTL:
7750
400
    case ISD::ROTR:
7751
400
      Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
7752
400
                               getShiftAmountOperand(Operands[0].getValueType(),
7753
400
                                                     Operands[1])));
7754
400
      break;
7755
72
    case ISD::SIGN_EXTEND_INREG:
7756
72
    case ISD::FP_ROUND_INREG: {
7757
72
      EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
7758
72
      Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
7759
72
                                Operands[0],
7760
72
                                getValueType(ExtVT)));
7761
72
    }
7762
16.1k
    }
7763
16.1k
  }
7764
6.35k
7765
6.36k
  
for (; 6.35k
i < ResNE6.36k
;
++i7
)
7766
7
    Scalars.push_back(getUNDEF(EltVT));
7767
6.35k
7768
6.35k
  EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
7769
6.35k
  return getBuildVector(VecVT, dl, Scalars);
7770
6.35k
}
7771
7772
bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
7773
                                                  LoadSDNode *Base,
7774
                                                  unsigned Bytes,
7775
4.83k
                                                  int Dist) const {
7776
4.83k
  if (
LD->isVolatile() || 4.83k
Base->isVolatile()4.78k
)
7777
91
    return false;
7778
4.74k
  
if (4.74k
LD->isIndexed() || 4.74k
Base->isIndexed()4.74k
)
7779
0
    return false;
7780
4.74k
  
if (4.74k
LD->getChain() != Base->getChain()4.74k
)
7781
333
    return false;
7782
4.41k
  EVT VT = LD->getValueType(0);
7783
4.41k
  if (VT.getSizeInBits() / 8 != Bytes)
7784
0
    return false;
7785
4.41k
7786
4.41k
  SDValue Loc = LD->getOperand(1);
7787
4.41k
  SDValue BaseLoc = Base->getOperand(1);
7788
4.41k
7789
4.41k
  auto BaseLocDecomp = BaseIndexOffset::match(BaseLoc, *this);
7790
4.41k
  auto LocDecomp = BaseIndexOffset::match(Loc, *this);
7791
4.41k
7792
4.41k
  int64_t Offset = 0;
7793
4.41k
  if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
7794
4.21k
    return (Dist * Bytes == Offset);
7795
191
  return false;
7796
191
}
7797
7798
/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
7799
/// it cannot be inferred.
7800
18.9M
unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
7801
18.9M
  // If this is a GlobalAddress + cst, return the alignment.
7802
18.9M
  const GlobalValue *GV;
7803
18.9M
  int64_t GVOffset = 0;
7804
18.9M
  if (
TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)18.9M
) {
7805
1.15M
    unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
7806
1.15M
    KnownBits Known(PtrWidth);
7807
1.15M
    llvm::computeKnownBits(GV, Known, getDataLayout());
7808
1.15M
    unsigned AlignBits = Known.countMinTrailingZeros();
7809
1.15M
    unsigned Align = AlignBits ? 
1 << std::min(31U, AlignBits)1.07M
:
082.1k
;
7810
1.15M
    if (Align)
7811
1.07M
      return MinAlign(Align, GVOffset);
7812
17.9M
  }
7813
17.9M
7814
17.9M
  // If this is a direct reference to a stack slot, use information about the
7815
17.9M
  // stack slot's alignment.
7816
17.9M
  int FrameIdx = 1 << 31;
7817
17.9M
  int64_t FrameOffset = 0;
7818
17.9M
  if (FrameIndexSDNode *
FI17.9M
= dyn_cast<FrameIndexSDNode>(Ptr)) {
7819
4.63M
    FrameIdx = FI->getIndex();
7820
17.9M
  } else 
if (13.2M
isBaseWithConstantOffset(Ptr) &&
7821
13.2M
             
isa<FrameIndexSDNode>(Ptr.getOperand(0))10.5M
) {
7822
1.43M
    // Handle FI+Cst
7823
1.43M
    FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
7824
1.43M
    FrameOffset = Ptr.getConstantOperandVal(1);
7825
1.43M
  }
7826
17.9M
7827
17.9M
  if (
FrameIdx != (1 << 31)17.9M
) {
7828
6.07M
    const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
7829
6.07M
    unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx),
7830
6.07M
                                    FrameOffset);
7831
6.07M
    return FIInfoAlign;
7832
6.07M
  }
7833
11.8M
7834
11.8M
  return 0;
7835
11.8M
}
7836
7837
/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
7838
/// which is split (or expanded) into two not necessarily identical pieces.
7839
143k
std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
7840
143k
  // Currently all types are split in half.
7841
143k
  EVT LoVT, HiVT;
7842
143k
  if (!VT.isVector())
7843
1.63k
    LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
7844
143k
  else
7845
141k
    LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
7846
143k
7847
143k
  return std::make_pair(LoVT, HiVT);
7848
143k
}
7849
7850
/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
7851
/// low/high part.
7852
std::pair<SDValue, SDValue>
7853
SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
7854
34.2k
                          const EVT &HiVT) {
7855
34.2k
  assert(LoVT.getVectorNumElements() + HiVT.getVectorNumElements() <=
7856
34.2k
         N.getValueType().getVectorNumElements() &&
7857
34.2k
         "More vector elements requested than available!");
7858
34.2k
  SDValue Lo, Hi;
7859
34.2k
  Lo = getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
7860
34.2k
               getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout())));
7861
34.2k
  Hi = getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
7862
34.2k
               getConstant(LoVT.getVectorNumElements(), DL,
7863
34.2k
                           TLI->getVectorIdxTy(getDataLayout())));
7864
34.2k
  return std::make_pair(Lo, Hi);
7865
34.2k
}
7866
7867
void SelectionDAG::ExtractVectorElements(SDValue Op,
7868
                                         SmallVectorImpl<SDValue> &Args,
7869
17.4k
                                         unsigned Start, unsigned Count) {
7870
17.4k
  EVT VT = Op.getValueType();
7871
17.4k
  if (Count == 0)
7872
4.46k
    Count = VT.getVectorNumElements();
7873
17.4k
7874
17.4k
  EVT EltVT = VT.getVectorElementType();
7875
17.4k
  EVT IdxTy = TLI->getVectorIdxTy(getDataLayout());
7876
17.4k
  SDLoc SL(Op);
7877
92.2k
  for (unsigned i = Start, e = Start + Count; 
i != e92.2k
;
++i74.8k
) {
7878
74.8k
    Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT,
7879
74.8k
                           Op, getConstant(i, SL, IdxTy)));
7880
74.8k
  }
7881
17.4k
}
7882
7883
// getAddressSpace - Return the address space this GlobalAddress belongs to.
7884
4.87k
unsigned GlobalAddressSDNode::getAddressSpace() const {
7885
4.87k
  return getGlobal()->getType()->getAddressSpace();
7886
4.87k
}
7887
7888
175k
Type *ConstantPoolSDNode::getType() const {
7889
175k
  if (isMachineConstantPoolEntry())
7890
343
    return Val.MachineCPVal->getType();
7891
175k
  return Val.ConstVal->getType();
7892
175k
}
7893
7894
bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
7895
                                        unsigned &SplatBitSize,
7896
                                        bool &HasAnyUndefs,
7897
                                        unsigned MinSplatBits,
7898
205k
                                        bool IsBigEndian) const {
7899
205k
  EVT VT = getValueType(0);
7900
205k
  assert(VT.isVector() && "Expected a vector type");
7901
205k
  unsigned VecWidth = VT.getSizeInBits();
7902
205k
  if (MinSplatBits > VecWidth)
7903
0
    return false;
7904
205k
7905
205k
  // FIXME: The widths are based on this node's type, but build vectors can
7906
205k
  // truncate their operands.
7907
205k
  SplatValue = APInt(VecWidth, 0);
7908
205k
  SplatUndef = APInt(VecWidth, 0);
7909
205k
7910
205k
  // Get the bits. Bits with undefined values (when the corresponding element
7911
205k
  // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
7912
205k
  // in SplatValue. If any of the values are not constant, give up and return
7913
205k
  // false.
7914
205k
  unsigned int NumOps = getNumOperands();
7915
205k
  assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
7916
205k
  unsigned EltWidth = VT.getScalarSizeInBits();
7917
205k
7918
849k
  for (unsigned j = 0; 
j < NumOps849k
;
++j643k
) {
7919
675k
    unsigned i = IsBigEndian ? 
NumOps - 1 - j28.4k
:
j647k
;
7920
675k
    SDValue OpVal = getOperand(i);
7921
675k
    unsigned BitPos = j * EltWidth;
7922
675k
7923
675k
    if (OpVal.isUndef())
7924
13.6k
      SplatUndef.setBits(BitPos, BitPos + EltWidth);
7925
662k
    else 
if (auto *662k
CN662k
= dyn_cast<ConstantSDNode>(OpVal))
7926
592k
      SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
7927
69.1k
    else 
if (auto *69.1k
CN69.1k
= dyn_cast<ConstantFPSDNode>(OpVal))
7928
36.9k
      SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
7929
69.1k
    else
7930
32.1k
      return false;
7931
675k
  }
7932
205k
7933
205k
  // The build_vector is all constants or undefs. Find the smallest element
7934
205k
  // size that splats the vector.
7935
173k
  HasAnyUndefs = (SplatUndef != 0);
7936
173k
7937
173k
  // FIXME: This does not work for vectors with elements less than 8 bits.
7938
382k
  while (
VecWidth > 8382k
) {
7939
349k
    unsigned HalfSize = VecWidth / 2;
7940
349k
    APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize);
7941
349k
    APInt LowValue = SplatValue.trunc(HalfSize);
7942
349k
    APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize);
7943
349k
    APInt LowUndef = SplatUndef.trunc(HalfSize);
7944
349k
7945
349k
    // If the two halves do not match (ignoring undef bits), stop here.
7946
349k
    if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
7947
209k
        MinSplatBits > HalfSize)
7948
140k
      break;
7949
208k
7950
208k
    SplatValue = HighValue | LowValue;
7951
208k
    SplatUndef = HighUndef & LowUndef;
7952
208k
7953
208k
    VecWidth = HalfSize;
7954
208k
  }
7955
173k
7956
173k
  SplatBitSize = VecWidth;
7957
173k
  return true;
7958
205k
}
7959
7960
200k
SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
7961
200k
  if (
UndefElements200k
) {
7962
178k
    UndefElements->clear();
7963
178k
    UndefElements->resize(getNumOperands());
7964
178k
  }
7965
200k
  SDValue Splatted;
7966
1.28M
  for (unsigned i = 0, e = getNumOperands(); 
i != e1.28M
;
++i1.08M
) {
7967
1.10M
    SDValue Op = getOperand(i);
7968
1.10M
    if (
Op.isUndef()1.10M
) {
7969
8.73k
      if (UndefElements)
7970
8.49k
        (*UndefElements)[i] = true;
7971
1.10M
    } else 
if (1.09M
!Splatted1.09M
) {
7972
200k
      Splatted = Op;
7973
1.09M
    } else 
if (899k
Splatted != Op899k
) {
7974
25.1k
      return SDValue();
7975
25.1k
    }
7976
1.10M
  }
7977
200k
7978
175k
  
if (175k
!Splatted175k
) {
7979
49
    assert(getOperand(0).isUndef() &&
7980
49
           "Can only have a splat without a constant for all undefs.");
7981
49
    return getOperand(0);
7982
49
  }
7983
175k
7984
175k
  return Splatted;
7985
175k
}
7986
7987
ConstantSDNode *
7988
160k
BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
7989
160k
  return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
7990
160k
}
7991
7992
ConstantFPSDNode *
7993
12.5k
BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
7994
12.5k
  return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
7995
12.5k
}
7996
7997
int32_t
7998
BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
7999
117
                                                   uint32_t BitWidth) const {
8000
117
  if (ConstantFPSDNode *CN =
8001
98
          dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
8002
98
    bool IsExact;
8003
98
    APSInt IntVal(BitWidth);
8004
98
    const APFloat &APF = CN->getValueAPF();
8005
98
    if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
8006
98
            APFloat::opOK ||
8007
92
        !IsExact)
8008
6
      return -1;
8009
92
8010
92
    return IntVal.exactLogBase2();
8011
92
  }
8012
19
  return -1;
8013
19
}
8014
8015
241k
bool BuildVectorSDNode::isConstant() const {
8016
584k
  for (const SDValue &Op : op_values()) {
8017
584k
    unsigned Opc = Op.getOpcode();
8018
584k
    if (
Opc != ISD::UNDEF && 584k
Opc != ISD::Constant580k
&&
Opc != ISD::ConstantFP231k
)
8019
181k
      return false;
8020
60.7k
  }
8021
60.7k
  return true;
8022
60.7k
}
8023
8024
61.7k
bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
8025
61.7k
  // Find the first non-undef value in the shuffle mask.
8026
61.7k
  unsigned i, e;
8027
64.8k
  for (i = 0, e = VT.getVectorNumElements(); 
i != e && 64.8k
Mask[i] < 064.8k
;
++i3.10k
)
8028
3.10k
    /* search */;
8029
61.7k
8030
61.7k
  assert(i != e && "VECTOR_SHUFFLE node with all undef indices!");
8031
61.7k
8032
61.7k
  // Make sure all remaining elements are either undef or the same as the first
8033
61.7k
  // non-undef value.
8034
216k
  for (int Idx = Mask[i]; 
i != e216k
;
++i154k
)
8035
199k
    
if (199k
Mask[i] >= 0 && 199k
Mask[i] != Idx183k
)
8036
44.8k
      return false;
8037
16.9k
  return true;
8038
61.7k
}
8039
8040
// \brief Returns the SDNode if it is a constant integer BuildVector
8041
// or constant integer.
8042
16.1M
SDNode *SelectionDAG::isConstantIntBuildVectorOrConstantInt(SDValue N) {
8043
16.1M
  if (isa<ConstantSDNode>(N))
8044
1.47M
    return N.getNode();
8045
14.6M
  
if (14.6M
ISD::isBuildVectorOfConstantSDNodes(N.getNode())14.6M
)
8046
7.49k
    return N.getNode();
8047
14.6M
  // Treat a GlobalAddress supporting constant offset folding as a
8048
14.6M
  // constant integer.
8049
14.6M
  
if (GlobalAddressSDNode *14.6M
GA14.6M
= dyn_cast<GlobalAddressSDNode>(N))
8050
357k
    
if (357k
GA->getOpcode() == ISD::GlobalAddress &&
8051
357k
        TLI->isOffsetFoldingLegal(GA))
8052
885
      return GA;
8053
14.6M
  return nullptr;
8054
14.6M
}
8055
8056
5.43k
SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
8057
5.43k
  if (isa<ConstantFPSDNode>(N))
8058
226
    return N.getNode();
8059
5.20k
8060
5.20k
  
if (5.20k
ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())5.20k
)
8061
4
    return N.getNode();
8062
5.20k
8063
5.20k
  return nullptr;
8064
5.20k
}
8065
8066
#ifndef NDEBUG
8067
static void checkForCyclesHelper(const SDNode *N,
8068
                                 SmallPtrSetImpl<const SDNode*> &Visited,
8069
                                 SmallPtrSetImpl<const SDNode*> &Checked,
8070
                                 const llvm::SelectionDAG *DAG) {
8071
  // If this node has already been checked, don't check it again.
8072
  if (Checked.count(N))
8073
    return;
8074
8075
  // If a node has already been visited on this depth-first walk, reject it as
8076
  // a cycle.
8077
  if (!Visited.insert(N).second) {
8078
    errs() << "Detected cycle in SelectionDAG\n";
8079
    dbgs() << "Offending node:\n";
8080
    N->dumprFull(DAG); dbgs() << "\n";
8081
    abort();
8082
  }
8083
8084
  for (const SDValue &Op : N->op_values())
8085
    checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
8086
8087
  Checked.insert(N);
8088
  Visited.erase(N);
8089
}
8090
#endif
8091
8092
void llvm::checkForCycles(const llvm::SDNode *N,
8093
                          const llvm::SelectionDAG *DAG,
8094
455M
                          bool force) {
8095
#ifndef NDEBUG
8096
  bool check = force;
8097
#ifdef EXPENSIVE_CHECKS
8098
  check = true;
8099
#endif  // EXPENSIVE_CHECKS
8100
  if (check) {
8101
    assert(N && "Checking nonexistent SDNode");
8102
    SmallPtrSet<const SDNode*, 32> visited;
8103
    SmallPtrSet<const SDNode*, 32> checked;
8104
    checkForCyclesHelper(N, visited, checked, DAG);
8105
  }
8106
#endif  // !NDEBUG
8107
}
8108
8109
41.8M
void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
8110
41.8M
  checkForCycles(DAG->getRoot().getNode(), DAG, force);
8111
41.8M
}