Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This implements the SelectionDAG class.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "llvm/CodeGen/SelectionDAG.h"
14
#include "SDNodeDbgValue.h"
15
#include "llvm/ADT/APFloat.h"
16
#include "llvm/ADT/APInt.h"
17
#include "llvm/ADT/APSInt.h"
18
#include "llvm/ADT/ArrayRef.h"
19
#include "llvm/ADT/BitVector.h"
20
#include "llvm/ADT/FoldingSet.h"
21
#include "llvm/ADT/None.h"
22
#include "llvm/ADT/STLExtras.h"
23
#include "llvm/ADT/SmallPtrSet.h"
24
#include "llvm/ADT/SmallVector.h"
25
#include "llvm/ADT/Triple.h"
26
#include "llvm/ADT/Twine.h"
27
#include "llvm/Analysis/ValueTracking.h"
28
#include "llvm/CodeGen/ISDOpcodes.h"
29
#include "llvm/CodeGen/MachineBasicBlock.h"
30
#include "llvm/CodeGen/MachineConstantPool.h"
31
#include "llvm/CodeGen/MachineFrameInfo.h"
32
#include "llvm/CodeGen/MachineFunction.h"
33
#include "llvm/CodeGen/MachineMemOperand.h"
34
#include "llvm/CodeGen/RuntimeLibcalls.h"
35
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
36
#include "llvm/CodeGen/SelectionDAGNodes.h"
37
#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
38
#include "llvm/CodeGen/TargetLowering.h"
39
#include "llvm/CodeGen/TargetRegisterInfo.h"
40
#include "llvm/CodeGen/TargetSubtargetInfo.h"
41
#include "llvm/CodeGen/ValueTypes.h"
42
#include "llvm/IR/Constant.h"
43
#include "llvm/IR/Constants.h"
44
#include "llvm/IR/DataLayout.h"
45
#include "llvm/IR/DebugInfoMetadata.h"
46
#include "llvm/IR/DebugLoc.h"
47
#include "llvm/IR/DerivedTypes.h"
48
#include "llvm/IR/Function.h"
49
#include "llvm/IR/GlobalValue.h"
50
#include "llvm/IR/Metadata.h"
51
#include "llvm/IR/Type.h"
52
#include "llvm/IR/Value.h"
53
#include "llvm/Support/Casting.h"
54
#include "llvm/Support/CodeGen.h"
55
#include "llvm/Support/Compiler.h"
56
#include "llvm/Support/Debug.h"
57
#include "llvm/Support/ErrorHandling.h"
58
#include "llvm/Support/KnownBits.h"
59
#include "llvm/Support/MachineValueType.h"
60
#include "llvm/Support/ManagedStatic.h"
61
#include "llvm/Support/MathExtras.h"
62
#include "llvm/Support/Mutex.h"
63
#include "llvm/Support/raw_ostream.h"
64
#include "llvm/Target/TargetMachine.h"
65
#include "llvm/Target/TargetOptions.h"
66
#include <algorithm>
67
#include <cassert>
68
#include <cstdint>
69
#include <cstdlib>
70
#include <limits>
71
#include <set>
72
#include <string>
73
#include <utility>
74
#include <vector>
75
76
using namespace llvm;
77
78
/// makeVTList - Return an instance of the SDVTList struct initialized with the
79
/// specified members.
80
63.2M
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
81
63.2M
  SDVTList Res = {VTs, NumVTs};
82
63.2M
  return Res;
83
63.2M
}
84
85
// Default null implementations of the callbacks.
86
14.8k
void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
87
26.2M
void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
88
17.7M
void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
89
90
0
void SelectionDAG::DAGNodeDeletedListener::anchor() {}
91
92
#define DEBUG_TYPE "selectiondag"
93
94
static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
95
       cl::Hidden, cl::init(true),
96
       cl::desc("Gang up loads and stores generated by inlining of memcpy"));
97
98
static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
99
       cl::desc("Number limit for gluing ld/st of memcpy."),
100
       cl::Hidden, cl::init(0));
101
102
35.1M
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
103
35.1M
  LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
104
35.1M
}
105
106
//===----------------------------------------------------------------------===//
107
//                              ConstantFPSDNode Class
108
//===----------------------------------------------------------------------===//
109
110
/// isExactlyValue - We don't rely on operator== working on double values, as
111
/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
112
/// As such, this method can be used to do an exact bit-for-bit comparison of
113
/// two floating point values.
114
3
bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
115
3
  return getValueAPF().bitwiseIsEqual(V);
116
3
}
117
118
bool ConstantFPSDNode::isValueValidForType(EVT VT,
119
9.38k
                                           const APFloat& Val) {
120
9.38k
  assert(VT.isFloatingPoint() && "Can only convert between FP types");
121
9.38k
122
9.38k
  // convert modifies in place, so make a copy.
123
9.38k
  APFloat Val2 = APFloat(Val);
124
9.38k
  bool losesInfo;
125
9.38k
  (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT),
126
9.38k
                      APFloat::rmNearestTiesToEven,
127
9.38k
                      &losesInfo);
128
9.38k
  return !losesInfo;
129
9.38k
}
130
131
//===----------------------------------------------------------------------===//
132
//                              ISD Namespace
133
//===----------------------------------------------------------------------===//
134
135
168k
bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
136
168k
  auto *BV = dyn_cast<BuildVectorSDNode>(N);
137
168k
  if (!BV)
138
151k
    return false;
139
16.5k
140
16.5k
  APInt SplatUndef;
141
16.5k
  unsigned SplatBitSize;
142
16.5k
  bool HasUndefs;
143
16.5k
  unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
144
16.5k
  return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
145
16.5k
                             EltSize) &&
146
16.5k
         
EltSize == SplatBitSize15.8k
;
147
16.5k
}
148
149
// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
150
// specializations of the more general isConstantSplatVector()?
151
152
626k
bool ISD::isBuildVectorAllOnes(const SDNode *N) {
153
626k
  // Look through a bit convert.
154
689k
  while (N->getOpcode() == ISD::BITCAST)
155
63.6k
    N = N->getOperand(0).getNode();
156
626k
157
626k
  if (N->getOpcode() != ISD::BUILD_VECTOR) 
return false480k
;
158
145k
159
145k
  unsigned i = 0, e = N->getNumOperands();
160
145k
161
145k
  // Skip over all of the undef values.
162
148k
  while (i != e && N->getOperand(i).isUndef())
163
3.09k
    ++i;
164
145k
165
145k
  // Do not accept an all-undef vector.
166
145k
  if (i == e) 
return false0
;
167
145k
168
145k
  // Do not accept build_vectors that aren't all constants or which have non-~0
169
145k
  // elements. We have to be a bit careful here, as the type of the constant
170
145k
  // may not be the same as the type of the vector elements due to type
171
145k
  // legalization (the elements are promoted to a legal type for the target and
172
145k
  // a vector of a type may be legal when the base element type is not).
173
145k
  // We only want to check enough bits to cover the vector elements, because
174
145k
  // we care if the resultant vector is all ones, not whether the individual
175
145k
  // constants are.
176
145k
  SDValue NotZero = N->getOperand(i);
177
145k
  unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
178
145k
  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
179
115k
    if (CN->getAPIntValue().countTrailingOnes() < EltSize)
180
75.3k
      return false;
181
29.4k
  } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
182
4.55k
    if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
183
4.43k
      return false;
184
24.9k
  } else
185
24.9k
    return false;
186
40.7k
187
40.7k
  // Okay, we have at least one ~0 value, check to see if the rest match or are
188
40.7k
  // undefs. Even with the above element type twiddling, this should be OK, as
189
40.7k
  // the same type legalization should have applied to all the elements.
190
240k
  
for (++i; 40.7k
i != e;
++i199k
)
191
202k
    if (N->getOperand(i) != NotZero && 
!N->getOperand(i).isUndef()3.02k
)
192
2.74k
      return false;
193
40.7k
  
return true37.9k
;
194
40.7k
}
195
196
1.81M
bool ISD::isBuildVectorAllZeros(const SDNode *N) {
197
1.81M
  // Look through a bit convert.
198
1.97M
  while (N->getOpcode() == ISD::BITCAST)
199
160k
    N = N->getOperand(0).getNode();
200
1.81M
201
1.81M
  if (N->getOpcode() != ISD::BUILD_VECTOR) 
return false1.39M
;
202
426k
203
426k
  bool IsAllUndef = true;
204
2.34M
  for (const SDValue &Op : N->op_values()) {
205
2.34M
    if (Op.isUndef())
206
17.6k
      continue;
207
2.32M
    IsAllUndef = false;
208
2.32M
    // Do not accept build_vectors that aren't all constants or which have non-0
209
2.32M
    // elements. We have to be a bit careful here, as the type of the constant
210
2.32M
    // may not be the same as the type of the vector elements due to type
211
2.32M
    // legalization (the elements are promoted to a legal type for the target
212
2.32M
    // and a vector of a type may be legal when the base element type is not).
213
2.32M
    // We only want to check enough bits to cover the vector elements, because
214
2.32M
    // we care if the resultant vector is all zeros, not whether the individual
215
2.32M
    // constants are.
216
2.32M
    unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
217
2.32M
    if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
218
2.20M
      if (CN->getAPIntValue().countTrailingZeros() < EltSize)
219
186k
        return false;
220
121k
    } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
221
89.6k
      if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
222
4.36k
        return false;
223
31.7k
    } else
224
31.7k
      return false;
225
2.32M
  }
226
426k
227
426k
  // Do not accept an all-undef vector.
228
426k
  
if (203k
IsAllUndef203k
)
229
0
    return false;
230
203k
  return true;
231
203k
}
232
233
8.87M
bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
234
8.87M
  if (N->getOpcode() != ISD::BUILD_VECTOR)
235
8.72M
    return false;
236
158k
237
1.32M
  
for (const SDValue &Op : N->op_values())158k
{
238
1.32M
    if (Op.isUndef())
239
44.7k
      continue;
240
1.28M
    if (!isa<ConstantSDNode>(Op))
241
33.5k
      return false;
242
1.28M
  }
243
158k
  
return true124k
;
244
158k
}
245
246
1.52M
bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
247
1.52M
  if (N->getOpcode() != ISD::BUILD_VECTOR)
248
1.50M
    return false;
249
25.0k
250
63.1k
  
for (const SDValue &Op : N->op_values())25.0k
{
251
63.1k
    if (Op.isUndef())
252
879
      continue;
253
62.2k
    if (!isa<ConstantFPSDNode>(Op))
254
5.32k
      return false;
255
62.2k
  }
256
25.0k
  
return true19.7k
;
257
25.0k
}
258
259
574k
bool ISD::allOperandsUndef(const SDNode *N) {
260
574k
  // Return false if the node has no operands.
261
574k
  // This is "logically inconsistent" with the definition of "all" but
262
574k
  // is probably the desired behavior.
263
574k
  if (N->getNumOperands() == 0)
264
0
    return false;
265
583k
  
return all_of(N->op_values(), [](SDValue Op) 574k
{ return Op.isUndef(); });
266
574k
}
267
268
bool ISD::matchUnaryPredicate(SDValue Op,
269
                              std::function<bool(ConstantSDNode *)> Match,
270
1.25M
                              bool AllowUndefs) {
271
1.25M
  // FIXME: Add support for scalar UNDEF cases?
272
1.25M
  if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
273
1.05M
    return Match(Cst);
274
204k
275
204k
  // FIXME: Add support for vector UNDEF cases?
276
204k
  if (ISD::BUILD_VECTOR != Op.getOpcode())
277
136k
    return false;
278
67.2k
279
67.2k
  EVT SVT = Op.getValueType().getScalarType();
280
81.6k
  for (unsigned i = 0, e = Op.getNumOperands(); i != e; 
++i14.3k
) {
281
79.9k
    if (AllowUndefs && 
Op.getOperand(i).isUndef()66.9k
) {
282
1.04k
      if (!Match(nullptr))
283
0
        return false;
284
1.04k
      continue;
285
1.04k
    }
286
78.9k
287
78.9k
    auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
288
78.9k
    if (!Cst || 
Cst->getValueType(0) != SVT77.2k
||
!Match(Cst)75.6k
)
289
65.5k
      return false;
290
78.9k
  }
291
67.2k
  
return true1.71k
;
292
67.2k
}
293
294
bool ISD::matchBinaryPredicate(
295
    SDValue LHS, SDValue RHS,
296
    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
297
71.2k
    bool AllowUndefs, bool AllowTypeMismatch) {
298
71.2k
  if (!AllowTypeMismatch && 
LHS.getValueType() != RHS.getValueType()70.8k
)
299
1.70k
    return false;
300
69.5k
301
69.5k
  // TODO: Add support for scalar UNDEF cases?
302
69.5k
  if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
303
33.6k
    if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
304
20.5k
      return Match(LHSCst, RHSCst);
305
49.0k
306
49.0k
  // TODO: Add support for vector UNDEF cases?
307
49.0k
  if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
308
49.0k
      
ISD::BUILD_VECTOR != RHS.getOpcode()2.59k
)
309
48.0k
    return false;
310
953
311
953
  EVT SVT = LHS.getValueType().getScalarType();
312
11.9k
  for (unsigned i = 0, e = LHS.getNumOperands(); i != e; 
++i11.0k
) {
313
11.2k
    SDValue LHSOp = LHS.getOperand(i);
314
11.2k
    SDValue RHSOp = RHS.getOperand(i);
315
11.2k
    bool LHSUndef = AllowUndefs && 
LHSOp.isUndef()3.33k
;
316
11.2k
    bool RHSUndef = AllowUndefs && 
RHSOp.isUndef()3.33k
;
317
11.2k
    auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
318
11.2k
    auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
319
11.2k
    if ((!LHSCst && 
!LHSUndef88
) || (!RHSCst &&
!RHSUndef84
))
320
0
      return false;
321
11.2k
    if (!AllowTypeMismatch && 
(11.0k
LHSOp.getValueType() != SVT11.0k
||
322
11.0k
                               LHSOp.getValueType() != RHSOp.getValueType()))
323
0
      return false;
324
11.2k
    if (!Match(LHSCst, RHSCst))
325
253
      return false;
326
11.2k
  }
327
953
  
return true700
;
328
953
}
329
330
455
ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
331
455
  switch (ExtType) {
332
455
  case ISD::EXTLOAD:
333
33
    return IsFP ? 
ISD::FP_EXTEND0
: ISD::ANY_EXTEND;
334
455
  case ISD::SEXTLOAD:
335
172
    return ISD::SIGN_EXTEND;
336
455
  case ISD::ZEXTLOAD:
337
250
    return ISD::ZERO_EXTEND;
338
455
  default:
339
0
    break;
340
0
  }
341
0
342
0
  llvm_unreachable("Invalid LoadExtType");
343
0
}
344
345
1.24M
ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
346
1.24M
  // To perform this operation, we just need to swap the L and G bits of the
347
1.24M
  // operation.
348
1.24M
  unsigned OldL = (Operation >> 2) & 1;
349
1.24M
  unsigned OldG = (Operation >> 1) & 1;
350
1.24M
  return ISD::CondCode((Operation & ~6) |  // Keep the N, U, E bits
351
1.24M
                       (OldL << 1) |       // New G bit
352
1.24M
                       (OldG << 2));       // New L bit.
353
1.24M
}
354
355
263k
ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) {
356
263k
  unsigned Operation = Op;
357
263k
  if (isInteger)
358
258k
    Operation ^= 7;   // Flip L, G, E bits, but not U.
359
4.71k
  else
360
4.71k
    Operation ^= 15;  // Flip all of the condition bits.
361
263k
362
263k
  if (Operation > ISD::SETTRUE2)
363
470
    Operation &= ~8;  // Don't let N and U bits get set.
364
263k
365
263k
  return ISD::CondCode(Operation);
366
263k
}
367
368
/// For an integer comparison, return 1 if the comparison is a signed operation
369
/// and 2 if the result is an unsigned comparison. Return zero if the operation
370
/// does not depend on the sign of the input (setne and seteq).
371
50
static int isSignedOp(ISD::CondCode Opcode) {
372
50
  switch (Opcode) {
373
50
  
default: 0
llvm_unreachable0
("Illegal integer setcc operation!");
374
50
  case ISD::SETEQ:
375
24
  case ISD::SETNE: return 0;
376
24
  case ISD::SETLT:
377
12
  case ISD::SETLE:
378
12
  case ISD::SETGT:
379
12
  case ISD::SETGE: return 1;
380
14
  case ISD::SETULT:
381
14
  case ISD::SETULE:
382
14
  case ISD::SETUGT:
383
14
  case ISD::SETUGE: return 2;
384
50
  }
385
50
}
386
387
ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
388
52
                                       bool IsInteger) {
389
52
  if (IsInteger && 
(isSignedOp(Op1) | isSignedOp(Op2)) == 320
)
390
10
    // Cannot fold a signed integer setcc with an unsigned integer setcc.
391
10
    return ISD::SETCC_INVALID;
392
42
393
42
  unsigned Op = Op1 | Op2;  // Combine all of the condition bits.
394
42
395
42
  // If the N and U bits get set, then the resultant comparison DOES suddenly
396
42
  // care about orderedness, and it is true when ordered.
397
42
  if (Op > ISD::SETTRUE2)
398
31
    Op &= ~16;     // Clear the U bit if the N bit is set.
399
42
400
42
  // Canonicalize illegal integer setcc's.
401
42
  if (IsInteger && 
Op == ISD::SETUNE10
) // e.g. SETUGT | SETULT
402
0
    Op = ISD::SETNE;
403
42
404
42
  return ISD::CondCode(Op);
405
42
}
406
407
ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
408
45
                                        bool IsInteger) {
409
45
  if (IsInteger && 
(isSignedOp(Op1) | isSignedOp(Op2)) == 35
)
410
0
    // Cannot fold a signed setcc with an unsigned setcc.
411
0
    return ISD::SETCC_INVALID;
412
45
413
45
  // Combine all of the condition bits.
414
45
  ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
415
45
416
45
  // Canonicalize illegal integer setcc's.
417
45
  if (IsInteger) {
418
5
    switch (Result) {
419
5
    
default: break2
;
420
5
    
case ISD::SETUO : Result = ISD::SETFALSE; break0
; // SETUGT & SETULT
421
5
    case ISD::SETOEQ:                                 // SETEQ  & SETU[LG]E
422
1
    case ISD::SETUEQ: Result = ISD::SETEQ   ; break;  // SETUGE & SETULE
423
2
    case ISD::SETOLT: Result = ISD::SETULT  ; break;  // SETULT & SETNE
424
1
    
case ISD::SETOGT: Result = ISD::SETUGT ; break0
; // SETUGT & SETNE
425
45
    }
426
45
  }
427
45
428
45
  return Result;
429
45
}
430
431
//===----------------------------------------------------------------------===//
432
//                           SDNode Profile Support
433
//===----------------------------------------------------------------------===//
434
435
/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
436
127M
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)  {
437
127M
  ID.AddInteger(OpC);
438
127M
}
439
440
/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
441
/// solely with their pointer.
442
127M
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
443
127M
  ID.AddPointer(VTList.VTs);
444
127M
}
445
446
/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
447
static void AddNodeIDOperands(FoldingSetNodeID &ID,
448
70.7M
                              ArrayRef<SDValue> Ops) {
449
87.6M
  for (auto& Op : Ops) {
450
87.6M
    ID.AddPointer(Op.getNode());
451
87.6M
    ID.AddInteger(Op.getResNo());
452
87.6M
  }
453
70.7M
}
454
455
/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
456
static void AddNodeIDOperands(FoldingSetNodeID &ID,
457
56.8M
                              ArrayRef<SDUse> Ops) {
458
91.3M
  for (auto& Op : Ops) {
459
91.3M
    ID.AddPointer(Op.getNode());
460
91.3M
    ID.AddInteger(Op.getResNo());
461
91.3M
  }
462
56.8M
}
463
464
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned short OpC,
465
70.7M
                          SDVTList VTList, ArrayRef<SDValue> OpList) {
466
70.7M
  AddNodeIDOpcode(ID, OpC);
467
70.7M
  AddNodeIDValueTypes(ID, VTList);
468
70.7M
  AddNodeIDOperands(ID, OpList);
469
70.7M
}
470
471
/// If this is an SDNode with special info, add this info to the NodeID data.
472
57.6M
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
473
57.6M
  switch (N->getOpcode()) {
474
57.6M
  case ISD::TargetExternalSymbol:
475
0
  case ISD::ExternalSymbol:
476
0
  case ISD::MCSymbol:
477
0
    llvm_unreachable("Should only be used on nodes with operands");
478
28.2M
  default: break;  // Normal nodes don't need extra info.
479
15.3M
  case ISD::TargetConstant:
480
15.3M
  case ISD::Constant: {
481
15.3M
    const ConstantSDNode *C = cast<ConstantSDNode>(N);
482
15.3M
    ID.AddPointer(C->getConstantIntValue());
483
15.3M
    ID.AddBoolean(C->isOpaque());
484
15.3M
    break;
485
15.3M
  }
486
15.3M
  case ISD::TargetConstantFP:
487
72.8k
  case ISD::ConstantFP:
488
72.8k
    ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
489
72.8k
    break;
490
798k
  case ISD::TargetGlobalAddress:
491
798k
  case ISD::GlobalAddress:
492
798k
  case ISD::TargetGlobalTLSAddress:
493
798k
  case ISD::GlobalTLSAddress: {
494
798k
    const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
495
798k
    ID.AddPointer(GA->getGlobal());
496
798k
    ID.AddInteger(GA->getOffset());
497
798k
    ID.AddInteger(GA->getTargetFlags());
498
798k
    break;
499
798k
  }
500
798k
  case ISD::BasicBlock:
501
552k
    ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
502
552k
    break;
503
6.74M
  case ISD::Register:
504
6.74M
    ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
505
6.74M
    break;
506
798k
  case ISD::RegisterMask:
507
282k
    ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
508
282k
    break;
509
798k
  case ISD::SRCVALUE:
510
1.63k
    ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
511
1.63k
    break;
512
798k
  case ISD::FrameIndex:
513
538k
  case ISD::TargetFrameIndex:
514
538k
    ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
515
538k
    break;
516
538k
  case ISD::LIFETIME_START:
517
109k
  case ISD::LIFETIME_END:
518
109k
    if (cast<LifetimeSDNode>(N)->hasOffset()) {
519
108k
      ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
520
108k
      ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
521
108k
    }
522
109k
    break;
523
109k
  case ISD::JumpTable:
524
1.17k
  case ISD::TargetJumpTable:
525
1.17k
    ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
526
1.17k
    ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
527
1.17k
    break;
528
67.8k
  case ISD::ConstantPool:
529
67.8k
  case ISD::TargetConstantPool: {
530
67.8k
    const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
531
67.8k
    ID.AddInteger(CP->getAlignment());
532
67.8k
    ID.AddInteger(CP->getOffset());
533
67.8k
    if (CP->isMachineConstantPoolEntry())
534
82
      CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
535
67.7k
    else
536
67.7k
      ID.AddPointer(CP->getConstVal());
537
67.8k
    ID.AddInteger(CP->getTargetFlags());
538
67.8k
    break;
539
67.8k
  }
540
67.8k
  case ISD::TargetIndex: {
541
0
    const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
542
0
    ID.AddInteger(TI->getIndex());
543
0
    ID.AddInteger(TI->getOffset());
544
0
    ID.AddInteger(TI->getTargetFlags());
545
0
    break;
546
67.8k
  }
547
1.88M
  case ISD::LOAD: {
548
1.88M
    const LoadSDNode *LD = cast<LoadSDNode>(N);
549
1.88M
    ID.AddInteger(LD->getMemoryVT().getRawBits());
550
1.88M
    ID.AddInteger(LD->getRawSubclassData());
551
1.88M
    ID.AddInteger(LD->getPointerInfo().getAddrSpace());
552
1.88M
    break;
553
67.8k
  }
554
2.81M
  case ISD::STORE: {
555
2.81M
    const StoreSDNode *ST = cast<StoreSDNode>(N);
556
2.81M
    ID.AddInteger(ST->getMemoryVT().getRawBits());
557
2.81M
    ID.AddInteger(ST->getRawSubclassData());
558
2.81M
    ID.AddInteger(ST->getPointerInfo().getAddrSpace());
559
2.81M
    break;
560
67.8k
  }
561
67.8k
  case ISD::MLOAD: {
562
872
    const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
563
872
    ID.AddInteger(MLD->getMemoryVT().getRawBits());
564
872
    ID.AddInteger(MLD->getRawSubclassData());
565
872
    ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
566
872
    break;
567
67.8k
  }
568
67.8k
  case ISD::MSTORE: {
569
828
    const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
570
828
    ID.AddInteger(MST->getMemoryVT().getRawBits());
571
828
    ID.AddInteger(MST->getRawSubclassData());
572
828
    ID.AddInteger(MST->getPointerInfo().getAddrSpace());
573
828
    break;
574
67.8k
  }
575
67.8k
  case ISD::MGATHER: {
576
711
    const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
577
711
    ID.AddInteger(MG->getMemoryVT().getRawBits());
578
711
    ID.AddInteger(MG->getRawSubclassData());
579
711
    ID.AddInteger(MG->getPointerInfo().getAddrSpace());
580
711
    break;
581
67.8k
  }
582
67.8k
  case ISD::MSCATTER: {
583
143
    const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
584
143
    ID.AddInteger(MS->getMemoryVT().getRawBits());
585
143
    ID.AddInteger(MS->getRawSubclassData());
586
143
    ID.AddInteger(MS->getPointerInfo().getAddrSpace());
587
143
    break;
588
67.8k
  }
589
67.8k
  case ISD::ATOMIC_CMP_SWAP:
590
29.6k
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
591
29.6k
  case ISD::ATOMIC_SWAP:
592
29.6k
  case ISD::ATOMIC_LOAD_ADD:
593
29.6k
  case ISD::ATOMIC_LOAD_SUB:
594
29.6k
  case ISD::ATOMIC_LOAD_AND:
595
29.6k
  case ISD::ATOMIC_LOAD_CLR:
596
29.6k
  case ISD::ATOMIC_LOAD_OR:
597
29.6k
  case ISD::ATOMIC_LOAD_XOR:
598
29.6k
  case ISD::ATOMIC_LOAD_NAND:
599
29.6k
  case ISD::ATOMIC_LOAD_MIN:
600
29.6k
  case ISD::ATOMIC_LOAD_MAX:
601
29.6k
  case ISD::ATOMIC_LOAD_UMIN:
602
29.6k
  case ISD::ATOMIC_LOAD_UMAX:
603
29.6k
  case ISD::ATOMIC_LOAD:
604
29.6k
  case ISD::ATOMIC_STORE: {
605
29.6k
    const AtomicSDNode *AT = cast<AtomicSDNode>(N);
606
29.6k
    ID.AddInteger(AT->getMemoryVT().getRawBits());
607
29.6k
    ID.AddInteger(AT->getRawSubclassData());
608
29.6k
    ID.AddInteger(AT->getPointerInfo().getAddrSpace());
609
29.6k
    break;
610
29.6k
  }
611
29.6k
  case ISD::PREFETCH: {
612
870
    const MemSDNode *PF = cast<MemSDNode>(N);
613
870
    ID.AddInteger(PF->getPointerInfo().getAddrSpace());
614
870
    break;
615
29.6k
  }
616
120k
  case ISD::VECTOR_SHUFFLE: {
617
120k
    const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
618
120k
    for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements();
619
1.69M
         i != e; 
++i1.57M
)
620
1.57M
      ID.AddInteger(SVN->getMaskElt(i));
621
120k
    break;
622
29.6k
  }
623
29.6k
  case ISD::TargetBlockAddress:
624
70
  case ISD::BlockAddress: {
625
70
    const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
626
70
    ID.AddPointer(BA->getBlockAddress());
627
70
    ID.AddInteger(BA->getOffset());
628
70
    ID.AddInteger(BA->getTargetFlags());
629
70
    break;
630
57.6M
  }
631
57.6M
  } // end switch (N->getOpcode())
632
57.6M
633
57.6M
  // Target specific memory nodes could also have address spaces to check.
634
57.6M
  if (N->isTargetMemoryOpcode())
635
14.4k
    ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace());
636
57.6M
}
637
638
/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
639
/// data.
640
56.8M
static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
641
56.8M
  AddNodeIDOpcode(ID, N->getOpcode());
642
56.8M
  // Add the return value info.
643
56.8M
  AddNodeIDValueTypes(ID, N->getVTList());
644
56.8M
  // Add the operand info.
645
56.8M
  AddNodeIDOperands(ID, N->ops());
646
56.8M
647
56.8M
  // Handle SDNode leafs with special info.
648
56.8M
  AddNodeIDCustom(ID, N);
649
56.8M
}
650
651
//===----------------------------------------------------------------------===//
652
//                              SelectionDAG Class
653
//===----------------------------------------------------------------------===//
654
655
/// doNotCSE - Return true if CSE should not be performed for this node.
656
10.8M
static bool doNotCSE(SDNode *N) {
657
10.8M
  if (N->getValueType(0) == MVT::Glue)
658
10.5k
    return true; // Never CSE anything that produces a flag.
659
10.8M
660
10.8M
  switch (N->getOpcode()) {
661
10.8M
  
default: break10.7M
;
662
10.8M
  case ISD::HANDLENODE:
663
104k
  case ISD::EH_LABEL:
664
104k
    return true;   // Never CSE these nodes.
665
10.7M
  }
666
10.7M
667
10.7M
  // Check that remaining values produced are not flags.
668
12.6M
  
for (unsigned i = 1, e = N->getNumValues(); 10.7M
i != e;
++i1.91M
)
669
4.04M
    if (N->getValueType(i) == MVT::Glue)
670
2.12M
      return true; // Never CSE anything that produces a flag.
671
10.7M
672
10.7M
  
return false8.63M
;
673
10.7M
}
674
675
/// RemoveDeadNodes - This method deletes all unreachable nodes in the
676
/// SelectionDAG.
677
6.02M
void SelectionDAG::RemoveDeadNodes() {
678
6.02M
  // Create a dummy node (which is not added to allnodes), that adds a reference
679
6.02M
  // to the root node, preventing it from being deleted.
680
6.02M
  HandleSDNode Dummy(getRoot());
681
6.02M
682
6.02M
  SmallVector<SDNode*, 128> DeadNodes;
683
6.02M
684
6.02M
  // Add all obviously-dead nodes to the DeadNodes worklist.
685
6.02M
  for (SDNode &Node : allnodes())
686
130M
    if (Node.use_empty())
687
1.25M
      DeadNodes.push_back(&Node);
688
6.02M
689
6.02M
  RemoveDeadNodes(DeadNodes);
690
6.02M
691
6.02M
  // If the root changed (e.g. it was a dead load, update the root).
692
6.02M
  setRoot(Dummy.getValue());
693
6.02M
}
694
695
/// RemoveDeadNodes - This method deletes the unreachable nodes in the
696
/// given list, and any nodes that become unreachable as a result.
697
13.1M
void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
698
13.1M
699
13.1M
  // Process the worklist, deleting the nodes and adding their uses to the
700
13.1M
  // worklist.
701
21.2M
  while (!DeadNodes.empty()) {
702
8.13M
    SDNode *N = DeadNodes.pop_back_val();
703
8.13M
    // Skip to next node if we've already managed to delete the node. This could
704
8.13M
    // happen if replacing a node causes a node previously added to the node to
705
8.13M
    // be deleted.
706
8.13M
    if (N->getOpcode() == ISD::DELETED_NODE)
707
7
      continue;
708
8.13M
709
18.0M
    
for (DAGUpdateListener *DUL = UpdateListeners; 8.13M
DUL;
DUL = DUL->Next9.91M
)
710
9.91M
      DUL->NodeDeleted(N, nullptr);
711
8.13M
712
8.13M
    // Take the node out of the appropriate CSE map.
713
8.13M
    RemoveNodeFromCSEMaps(N);
714
8.13M
715
8.13M
    // Next, brutally remove the operand list.  This is safe to do, as there are
716
8.13M
    // no cycles in the graph.
717
18.1M
    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
718
9.97M
      SDUse &Use = *I++;
719
9.97M
      SDNode *Operand = Use.getNode();
720
9.97M
      Use.set(SDValue());
721
9.97M
722
9.97M
      // Now that we removed this operand, see if there are no uses of it left.
723
9.97M
      if (Operand->use_empty())
724
2.84M
        DeadNodes.push_back(Operand);
725
9.97M
    }
726
8.13M
727
8.13M
    DeallocateNode(N);
728
8.13M
  }
729
13.1M
}
730
731
755k
void SelectionDAG::RemoveDeadNode(SDNode *N){
732
755k
  SmallVector<SDNode*, 16> DeadNodes(1, N);
733
755k
734
755k
  // Create a dummy node that adds a reference to the root node, preventing
735
755k
  // it from being deleted.  (This matters if the root is an operand of the
736
755k
  // dead node.)
737
755k
  HandleSDNode Dummy(getRoot());
738
755k
739
755k
  RemoveDeadNodes(DeadNodes);
740
755k
}
741
742
12.8M
void SelectionDAG::DeleteNode(SDNode *N) {
743
12.8M
  // First take this out of the appropriate CSE map.
744
12.8M
  RemoveNodeFromCSEMaps(N);
745
12.8M
746
12.8M
  // Finally, remove uses due to operands of this node, remove from the
747
12.8M
  // AllNodes list, and delete the node.
748
12.8M
  DeleteNodeNotInCSEMaps(N);
749
12.8M
}
750
751
12.8M
void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
752
12.8M
  assert(N->getIterator() != AllNodes.begin() &&
753
12.8M
         "Cannot delete the entry node!");
754
12.8M
  assert(N->use_empty() && "Cannot delete a node that is not dead!");
755
12.8M
756
12.8M
  // Drop all of the operands and decrement used node's use counts.
757
12.8M
  N->DropOperands();
758
12.8M
759
12.8M
  DeallocateNode(N);
760
12.8M
}
761
762
46.6M
void SDDbgInfo::erase(const SDNode *Node) {
763
46.6M
  DbgValMapType::iterator I = DbgValMap.find(Node);
764
46.6M
  if (I == DbgValMap.end())
765
46.6M
    return;
766
493
  for (auto &Val: I->second)
767
540
    Val->setIsInvalidated();
768
493
  DbgValMap.erase(I);
769
493
}
770
771
46.6M
void SelectionDAG::DeallocateNode(SDNode *N) {
772
46.6M
  // If we have operands, deallocate them.
773
46.6M
  removeOperands(N);
774
46.6M
775
46.6M
  NodeAllocator.Deallocate(AllNodes.remove(N));
776
46.6M
777
46.6M
  // Set the opcode to DELETED_NODE to help catch bugs when node
778
46.6M
  // memory is reallocated.
779
46.6M
  // FIXME: There are places in SDag that have grown a dependency on the opcode
780
46.6M
  // value in the released node.
781
46.6M
  __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
782
46.6M
  N->NodeType = ISD::DELETED_NODE;
783
46.6M
784
46.6M
  // If any of the SDDbgValue nodes refer to this SDNode, invalidate
785
46.6M
  // them and forget about that node.
786
46.6M
  DbgInfo->erase(N);
787
46.6M
}
788
789
#ifndef NDEBUG
790
/// VerifySDNode - Sanity check the given SDNode.  Aborts if it is invalid.
791
static void VerifySDNode(SDNode *N) {
792
  switch (N->getOpcode()) {
793
  default:
794
    break;
795
  case ISD::BUILD_PAIR: {
796
    EVT VT = N->getValueType(0);
797
    assert(N->getNumValues() == 1 && "Too many results!");
798
    assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
799
           "Wrong return type!");
800
    assert(N->getNumOperands() == 2 && "Wrong number of operands!");
801
    assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
802
           "Mismatched operand types!");
803
    assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
804
           "Wrong operand type!");
805
    assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
806
           "Wrong return type size");
807
    break;
808
  }
809
  case ISD::BUILD_VECTOR: {
810
    assert(N->getNumValues() == 1 && "Too many results!");
811
    assert(N->getValueType(0).isVector() && "Wrong return type!");
812
    assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
813
           "Wrong number of operands!");
814
    EVT EltVT = N->getValueType(0).getVectorElementType();
815
    for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) {
816
      assert((I->getValueType() == EltVT ||
817
             (EltVT.isInteger() && I->getValueType().isInteger() &&
818
              EltVT.bitsLE(I->getValueType()))) &&
819
            "Wrong operand type!");
820
      assert(I->getValueType() == N->getOperand(0).getValueType() &&
821
             "Operands must all have the same type");
822
    }
823
    break;
824
  }
825
  }
826
}
827
#endif // NDEBUG
828
829
/// Insert a newly allocated node into the DAG.
830
///
831
/// Handles insertion into the all nodes list and CSE map, as well as
832
/// verification and other common operations when a new node is allocated.
833
47.9M
void SelectionDAG::InsertNode(SDNode *N) {
834
47.9M
  AllNodes.push_back(N);
835
#ifndef NDEBUG
836
  N->PersistentId = NextPersistentId++;
837
  VerifySDNode(N);
838
#endif
839
70.5M
  for (DAGUpdateListener *DUL = UpdateListeners; DUL; 
DUL = DUL->Next22.6M
)
840
22.6M
    DUL->NodeInserted(N);
841
47.9M
}
842
843
/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
844
/// correspond to it.  This is useful when we're about to delete or repurpose
845
/// the node.  We don't want future request for structurally identical nodes
846
/// to return N anymore.
847
39.4M
bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
848
39.4M
  bool Erased = false;
849
39.4M
  switch (N->getOpcode()) {
850
39.4M
  
case ISD::HANDLENODE: return false97.6k
; // noop.
851
39.4M
  case ISD::CONDCODE:
852
952k
    assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
853
952k
           "Cond code doesn't exist!");
854
952k
    Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
855
952k
    CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
856
952k
    break;
857
39.4M
  case ISD::ExternalSymbol:
858
14.8k
    Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
859
14.8k
    break;
860
39.4M
  case ISD::TargetExternalSymbol: {
861
103
    ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
862
103
    Erased = TargetExternalSymbols.erase(
863
103
               std::pair<std::string,unsigned char>(ESN->getSymbol(),
864
103
                                                    ESN->getTargetFlags()));
865
103
    break;
866
39.4M
  }
867
39.4M
  case ISD::MCSymbol: {
868
0
    auto *MCSN = cast<MCSymbolSDNode>(N);
869
0
    Erased = MCSymbols.erase(MCSN->getMCSymbol());
870
0
    break;
871
39.4M
  }
872
39.4M
  case ISD::VALUETYPE: {
873
213k
    EVT VT = cast<VTSDNode>(N)->getVT();
874
213k
    if (VT.isExtended()) {
875
67.4k
      Erased = ExtendedValueTypeNodes.erase(VT);
876
146k
    } else {
877
146k
      Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
878
146k
      ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
879
146k
    }
880
213k
    break;
881
39.4M
  }
882
39.4M
  default:
883
38.1M
    // Remove it from the CSE Map.
884
38.1M
    assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
885
38.1M
    assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
886
38.1M
    Erased = CSEMap.RemoveNode(N);
887
38.1M
    break;
888
39.3M
  }
889
#ifndef NDEBUG
890
  // Verify that the node was actually in one of the CSE maps, unless it has a
891
  // flag result (which cannot be CSE'd) or is one of the special cases that are
892
  // not subject to CSE.
893
  if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
894
      !N->isMachineOpcode() && !doNotCSE(N)) {
895
    N->dump(this);
896
    dbgs() << "\n";
897
    llvm_unreachable("Node is not in map!");
898
  }
899
#endif
900
39.3M
  return Erased;
901
39.3M
}
902
903
/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
904
/// maps and modified in place. Add it back to the CSE maps, unless an identical
905
/// node already exists, in which case transfer all its users to the existing
906
/// node. This transfer can potentially trigger recursive merging.
907
void
908
10.0M
SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
909
10.0M
  // For node types that aren't CSE'd, just act as if no identical node
910
10.0M
  // already exists.
911
10.0M
  if (!doNotCSE(N)) {
912
7.86M
    SDNode *Existing = CSEMap.GetOrInsertNode(N);
913
7.86M
    if (Existing != N) {
914
18.5k
      // If there was already an existing matching node, use ReplaceAllUsesWith
915
18.5k
      // to replace the dead one with the existing one.  This can cause
916
18.5k
      // recursive merging of other unrelated nodes down the line.
917
18.5k
      ReplaceAllUsesWith(N, Existing);
918
18.5k
919
18.5k
      // N is now dead. Inform the listeners and delete it.
920
87.2k
      for (DAGUpdateListener *DUL = UpdateListeners; DUL; 
DUL = DUL->Next68.7k
)
921
68.7k
        DUL->NodeDeleted(N, Existing);
922
18.5k
      DeleteNodeNotInCSEMaps(N);
923
18.5k
      return;
924
18.5k
    }
925
10.0M
  }
926
10.0M
927
10.0M
  // If the node doesn't already exist, we updated it.  Inform listeners.
928
37.3M
  
for (DAGUpdateListener *DUL = UpdateListeners; 10.0M
DUL;
DUL = DUL->Next27.2M
)
929
27.2M
    DUL->NodeUpdated(N);
930
10.0M
}
931
932
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
933
/// were replaced with those specified.  If this node is never memoized,
934
/// return null, otherwise return a pointer to the slot it would take.  If a
935
/// node already exists with these operands, the slot will be non-null.
936
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
937
15.8k
                                           void *&InsertPos) {
938
15.8k
  if (doNotCSE(N))
939
0
    return nullptr;
940
15.8k
941
15.8k
  SDValue Ops[] = { Op };
942
15.8k
  FoldingSetNodeID ID;
943
15.8k
  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
944
15.8k
  AddNodeIDCustom(ID, N);
945
15.8k
  SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
946
15.8k
  if (Node)
947
0
    Node->intersectFlagsWith(N->getFlags());
948
15.8k
  return Node;
949
15.8k
}
950
951
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
952
/// were replaced with those specified.  If this node is never memoized,
953
/// return null, otherwise return a pointer to the slot it would take.  If a
954
/// node already exists with these operands, the slot will be non-null.
955
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
956
                                           SDValue Op1, SDValue Op2,
957
84.3k
                                           void *&InsertPos) {
958
84.3k
  if (doNotCSE(N))
959
0
    return nullptr;
960
84.3k
961
84.3k
  SDValue Ops[] = { Op1, Op2 };
962
84.3k
  FoldingSetNodeID ID;
963
84.3k
  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
964
84.3k
  AddNodeIDCustom(ID, N);
965
84.3k
  SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
966
84.3k
  if (Node)
967
1.09k
    Node->intersectFlagsWith(N->getFlags());
968
84.3k
  return Node;
969
84.3k
}
970
971
/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
972
/// were replaced with those specified.  If this node is never memoized,
973
/// return null, otherwise return a pointer to the slot it would take.  If a
974
/// node already exists with these operands, the slot will be non-null.
975
SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
976
682k
                                           void *&InsertPos) {
977
682k
  if (doNotCSE(N))
978
16.9k
    return nullptr;
979
665k
980
665k
  FoldingSetNodeID ID;
981
665k
  AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
982
665k
  AddNodeIDCustom(ID, N);
983
665k
  SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
984
665k
  if (Node)
985
286
    Node->intersectFlagsWith(N->getFlags());
986
665k
  return Node;
987
665k
}
988
989
372k
unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
990
372k
  Type *Ty = VT == MVT::iPTR ?
991
0
                   PointerType::get(Type::getInt8Ty(*getContext()), 0) :
992
372k
                   VT.getTypeForEVT(*getContext());
993
372k
994
372k
  return getDataLayout().getABITypeAlignment(Ty);
995
372k
}
996
997
// EntryNode could meaningfully have debug info if we can find it...
998
SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL)
999
    : TM(tm), OptLevel(OL),
1000
      EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)),
1001
38.1k
      Root(getEntryNode()) {
1002
38.1k
  InsertNode(&EntryNode);
1003
38.1k
  DbgInfo = new SDDbgInfo();
1004
38.1k
}
1005
1006
void SelectionDAG::init(MachineFunction &NewMF,
1007
                        OptimizationRemarkEmitter &NewORE,
1008
                        Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
1009
278k
                        LegacyDivergenceAnalysis * Divergence) {
1010
278k
  MF = &NewMF;
1011
278k
  SDAGISelPass = PassPtr;
1012
278k
  ORE = &NewORE;
1013
278k
  TLI = getSubtarget().getTargetLowering();
1014
278k
  TSI = getSubtarget().getSelectionDAGInfo();
1015
278k
  LibInfo = LibraryInfo;
1016
278k
  Context = &MF->getFunction().getContext();
1017
278k
  DA = Divergence;
1018
278k
}
1019
1020
37.9k
SelectionDAG::~SelectionDAG() {
1021
37.9k
  assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1022
37.9k
  allnodes_clear();
1023
37.9k
  OperandRecycler.clear(OperandAllocator);
1024
37.9k
  delete DbgInfo;
1025
37.9k
}
1026
1027
1.28M
void SelectionDAG::allnodes_clear() {
1028
1.28M
  assert(&*AllNodes.begin() == &EntryNode);
1029
1.28M
  AllNodes.remove(AllNodes.begin());
1030
26.8M
  while (!AllNodes.empty())
1031
25.5M
    DeallocateNode(&AllNodes.front());
1032
#ifndef NDEBUG
1033
  NextPersistentId = 0;
1034
#endif
1035
}
1036
1037
SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1038
11.7M
                                          void *&InsertPos) {
1039
11.7M
  SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1040
11.7M
  if (N) {
1041
4.00M
    switch (N->getOpcode()) {
1042
4.00M
    default: break;
1043
4.00M
    case ISD::Constant:
1044
0
    case ISD::ConstantFP:
1045
0
      llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1046
11.7M
                       "debug location.  Use another overload.");
1047
11.7M
    }
1048
11.7M
  }
1049
11.7M
  return N;
1050
11.7M
}
1051
1052
SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1053
59.0M
                                          const SDLoc &DL, void *&InsertPos) {
1054
59.0M
  SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1055
59.0M
  if (N) {
1056
15.7M
    switch (N->getOpcode()) {
1057
15.7M
    case ISD::Constant:
1058
4.76M
    case ISD::ConstantFP:
1059
4.76M
      // Erase debug location from the node if the node is used at several
1060
4.76M
      // different places. Do not propagate one location to all uses as it
1061
4.76M
      // will cause a worse single stepping debugging experience.
1062
4.76M
      if (N->getDebugLoc() != DL.getDebugLoc())
1063
768k
        N->setDebugLoc(DebugLoc());
1064
4.76M
      break;
1065
11.0M
    default:
1066
11.0M
      // When the node's point of use is located earlier in the instruction
1067
11.0M
      // sequence than its prior point of use, update its debug info to the
1068
11.0M
      // earlier location.
1069
11.0M
      if (DL.getIROrder() && 
DL.getIROrder() < N->getIROrder()6.53M
)
1070
136k
        N->setDebugLoc(DL.getDebugLoc());
1071
11.0M
      break;
1072
59.0M
    }
1073
59.0M
  }
1074
59.0M
  return N;
1075
59.0M
}
1076
1077
1.24M
void SelectionDAG::clear() {
1078
1.24M
  allnodes_clear();
1079
1.24M
  OperandRecycler.clear(OperandAllocator);
1080
1.24M
  OperandAllocator.Reset();
1081
1.24M
  CSEMap.clear();
1082
1.24M
1083
1.24M
  ExtendedValueTypeNodes.clear();
1084
1.24M
  ExternalSymbols.clear();
1085
1.24M
  TargetExternalSymbols.clear();
1086
1.24M
  MCSymbols.clear();
1087
1.24M
  std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1088
1.24M
            static_cast<CondCodeSDNode*>(nullptr));
1089
1.24M
  std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1090
1.24M
            static_cast<SDNode*>(nullptr));
1091
1.24M
1092
1.24M
  EntryNode.UseList = nullptr;
1093
1.24M
  InsertNode(&EntryNode);
1094
1.24M
  Root = getEntryNode();
1095
1.24M
  DbgInfo->clear();
1096
1.24M
}
1097
1098
91
SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1099
91
  return VT.bitsGT(Op.getValueType())
1100
91
             ? 
getNode(ISD::FP_EXTEND, DL, VT, Op)11
1101
91
             : 
getNode(ISD::FP_ROUND, DL, VT, Op, getIntPtrConstant(0, DL))80
;
1102
91
}
1103
1104
139k
SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1105
139k
  return VT.bitsGT(Op.getValueType()) ?
1106
75.2k
    getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1107
139k
    
getNode(ISD::TRUNCATE, DL, VT, Op)64.2k
;
1108
139k
}
1109
1110
1.17M
SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1111
1.17M
  return VT.bitsGT(Op.getValueType()) ?
1112
56.3k
    getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1113
1.17M
    
getNode(ISD::TRUNCATE, DL, VT, Op)1.12M
;
1114
1.17M
}
1115
1116
362k
SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1117
362k
  return VT.bitsGT(Op.getValueType()) ?
1118
22.8k
    getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1119
362k
    
getNode(ISD::TRUNCATE, DL, VT, Op)339k
;
1120
362k
}
1121
1122
SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1123
8.09k
                                        EVT OpVT) {
1124
8.09k
  if (VT.bitsLE(Op.getValueType()))
1125
7.58k
    return getNode(ISD::TRUNCATE, SL, VT, Op);
1126
517
1127
517
  TargetLowering::BooleanContent BType = TLI->getBooleanContents(OpVT);
1128
517
  return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1129
517
}
1130
1131
496k
SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1132
496k
  assert(!VT.isVector() &&
1133
496k
         "getZeroExtendInReg should use the vector element type instead of "
1134
496k
         "the vector type!");
1135
496k
  if (Op.getValueType().getScalarType() == VT) 
return Op1.00k
;
1136
495k
  unsigned BitWidth = Op.getScalarValueSizeInBits();
1137
495k
  APInt Imm = APInt::getLowBitsSet(BitWidth,
1138
495k
                                   VT.getSizeInBits());
1139
495k
  return getNode(ISD::AND, DL, Op.getValueType(), Op,
1140
495k
                 getConstant(Imm, DL, Op.getValueType()));
1141
495k
}
1142
1143
104k
SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1144
104k
  // Only unsigned pointer semantics are supported right now. In the future this
1145
104k
  // might delegate to TLI to check pointer signedness.
1146
104k
  return getZExtOrTrunc(Op, DL, VT);
1147
104k
}
1148
1149
0
SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1150
0
  // Only unsigned pointer semantics are supported right now. In the future this
1151
0
  // might delegate to TLI to check pointer signedness.
1152
0
  return getZeroExtendInReg(Op, DL, VT);
1153
0
}
1154
1155
/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1156
8.22k
SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1157
8.22k
  EVT EltVT = VT.getScalarType();
1158
8.22k
  SDValue NegOne =
1159
8.22k
    getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), DL, VT);
1160
8.22k
  return getNode(ISD::XOR, DL, VT, Val, NegOne);
1161
8.22k
}
1162
1163
226
SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1164
226
  SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1165
226
  return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1166
226
}
1167
1168
SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1169
20.4k
                                      EVT OpVT) {
1170
20.4k
  if (!V)
1171
17.8k
    return getConstant(0, DL, VT);
1172
2.56k
1173
2.56k
  switch (TLI->getBooleanContents(OpVT)) {
1174
2.56k
  case TargetLowering::ZeroOrOneBooleanContent:
1175
2.26k
  case TargetLowering::UndefinedBooleanContent:
1176
2.26k
    return getConstant(1, DL, VT);
1177
2.26k
  case TargetLowering::ZeroOrNegativeOneBooleanContent:
1178
298
    return getAllOnesConstant(DL, VT);
1179
0
  }
1180
0
  llvm_unreachable("Unexpected boolean content enum!");
1181
0
}
1182
1183
SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1184
13.8M
                                  bool isT, bool isO) {
1185
13.8M
  EVT EltVT = VT.getScalarType();
1186
13.8M
  assert((EltVT.getSizeInBits() >= 64 ||
1187
13.8M
         (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1188
13.8M
         "getConstant with a uint64_t value that doesn't fit in the type!");
1189
13.8M
  return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1190
13.8M
}
1191
1192
SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1193
17.4M
                                  bool isT, bool isO) {
1194
17.4M
  return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1195
17.4M
}
1196
1197
SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1198
20.0M
                                  EVT VT, bool isT, bool isO) {
1199
20.0M
  assert(VT.isInteger() && "Cannot create FP integer constant!");
1200
20.0M
1201
20.0M
  EVT EltVT = VT.getScalarType();
1202
20.0M
  const ConstantInt *Elt = &Val;
1203
20.0M
1204
20.0M
  // In some cases the vector type is legal but the element type is illegal and
1205
20.0M
  // needs to be promoted, for example v8i8 on ARM.  In this case, promote the
1206
20.0M
  // inserted value (the type does not need to match the vector element type).
1207
20.0M
  // Any extra bits introduced will be truncated away.
1208
20.0M
  if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1209
113k
      TargetLowering::TypePromoteInteger) {
1210
9.75k
   EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1211
9.75k
   APInt NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1212
9.75k
   Elt = ConstantInt::get(*getContext(), NewVal);
1213
9.75k
  }
1214
20.0M
  // In other cases the element type is illegal and needs to be expanded, for
1215
20.0M
  // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1216
20.0M
  // the value into n parts and use a vector type with n-times the elements.
1217
20.0M
  // Then bitcast to the type requested.
1218
20.0M
  // Legalizing constants too early makes the DAGCombiner's job harder so we
1219
20.0M
  // only legalize if the DAG tells us we must produce legal types.
1220
20.0M
  else if (NewNodesMustHaveLegalTypes && 
VT.isVector()10.7M
&&
1221
20.0M
           TLI->getTypeAction(*getContext(), EltVT) ==
1222
75.7k
           TargetLowering::TypeExpandInteger) {
1223
579
    const APInt &NewVal = Elt->getValue();
1224
579
    EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1225
579
    unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1226
579
    unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1227
579
    EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1228
579
1229
579
    // Check the temporary vector is the correct size. If this fails then
1230
579
    // getTypeToTransformTo() probably returned a type whose size (in bits)
1231
579
    // isn't a power-of-2 factor of the requested type size.
1232
579
    assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1233
579
1234
579
    SmallVector<SDValue, 2> EltParts;
1235
1.73k
    for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); 
++i1.15k
) {
1236
1.15k
      EltParts.push_back(getConstant(NewVal.lshr(i * ViaEltSizeInBits)
1237
1.15k
                                           .zextOrTrunc(ViaEltSizeInBits), DL,
1238
1.15k
                                     ViaEltVT, isT, isO));
1239
1.15k
    }
1240
579
1241
579
    // EltParts is currently in little endian order. If we actually want
1242
579
    // big-endian order then reverse it now.
1243
579
    if (getDataLayout().isBigEndian())
1244
57
      std::reverse(EltParts.begin(), EltParts.end());
1245
579
1246
579
    // The elements must be reversed when the element order is different
1247
579
    // to the endianness of the elements (because the BITCAST is itself a
1248
579
    // vector shuffle in this situation). However, we do not need any code to
1249
579
    // perform this reversal because getConstant() is producing a vector
1250
579
    // splat.
1251
579
    // This situation occurs in MIPS MSA.
1252
579
1253
579
    SmallVector<SDValue, 8> Ops;
1254
1.83k
    for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; 
++i1.25k
)
1255
1.25k
      Ops.insert(Ops.end(), EltParts.begin(), EltParts.end());
1256
579
1257
579
    SDValue V = getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1258
579
    return V;
1259
579
  }
1260
20.0M
1261
20.0M
  assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1262
20.0M
         "APInt size does not match type size!");
1263
20.0M
  unsigned Opc = isT ? 
ISD::TargetConstant9.95M
:
ISD::Constant10.1M
;
1264
20.0M
  FoldingSetNodeID ID;
1265
20.0M
  AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1266
20.0M
  ID.AddPointer(Elt);
1267
20.0M
  ID.AddBoolean(isO);
1268
20.0M
  void *IP = nullptr;
1269
20.0M
  SDNode *N = nullptr;
1270
20.0M
  if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1271
10.3M
    if (!VT.isVector())
1272
10.2M
      return SDValue(N, 0);
1273
9.78M
1274
9.78M
  if (!N) {
1275
9.72M
    N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1276
9.72M
    CSEMap.InsertNode(N, IP);
1277
9.72M
    InsertNode(N);
1278
9.72M
    NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1279
9.72M
  }
1280
9.78M
1281
9.78M
  SDValue Result(N, 0);
1282
9.78M
  if (VT.isVector())
1283
112k
    Result = getSplatBuildVector(VT, DL, Result);
1284
9.78M
1285
9.78M
  return Result;
1286
9.78M
}
1287
1288
SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1289
1.93M
                                        bool isTarget) {
1290
1.93M
  return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1291
1.93M
}
1292
1293
SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1294
210
                                             const SDLoc &DL, bool LegalTypes) {
1295
210
  EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1296
210
  return getConstant(Val, DL, ShiftVT);
1297
210
}
1298
1299
SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1300
19.6k
                                    bool isTarget) {
1301
19.6k
  return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1302
19.6k
}
1303
1304
SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1305
78.8k
                                    EVT VT, bool isTarget) {
1306
78.8k
  assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1307
78.8k
1308
78.8k
  EVT EltVT = VT.getScalarType();
1309
78.8k
1310
78.8k
  // Do the map lookup using the actual bit pattern for the floating point
1311
78.8k
  // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1312
78.8k
  // we don't have issues with SNANs.
1313
78.8k
  unsigned Opc = isTarget ? 
ISD::TargetConstantFP3.41k
:
ISD::ConstantFP75.4k
;
1314
78.8k
  FoldingSetNodeID ID;
1315
78.8k
  AddNodeIDNode(ID, Opc, getVTList(EltVT), None);
1316
78.8k
  ID.AddPointer(&V);
1317
78.8k
  void *IP = nullptr;
1318
78.8k
  SDNode *N = nullptr;
1319
78.8k
  if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1320
8.12k
    if (!VT.isVector())
1321
6.53k
      return SDValue(N, 0);
1322
72.3k
1323
72.3k
  if (!N) {
1324
70.7k
    N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1325
70.7k
    CSEMap.InsertNode(N, IP);
1326
70.7k
    InsertNode(N);
1327
70.7k
  }
1328
72.3k
1329
72.3k
  SDValue Result(N, 0);
1330
72.3k
  if (VT.isVector())
1331
3.34k
    Result = getSplatBuildVector(VT, DL, Result);
1332
72.3k
  NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1333
72.3k
  return Result;
1334
72.3k
}
1335
1336
SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1337
9.20k
                                    bool isTarget) {
1338
9.20k
  EVT EltVT = VT.getScalarType();
1339
9.20k
  if (EltVT == MVT::f32)
1340
5.82k
    return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1341
3.38k
  else if (EltVT == MVT::f64)
1342
3.03k
    return getConstantFP(APFloat(Val), DL, VT, isTarget);
1343
349
  else if (EltVT == MVT::f80 || 
EltVT == MVT::f128201
||
EltVT == MVT::ppcf128194
||
1344
349
           
EltVT == MVT::f16192
) {
1345
349
    bool Ignored;
1346
349
    APFloat APF = APFloat(Val);
1347
349
    APF.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven,
1348
349
                &Ignored);
1349
349
    return getConstantFP(APF, DL, VT, isTarget);
1350
349
  } else
1351
349
    llvm_unreachable("Unsupported type in getConstantFP");
1352
9.20k
}
1353
1354
SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1355
                                       EVT VT, int64_t Offset, bool isTargetGA,
1356
1.84M
                                       unsigned char TargetFlags) {
1357
1.84M
  assert((TargetFlags == 0 || isTargetGA) &&
1358
1.84M
         "Cannot set target flags on target-independent globals");
1359
1.84M
1360
1.84M
  // Truncate (with sign-extension) the offset value to the pointer size.
1361
1.84M
  unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1362
1.84M
  if (BitWidth < 64)
1363
285k
    Offset = SignExtend64(Offset, BitWidth);
1364
1.84M
1365
1.84M
  unsigned Opc;
1366
1.84M
  if (GV->isThreadLocal())
1367
3.38k
    Opc = isTargetGA ? 
ISD::TargetGlobalTLSAddress1.94k
:
ISD::GlobalTLSAddress1.43k
;
1368
1.83M
  else
1369
1.83M
    Opc = isTargetGA ? 
ISD::TargetGlobalAddress1.06M
:
ISD::GlobalAddress772k
;
1370
1.84M
1371
1.84M
  FoldingSetNodeID ID;
1372
1.84M
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1373
1.84M
  ID.AddPointer(GV);
1374
1.84M
  ID.AddInteger(Offset);
1375
1.84M
  ID.AddInteger(TargetFlags);
1376
1.84M
  void *IP = nullptr;
1377
1.84M
  if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1378
50.4k
    return SDValue(E, 0);
1379
1.79M
1380
1.79M
  auto *N = newSDNode<GlobalAddressSDNode>(
1381
1.79M
      Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1382
1.79M
  CSEMap.InsertNode(N, IP);
1383
1.79M
    InsertNode(N);
1384
1.79M
  return SDValue(N, 0);
1385
1.79M
}
1386
1387
673k
SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1388
673k
  unsigned Opc = isTarget ? 
ISD::TargetFrameIndex422k
:
ISD::FrameIndex250k
;
1389
673k
  FoldingSetNodeID ID;
1390
673k
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1391
673k
  ID.AddInteger(FI);
1392
673k
  void *IP = nullptr;
1393
673k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1394
186k
    return SDValue(E, 0);
1395
487k
1396
487k
  auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1397
487k
  CSEMap.InsertNode(N, IP);
1398
487k
  InsertNode(N);
1399
487k
  return SDValue(N, 0);
1400
487k
}
1401
1402
SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1403
8.21k
                                   unsigned char TargetFlags) {
1404
8.21k
  assert((TargetFlags == 0 || isTarget) &&
1405
8.21k
         "Cannot set target flags on target-independent jump tables");
1406
8.21k
  unsigned Opc = isTarget ? 
ISD::TargetJumpTable5.94k
:
ISD::JumpTable2.26k
;
1407
8.21k
  FoldingSetNodeID ID;
1408
8.21k
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1409
8.21k
  ID.AddInteger(JTI);
1410
8.21k
  ID.AddInteger(TargetFlags);
1411
8.21k
  void *IP = nullptr;
1412
8.21k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1413
99
    return SDValue(E, 0);
1414
8.11k
1415
8.11k
  auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1416
8.11k
  CSEMap.InsertNode(N, IP);
1417
8.11k
  InsertNode(N);
1418
8.11k
  return SDValue(N, 0);
1419
8.11k
}
1420
1421
SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1422
                                      unsigned Alignment, int Offset,
1423
                                      bool isTarget,
1424
207k
                                      unsigned char TargetFlags) {
1425
207k
  assert((TargetFlags == 0 || isTarget) &&
1426
207k
         "Cannot set target flags on target-independent globals");
1427
207k
  if (Alignment == 0)
1428
70.6k
    Alignment = MF->getFunction().hasOptSize()
1429
70.6k
                    ? 
getDataLayout().getABITypeAlignment(C->getType())1.98k
1430
70.6k
                    : 
getDataLayout().getPrefTypeAlignment(C->getType())68.6k
;
1431
207k
  unsigned Opc = isTarget ? 
ISD::TargetConstantPool136k
:
ISD::ConstantPool71.1k
;
1432
207k
  FoldingSetNodeID ID;
1433
207k
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1434
207k
  ID.AddInteger(Alignment);
1435
207k
  ID.AddInteger(Offset);
1436
207k
  ID.AddPointer(C);
1437
207k
  ID.AddInteger(TargetFlags);
1438
207k
  void *IP = nullptr;
1439
207k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1440
4.88k
    return SDValue(E, 0);
1441
202k
1442
202k
  auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1443
202k
                                          TargetFlags);
1444
202k
  CSEMap.InsertNode(N, IP);
1445
202k
  InsertNode(N);
1446
202k
  return SDValue(N, 0);
1447
202k
}
1448
1449
SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1450
                                      unsigned Alignment, int Offset,
1451
                                      bool isTarget,
1452
265
                                      unsigned char TargetFlags) {
1453
265
  assert((TargetFlags == 0 || isTarget) &&
1454
265
         "Cannot set target flags on target-independent globals");
1455
265
  if (Alignment == 0)
1456
0
    Alignment = getDataLayout().getPrefTypeAlignment(C->getType());
1457
265
  unsigned Opc = isTarget ? 
ISD::TargetConstantPool248
:
ISD::ConstantPool17
;
1458
265
  FoldingSetNodeID ID;
1459
265
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1460
265
  ID.AddInteger(Alignment);
1461
265
  ID.AddInteger(Offset);
1462
265
  C->addSelectionDAGCSEId(ID);
1463
265
  ID.AddInteger(TargetFlags);
1464
265
  void *IP = nullptr;
1465
265
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1466
0
    return SDValue(E, 0);
1467
265
1468
265
  auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, Alignment,
1469
265
                                          TargetFlags);
1470
265
  CSEMap.InsertNode(N, IP);
1471
265
  InsertNode(N);
1472
265
  return SDValue(N, 0);
1473
265
}
1474
1475
SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset,
1476
0
                                     unsigned char TargetFlags) {
1477
0
  FoldingSetNodeID ID;
1478
0
  AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), None);
1479
0
  ID.AddInteger(Index);
1480
0
  ID.AddInteger(Offset);
1481
0
  ID.AddInteger(TargetFlags);
1482
0
  void *IP = nullptr;
1483
0
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1484
0
    return SDValue(E, 0);
1485
0
1486
0
  auto *N = newSDNode<TargetIndexSDNode>(Index, VT, Offset, TargetFlags);
1487
0
  CSEMap.InsertNode(N, IP);
1488
0
  InsertNode(N);
1489
0
  return SDValue(N, 0);
1490
0
}
1491
1492
1.23M
SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
1493
1.23M
  FoldingSetNodeID ID;
1494
1.23M
  AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), None);
1495
1.23M
  ID.AddPointer(MBB);
1496
1.23M
  void *IP = nullptr;
1497
1.23M
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1498
56
    return SDValue(E, 0);
1499
1.23M
1500
1.23M
  auto *N = newSDNode<BasicBlockSDNode>(MBB);
1501
1.23M
  CSEMap.InsertNode(N, IP);
1502
1.23M
  InsertNode(N);
1503
1.23M
  return SDValue(N, 0);
1504
1.23M
}
1505
1506
313k
SDValue SelectionDAG::getValueType(EVT VT) {
1507
313k
  if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1508
229k
      ValueTypeNodes.size())
1509
13.5k
    ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1510
313k
1511
313k
  SDNode *&N = VT.isExtended() ?
1512
229k
    
ExtendedValueTypeNodes[VT]83.3k
: ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1513
313k
1514
313k
  if (N) 
return SDValue(N, 0)99.8k
;
1515
213k
  N = newSDNode<VTSDNode>(VT);
1516
213k
  InsertNode(N);
1517
213k
  return SDValue(N, 0);
1518
213k
}
1519
1520
18.9k
SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
1521
18.9k
  SDNode *&N = ExternalSymbols[Sym];
1522
18.9k
  if (N) 
return SDValue(N, 0)4.11k
;
1523
14.8k
  N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1524
14.8k
  InsertNode(N);
1525
14.8k
  return SDValue(N, 0);
1526
14.8k
}
1527
1528
110
SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
1529
110
  SDNode *&N = MCSymbols[Sym];
1530
110
  if (N)
1531
12
    return SDValue(N, 0);
1532
98
  N = newSDNode<MCSymbolSDNode>(Sym, VT);
1533
98
  InsertNode(N);
1534
98
  return SDValue(N, 0);
1535
98
}
1536
1537
SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
1538
42.2k
                                              unsigned char TargetFlags) {
1539
42.2k
  SDNode *&N =
1540
42.2k
    TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym,
1541
42.2k
                                                               TargetFlags)];
1542
42.2k
  if (N) 
return SDValue(N, 0)14.3k
;
1543
27.8k
  N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1544
27.8k
  InsertNode(N);
1545
27.8k
  return SDValue(N, 0);
1546
27.8k
}
1547
1548
1.12M
SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
1549
1.12M
  if ((unsigned)Cond >= CondCodeNodes.size())
1550
26.8k
    CondCodeNodes.resize(Cond+1);
1551
1.12M
1552
1.12M
  if (!CondCodeNodes[Cond]) {
1553
952k
    auto *N = newSDNode<CondCodeSDNode>(Cond);
1554
952k
    CondCodeNodes[Cond] = N;
1555
952k
    InsertNode(N);
1556
952k
  }
1557
1.12M
1558
1.12M
  return SDValue(CondCodeNodes[Cond], 0);
1559
1.12M
}
1560
1561
/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
1562
/// point at N1 to point at N2 and indices that point at N2 to point at N1.
1563
2.16k
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
1564
2.16k
  std::swap(N1, N2);
1565
2.16k
  ShuffleVectorSDNode::commuteMask(M);
1566
2.16k
}
1567
1568
SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
1569
195k
                                       SDValue N2, ArrayRef<int> Mask) {
1570
195k
  assert(VT.getVectorNumElements() == Mask.size() &&
1571
195k
           "Must have the same number of vector elements as mask elements!");
1572
195k
  assert(VT == N1.getValueType() && VT == N2.getValueType() &&
1573
195k
         "Invalid VECTOR_SHUFFLE");
1574
195k
1575
195k
  // Canonicalize shuffle undef, undef -> undef
1576
195k
  if (N1.isUndef() && 
N2.isUndef()714
)
1577
164
    return getUNDEF(VT);
1578
195k
1579
195k
  // Validate that all indices in Mask are within the range of the elements
1580
195k
  // input to the shuffle.
1581
195k
  int NElts = Mask.size();
1582
195k
  assert(llvm::all_of(Mask,
1583
195k
                      [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
1584
195k
         "Index out of range");
1585
195k
1586
195k
  // Copy the mask so we can do any needed cleanup.
1587
195k
  SmallVector<int, 8> MaskVec(Mask.begin(), Mask.end());
1588
195k
1589
195k
  // Canonicalize shuffle v, v -> v, undef
1590
195k
  if (N1 == N2) {
1591
14.2k
    N2 = getUNDEF(VT);
1592
120k
    for (int i = 0; i != NElts; 
++i106k
)
1593
106k
      if (MaskVec[i] >= NElts) 
MaskVec[i] -= NElts7.31k
;
1594
14.2k
  }
1595
195k
1596
195k
  // Canonicalize shuffle undef, v -> v, undef.  Commute the shuffle mask.
1597
195k
  if (N1.isUndef())
1598
550
    commuteShuffle(N1, N2, MaskVec);
1599
195k
1600
195k
  if (TLI->hasVectorBlend()) {
1601
158k
    // If shuffling a splat, try to blend the splat instead. We do this here so
1602
158k
    // that even when this arises during lowering we don't have to re-handle it.
1603
158k
    auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
1604
12.9k
      BitVector UndefElements;
1605
12.9k
      SDValue Splat = BV->getSplatValue(&UndefElements);
1606
12.9k
      if (!Splat)
1607
1.84k
        return;
1608
11.1k
1609
133k
      
for (int i = 0; 11.1k
i < NElts;
++i122k
) {
1610
122k
        if (MaskVec[i] < Offset || 
MaskVec[i] >= (Offset + NElts)78.4k
)
1611
51.3k
          continue;
1612
70.6k
1613
70.6k
        // If this input comes from undef, mark it as such.
1614
70.6k
        if (UndefElements[MaskVec[i] - Offset]) {
1615
39
          MaskVec[i] = -1;
1616
39
          continue;
1617
39
        }
1618
70.5k
1619
70.5k
        // If we can blend a non-undef lane, use that instead.
1620
70.5k
        if (!UndefElements[i])
1621
67.8k
          MaskVec[i] = i + Offset;
1622
70.5k
      }
1623
11.1k
    };
1624
158k
    if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
1625
4.37k
      BlendSplat(N1BV, 0);
1626
158k
    if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
1627
8.62k
      BlendSplat(N2BV, NElts);
1628
158k
  }
1629
195k
1630
195k
  // Canonicalize all index into lhs, -> shuffle lhs, undef
1631
195k
  // Canonicalize all index into rhs, -> shuffle rhs, undef
1632
195k
  bool AllLHS = true, AllRHS = true;
1633
195k
  bool N2Undef = N2.isUndef();
1634
2.23M
  for (int i = 0; i != NElts; 
++i2.04M
) {
1635
2.04M
    if (MaskVec[i] >= NElts) {
1636
253k
      if (N2Undef)
1637
37.8k
        MaskVec[i] = -1;
1638
215k
      else
1639
215k
        AllLHS = false;
1640
1.78M
    } else if (MaskVec[i] >= 0) {
1641
1.00M
      AllRHS = false;
1642
1.00M
    }
1643
2.04M
  }
1644
195k
  if (AllLHS && 
AllRHS140k
)
1645
216
    return getUNDEF(VT);
1646
195k
  if (AllLHS && 
!N2Undef140k
)
1647
4.18k
    N2 = getUNDEF(VT);
1648
195k
  if (AllRHS) {
1649
1.61k
    N1 = getUNDEF(VT);
1650
1.61k
    commuteShuffle(N1, N2, MaskVec);
1651
1.61k
  }
1652
195k
  // Reset our undef status after accounting for the mask.
1653
195k
  N2Undef = N2.isUndef();
1654
195k
  // Re-check whether both sides ended up undef.
1655
195k
  if (N1.isUndef() && 
N2Undef0
)
1656
0
    return getUNDEF(VT);
1657
195k
1658
195k
  // If Identity shuffle return that node.
1659
195k
  bool Identity = true, AllSame = true;
1660
2.23M
  for (int i = 0; i != NElts; 
++i2.04M
) {
1661
2.04M
    if (MaskVec[i] >= 0 && 
MaskVec[i] != i1.21M
)
Identity = false876k
;
1662
2.04M
    if (MaskVec[i] != MaskVec[0]) 
AllSame = false1.66M
;
1663
2.04M
  }
1664
195k
  if (Identity && 
NElts33.0k
)
1665
33.0k
    return N1;
1666
162k
1667
162k
  // Shuffling a constant splat doesn't change the result.
1668
162k
  if (N2Undef) {
1669
109k
    SDValue V = N1;
1670
109k
1671
109k
    // Look through any bitcasts. We check that these don't change the number
1672
109k
    // (and size) of elements and just changes their types.
1673
137k
    while (V.getOpcode() == ISD::BITCAST)
1674
27.8k
      V = V->getOperand(0);
1675
109k
1676
109k
    // A splat should always show up as a build vector node.
1677
109k
    if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
1678
2.70k
      BitVector UndefElements;
1679
2.70k
      SDValue Splat = BV->getSplatValue(&UndefElements);
1680
2.70k
      // If this is a splat of an undef, shuffling it is also undef.
1681
2.70k
      if (Splat && 
Splat.isUndef()363
)
1682
0
        return getUNDEF(VT);
1683
2.70k
1684
2.70k
      bool SameNumElts =
1685
2.70k
          V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
1686
2.70k
1687
2.70k
      // We only have a splat which can skip shuffles if there is a splatted
1688
2.70k
      // value and no undef lanes rearranged by the shuffle.
1689
2.70k
      if (Splat && 
UndefElements.none()363
) {
1690
263
        // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
1691
263
        // number of elements match or the value splatted is a zero constant.
1692
263
        if (SameNumElts)
1693
240
          return N1;
1694
23
        if (auto *C = dyn_cast<ConstantSDNode>(Splat))
1695
21
          if (C->isNullValue())
1696
14
            return N1;
1697
2.44k
      }
1698
2.44k
1699
2.44k
      // If the shuffle itself creates a splat, build the vector directly.
1700
2.44k
      if (AllSame && 
SameNumElts1.13k
) {
1701
1.04k
        EVT BuildVT = BV->getValueType(0);
1702
1.04k
        const SDValue &Splatted = BV->getOperand(MaskVec[0]);
1703
1.04k
        SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
1704
1.04k
1705
1.04k
        // We may have jumped through bitcasts, so the type of the
1706
1.04k
        // BUILD_VECTOR may not match the type of the shuffle.
1707
1.04k
        if (BuildVT != VT)
1708
0
          NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
1709
1.04k
        return NewBV;
1710
1.04k
      }
1711
160k
    }
1712
109k
  }
1713
160k
1714
160k
  FoldingSetNodeID ID;
1715
160k
  SDValue Ops[2] = { N1, N2 };
1716
160k
  AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops);
1717
2.01M
  for (int i = 0; i != NElts; 
++i1.85M
)
1718
1.85M
    ID.AddInteger(MaskVec[i]);
1719
160k
1720
160k
  void* IP = nullptr;
1721
160k
  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1722
5.23k
    return SDValue(E, 0);
1723
155k
1724
155k
  // Allocate the mask array for the node out of the BumpPtrAllocator, since
1725
155k
  // SDNode doesn't have access to it.  This memory will be "leaked" when
1726
155k
  // the node is deallocated, but recovered when the NodeAllocator is released.
1727
155k
  int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
1728
155k
  llvm::copy(MaskVec, MaskAlloc);
1729
155k
1730
155k
  auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
1731
155k
                                           dl.getDebugLoc(), MaskAlloc);
1732
155k
  createOperands(N, Ops);
1733
155k
1734
155k
  CSEMap.InsertNode(N, IP);
1735
155k
  InsertNode(N);
1736
155k
  SDValue V = SDValue(N, 0);
1737
155k
  NewSDValueDbgMsg(V, "Creating new node: ", this);
1738
155k
  return V;
1739
155k
}
1740
1741
923
SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
1742
923
  EVT VT = SV.getValueType(0);
1743
923
  SmallVector<int, 8> MaskVec(SV.getMask().begin(), SV.getMask().end());
1744
923
  ShuffleVectorSDNode::commuteMask(MaskVec);
1745
923
1746
923
  SDValue Op0 = SV.getOperand(0);
1747
923
  SDValue Op1 = SV.getOperand(1);
1748
923
  return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
1749
923
}
1750
1751
9.14M
SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) {
1752
9.14M
  FoldingSetNodeID ID;
1753
9.14M
  AddNodeIDNode(ID, ISD::Register, getVTList(VT), None);
1754
9.14M
  ID.AddInteger(RegNo);
1755
9.14M
  void *IP = nullptr;
1756
9.14M
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1757
3.68M
    return SDValue(E, 0);
1758
5.45M
1759
5.45M
  auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
1760
5.45M
  N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, DA);
1761
5.45M
  CSEMap.InsertNode(N, IP);
1762
5.45M
  InsertNode(N);
1763
5.45M
  return SDValue(N, 0);
1764
5.45M
}
1765
1766
455k
SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
1767
455k
  FoldingSetNodeID ID;
1768
455k
  AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), None);
1769
455k
  ID.AddPointer(RegMask);
1770
455k
  void *IP = nullptr;
1771
455k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1772
127k
    return SDValue(E, 0);
1773
327k
1774
327k
  auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
1775
327k
  CSEMap.InsertNode(N, IP);
1776
327k
  InsertNode(N);
1777
327k
  return SDValue(N, 0);
1778
327k
}
1779
1780
SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
1781
12.7k
                                 MCSymbol *Label) {
1782
12.7k
  return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
1783
12.7k
}
1784
1785
SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
1786
12.7k
                                   SDValue Root, MCSymbol *Label) {
1787
12.7k
  FoldingSetNodeID ID;
1788
12.7k
  SDValue Ops[] = { Root };
1789
12.7k
  AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
1790
12.7k
  ID.AddPointer(Label);
1791
12.7k
  void *IP = nullptr;
1792
12.7k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1793
0
    return SDValue(E, 0);
1794
12.7k
1795
12.7k
  auto *N =
1796
12.7k
      newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
1797
12.7k
  createOperands(N, Ops);
1798
12.7k
1799
12.7k
  CSEMap.InsertNode(N, IP);
1800
12.7k
  InsertNode(N);
1801
12.7k
  return SDValue(N, 0);
1802
12.7k
}
1803
1804
SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
1805
                                      int64_t Offset,
1806
                                      bool isTarget,
1807
305
                                      unsigned char TargetFlags) {
1808
305
  unsigned Opc = isTarget ? 
ISD::TargetBlockAddress170
:
ISD::BlockAddress135
;
1809
305
1810
305
  FoldingSetNodeID ID;
1811
305
  AddNodeIDNode(ID, Opc, getVTList(VT), None);
1812
305
  ID.AddPointer(BA);
1813
305
  ID.AddInteger(Offset);
1814
305
  ID.AddInteger(TargetFlags);
1815
305
  void *IP = nullptr;
1816
305
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1817
2
    return SDValue(E, 0);
1818
303
1819
303
  auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
1820
303
  CSEMap.InsertNode(N, IP);
1821
303
  InsertNode(N);
1822
303
  return SDValue(N, 0);
1823
303
}
1824
1825
2.10k
SDValue SelectionDAG::getSrcValue(const Value *V) {
1826
2.10k
  assert((!V || V->getType()->isPointerTy()) &&
1827
2.10k
         "SrcValue is not a pointer?");
1828
2.10k
1829
2.10k
  FoldingSetNodeID ID;
1830
2.10k
  AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), None);
1831
2.10k
  ID.AddPointer(V);
1832
2.10k
1833
2.10k
  void *IP = nullptr;
1834
2.10k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1835
455
    return SDValue(E, 0);
1836
1.64k
1837
1.64k
  auto *N = newSDNode<SrcValueSDNode>(V);
1838
1.64k
  CSEMap.InsertNode(N, IP);
1839
1.64k
  InsertNode(N);
1840
1.64k
  return SDValue(N, 0);
1841
1.64k
}
1842
1843
20.9k
SDValue SelectionDAG::getMDNode(const MDNode *MD) {
1844
20.9k
  FoldingSetNodeID ID;
1845
20.9k
  AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), None);
1846
20.9k
  ID.AddPointer(MD);
1847
20.9k
1848
20.9k
  void *IP = nullptr;
1849
20.9k
  if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1850
0
    return SDValue(E, 0);
1851
20.9k
1852
20.9k
  auto *N = newSDNode<MDNodeSDNode>(MD);
1853
20.9k
  CSEMap.InsertNode(N, IP);
1854
20.9k
  InsertNode(N);
1855
20.9k
  return SDValue(N, 0);
1856
20.9k
}
1857
1858
482k
SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
1859
482k
  if (VT == V.getValueType())
1860
189k
    return V;
1861
293k
1862
293k
  return getNode(ISD::BITCAST, SDLoc(V), VT, V);
1863
293k
}
1864
1865
SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
1866
228
                                       unsigned SrcAS, unsigned DestAS) {
1867
228
  SDValue Ops[] = {Ptr};
1868
228
  FoldingSetNodeID ID;
1869
228
  AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), Ops);
1870
228
  ID.AddInteger(SrcAS);
1871
228
  ID.AddInteger(DestAS);
1872
228
1873
228
  void *IP = nullptr;
1874
228
  if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
1875
0
    return SDValue(E, 0);
1876
228
1877
228
  auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
1878
228
                                           VT, SrcAS, DestAS);
1879
228
  createOperands(N, Ops);
1880
228
1881
228
  CSEMap.InsertNode(N, IP);
1882
228
  InsertNode(N);
1883
228
  return SDValue(N, 0);
1884
228
}
1885
1886
/// getShiftAmountOperand - Return the specified value casted to
1887
/// the target's desired shift amount type.
1888
535k
SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
1889
535k
  EVT OpTy = Op.getValueType();
1890
535k
  EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
1891
535k
  if (OpTy == ShTy || 
OpTy.isVector()28.6k
)
return Op506k
;
1892
28.6k
1893
28.6k
  return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
1894
28.6k
}
1895
1896
121
SDValue SelectionDAG::expandVAArg(SDNode *Node) {
1897
121
  SDLoc dl(Node);
1898
121
  const TargetLowering &TLI = getTargetLoweringInfo();
1899
121
  const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
1900
121
  EVT VT = Node->getValueType(0);
1901
121
  SDValue Tmp1 = Node->getOperand(0);
1902
121
  SDValue Tmp2 = Node->getOperand(1);
1903
121
  unsigned Align = Node->getConstantOperandVal(3);
1904
121
1905
121
  SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
1906
121
                               Tmp2, MachinePointerInfo(V));
1907
121
  SDValue VAList = VAListLoad;
1908
121
1909
121
  if (Align > TLI.getMinStackArgumentAlignment()) {
1910
98
    assert(((Align & (Align-1)) == 0) && "Expected Align to be a power of 2");
1911
98
1912
98
    VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1913
98
                     getConstant(Align - 1, dl, VAList.getValueType()));
1914
98
1915
98
    VAList = getNode(ISD::AND, dl, VAList.getValueType(), VAList,
1916
98
                     getConstant(-(int64_t)Align, dl, VAList.getValueType()));
1917
98
  }
1918
121
1919
121
  // Increment the pointer, VAList, to the next vaarg
1920
121
  Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
1921
121
                 getConstant(getDataLayout().getTypeAllocSize(
1922
121
                                               VT.getTypeForEVT(*getContext())),
1923
121
                             dl, VAList.getValueType()));
1924
121
  // Store the incremented VAList to the legalized pointer
1925
121
  Tmp1 =
1926
121
      getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
1927
121
  // Load the actual argument out of the pointer VAList
1928
121
  return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
1929
121
}
1930
1931
104
SDValue SelectionDAG::expandVACopy(SDNode *Node) {
1932
104
  SDLoc dl(Node);
1933
104
  const TargetLowering &TLI = getTargetLoweringInfo();
1934
104
  // This defaults to loading a pointer from the input and storing it to the
1935
104
  // output, returning the chain.
1936
104
  const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
1937
104
  const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
1938
104
  SDValue Tmp1 =
1939
104
      getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
1940
104
              Node->getOperand(2), MachinePointerInfo(VS));
1941
104
  return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
1942
104
                  MachinePointerInfo(VD));
1943
104
}
1944
1945
3.90k
SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
1946
3.90k
  MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1947
3.90k
  unsigned ByteSize = VT.getStoreSize();
1948
3.90k
  Type *Ty = VT.getTypeForEVT(*getContext());
1949
3.90k
  unsigned StackAlign =
1950
3.90k
      std::max((unsigned)getDataLayout().getPrefTypeAlignment(Ty), minAlign);
1951
3.90k
1952
3.90k
  int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
1953
3.90k
  return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1954
3.90k
}
1955
1956
3.39k
SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
1957
3.39k
  unsigned Bytes = std::max(VT1.getStoreSize(), VT2.getStoreSize());
1958
3.39k
  Type *Ty1 = VT1.getTypeForEVT(*getContext());
1959
3.39k
  Type *Ty2 = VT2.getTypeForEVT(*getContext());
1960
3.39k
  const DataLayout &DL = getDataLayout();
1961
3.39k
  unsigned Align =
1962
3.39k
      std::max(DL.getPrefTypeAlignment(Ty1), DL.getPrefTypeAlignment(Ty2));
1963
3.39k
1964
3.39k
  MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
1965
3.39k
  int FrameIdx = MFI.CreateStackObject(Bytes, Align, false);
1966
3.39k
  return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
1967
3.39k
}
1968
1969
SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
1970
2.76M
                                ISD::CondCode Cond, const SDLoc &dl) {
1971
2.76M
  EVT OpVT = N1.getValueType();
1972
2.76M
1973
2.76M
  // These setcc operations always fold.
1974
2.76M
  switch (Cond) {
1975
2.76M
  
default: break2.70M
;
1976
2.76M
  case ISD::SETFALSE:
1977
53
  case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
1978
58
  case ISD::SETTRUE:
1979
58
  case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
1980
58
1981
59.1k
  case ISD::SETOEQ:
1982
59.1k
  case ISD::SETOGT:
1983
59.1k
  case ISD::SETOGE:
1984
59.1k
  case ISD::SETOLT:
1985
59.1k
  case ISD::SETOLE:
1986
59.1k
  case ISD::SETONE:
1987
59.1k
  case ISD::SETO:
1988
59.1k
  case ISD::SETUO:
1989
59.1k
  case ISD::SETUEQ:
1990
59.1k
  case ISD::SETUNE:
1991
59.1k
    assert(!OpVT.isInteger() && "Illegal setcc for integer!");
1992
59.1k
    break;
1993
2.76M
  }
1994
2.76M
1995
2.76M
  if (OpVT.isInteger()) {
1996
2.68M
    // For EQ and NE, we can always pick a value for the undef to make the
1997
2.68M
    // predicate pass or fail, so we can return undef.
1998
2.68M
    // Matches behavior in llvm::ConstantFoldCompareInstruction.
1999
2.68M
    // icmp eq/ne X, undef -> undef.
2000
2.68M
    if ((N1.isUndef() || 
N2.isUndef()2.68M
) &&
2001
2.68M
        
(755
Cond == ISD::SETEQ755
||
Cond == ISD::SETNE384
))
2002
391
      return getUNDEF(VT);
2003
2.68M
2004
2.68M
    // If both operands are undef, we can return undef for int comparison.
2005
2.68M
    // icmp undef, undef -> undef.
2006
2.68M
    if (N1.isUndef() && 
N2.isUndef()268
)
2007
22
      return getUNDEF(VT);
2008
2.68M
2009
2.68M
    // icmp X, X -> true/false
2010
2.68M
    // icmp X, undef -> true/false because undef could be X.
2011
2.68M
    if (N1 == N2)
2012
1.50k
      return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2013
2.76M
  }
2014
2.76M
2015
2.76M
  if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2016
2.06M
    const APInt &C2 = N2C->getAPIntValue();
2017
2.06M
    if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2018
17.4k
      const APInt &C1 = N1C->getAPIntValue();
2019
17.4k
2020
17.4k
      switch (Cond) {
2021
17.4k
      
default: 0
llvm_unreachable0
("Unknown integer setcc!");
2022
17.4k
      
case ISD::SETEQ: return getBoolConstant(C1 == C2, dl, VT, OpVT)15.2k
;
2023
17.4k
      
case ISD::SETNE: return getBoolConstant(C1 != C2, dl, VT, OpVT)26
;
2024
17.4k
      
case ISD::SETULT: return getBoolConstant(C1.ult(C2), dl, VT, OpVT)16
;
2025
17.4k
      
case ISD::SETUGT: return getBoolConstant(C1.ugt(C2), dl, VT, OpVT)14
;
2026
17.4k
      
case ISD::SETULE: return getBoolConstant(C1.ule(C2), dl, VT, OpVT)37
;
2027
17.4k
      
case ISD::SETUGE: return getBoolConstant(C1.uge(C2), dl, VT, OpVT)16
;
2028
17.4k
      
case ISD::SETLT: return getBoolConstant(C1.slt(C2), dl, VT, OpVT)2.00k
;
2029
17.4k
      
case ISD::SETGT: return getBoolConstant(C1.sgt(C2), dl, VT, OpVT)154
;
2030
17.4k
      
case ISD::SETLE: return getBoolConstant(C1.sle(C2), dl, VT, OpVT)2
;
2031
17.4k
      
case ISD::SETGE: return getBoolConstant(C1.sge(C2), dl, VT, OpVT)0
;
2032
2.74M
      }
2033
2.74M
    }
2034
2.06M
  }
2035
2.74M
2036
2.74M
  auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2037
2.74M
  auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2038
2.74M
2039
2.74M
  if (N1CFP && 
N2CFP563
) {
2040
253
    APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2041
253
    switch (Cond) {
2042
253
    
default: break0
;
2043
253
    
case ISD::SETEQ: if (0
R==APFloat::cmpUnordered0
)
2044
0
                        return getUNDEF(VT);
2045
0
                      LLVM_FALLTHROUGH;
2046
97
    case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2047
97
                                             OpVT);
2048
0
    case ISD::SETNE:  if (R==APFloat::cmpUnordered)
2049
0
                        return getUNDEF(VT);
2050
0
                      LLVM_FALLTHROUGH;
2051
0
    case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2052
0
                                             R==APFloat::cmpLessThan, dl, VT,
2053
0
                                             OpVT);
2054
3
    case ISD::SETLT:  if (R==APFloat::cmpUnordered)
2055
0
                        return getUNDEF(VT);
2056
3
                      LLVM_FALLTHROUGH;
2057
26
    case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2058
26
                                             OpVT);
2059
3
    
case ISD::SETGT: if (0
R==APFloat::cmpUnordered0
)
2060
0
                        return getUNDEF(VT);
2061
0
                      LLVM_FALLTHROUGH;
2062
10
    case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl,
2063
10
                                             VT, OpVT);
2064
0
    case ISD::SETLE:  if (R==APFloat::cmpUnordered)
2065
0
                        return getUNDEF(VT);
2066
0
                      LLVM_FALLTHROUGH;
2067
4
    case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan ||
2068
4
                                             R==APFloat::cmpEqual, dl, VT,
2069
4
                                             OpVT);
2070
0
    case ISD::SETGE:  if (R==APFloat::cmpUnordered)
2071
0
                        return getUNDEF(VT);
2072
0
                      LLVM_FALLTHROUGH;
2073
6
    case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2074
6
                                         R==APFloat::cmpEqual, dl, VT, OpVT);
2075
8
    case ISD::SETO:   return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2076
8
                                             OpVT);
2077
31
    case ISD::SETUO:  return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2078
31
                                             OpVT);
2079
9
    case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered ||
2080
9
                                             R==APFloat::cmpEqual, dl, VT,
2081
9
                                             OpVT);
2082
16
    case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2083
16
                                             OpVT);
2084
7
    case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered ||
2085
7
                                             
R==APFloat::cmpLessThan6
, dl, VT,
2086
7
                                             OpVT);
2087
36
    case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan ||
2088
36
                                             R==APFloat::cmpUnordered, dl, VT,
2089
36
                                             OpVT);
2090
1
    case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl,
2091
1
                                             VT, OpVT);
2092
2
    case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2093
2
                                             OpVT);
2094
2.74M
    }
2095
2.74M
  } else if (N1CFP && 
OpVT.isSimple()310
&&
!N2.isUndef()310
) {
2096
303
    // Ensure that the constant occurs on the RHS.
2097
303
    ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond);
2098
303
    if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2099
101
      return SDValue();
2100
202
    return getSetCC(dl, VT, N2, N1, SwappedCond);
2101
2.74M
  } else if ((N2CFP && 
N2CFP->getValueAPF().isNaN()33.0k
) ||
2102
2.74M
             
(2.74M
OpVT.isFloatingPoint()2.74M
&&
(79.0k
N1.isUndef()79.0k
||
N2.isUndef()79.0k
))) {
2103
280
    // If an operand is known to be a nan (or undef that could be a nan), we can
2104
280
    // fold it.
2105
280
    // Choosing NaN for the undef will always make unordered comparison succeed
2106
280
    // and ordered comparison fails.
2107
280
    // Matches behavior in llvm::ConstantFoldCompareInstruction.
2108
280
    switch (ISD::getUnorderedFlavor(Cond)) {
2109
280
    default:
2110
0
      llvm_unreachable("Unknown flavor!");
2111
280
    case 0: // Known false.
2112
18
      return getBoolConstant(false, dl, VT, OpVT);
2113
280
    case 1: // Known true.
2114
18
      return getBoolConstant(true, dl, VT, OpVT);
2115
280
    case 2: // Undefined.
2116
244
      return getUNDEF(VT);
2117
2.74M
    }
2118
2.74M
  }
2119
2.74M
2120
2.74M
  // Could not fold it.
2121
2.74M
  return SDValue();
2122
2.74M
}
2123
2124
/// See if the specified operand can be simplified with the knowledge that only
2125
/// the bits specified by DemandedBits are used.
2126
/// TODO: really we should be making this into the DAG equivalent of
2127
/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2128
781k
SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) {
2129
781k
  EVT VT = V.getValueType();
2130
781k
  APInt DemandedElts = VT.isVector()
2131
781k
                           ? 
APInt::getAllOnesValue(VT.getVectorNumElements())21.9k
2132
781k
                           : 
APInt(1, 1)759k
;
2133
781k
  return GetDemandedBits(V, DemandedBits, DemandedElts);
2134
781k
}
2135
2136
/// See if the specified operand can be simplified with the knowledge that only
2137
/// the bits specified by DemandedBits are used in the elements specified by
2138
/// DemandedElts.
2139
/// TODO: really we should be making this into the DAG equivalent of
2140
/// SimplifyMultipleUseDemandedBits and not generate any new nodes.
2141
SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits,
2142
781k
                                      const APInt &DemandedElts) {
2143
781k
  switch (V.getOpcode()) {
2144
781k
  default:
2145
608k
    break;
2146
781k
  case ISD::Constant: {
2147
53.8k
    auto *CV = cast<ConstantSDNode>(V.getNode());
2148
53.8k
    assert(CV && "Const value should be ConstSDNode.");
2149
53.8k
    const APInt &CVal = CV->getAPIntValue();
2150
53.8k
    APInt NewVal = CVal & DemandedBits;
2151
53.8k
    if (NewVal != CVal)
2152
388
      return getConstant(NewVal, SDLoc(V), V.getValueType());
2153
53.4k
    break;
2154
53.4k
  }
2155
53.4k
  case ISD::OR:
2156
11.8k
  case ISD::XOR:
2157
11.8k
    // If the LHS or RHS don't contribute bits to the or, drop them.
2158
11.8k
    if (MaskedValueIsZero(V.getOperand(0), DemandedBits))
2159
851
      return V.getOperand(1);
2160
10.9k
    if (MaskedValueIsZero(V.getOperand(1), DemandedBits))
2161
1.07k
      return V.getOperand(0);
2162
9.92k
    break;
2163
75.4k
  case ISD::SRL:
2164
75.4k
    // Only look at single-use SRLs.
2165
75.4k
    if (!V.getNode()->hasOneUse())
2166
7.55k
      break;
2167
67.9k
    if (auto *RHSC = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2168
66.4k
      // See if we can recursively simplify the LHS.
2169
66.4k
      unsigned Amt = RHSC->getZExtValue();
2170
66.4k
2171
66.4k
      // Watch out for shift count overflow though.
2172
66.4k
      if (Amt >= DemandedBits.getBitWidth())
2173
0
        break;
2174
66.4k
      APInt SrcDemandedBits = DemandedBits << Amt;
2175
66.4k
      if (SDValue SimplifyLHS =
2176
1.99k
              GetDemandedBits(V.getOperand(0), SrcDemandedBits))
2177
1.99k
        return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS,
2178
1.99k
                       V.getOperand(1));
2179
65.9k
    }
2180
65.9k
    break;
2181
65.9k
  case ISD::AND: {
2182
20.4k
    // X & -1 -> X (ignoring bits which aren't demanded).
2183
20.4k
    // Also handle the case where masked out bits in X are known to be zero.
2184
20.4k
    if (ConstantSDNode *RHSC = isConstOrConstSplat(V.getOperand(1))) {
2185
19.2k
      const APInt &AndVal = RHSC->getAPIntValue();
2186
19.2k
      if (DemandedBits.isSubsetOf(AndVal) ||
2187
19.2k
          DemandedBits.isSubsetOf(computeKnownBits(V.getOperand(0)).Zero |
2188
16.7k
                                  AndVal))
2189
4.19k
        return V.getOperand(0);
2190
16.2k
    }
2191
16.2k
    break;
2192
16.2k
  }
2193
16.2k
  case ISD::ANY_EXTEND: {
2194
3.15k
    SDValue Src = V.getOperand(0);
2195
3.15k
    unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
2196
3.15k
    // Being conservative here - only peek through if we only demand bits in the
2197
3.15k
    // non-extended source (even though the extended bits are technically
2198
3.15k
    // undef).
2199
3.15k
    if (DemandedBits.getActiveBits() > SrcBitWidth)
2200
101
      break;
2201
3.05k
    APInt SrcDemandedBits = DemandedBits.trunc(SrcBitWidth);
2202
3.05k
    if (SDValue DemandedSrc = GetDemandedBits(Src, SrcDemandedBits))
2203
195
      return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc);
2204
2.86k
    break;
2205
2.86k
  }
2206
8.16k
  case ISD::SIGN_EXTEND_INREG:
2207
8.16k
    EVT ExVT = cast<VTSDNode>(V.getOperand(1))->getVT();
2208
8.16k
    unsigned ExVTBits = ExVT.getScalarSizeInBits();
2209
8.16k
2210
8.16k
    // If none of the extended bits are demanded, eliminate the sextinreg.
2211
8.16k
    if (DemandedBits.getActiveBits() <= ExVTBits)
2212
64
      return V.getOperand(0);
2213
8.10k
2214
8.10k
    break;
2215
773k
  }
2216
773k
  return SDValue();
2217
773k
}
2218
2219
/// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
2220
/// use this predicate to simplify operations downstream.
2221
188k
bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2222
188k
  unsigned BitWidth = Op.getScalarValueSizeInBits();
2223
188k
  return MaskedValueIsZero(Op, APInt::getSignMask(BitWidth), Depth);
2224
188k
}
2225
2226
/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero.  We use
2227
/// this predicate to simplify operations downstream.  Mask is known to be zero
2228
/// for bits that V cannot have.
2229
bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2230
3.28M
                                     unsigned Depth) const {
2231
3.28M
  EVT VT = V.getValueType();
2232
3.28M
  APInt DemandedElts = VT.isVector()
2233
3.28M
                           ? 
APInt::getAllOnesValue(VT.getVectorNumElements())181k
2234
3.28M
                           : 
APInt(1, 1)3.10M
;
2235
3.28M
  return MaskedValueIsZero(V, Mask, DemandedElts, Depth);
2236
3.28M
}
2237
2238
/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2239
/// DemandedElts.  We use this predicate to simplify operations downstream.
2240
/// Mask is known to be zero for bits that V cannot have.
2241
bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2242
                                     const APInt &DemandedElts,
2243
3.31M
                                     unsigned Depth) const {
2244
3.31M
  return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2245
3.31M
}
2246
2247
/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2248
bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2249
1.35k
                                        unsigned Depth) const {
2250
1.35k
  return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2251
1.35k
}
2252
2253
/// isSplatValue - Return true if the vector V has the same value
2254
/// across all DemandedElts.
2255
bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2256
1.27M
                                APInt &UndefElts) {
2257
1.27M
  if (!DemandedElts)
2258
0
    return false; // No demanded elts, better to assume we don't know anything.
2259
1.27M
2260
1.27M
  EVT VT = V.getValueType();
2261
1.27M
  assert(VT.isVector() && "Vector type expected");
2262
1.27M
2263
1.27M
  unsigned NumElts = VT.getVectorNumElements();
2264
1.27M
  assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2265
1.27M
  UndefElts = APInt::getNullValue(NumElts);
2266
1.27M
2267
1.27M
  switch (V.getOpcode()) {
2268
1.27M
  case ISD::BUILD_VECTOR: {
2269
245k
    SDValue Scl;
2270
1.34M
    for (unsigned i = 0; i != NumElts; 
++i1.09M
) {
2271
1.13M
      SDValue Op = V.getOperand(i);
2272
1.13M
      if (Op.isUndef()) {
2273
6.93k
        UndefElts.setBit(i);
2274
6.93k
        continue;
2275
6.93k
      }
2276
1.13M
      if (!DemandedElts[i])
2277
1.30k
        continue;
2278
1.12M
      if (Scl && 
Scl != Op884k
)
2279
41.0k
        return false;
2280
1.08M
      Scl = Op;
2281
1.08M
    }
2282
245k
    
return true204k
;
2283
245k
  }
2284
245k
  case ISD::VECTOR_SHUFFLE: {
2285
5.68k
    // Check if this is a shuffle node doing a splat.
2286
5.68k
    // TODO: Do we need to handle shuffle(splat, undef, mask)?
2287
5.68k
    int SplatIndex = -1;
2288
5.68k
    ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2289
58.0k
    for (int i = 0; i != (int)NumElts; 
++i52.3k
) {
2290
54.7k
      int M = Mask[i];
2291
54.7k
      if (M < 0) {
2292
953
        UndefElts.setBit(i);
2293
953
        continue;
2294
953
      }
2295
53.7k
      if (!DemandedElts[i])
2296
2.69k
        continue;
2297
51.0k
      if (0 <= SplatIndex && 
SplatIndex != M45.3k
)
2298
2.37k
        return false;
2299
48.6k
      SplatIndex = M;
2300
48.6k
    }
2301
5.68k
    
return true3.31k
;
2302
5.68k
  }
2303
5.68k
  case ISD::EXTRACT_SUBVECTOR: {
2304
3.11k
    SDValue Src = V.getOperand(0);
2305
3.11k
    ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(V.getOperand(1));
2306
3.11k
    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2307
3.11k
    if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2308
3.11k
      // Offset the demanded elts by the subvector index.
2309
3.11k
      uint64_t Idx = SubIdx->getZExtValue();
2310
3.11k
      APInt UndefSrcElts;
2311
3.11k
      APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2312
3.11k
      if (isSplatValue(Src, DemandedSrc, UndefSrcElts)) {
2313
291
        UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2314
291
        return true;
2315
291
      }
2316
2.82k
    }
2317
2.82k
    break;
2318
2.82k
  }
2319
139k
  case ISD::ADD:
2320
139k
  case ISD::SUB:
2321
139k
  case ISD::AND: {
2322
139k
    APInt UndefLHS, UndefRHS;
2323
139k
    SDValue LHS = V.getOperand(0);
2324
139k
    SDValue RHS = V.getOperand(1);
2325
139k
    if (isSplatValue(LHS, DemandedElts, UndefLHS) &&
2326
139k
        
isSplatValue(RHS, DemandedElts, UndefRHS)9.11k
) {
2327
3.63k
      UndefElts = UndefLHS | UndefRHS;
2328
3.63k
      return true;
2329
3.63k
    }
2330
135k
    break;
2331
135k
  }
2332
1.01M
  }
2333
1.01M
2334
1.01M
  return false;
2335
1.01M
}
2336
2337
/// Helper wrapper to main isSplatValue function.
2338
1.20k
bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) {
2339
1.20k
  EVT VT = V.getValueType();
2340
1.20k
  assert(VT.isVector() && "Vector type expected");
2341
1.20k
  unsigned NumElts = VT.getVectorNumElements();
2342
1.20k
2343
1.20k
  APInt UndefElts;
2344
1.20k
  APInt DemandedElts = APInt::getAllOnesValue(NumElts);
2345
1.20k
  return isSplatValue(V, DemandedElts, UndefElts) &&
2346
1.20k
         
(232
AllowUndefs232
||
!UndefElts232
);
2347
1.20k
}
2348
2349
1.15M
SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
2350
1.15M
  V = peekThroughExtractSubvectors(V);
2351
1.15M
2352
1.15M
  EVT VT = V.getValueType();
2353
1.15M
  unsigned Opcode = V.getOpcode();
2354
1.15M
  switch (Opcode) {
2355
1.15M
  default: {
2356
1.12M
    APInt UndefElts;
2357
1.12M
    APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
2358
1.12M
    if (isSplatValue(V, DemandedElts, UndefElts)) {
2359
198k
      // Handle case where all demanded elements are UNDEF.
2360
198k
      if (DemandedElts.isSubsetOf(UndefElts)) {
2361
0
        SplatIdx = 0;
2362
0
        return getUNDEF(VT);
2363
0
      }
2364
198k
      SplatIdx = (UndefElts & DemandedElts).countTrailingOnes();
2365
198k
      return V;
2366
198k
    }
2367
922k
    break;
2368
922k
  }
2369
922k
  case ISD::VECTOR_SHUFFLE: {
2370
35.6k
    // Check if this is a shuffle node doing a splat.
2371
35.6k
    // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2372
35.6k
    // getTargetVShiftNode currently struggles without the splat source.
2373
35.6k
    auto *SVN = cast<ShuffleVectorSDNode>(V);
2374
35.6k
    if (!SVN->isSplat())
2375
23.0k
      break;
2376
12.5k
    int Idx = SVN->getSplatIndex();
2377
12.5k
    int NumElts = V.getValueType().getVectorNumElements();
2378
12.5k
    SplatIdx = Idx % NumElts;
2379
12.5k
    return V.getOperand(Idx / NumElts);
2380
12.5k
  }
2381
945k
  }
2382
945k
2383
945k
  return SDValue();
2384
945k
}
2385
2386
14.0k
SDValue SelectionDAG::getSplatValue(SDValue V) {
2387
14.0k
  int SplatIdx;
2388
14.0k
  if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx))
2389
3.04k
    return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V),
2390
3.04k
                   SrcVector.getValueType().getScalarType(), SrcVector,
2391
3.04k
                   getIntPtrConstant(SplatIdx, SDLoc(V)));
2392
11.0k
  return SDValue();
2393
11.0k
}
2394
2395
/// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
2396
/// is less than the element bit-width of the shift node, return it.
2397
3.24M
static const APInt *getValidShiftAmountConstant(SDValue V) {
2398
3.24M
  if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) {
2399
3.05M
    // Shifting more than the bitwidth is not valid.
2400
3.05M
    const APInt &ShAmt = SA->getAPIntValue();
2401
3.05M
    if (ShAmt.ult(V.getScalarValueSizeInBits()))
2402
3.05M
      return &ShAmt;
2403
192k
  }
2404
192k
  return nullptr;
2405
192k
}
2406
2407
/// Determine which bits of Op are known to be either zero or one and return
2408
/// them in Known. For vectors, the known bits are those that are shared by
2409
/// every vector element.
2410
14.3M
KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
2411
14.3M
  EVT VT = Op.getValueType();
2412
14.3M
  APInt DemandedElts = VT.isVector()
2413
14.3M
                           ? 
APInt::getAllOnesValue(VT.getVectorNumElements())340k
2414
14.3M
                           : 
APInt(1, 1)13.9M
;
2415
14.3M
  return computeKnownBits(Op, DemandedElts, Depth);
2416
14.3M
}
2417
2418
/// Determine which bits of Op are known to be either zero or one and return
2419
/// them in Known. The DemandedElts argument allows us to only collect the known
2420
/// bits that are shared by the requested vector elements.
2421
KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
2422
65.5M
                                         unsigned Depth) const {
2423
65.5M
  unsigned BitWidth = Op.getScalarValueSizeInBits();
2424
65.5M
2425
65.5M
  KnownBits Known(BitWidth);   // Don't know anything.
2426
65.5M
2427
65.5M
  if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2428
12.9M
    // We know all of the bits for a constant!
2429
12.9M
    Known.One = C->getAPIntValue();
2430
12.9M
    Known.Zero = ~Known.One;
2431
12.9M
    return Known;
2432
12.9M
  }
2433
52.5M
  if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
2434
16.0k
    // We know all of the bits for a constant fp!
2435
16.0k
    Known.One = C->getValueAPF().bitcastToAPInt();
2436
16.0k
    Known.Zero = ~Known.One;
2437
16.0k
    return Known;
2438
16.0k
  }
2439
52.5M
2440
52.5M
  if (Depth == 6)
2441
3.16M
    return Known;  // Limit search depth.
2442
49.3M
2443
49.3M
  KnownBits Known2;
2444
49.3M
  unsigned NumElts = DemandedElts.getBitWidth();
2445
49.3M
  assert((!Op.getValueType().isVector() ||
2446
49.3M
          NumElts == Op.getValueType().getVectorNumElements()) &&
2447
49.3M
         "Unexpected vector size");
2448
49.3M
2449
49.3M
  if (!DemandedElts)
2450
4.54k
    return Known;  // No demanded elts, better to assume we don't know anything.
2451
49.3M
2452
49.3M
  unsigned Opcode = Op.getOpcode();
2453
49.3M
  switch (Opcode) {
2454
49.3M
  case ISD::BUILD_VECTOR:
2455
1.24M
    // Collect the known bits that are shared by every demanded vector element.
2456
1.24M
    Known.Zero.setAllBits(); Known.One.setAllBits();
2457
9.24M
    for (unsigned i = 0, e = Op.getNumOperands(); i != e; 
++i7.99M
) {
2458
8.38M
      if (!DemandedElts[i])
2459
4.26M
        continue;
2460
4.11M
2461
4.11M
      SDValue SrcOp = Op.getOperand(i);
2462
4.11M
      Known2 = computeKnownBits(SrcOp, Depth + 1);
2463
4.11M
2464
4.11M
      // BUILD_VECTOR can implicitly truncate sources, we must handle this.
2465
4.11M
      if (SrcOp.getValueSizeInBits() != BitWidth) {
2466
352k
        assert(SrcOp.getValueSizeInBits() > BitWidth &&
2467
352k
               "Expected BUILD_VECTOR implicit truncation");
2468
352k
        Known2 = Known2.trunc(BitWidth);
2469
352k
      }
2470
4.11M
2471
4.11M
      // Known bits are the values that are shared by every demanded element.
2472
4.11M
      Known.One &= Known2.One;
2473
4.11M
      Known.Zero &= Known2.Zero;
2474
4.11M
2475
4.11M
      // If we don't know any bits, early out.
2476
4.11M
      if (Known.isUnknown())
2477
387k
        break;
2478
4.11M
    }
2479
1.24M
    break;
2480
49.3M
  case ISD::VECTOR_SHUFFLE: {
2481
492k
    // Collect the known bits that are shared by every vector element referenced
2482
492k
    // by the shuffle.
2483
492k
    APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
2484
492k
    Known.Zero.setAllBits(); Known.One.setAllBits();
2485
492k
    const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
2486
492k
    assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
2487
5.74M
    for (unsigned i = 0; i != NumElts; 
++i5.24M
) {
2488
5.37M
      if (!DemandedElts[i])
2489
3.31M
        continue;
2490
2.06M
2491
2.06M
      int M = SVN->getMaskElt(i);
2492
2.06M
      if (M < 0) {
2493
126k
        // For UNDEF elements, we don't know anything about the common state of
2494
126k
        // the shuffle result.
2495
126k
        Known.resetAll();
2496
126k
        DemandedLHS.clearAllBits();
2497
126k
        DemandedRHS.clearAllBits();
2498
126k
        break;
2499
126k
      }
2500
1.93M
2501
1.93M
      if ((unsigned)M < NumElts)
2502
1.70M
        DemandedLHS.setBit((unsigned)M % NumElts);
2503
226k
      else
2504
226k
        DemandedRHS.setBit((unsigned)M % NumElts);
2505
1.93M
    }
2506
492k
    // Known bits are the values that are shared by every demanded element.
2507
492k
    if (!!DemandedLHS) {
2508
315k
      SDValue LHS = Op.getOperand(0);
2509
315k
      Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
2510
315k
      Known.One &= Known2.One;
2511
315k
      Known.Zero &= Known2.Zero;
2512
315k
    }
2513
492k
    // If we don't know any bits, early out.
2514
492k
    if (Known.isUnknown())
2515
432k
      break;
2516
60.5k
    if (!!DemandedRHS) {
2517
51.7k
      SDValue RHS = Op.getOperand(1);
2518
51.7k
      Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
2519
51.7k
      Known.One &= Known2.One;
2520
51.7k
      Known.Zero &= Known2.Zero;
2521
51.7k
    }
2522
60.5k
    break;
2523
60.5k
  }
2524
138k
  case ISD::CONCAT_VECTORS: {
2525
138k
    // Split DemandedElts and test each of the demanded subvectors.
2526
138k
    Known.Zero.setAllBits(); Known.One.setAllBits();
2527
138k
    EVT SubVectorVT = Op.getOperand(0).getValueType();
2528
138k
    unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
2529
138k
    unsigned NumSubVectors = Op.getNumOperands();
2530
185k
    for (unsigned i = 0; i != NumSubVectors; 
++i46.9k
) {
2531
178k
      APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
2532
178k
      DemandedSub = DemandedSub.trunc(NumSubVectorElts);
2533
178k
      if (!!DemandedSub) {
2534
138k
        SDValue Sub = Op.getOperand(i);
2535
138k
        Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
2536
138k
        Known.One &= Known2.One;
2537
138k
        Known.Zero &= Known2.Zero;
2538
138k
      }
2539
178k
      // If we don't know any bits, early out.
2540
178k
      if (Known.isUnknown())
2541
131k
        break;
2542
178k
    }
2543
138k
    break;
2544
60.5k
  }
2545
76.8k
  case ISD::INSERT_SUBVECTOR: {
2546
76.8k
    // If we know the element index, demand any elements from the subvector and
2547
76.8k
    // the remainder from the src its inserted into, otherwise demand them all.
2548
76.8k
    SDValue Src = Op.getOperand(0);
2549
76.8k
    SDValue Sub = Op.getOperand(1);
2550
76.8k
    ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2551
76.8k
    unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2552
76.8k
    if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
2553
76.8k
      Known.One.setAllBits();
2554
76.8k
      Known.Zero.setAllBits();
2555
76.8k
      uint64_t Idx = SubIdx->getZExtValue();
2556
76.8k
      APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2557
76.8k
      if (!!DemandedSubElts) {
2558
62.7k
        Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
2559
62.7k
        if (Known.isUnknown())
2560
61.4k
          break; // early-out.
2561
15.3k
      }
2562
15.3k
      APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
2563
15.3k
      APInt DemandedSrcElts = DemandedElts & ~SubMask;
2564
15.3k
      if (!!DemandedSrcElts) {
2565
14.3k
        Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
2566
14.3k
        Known.One &= Known2.One;
2567
14.3k
        Known.Zero &= Known2.Zero;
2568
14.3k
      }
2569
15.3k
    } else {
2570
0
      Known = computeKnownBits(Sub, Depth + 1);
2571
0
      if (Known.isUnknown())
2572
0
        break; // early-out.
2573
0
      Known2 = computeKnownBits(Src, Depth + 1);
2574
0
      Known.One &= Known2.One;
2575
0
      Known.Zero &= Known2.Zero;
2576
0
    }
2577
76.8k
    
break15.3k
;
2578
76.8k
  }
2579
436k
  case ISD::EXTRACT_SUBVECTOR: {
2580
436k
    // If we know the element index, just demand that subvector elements,
2581
436k
    // otherwise demand them all.
2582
436k
    SDValue Src = Op.getOperand(0);
2583
436k
    ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2584
436k
    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2585
436k
    if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
2586
436k
      // Offset the demanded elts by the subvector index.
2587
436k
      uint64_t Idx = SubIdx->getZExtValue();
2588
436k
      APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2589
436k
      Known = computeKnownBits(Src, DemandedSrc, Depth + 1);
2590
436k
    } else {
2591
0
      Known = computeKnownBits(Src, Depth + 1);
2592
0
    }
2593
436k
    break;
2594
76.8k
  }
2595
76.8k
  case ISD::SCALAR_TO_VECTOR: {
2596
28.8k
    // We know about scalar_to_vector as much as we know about it source,
2597
28.8k
    // which becomes the first element of otherwise unknown vector.
2598
28.8k
    if (DemandedElts != 1)
2599
12.8k
      break;
2600
15.9k
2601
15.9k
    SDValue N0 = Op.getOperand(0);
2602
15.9k
    Known = computeKnownBits(N0, Depth + 1);
2603
15.9k
    if (N0.getValueSizeInBits() != BitWidth)
2604
326
      Known = Known.trunc(BitWidth);
2605
15.9k
2606
15.9k
    break;
2607
15.9k
  }
2608
1.49M
  case ISD::BITCAST: {
2609
1.49M
    SDValue N0 = Op.getOperand(0);
2610
1.49M
    EVT SubVT = N0.getValueType();
2611
1.49M
    unsigned SubBitWidth = SubVT.getScalarSizeInBits();
2612
1.49M
2613
1.49M
    // Ignore bitcasts from unsupported types.
2614
1.49M
    if (!(SubVT.isInteger() || 
SubVT.isFloatingPoint()111k
))
2615
292
      break;
2616
1.48M
2617
1.48M
    // Fast handling of 'identity' bitcasts.
2618
1.48M
    if (BitWidth == SubBitWidth) {
2619
83.4k
      Known = computeKnownBits(N0, DemandedElts, Depth + 1);
2620
83.4k
      break;
2621
83.4k
    }
2622
1.40M
2623
1.40M
    bool IsLE = getDataLayout().isLittleEndian();
2624
1.40M
2625
1.40M
    // Bitcast 'small element' vector to 'large element' scalar/vector.
2626
1.40M
    if ((BitWidth % SubBitWidth) == 0) {
2627
743k
      assert(N0.getValueType().isVector() && "Expected bitcast from vector");
2628
743k
2629
743k
      // Collect known bits for the (larger) output by collecting the known
2630
743k
      // bits from each set of sub elements and shift these into place.
2631
743k
      // We need to separately call computeKnownBits for each set of
2632
743k
      // sub elements as the knownbits for each is likely to be different.
2633
743k
      unsigned SubScale = BitWidth / SubBitWidth;
2634
743k
      APInt SubDemandedElts(NumElts * SubScale, 0);
2635
4.18M
      for (unsigned i = 0; i != NumElts; 
++i3.44M
)
2636
3.44M
        if (DemandedElts[i])
2637
3.09M
          SubDemandedElts.setBit(i * SubScale);
2638
743k
2639
2.82M
      for (unsigned i = 0; i != SubScale; 
++i2.08M
) {
2640
2.08M
        Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
2641
2.08M
                         Depth + 1);
2642
2.08M
        unsigned Shifts = IsLE ? 
i2.07M
:
SubScale - 1 - i8.49k
;
2643
2.08M
        Known.One |= Known2.One.zext(BitWidth).shl(SubBitWidth * Shifts);
2644
2.08M
        Known.Zero |= Known2.Zero.zext(BitWidth).shl(SubBitWidth * Shifts);
2645
2.08M
      }
2646
743k
    }
2647
1.40M
2648
1.40M
    // Bitcast 'large element' scalar/vector to 'small element' vector.
2649
1.40M
    if ((SubBitWidth % BitWidth) == 0) {
2650
663k
      assert(Op.getValueType().isVector() && "Expected bitcast to vector");
2651
663k
2652
663k
      // Collect known bits for the (smaller) output by collecting the known
2653
663k
      // bits from the overlapping larger input elements and extracting the
2654
663k
      // sub sections we actually care about.
2655
663k
      unsigned SubScale = SubBitWidth / BitWidth;
2656
663k
      APInt SubDemandedElts(NumElts / SubScale, 0);
2657
11.4M
      for (unsigned i = 0; i != NumElts; 
++i10.7M
)
2658
10.7M
        if (DemandedElts[i])
2659
3.36M
          SubDemandedElts.setBit(i / SubScale);
2660
663k
2661
663k
      Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
2662
663k
2663
663k
      Known.Zero.setAllBits(); Known.One.setAllBits();
2664
4.12M
      for (unsigned i = 0; i != NumElts; 
++i3.46M
)
2665
4.06M
        if (DemandedElts[i]) {
2666
1.05M
          unsigned Shifts = IsLE ? 
i1.04M
:
NumElts - 1 - i10.7k
;
2667
1.05M
          unsigned Offset = (Shifts % SubScale) * BitWidth;
2668
1.05M
          Known.One &= Known2.One.lshr(Offset).trunc(BitWidth);
2669
1.05M
          Known.Zero &= Known2.Zero.lshr(Offset).trunc(BitWidth);
2670
1.05M
          // If we don't know any bits, early out.
2671
1.05M
          if (Known.isUnknown())
2672
599k
            break;
2673
1.05M
        }
2674
663k
    }
2675
1.40M
    break;
2676
1.40M
  }
2677
2.28M
  case ISD::AND:
2678
2.28M
    // If either the LHS or the RHS are Zero, the result is zero.
2679
2.28M
    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2680
2.28M
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2681
2.28M
2682
2.28M
    // Output known-1 bits are only known if set in both the LHS & RHS.
2683
2.28M
    Known.One &= Known2.One;
2684
2.28M
    // Output known-0 are known to be clear if zero in either the LHS | RHS.
2685
2.28M
    Known.Zero |= Known2.Zero;
2686
2.28M
    break;
2687
1.40M
  case ISD::OR:
2688
704k
    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2689
704k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2690
704k
2691
704k
    // Output known-0 bits are only known if clear in both the LHS & RHS.
2692
704k
    Known.Zero &= Known2.Zero;
2693
704k
    // Output known-1 are known to be set if set in either the LHS | RHS.
2694
704k
    Known.One |= Known2.One;
2695
704k
    break;
2696
1.40M
  case ISD::XOR: {
2697
346k
    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2698
346k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2699
346k
2700
346k
    // Output known-0 bits are known if clear or set in both the LHS & RHS.
2701
346k
    APInt KnownZeroOut = (Known.Zero & Known2.Zero) | (Known.One & Known2.One);
2702
346k
    // Output known-1 are known to be set if set in only one of the LHS, RHS.
2703
346k
    Known.One = (Known.Zero & Known2.One) | (Known.One & Known2.Zero);
2704
346k
    Known.Zero = KnownZeroOut;
2705
346k
    break;
2706
1.40M
  }
2707
1.40M
  case ISD::MUL: {
2708
744k
    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2709
744k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2710
744k
2711
744k
    // If low bits are zero in either operand, output low known-0 bits.
2712
744k
    // Also compute a conservative estimate for high known-0 bits.
2713
744k
    // More trickiness is possible, but this is sufficient for the
2714
744k
    // interesting case of alignment computation.
2715
744k
    unsigned TrailZ = Known.countMinTrailingZeros() +
2716
744k
                      Known2.countMinTrailingZeros();
2717
744k
    unsigned LeadZ =  std::max(Known.countMinLeadingZeros() +
2718
744k
                               Known2.countMinLeadingZeros(),
2719
744k
                               BitWidth) - BitWidth;
2720
744k
2721
744k
    Known.resetAll();
2722
744k
    Known.Zero.setLowBits(std::min(TrailZ, BitWidth));
2723
744k
    Known.Zero.setHighBits(std::min(LeadZ, BitWidth));
2724
744k
    break;
2725
1.40M
  }
2726
1.40M
  case ISD::UDIV: {
2727
11.4k
    // For the purposes of computing leading zeros we can conservatively
2728
11.4k
    // treat a udiv as a logical right shift by the power of 2 known to
2729
11.4k
    // be less than the denominator.
2730
11.4k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2731
11.4k
    unsigned LeadZ = Known2.countMinLeadingZeros();
2732
11.4k
2733
11.4k
    Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2734
11.4k
    unsigned RHSMaxLeadingZeros = Known2.countMaxLeadingZeros();
2735
11.4k
    if (RHSMaxLeadingZeros != BitWidth)
2736
6.18k
      LeadZ = std::min(BitWidth, LeadZ + BitWidth - RHSMaxLeadingZeros - 1);
2737
11.4k
2738
11.4k
    Known.Zero.setHighBits(LeadZ);
2739
11.4k
    break;
2740
1.40M
  }
2741
1.40M
  case ISD::SELECT:
2742
272k
  case ISD::VSELECT:
2743
272k
    Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2744
272k
    // If we don't know any bits, early out.
2745
272k
    if (Known.isUnknown())
2746
106k
      break;
2747
166k
    Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
2748
166k
2749
166k
    // Only known if known in both the LHS and RHS.
2750
166k
    Known.One &= Known2.One;
2751
166k
    Known.Zero &= Known2.Zero;
2752
166k
    break;
2753
166k
  case ISD::SELECT_CC:
2754
119k
    Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
2755
119k
    // If we don't know any bits, early out.
2756
119k
    if (Known.isUnknown())
2757
47.7k
      break;
2758
71.6k
    Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
2759
71.6k
2760
71.6k
    // Only known if known in both the LHS and RHS.
2761
71.6k
    Known.One &= Known2.One;
2762
71.6k
    Known.Zero &= Known2.Zero;
2763
71.6k
    break;
2764
71.6k
  case ISD::SMULO:
2765
7.58k
  case ISD::UMULO:
2766
7.58k
  case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
2767
7.58k
    if (Op.getResNo() != 1)
2768
1.60k
      break;
2769
5.98k
    // The boolean result conforms to getBooleanContents.
2770
5.98k
    // If we know the result of a setcc has the top bits zero, use this info.
2771
5.98k
    // We know that we have an integer-based boolean since these operations
2772
5.98k
    // are only available for integer.
2773
5.98k
    if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
2774
5.98k
            TargetLowering::ZeroOrOneBooleanContent &&
2775
5.98k
        
BitWidth > 14.84k
)
2776
4.14k
      Known.Zero.setBitsFrom(1);
2777
5.98k
    break;
2778
894k
  case ISD::SETCC:
2779
894k
    // If we know the result of a setcc has the top bits zero, use this info.
2780
894k
    if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
2781
894k
            TargetLowering::ZeroOrOneBooleanContent &&
2782
894k
        
BitWidth > 1559k
)
2783
513k
      Known.Zero.setBitsFrom(1);
2784
894k
    break;
2785
1.67M
  case ISD::SHL:
2786
1.67M
    if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2787
1.54M
      Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2788
1.54M
      unsigned Shift = ShAmt->getZExtValue();
2789
1.54M
      Known.Zero <<= Shift;
2790
1.54M
      Known.One <<= Shift;
2791
1.54M
      // Low bits are known zero.
2792
1.54M
      Known.Zero.setLowBits(Shift);
2793
1.54M
    }
2794
1.67M
    break;
2795
1.42M
  case ISD::SRL:
2796
1.42M
    if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2797
1.37M
      Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2798
1.37M
      unsigned Shift = ShAmt->getZExtValue();
2799
1.37M
      Known.Zero.lshrInPlace(Shift);
2800
1.37M
      Known.One.lshrInPlace(Shift);
2801
1.37M
      // High bits are known zero.
2802
1.37M
      Known.Zero.setHighBits(Shift);
2803
1.37M
    } else 
if (auto *52.0k
BV52.0k
= dyn_cast<BuildVectorSDNode>(Op.getOperand(1))) {
2804
5.23k
      // If the shift amount is a vector of constants see if we can bound
2805
5.23k
      // the number of upper zero bits.
2806
5.23k
      unsigned ShiftAmountMin = BitWidth;
2807
52.2k
      for (unsigned i = 0; i != BV->getNumOperands(); 
++i47.0k
) {
2808
47.8k
        if (auto *C = dyn_cast<ConstantSDNode>(BV->getOperand(i))) {
2809
47.0k
          const APInt &ShAmt = C->getAPIntValue();
2810
47.0k
          if (ShAmt.ult(BitWidth)) {
2811
47.0k
            ShiftAmountMin = std::min<unsigned>(ShiftAmountMin,
2812
47.0k
                                                ShAmt.getZExtValue());
2813
47.0k
            continue;
2814
47.0k
          }
2815
811
        }
2816
811
        // Don't know anything.
2817
811
        ShiftAmountMin = 0;
2818
811
        break;
2819
811
      }
2820
5.23k
2821
5.23k
      Known.Zero.setHighBits(ShiftAmountMin);
2822
5.23k
    }
2823
1.42M
    break;
2824
141k
  case ISD::SRA:
2825
141k
    if (const APInt *ShAmt = getValidShiftAmountConstant(Op)) {
2826
130k
      Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2827
130k
      unsigned Shift = ShAmt->getZExtValue();
2828
130k
      // Sign extend known zero/one bit (else is unknown).
2829
130k
      Known.Zero.ashrInPlace(Shift);
2830
130k
      Known.One.ashrInPlace(Shift);
2831
130k
    }
2832
141k
    break;
2833
5.98k
  case ISD::FSHL:
2834
1.47k
  case ISD::FSHR:
2835
1.47k
    if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
2836
576
      unsigned Amt = C->getAPIntValue().urem(BitWidth);
2837
576
2838
576
      // For fshl, 0-shift returns the 1st arg.
2839
576
      // For fshr, 0-shift returns the 2nd arg.
2840
576
      if (Amt == 0) {
2841
0
        Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
2842
0
                                 DemandedElts, Depth + 1);
2843
0
        break;
2844
0
      }
2845
576
2846
576
      // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
2847
576
      // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
2848
576
      Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2849
576
      Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
2850
576
      if (Opcode == ISD::FSHL) {
2851
364
        Known.One <<= Amt;
2852
364
        Known.Zero <<= Amt;
2853
364
        Known2.One.lshrInPlace(BitWidth - Amt);
2854
364
        Known2.Zero.lshrInPlace(BitWidth - Amt);
2855
364
      } else {
2856
212
        Known.One <<= BitWidth - Amt;
2857
212
        Known.Zero <<= BitWidth - Amt;
2858
212
        Known2.One.lshrInPlace(Amt);
2859
212
        Known2.Zero.lshrInPlace(Amt);
2860
212
      }
2861
576
      Known.One |= Known2.One;
2862
576
      Known.Zero |= Known2.Zero;
2863
576
    }
2864
1.47k
    break;
2865
78.0k
  case ISD::SIGN_EXTEND_INREG: {
2866
78.0k
    EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2867
78.0k
    unsigned EBits = EVT.getScalarSizeInBits();
2868
78.0k
2869
78.0k
    // Sign extension.  Compute the demanded bits in the result that are not
2870
78.0k
    // present in the input.
2871
78.0k
    APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits);
2872
78.0k
2873
78.0k
    APInt InSignMask = APInt::getSignMask(EBits);
2874
78.0k
    APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits);
2875
78.0k
2876
78.0k
    // If the sign extended bits are demanded, we know that the sign
2877
78.0k
    // bit is demanded.
2878
78.0k
    InSignMask = InSignMask.zext(BitWidth);
2879
78.0k
    if (NewBits.getBoolValue())
2880
78.0k
      InputDemandedBits |= InSignMask;
2881
78.0k
2882
78.0k
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2883
78.0k
    Known.One &= InputDemandedBits;
2884
78.0k
    Known.Zero &= InputDemandedBits;
2885
78.0k
2886
78.0k
    // If the sign bit of the input is known set or clear, then we know the
2887
78.0k
    // top bits of the result.
2888
78.0k
    if (Known.Zero.intersects(InSignMask)) {        // Input sign bit known clear
2889
4
      Known.Zero |= NewBits;
2890
4
      Known.One  &= ~NewBits;
2891
78.0k
    } else if (Known.One.intersects(InSignMask)) {  // Input sign bit known set
2892
4
      Known.One  |= NewBits;
2893
4
      Known.Zero &= ~NewBits;
2894
78.0k
    } else {                              // Input sign bit unknown
2895
78.0k
      Known.Zero &= ~NewBits;
2896
78.0k
      Known.One  &= ~NewBits;
2897
78.0k
    }
2898
78.0k
    break;
2899
1.47k
  }
2900
6.97k
  case ISD::CTTZ:
2901
6.97k
  case ISD::CTTZ_ZERO_UNDEF: {
2902
6.97k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2903
6.97k
    // If we have a known 1, its position is our upper bound.
2904
6.97k
    unsigned PossibleTZ = Known2.countMaxTrailingZeros();
2905
6.97k
    unsigned LowBits = Log2_32(PossibleTZ) + 1;
2906
6.97k
    Known.Zero.setBitsFrom(LowBits);
2907
6.97k
    break;
2908
6.97k
  }
2909
46.3k
  case ISD::CTLZ:
2910
46.3k
  case ISD::CTLZ_ZERO_UNDEF: {
2911
46.3k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2912
46.3k
    // If we have a known 1, its position is our upper bound.
2913
46.3k
    unsigned PossibleLZ = Known2.countMaxLeadingZeros();
2914
46.3k
    unsigned LowBits = Log2_32(PossibleLZ) + 1;
2915
46.3k
    Known.Zero.setBitsFrom(LowBits);
2916
46.3k
    break;
2917
46.3k
  }
2918
46.3k
  case ISD::CTPOP: {
2919
4.73k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2920
4.73k
    // If we know some of the bits are zero, they can't be one.
2921
4.73k
    unsigned PossibleOnes = Known2.countMaxPopulation();
2922
4.73k
    Known.Zero.setBitsFrom(Log2_32(PossibleOnes) + 1);
2923
4.73k
    break;
2924
46.3k
  }
2925
4.21M
  case ISD::LOAD: {
2926
4.21M
    LoadSDNode *LD = cast<LoadSDNode>(Op);
2927
4.21M
    const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
2928
4.21M
    if (ISD::isNON_EXTLoad(LD) && 
Cst3.39M
) {
2929
179k
      // Determine any common known bits from the loaded constant pool value.
2930
179k
      Type *CstTy = Cst->getType();
2931
179k
      if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) {
2932
179k
        // If its a vector splat, then we can (quickly) reuse the scalar path.
2933
179k
        // NOTE: We assume all elements match and none are UNDEF.
2934
179k
        if (CstTy->isVectorTy()) {
2935
179k
          if (const Constant *Splat = Cst->getSplatValue()) {
2936
91.2k
            Cst = Splat;
2937
91.2k
            CstTy = Cst->getType();
2938
91.2k
          }
2939
179k
        }
2940
179k
        // TODO - do we need to handle different bitwidths?
2941
179k
        if (CstTy->isVectorTy() && 
BitWidth == CstTy->getScalarSizeInBits()88.0k
) {
2942
87.0k
          // Iterate across all vector elements finding common known bits.
2943
87.0k
          Known.One.setAllBits();
2944
87.0k
          Known.Zero.setAllBits();
2945
1.12M
          for (unsigned i = 0; i != NumElts; 
++i1.04M
) {
2946
1.04M
            if (!DemandedElts[i])
2947
372k
              continue;
2948
676k
            if (Constant *Elt = Cst->getAggregateElement(i)) {
2949
676k
              if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
2950
667k
                const APInt &Value = CInt->getValue();
2951
667k
                Known.One &= Value;
2952
667k
                Known.Zero &= ~Value;
2953
667k
                continue;
2954
667k
              }
2955
9.19k
              if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
2956
844
                APInt Value = CFP->getValueAPF().bitcastToAPInt();
2957
844
                Known.One &= Value;
2958
844
                Known.Zero &= ~Value;
2959
844
                continue;
2960
844
              }
2961
8.34k
            }
2962
8.34k
            Known.One.clearAllBits();
2963
8.34k
            Known.Zero.clearAllBits();
2964
8.34k
            break;
2965
8.34k
          }
2966
92.4k
        } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
2967
91.0k
          if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
2968
85.6k
            const APInt &Value = CInt->getValue();
2969
85.6k
            Known.One = Value;
2970
85.6k
            Known.Zero = ~Value;
2971
85.6k
          } else 
if (auto *5.38k
CFP5.38k
= dyn_cast<ConstantFP>(Cst)) {
2972
5.38k
            APInt Value = CFP->getValueAPF().bitcastToAPInt();
2973
5.38k
            Known.One = Value;
2974
5.38k
            Known.Zero = ~Value;
2975
5.38k
          }
2976
91.0k
        }
2977
179k
      }
2978
4.03M
    } else if (ISD::isZEXTLoad(Op.getNode()) && 
Op.getResNo() == 0336k
) {
2979
333k
      // If this is a ZEXTLoad and we are looking at the loaded value.
2980
333k
      EVT VT = LD->getMemoryVT();
2981
333k
      unsigned MemBits = VT.getScalarSizeInBits();
2982
333k
      Known.Zero.setBitsFrom(MemBits);
2983
3.70M
    } else if (const MDNode *Ranges = LD->getRanges()) {
2984
36.2k
      if (LD->getExtensionType() == ISD::NON_EXTLOAD)
2985
10.4k
        computeKnownBitsFromRangeMetadata(*Ranges, Known);
2986
36.2k
    }
2987
4.21M
    break;
2988
46.3k
  }
2989
46.3k
  case ISD::ZERO_EXTEND_VECTOR_INREG: {
2990
22.7k
    EVT InVT = Op.getOperand(0).getValueType();
2991
22.7k
    APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
2992
22.7k
    Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
2993
22.7k
    Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
2994
22.7k
    break;
2995
46.3k
  }
2996
499k
  case ISD::ZERO_EXTEND: {
2997
499k
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
2998
499k
    Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */);
2999
499k
    break;
3000
46.3k
  }
3001
46.3k
  case ISD::SIGN_EXTEND_VECTOR_INREG: {
3002
2.49k
    EVT InVT = Op.getOperand(0).getValueType();
3003
2.49k
    APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
3004
2.49k
    Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3005
2.49k
    // If the sign bit is known to be zero or one, then sext will extend
3006
2.49k
    // it to the top bits, else it will just zext.
3007
2.49k
    Known = Known.sext(BitWidth);
3008
2.49k
    break;
3009
46.3k
  }
3010
312k
  case ISD::SIGN_EXTEND: {
3011
312k
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3012
312k
    // If the sign bit is known to be zero or one, then sext will extend
3013
312k
    // it to the top bits, else it will just zext.
3014
312k
    Known = Known.sext(BitWidth);
3015
312k
    break;
3016
46.3k
  }
3017
218k
  case ISD::ANY_EXTEND: {
3018
218k
    Known = computeKnownBits(Op.getOperand(0), Depth+1);
3019
218k
    Known = Known.zext(BitWidth, false /* ExtendedBitsAreKnownZero */);
3020
218k
    break;
3021
46.3k
  }
3022
1.06M
  case ISD::TRUNCATE: {
3023
1.06M
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3024
1.06M
    Known = Known.trunc(BitWidth);
3025
1.06M
    break;
3026
46.3k
  }
3027
880k
  case ISD::AssertZext: {
3028
880k
    EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3029
880k
    APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits());
3030
880k
    Known = computeKnownBits(Op.getOperand(0), Depth+1);
3031
880k
    Known.Zero |= (~InMask);
3032
880k
    Known.One  &= (~Known.Zero);
3033
880k
    break;
3034
46.3k
  }
3035
46.3k
  case ISD::FGETSIGN:
3036
34
    // All bits are zero except the low bit.
3037
34
    Known.Zero.setBitsFrom(1);
3038
34
    break;
3039
46.3k
  case ISD::USUBO:
3040
20.7k
  case ISD::SSUBO:
3041
20.7k
    if (Op.getResNo() == 1) {
3042
11.6k
      // If we know the result of a setcc has the top bits zero, use this info.
3043
11.6k
      if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3044
11.6k
              TargetLowering::ZeroOrOneBooleanContent &&
3045
11.6k
          
BitWidth > 14.96k
)
3046
2.75k
        Known.Zero.setBitsFrom(1);
3047
11.6k
      break;
3048
11.6k
    }
3049
9.07k
    LLVM_FALLTHROUGH;
3050
519k
  case ISD::SUB:
3051
519k
  case ISD::SUBC: {
3052
519k
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3053
519k
    Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3054
519k
    Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false,
3055
519k
                                        Known, Known2);
3056
519k
    break;
3057
519k
  }
3058
519k
  case ISD::UADDO:
3059
48.1k
  case ISD::SADDO:
3060
48.1k
  case ISD::ADDCARRY:
3061
48.1k
    if (Op.getResNo() == 1) {
3062
16.1k
      // If we know the result of a setcc has the top bits zero, use this info.
3063
16.1k
      if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3064
16.1k
              TargetLowering::ZeroOrOneBooleanContent &&
3065
16.1k
          
BitWidth > 113.7k
)
3066
10.7k
        Known.Zero.setBitsFrom(1);
3067
16.1k
      break;
3068
16.1k
    }
3069
32.0k
    LLVM_FALLTHROUGH;
3070
6.63M
  case ISD::ADD:
3071
6.63M
  case ISD::ADDC:
3072
6.63M
  case ISD::ADDE: {
3073
6.63M
    assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3074
6.63M
3075
6.63M
    // With ADDE and ADDCARRY, a carry bit may be added in.
3076
6.63M
    KnownBits Carry(1);
3077
6.63M
    if (Opcode == ISD::ADDE)
3078
3.78k
      // Can't track carry from glue, set carry to unknown.
3079
3.78k
      Carry.resetAll();
3080
6.62M
    else if (Opcode == ISD::ADDCARRY)
3081
17.0k
      // TODO: Compute known bits for the carry operand. Not sure if it is worth
3082
17.0k
      // the trouble (how often will we find a known carry bit). And I haven't
3083
17.0k
      // tested this very much yet, but something like this might work:
3084
17.0k
      //   Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3085
17.0k
      //   Carry = Carry.zextOrTrunc(1, false);
3086
17.0k
      Carry.resetAll();
3087
6.61M
    else
3088
6.61M
      Carry.setAllZero();
3089
6.63M
3090
6.63M
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3091
6.63M
    Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3092
6.63M
    Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3093
6.63M
    break;
3094
6.63M
  }
3095
6.63M
  case ISD::SREM:
3096
2.16k
    if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
3097
929
      const APInt &RA = Rem->getAPIntValue().abs();
3098
929
      if (RA.isPowerOf2()) {
3099
52
        APInt LowBits = RA - 1;
3100
52
        Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3101
52
3102
52
        // The low bits of the first operand are unchanged by the srem.
3103
52
        Known.Zero = Known2.Zero & LowBits;
3104
52
        Known.One = Known2.One & LowBits;
3105
52
3106
52
        // If the first operand is non-negative or has all low bits zero, then
3107
52
        // the upper bits are all zero.
3108
52
        if (Known2.Zero[BitWidth-1] || ((Known2.Zero & LowBits) == LowBits))
3109
8
          Known.Zero |= ~LowBits;
3110
52
3111
52
        // If the first operand is negative and not all low bits are zero, then
3112
52
        // the upper bits are all one.
3113
52
        if (Known2.One[BitWidth-1] && 
((Known2.One & LowBits) != 0)3
)
3114
3
          Known.One |= ~LowBits;
3115
52
        assert((Known.Zero & Known.One) == 0&&"Bits known to be one AND zero?");
3116
52
      }
3117
929
    }
3118
2.16k
    break;
3119
6.63M
  case ISD::UREM: {
3120
5.60k
    if (ConstantSDNode *Rem = isConstOrConstSplat(Op.getOperand(1))) {
3121
3.97k
      const APInt &RA = Rem->getAPIntValue();
3122
3.97k
      if (RA.isPowerOf2()) {
3123
210
        APInt LowBits = (RA - 1);
3124
210
        Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3125
210
3126
210
        // The upper bits are all zero, the lower ones are unchanged.
3127
210
        Known.Zero = Known2.Zero | ~LowBits;
3128
210
        Known.One = Known2.One & LowBits;
3129
210
        break;
3130
210
      }
3131
5.39k
    }
3132
5.39k
3133
5.39k
    // Since the result is less than or equal to either operand, any leading
3134
5.39k
    // zero bits in either operand must also exist in the result.
3135
5.39k
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3136
5.39k
    Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3137
5.39k
3138
5.39k
    uint32_t Leaders =
3139
5.39k
        std::max(Known.countMinLeadingZeros(), Known2.countMinLeadingZeros());
3140
5.39k
    Known.resetAll();
3141
5.39k
    Known.Zero.setHighBits(Leaders);
3142
5.39k
    break;
3143
5.39k
  }
3144
5.39k
  case ISD::EXTRACT_ELEMENT: {
3145
144
    Known = computeKnownBits(Op.getOperand(0), Depth+1);
3146
144
    const unsigned Index = Op.getConstantOperandVal(1);
3147
144
    const unsigned EltBitWidth = Op.getValueSizeInBits();
3148
144
3149
144
    // Remove low part of known bits mask
3150
144
    Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3151
144
    Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3152
144
3153
144
    // Remove high part of known bit mask
3154
144
    Known = Known.trunc(EltBitWidth);
3155
144
    break;
3156
5.39k
  }
3157
954k
  case ISD::EXTRACT_VECTOR_ELT: {
3158
954k
    SDValue InVec = Op.getOperand(0);
3159
954k
    SDValue EltNo = Op.getOperand(1);
3160
954k
    EVT VecVT = InVec.getValueType();
3161
954k
    const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3162
954k
    const unsigned NumSrcElts = VecVT.getVectorNumElements();
3163
954k
    // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3164
954k
    // anything about the extended bits.
3165
954k
    if (BitWidth > EltBitWidth)
3166
253k
      Known = Known.trunc(EltBitWidth);
3167
954k
    ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3168
954k
    if (ConstEltNo && 
ConstEltNo->getAPIntValue().ult(NumSrcElts)953k
) {
3169
953k
      // If we know the element index, just demand that vector element.
3170
953k
      unsigned Idx = ConstEltNo->getZExtValue();
3171
953k
      APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
3172
953k
      Known = computeKnownBits(InVec, DemandedElt, Depth + 1);
3173
953k
    } else {
3174
615
      // Unknown element index, so ignore DemandedElts and demand them all.
3175
615
      Known = computeKnownBits(InVec, Depth + 1);
3176
615
    }
3177
954k
    if (BitWidth > EltBitWidth)
3178
253k
      Known = Known.zext(BitWidth, false /* => any extend */);
3179
954k
    break;
3180
5.39k
  }
3181
15.9k
  case ISD::INSERT_VECTOR_ELT: {
3182
15.9k
    SDValue InVec = Op.getOperand(0);
3183
15.9k
    SDValue InVal = Op.getOperand(1);
3184
15.9k
    SDValue EltNo = Op.getOperand(2);
3185
15.9k
3186
15.9k
    ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3187
15.9k
    if (CEltNo && 
CEltNo->getAPIntValue().ult(NumElts)15.9k
) {
3188
15.9k
      // If we know the element index, split the demand between the
3189
15.9k
      // source vector and the inserted element.
3190
15.9k
      Known.Zero = Known.One = APInt::getAllOnesValue(BitWidth);
3191
15.9k
      unsigned EltIdx = CEltNo->getZExtValue();
3192
15.9k
3193
15.9k
      // If we demand the inserted element then add its common known bits.
3194
15.9k
      if (DemandedElts[EltIdx]) {
3195
13.9k
        Known2 = computeKnownBits(InVal, Depth + 1);
3196
13.9k
        Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
3197
13.9k
        Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
3198
13.9k
      }
3199
15.9k
3200
15.9k
      // If we demand the source vector then add its common known bits, ensuring
3201
15.9k
      // that we don't demand the inserted element.
3202
15.9k
      APInt VectorElts = DemandedElts & ~(APInt::getOneBitSet(NumElts, EltIdx));
3203
15.9k
      if (!!VectorElts) {
3204
11.0k
        Known2 = computeKnownBits(InVec, VectorElts, Depth + 1);
3205
11.0k
        Known.One &= Known2.One;
3206
11.0k
        Known.Zero &= Known2.Zero;
3207
11.0k
      }
3208
15.9k
    } else {
3209
6
      // Unknown element index, so ignore DemandedElts and demand them all.
3210
6
      Known = computeKnownBits(InVec, Depth + 1);
3211
6
      Known2 = computeKnownBits(InVal, Depth + 1);
3212
6
      Known.One &= Known2.One.zextOrTrunc(Known.One.getBitWidth());
3213
6
      Known.Zero &= Known2.Zero.zextOrTrunc(Known.Zero.getBitWidth());
3214
6
    }
3215
15.9k
    break;
3216
5.39k
  }
3217
5.39k
  case ISD::BITREVERSE: {
3218
936
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3219
936
    Known.Zero = Known2.Zero.reverseBits();
3220
936
    Known.One = Known2.One.reverseBits();
3221
936
    break;
3222
5.39k
  }
3223
6.98k
  case ISD::BSWAP: {
3224
6.98k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3225
6.98k
    Known.Zero = Known2.Zero.byteSwap();
3226
6.98k
    Known.One = Known2.One.byteSwap();
3227
6.98k
    break;
3228
5.39k
  }
3229
6.11k
  case ISD::ABS: {
3230
6.11k
    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3231
6.11k
3232
6.11k
    // If the source's MSB is zero then we know the rest of the bits already.
3233
6.11k
    if (Known2.isNonNegative()) {
3234
0
      Known.Zero = Known2.Zero;
3235
0
      Known.One = Known2.One;
3236
0
      break;
3237
0
    }
3238
6.11k
3239
6.11k
    // We only know that the absolute values's MSB will be zero iff there is
3240
6.11k
    // a set bit that isn't the sign bit (otherwise it could be INT_MIN).
3241
6.11k
    Known2.One.clearSignBit();
3242
6.11k
    if (Known2.One.getBoolValue()) {
3243
2
      Known.Zero = APInt::getSignMask(BitWidth);
3244
2
      break;
3245
2
    }
3246
6.10k
    break;
3247
6.10k
  }
3248
35.6k
  case ISD::UMIN: {
3249
35.6k
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3250
35.6k
    Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3251
35.6k
3252
35.6k
    // UMIN - we know that the result will have the maximum of the
3253
35.6k
    // known zero leading bits of the inputs.
3254
35.6k
    unsigned LeadZero = Known.countMinLeadingZeros();
3255
35.6k
    LeadZero = std::max(LeadZero, Known2.countMinLeadingZeros());
3256
35.6k
3257
35.6k
    Known.Zero &= Known2.Zero;
3258
35.6k
    Known.One &= Known2.One;
3259
35.6k
    Known.Zero.setHighBits(LeadZero);
3260
35.6k
    break;
3261
6.10k
  }
3262
26.5k
  case ISD::UMAX: {
3263
26.5k
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3264
26.5k
    Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3265
26.5k
3266
26.5k
    // UMAX - we know that the result will have the maximum of the
3267
26.5k
    // known one leading bits of the inputs.
3268
26.5k
    unsigned LeadOne = Known.countMinLeadingOnes();
3269
26.5k
    LeadOne = std::max(LeadOne, Known2.countMinLeadingOnes());
3270
26.5k
3271
26.5k
    Known.Zero &= Known2.Zero;
3272
26.5k
    Known.One &= Known2.One;
3273
26.5k
    Known.One.setHighBits(LeadOne);
3274
26.5k
    break;
3275
6.10k
  }
3276
80.2k
  case ISD::SMIN:
3277
80.2k
  case ISD::SMAX: {
3278
80.2k
    // If we have a clamp pattern, we know that the number of sign bits will be
3279
80.2k
    // the minimum of the clamp min/max range.
3280
80.2k
    bool IsMax = (Opcode == ISD::SMAX);
3281
80.2k
    ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3282
80.2k
    if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3283
13.1k
      if (Op.getOperand(0).getOpcode() == (IsMax ? 
ISD::SMIN9.78k
:
ISD::SMAX3.39k
))
3284
8.12k
        CstHigh =
3285
8.12k
            isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3286
80.2k
    if (CstLow && 
CstHigh13.1k
) {
3287
8.10k
      if (!IsMax)
3288
370
        std::swap(CstLow, CstHigh);
3289
8.10k
3290
8.10k
      const APInt &ValueLow = CstLow->getAPIntValue();
3291
8.10k
      const APInt &ValueHigh = CstHigh->getAPIntValue();
3292
8.10k
      if (ValueLow.sle(ValueHigh)) {
3293
8.10k
        unsigned LowSignBits = ValueLow.getNumSignBits();
3294
8.10k
        unsigned HighSignBits = ValueHigh.getNumSignBits();
3295
8.10k
        unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3296
8.10k
        if (ValueLow.isNegative() && 
ValueHigh.isNegative()2.65k
) {
3297
2
          Known.One.setHighBits(MinSignBits);
3298
2
          break;
3299
2
        }
3300
8.10k
        if (ValueLow.isNonNegative() && 
ValueHigh.isNonNegative()5.44k
) {
3301
5.44k
          Known.Zero.setHighBits(MinSignBits);
3302
5.44k
          break;
3303
5.44k
        }
3304
74.7k
      }
3305
8.10k
    }
3306
74.7k
3307
74.7k
    // Fallback - just get the shared known bits of the operands.
3308
74.7k
    Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3309
74.7k
    if (Known.isUnknown()) 
break74.6k
; // Early-out
3310
106
    Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3311
106
    Known.Zero &= Known2.Zero;
3312
106
    Known.One &= Known2.One;
3313
106
    break;
3314
106
  }
3315
2.70M
  case ISD::FrameIndex:
3316
2.70M
  case ISD::TargetFrameIndex:
3317
2.70M
    TLI->computeKnownBitsForFrameIndex(Op, Known, DemandedElts, *this, Depth);
3318
2.70M
    break;
3319
2.70M
3320
17.9M
  default:
3321
17.9M
    if (Opcode < ISD::BUILTIN_OP_END)
3322
15.6M
      break;
3323
2.29M
    LLVM_FALLTHROUGH;
3324
2.75M
  case ISD::INTRINSIC_WO_CHAIN:
3325
2.75M
  case ISD::INTRINSIC_W_CHAIN:
3326
2.75M
  case ISD::INTRINSIC_VOID:
3327
2.75M
    // Allow the target to implement this method for its nodes.
3328
2.75M
    TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
3329
2.75M
    break;
3330
49.3M
  }
3331
49.3M
3332
49.3M
  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
3333
49.3M
  return Known;
3334
49.3M
}
3335
3336
SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
3337
14.7k
                                                             SDValue N1) const {
3338
14.7k
  // X + 0 never overflow
3339
14.7k
  if (isNullConstant(N1))
3340
0
    return OFK_Never;
3341
14.7k
3342
14.7k
  KnownBits N1Known = computeKnownBits(N1);
3343
14.7k
  if (N1Known.Zero.getBoolValue()) {
3344
7.41k
    KnownBits N0Known = computeKnownBits(N0);
3345
7.41k
3346
7.41k
    bool overflow;
3347
7.41k
    (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow);
3348
7.41k
    if (!overflow)
3349
273
      return OFK_Never;
3350
14.4k
  }
3351
14.4k
3352
14.4k
  // mulhi + 1 never overflow
3353
14.4k
  if (N0.getOpcode() == ISD::UMUL_LOHI && 
N0.getResNo() == 12.66k
&&
3354
14.4k
      
(~N1Known.Zero & 0x01) == ~N1Known.Zero410
)
3355
237
    return OFK_Never;
3356
14.2k
3357
14.2k
  if (N1.getOpcode() == ISD::UMUL_LOHI && 
N1.getResNo() == 11.46k
) {
3358
339
    KnownBits N0Known = computeKnownBits(N0);
3359
339
3360
339
    if ((~N0Known.Zero & 0x01) == ~N0Known.Zero)
3361
0
      return OFK_Never;
3362
14.2k
  }
3363
14.2k
3364
14.2k
  return OFK_Sometime;
3365
14.2k
}
3366
3367
72.5k
bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val) const {
3368
72.5k
  EVT OpVT = Val.getValueType();
3369
72.5k
  unsigned BitWidth = OpVT.getScalarSizeInBits();
3370
72.5k
3371
72.5k
  // Is the constant a known power of 2?
3372
72.5k
  if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val))
3373
62.7k
    return Const->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3374
9.78k
3375
9.78k
  // A left-shift of a constant one will have exactly one bit set because
3376
9.78k
  // shifting the bit off the end is undefined.
3377
9.78k
  if (Val.getOpcode() == ISD::SHL) {
3378
45
    auto *C = isConstOrConstSplat(Val.getOperand(0));
3379
45
    if (C && 
C->getAPIntValue() == 142
)
3380
39
      return true;
3381
9.74k
  }
3382
9.74k
3383
9.74k
  // Similarly, a logical right-shift of a constant sign-bit will have exactly
3384
9.74k
  // one bit set.
3385
9.74k
  if (Val.getOpcode() == ISD::SRL) {
3386
23
    auto *C = isConstOrConstSplat(Val.getOperand(0));
3387
23
    if (C && 
C->getAPIntValue().isSignMask()5
)
3388
5
      return true;
3389
9.73k
  }
3390
9.73k
3391
9.73k
  // Are all operands of a build vector constant powers of two?
3392
9.73k
  if (Val.getOpcode() == ISD::BUILD_VECTOR)
3393
30.6k
    
if (7.32k
llvm::all_of(Val->ops(), [BitWidth](SDValue E) 7.32k
{
3394
30.6k
          if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3395
30.4k
            return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3396
207
          return false;
3397
207
        }))
3398
1.81k
      return true;
3399
7.92k
3400
7.92k
  // More could be done here, though the above checks are enough
3401
7.92k
  // to handle some common cases.
3402
7.92k
3403
7.92k
  // Fall back to computeKnownBits to catch other known cases.
3404
7.92k
  KnownBits Known = computeKnownBits(Val);
3405
7.92k
  return (Known.countMaxPopulation() == 1) && 
(Known.countMinPopulation() == 1)311
;
3406
7.92k
}
3407
3408
1.62M
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
3409
1.62M
  EVT VT = Op.getValueType();
3410
1.62M
  APInt DemandedElts = VT.isVector()
3411
1.62M
                           ? 
APInt::getAllOnesValue(VT.getVectorNumElements())147k
3412
1.62M
                           : 
APInt(1, 1)1.47M
;
3413
1.62M
  return ComputeNumSignBits(Op, DemandedElts, Depth);
3414
1.62M
}
3415
3416
unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
3417
1.96M
                                          unsigned Depth) const {
3418
1.96M
  EVT VT = Op.getValueType();
3419
1.96M
  assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
3420
1.96M
  unsigned VTBits = VT.getScalarSizeInBits();
3421
1.96M
  unsigned NumElts = DemandedElts.getBitWidth();
3422
1.96M
  unsigned Tmp, Tmp2;
3423
1.96M
  unsigned FirstAnswer = 1;
3424
1.96M
3425
1.96M
  if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3426
303k
    const APInt &Val = C->getAPIntValue();
3427
303k
    return Val.getNumSignBits();
3428
303k
  }
3429
1.65M
3430
1.65M
  if (Depth == 6)
3431
1.91k
    return 1;  // Limit search depth.
3432
1.65M
3433
1.65M
  if (!DemandedElts)
3434
0
    return 1;  // No demanded elts, better to assume we don't know anything.
3435
1.65M
3436
1.65M
  unsigned Opcode = Op.getOpcode();
3437
1.65M
  switch (Opcode) {
3438
1.65M
  
default: break938k
;
3439
1.65M
  case ISD::AssertSext:
3440
22.8k
    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3441
22.8k
    return VTBits-Tmp+1;
3442
1.65M
  case ISD::AssertZext:
3443
30.4k
    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
3444
30.4k
    return VTBits-Tmp;
3445
1.65M
3446
1.65M
  case ISD::BUILD_VECTOR:
3447
15.5k
    Tmp = VTBits;
3448
76.3k
    for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && 
(Tmp > 1)68.2k
;
++i60.7k
) {
3449
60.7k
      if (!DemandedElts[i])
3450
15.2k
        continue;
3451
45.4k
3452
45.4k
      SDValue SrcOp = Op.getOperand(i);
3453
45.4k
      Tmp2 = ComputeNumSignBits(Op.getOperand(i), Depth + 1);
3454
45.4k
3455
45.4k
      // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3456
45.4k
      if (SrcOp.getValueSizeInBits() != VTBits) {
3457
1.83k
        assert(SrcOp.getValueSizeInBits() > VTBits &&
3458
1.83k
               "Expected BUILD_VECTOR implicit truncation");
3459
1.83k
        unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
3460
1.83k
        Tmp2 = (Tmp2 > ExtraBits ? 
Tmp2 - ExtraBits1.79k
:
143
);
3461
1.83k
      }
3462
45.4k
      Tmp = std::min(Tmp, Tmp2);
3463
45.4k
    }
3464
15.5k
    return Tmp;
3465
1.65M
3466
1.65M
  case ISD::VECTOR_SHUFFLE: {
3467
5.81k
    // Collect the minimum number of sign bits that are shared by every vector
3468
5.81k
    // element referenced by the shuffle.
3469
5.81k
    APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
3470
5.81k
    const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3471
5.81k
    assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3472
36.7k
    for (unsigned i = 0; i != NumElts; 
++i30.8k
) {
3473
31.3k
      int M = SVN->getMaskElt(i);
3474
31.3k
      if (!DemandedElts[i])
3475
4.85k
        continue;
3476
26.4k
      // For UNDEF elements, we don't know anything about the common state of
3477
26.4k
      // the shuffle result.
3478
26.4k
      if (M < 0)
3479
455
        return 1;
3480
26.0k
      if ((unsigned)M < NumElts)
3481
23.9k
        DemandedLHS.setBit((unsigned)M % NumElts);
3482
2.12k
      else
3483
2.12k
        DemandedRHS.setBit((unsigned)M % NumElts);
3484
26.0k
    }
3485
5.81k
    Tmp = std::numeric_limits<unsigned>::max();
3486
5.36k
    if (!!DemandedLHS)
3487
5.35k
      Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
3488
5.36k
    if (!!DemandedRHS) {
3489
699
      Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
3490
699
      Tmp = std::min(Tmp, Tmp2);
3491
699
    }
3492
5.36k
    // If we don't know anything, early out and try computeKnownBits fall-back.
3493
5.36k
    if (Tmp == 1)
3494
1.41k
      break;
3495
3.94k
    assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3496
3.94k
    return Tmp;
3497
3.94k
  }
3498
3.94k
3499
37.2k
  case ISD::BITCAST: {
3500
37.2k
    SDValue N0 = Op.getOperand(0);
3501
37.2k
    EVT SrcVT = N0.getValueType();
3502
37.2k
    unsigned SrcBits = SrcVT.getScalarSizeInBits();
3503
37.2k
3504
37.2k
    // Ignore bitcasts from unsupported types..
3505
37.2k
    if (!(SrcVT.isInteger() || 
SrcVT.isFloatingPoint()5.62k
))
3506
9
      break;
3507
37.2k
3508
37.2k
    // Fast handling of 'identity' bitcasts.
3509
37.2k
    if (VTBits == SrcBits)
3510
4.46k
      return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
3511
32.7k
3512
32.7k
    bool IsLE = getDataLayout().isLittleEndian();
3513
32.7k
3514
32.7k
    // Bitcast 'large element' scalar/vector to 'small element' vector.
3515
32.7k
    if ((SrcBits % VTBits) == 0) {
3516
15.2k
      assert(VT.isVector() && "Expected bitcast to vector");
3517
15.2k
3518
15.2k
      unsigned Scale = SrcBits / VTBits;
3519
15.2k
      APInt SrcDemandedElts(NumElts / Scale, 0);
3520
137k
      for (unsigned i = 0; i != NumElts; 
++i122k
)
3521
122k
        if (DemandedElts[i])
3522
108k
          SrcDemandedElts.setBit(i / Scale);
3523
15.2k
3524
15.2k
      // Fast case - sign splat can be simply split across the small elements.
3525
15.2k
      Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
3526
15.2k
      if (Tmp == SrcBits)
3527
6.68k
        return VTBits;
3528
8.57k
3529
8.57k
      // Slow case - determine how far the sign extends into each sub-element.
3530
8.57k
      Tmp2 = VTBits;
3531
18.7k
      for (unsigned i = 0; i != NumElts; 
++i10.1k
)
3532
15.4k
        if (DemandedElts[i]) {
3533
11.7k
          unsigned SubOffset = i % Scale;
3534
11.7k
          SubOffset = (IsLE ? 
((Scale - 1) - SubOffset)11.5k
:
SubOffset160
);
3535
11.7k
          SubOffset = SubOffset * VTBits;
3536
11.7k
          if (Tmp <= SubOffset)
3537
5.27k
            return 1;
3538
6.47k
          Tmp2 = std::min(Tmp2, Tmp - SubOffset);
3539
6.47k
        }
3540
8.57k
      
return Tmp23.30k
;
3541
17.4k
    }
3542
17.4k
    break;
3543
17.4k
  }
3544
17.4k
3545
17.4k
  case ISD::SIGN_EXTEND:
3546
16.1k
    Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
3547
16.1k
    return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
3548
17.4k
  case ISD::SIGN_EXTEND_INREG:
3549
6.65k
    // Max of the input and what this extends.
3550
6.65k
    Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
3551
6.65k
    Tmp = VTBits-Tmp+1;
3552
6.65k
    Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3553
6.65k
    return std::max(Tmp, Tmp2);
3554
17.4k
  case ISD::SIGN_EXTEND_VECTOR_INREG: {
3555
160
    SDValue Src = Op.getOperand(0);
3556
160
    EVT SrcVT = Src.getValueType();
3557
160
    APInt DemandedSrcElts = DemandedElts.zextOrSelf(SrcVT.getVectorNumElements());
3558
160
    Tmp = VTBits - SrcVT.getScalarSizeInBits();
3559
160
    return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
3560
17.4k
  }
3561
17.4k
3562
17.4k
  case ISD::SRA:
3563
6.92k
    Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3564
6.92k
    // SRA X, C   -> adds C sign bits.
3565
6.92k
    if (ConstantSDNode *C =
3566
6.17k
            isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3567
6.17k
      APInt ShiftVal = C->getAPIntValue();
3568
6.17k
      ShiftVal += Tmp;
3569
6.17k
      Tmp = ShiftVal.uge(VTBits) ? 
VTBits981
:
ShiftVal.getZExtValue()5.19k
;
3570
6.17k
    }
3571
6.92k
    return Tmp;
3572
38.0k
  case ISD::SHL:
3573
38.0k
    if (ConstantSDNode *C =
3574
32.9k
            isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
3575
32.9k
      // shl destroys sign bits.
3576
32.9k
      Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3577
32.9k
      if (C->getAPIntValue().uge(VTBits) ||      // Bad shift.
3578
32.9k
          C->getAPIntValue().uge(Tmp)) 
break18.8k
; // Shifted all sign bits out.
3579
14.0k
      return Tmp - C->getZExtValue();
3580
14.0k
    }
3581
5.10k
    break;
3582
78.1k
  case ISD::AND:
3583
78.1k
  case ISD::OR:
3584
78.1k
  case ISD::XOR:    // NOT is handled here.
3585
78.1k
    // Logical binary ops preserve the number of sign bits at the worst.
3586
78.1k
    Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
3587
78.1k
    if (Tmp != 1) {
3588
34.5k
      Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3589
34.5k
      FirstAnswer = std::min(Tmp, Tmp2);
3590
34.5k
      // We computed what we know about the sign bits as our first
3591
34.5k
      // answer. Now proceed to the generic code that uses
3592
34.5k
      // computeKnownBits, and pick whichever answer is better.
3593
34.5k
    }
3594
78.1k
    break;
3595
78.1k
3596
78.1k
  case ISD::SELECT:
3597
2.31k
  case ISD::VSELECT:
3598
2.31k
    Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
3599
2.31k
    if (Tmp == 1) 
return 11.90k
; // Early out.
3600
407
    Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3601
407
    return std::min(Tmp, Tmp2);
3602
407
  case ISD::SELECT_CC:
3603
254
    Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
3604
254
    if (Tmp == 1) 
return 1142
; // Early out.
3605
112
    Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
3606
112
    return std::min(Tmp, Tmp2);
3607
112
3608
1.62k
  case ISD::SMIN:
3609
1.62k
  case ISD::SMAX: {
3610
1.62k
    // If we have a clamp pattern, we know that the number of sign bits will be
3611
1.62k
    // the minimum of the clamp min/max range.
3612
1.62k
    bool IsMax = (Opcode == ISD::SMAX);
3613
1.62k
    ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3614
1.62k
    if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3615
1.17k
      if (Op.getOperand(0).getOpcode() == (IsMax ? 
ISD::SMIN1.15k
:
ISD::SMAX16
))
3616
1.14k
        CstHigh =
3617
1.14k
            isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3618
1.62k
    if (CstLow && 
CstHigh1.17k
) {
3619
1.14k
      if (!IsMax)
3620
4
        std::swap(CstLow, CstHigh);
3621
1.14k
      if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
3622
1.14k
        Tmp = CstLow->getAPIntValue().getNumSignBits();
3623
1.14k
        Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
3624
1.14k
        return std::min(Tmp, Tmp2);
3625
1.14k
      }
3626
484
    }
3627
484
3628
484
    // Fallback - just get the minimum number of sign bits of the operands.
3629
484
    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3630
484
    if (Tmp == 1)
3631
298
      return 1;  // Early out.
3632
186
    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3633
186
    return std::min(Tmp, Tmp2);
3634
186
  }
3635
383
  case ISD::UMIN:
3636
383
  case ISD::UMAX:
3637
383
    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3638
383
    if (Tmp == 1)
3639
366
      return 1;  // Early out.
3640
17
    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
3641
17
    return std::min(Tmp, Tmp2);
3642
2.15k
  case ISD::SADDO:
3643
2.15k
  case ISD::UADDO:
3644
2.15k
  case ISD::SSUBO:
3645
2.15k
  case ISD::USUBO:
3646
2.15k
  case ISD::SMULO:
3647
2.15k
  case ISD::UMULO:
3648
2.15k
    if (Op.getResNo() != 1)
3649
93
      break;
3650
2.05k
    // The boolean result conforms to getBooleanContents.  Fall through.
3651
2.05k
    // If setcc returns 0/-1, all bits are sign bits.
3652
2.05k
    // We know that we have an integer-based boolean since these operations
3653
2.05k
    // are only available for integer.
3654
2.05k
    if (TLI->getBooleanContents(VT.isVector(), false) ==
3655
2.05k
        TargetLowering::ZeroOrNegativeOneBooleanContent)
3656
692
      return VTBits;
3657
1.36k
    break;
3658
41.4k
  case ISD::SETCC:
3659
41.4k
    // If setcc returns 0/-1, all bits are sign bits.
3660
41.4k
    if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3661
41.4k
        TargetLowering::ZeroOrNegativeOneBooleanContent)
3662
39.7k
      return VTBits;
3663
1.72k
    break;
3664
1.72k
  case ISD::ROTL:
3665
215
  case ISD::ROTR:
3666
215
    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
3667
123
      unsigned RotAmt = C->getAPIntValue().urem(VTBits);
3668
123
3669
123
      // Handle rotate right by N like a rotate left by 32-N.
3670
123
      if (Opcode == ISD::ROTR)
3671
39
        RotAmt = (VTBits - RotAmt) % VTBits;
3672
123
3673
123
      // If we aren't rotating out all of the known-in sign bits, return the
3674
123
      // number that are left.  This handles rotl(sext(x), 1) for example.
3675
123
      Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3676
123
      if (Tmp > (RotAmt + 1)) 
return (Tmp - RotAmt)0
;
3677
215
    }
3678
215
    break;
3679
231k
  case ISD::ADD:
3680
231k
  case ISD::ADDC:
3681
231k
    // Add can have at most one carry bit.  Thus we know that the output
3682
231k
    // is, at worst, one more bit than the inputs.
3683
231k
    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3684
231k
    if (Tmp == 1) 
return 1209k
; // Early out.
3685
21.7k
3686
21.7k
    // Special case decrementing a value (ADD X, -1):
3687
21.7k
    if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
3688
12.0k
      if (CRHS->isAllOnesValue()) {
3689
946
        KnownBits Known = computeKnownBits(Op.getOperand(0), Depth+1);
3690
946
3691
946
        // If the input is known to be 0 or 1, the output is 0/-1, which is all
3692
946
        // sign bits set.
3693
946
        if ((Known.Zero | 1).isAllOnesValue())
3694
16
          return VTBits;
3695
930
3696
930
        // If we are subtracting one from a positive number, there is no carry
3697
930
        // out of the result.
3698
930
        if (Known.isNonNegative())
3699
631
          return Tmp;
3700
21.0k
      }
3701
21.0k
3702
21.0k
    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3703
21.0k
    if (Tmp2 == 1) 
return 13.71k
;
3704
17.3k
    return std::min(Tmp, Tmp2)-1;
3705
17.3k
3706
35.8k
  case ISD::SUB:
3707
35.8k
    Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1);
3708
35.8k
    if (Tmp2 == 1) 
return 123.7k
;
3709
12.1k
3710
12.1k
    // Handle NEG.
3711
12.1k
    if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0)))
3712
6.47k
      if (CLHS->isNullValue()) {
3713
555
        KnownBits Known = computeKnownBits(Op.getOperand(1), Depth+1);
3714
555
        // If the input is known to be 0 or 1, the output is 0/-1, which is all
3715
555
        // sign bits set.
3716
555
        if ((Known.Zero | 1).isAllOnesValue())
3717
29
          return VTBits;
3718
526
3719
526
        // If the input is known to be positive (the sign bit is known clear),
3720
526
        // the output of the NEG has the same number of sign bits as the input.
3721
526
        if (Known.isNonNegative())
3722
395
          return Tmp2;
3723
11.7k
3724
11.7k
        // Otherwise, we treat this like a SUB.
3725
11.7k
      }
3726
11.7k
3727
11.7k
    // Sub can have at most one carry bit.  Thus we know that the output
3728
11.7k
    // is, at worst, one more bit than the inputs.
3729
11.7k
    Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3730
11.7k
    if (Tmp == 1) 
return 13.00k
; // Early out.
3731
8.70k
    return std::min(Tmp, Tmp2)-1;
3732
58.3k
  case ISD::TRUNCATE: {
3733
58.3k
    // Check if the sign bits of source go down as far as the truncated value.
3734
58.3k
    unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
3735
58.3k
    unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
3736
58.3k
    if (NumSrcSignBits > (NumSrcBits - VTBits))
3737
25.9k
      return NumSrcSignBits - (NumSrcBits - VTBits);
3738
32.3k
    break;
3739
32.3k
  }
3740
32.3k
  case ISD::EXTRACT_ELEMENT: {
3741
30
    const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
3742
30
    const int BitWidth = Op.getValueSizeInBits();
3743
30
    const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
3744
30
3745
30
    // Get reverse index (starting from 1), Op1 value indexes elements from
3746
30
    // little end. Sign starts at big end.
3747
30
    const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
3748
30
3749
30
    // If the sign portion ends in our element the subtraction gives correct
3750
30
    // result. Otherwise it gives either negative or > bitwidth result
3751
30
    return std::max(std::min(KnownSign - rIndex * BitWidth, BitWidth), 0);
3752
32.3k
  }
3753
32.3k
  case ISD::INSERT_VECTOR_ELT: {
3754
798
    SDValue InVec = Op.getOperand(0);
3755
798
    SDValue InVal = Op.getOperand(1);
3756
798
    SDValue EltNo = Op.getOperand(2);
3757
798
3758
798
    ConstantSDNode *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3759
798
    if (CEltNo && 
CEltNo->getAPIntValue().ult(NumElts)790
) {
3760
790
      // If we know the element index, split the demand between the
3761
790
      // source vector and the inserted element.
3762
790
      unsigned EltIdx = CEltNo->getZExtValue();
3763
790
3764
790
      // If we demand the inserted element then get its sign bits.
3765
790
      Tmp = std::numeric_limits<unsigned>::max();
3766
790
      if (DemandedElts[EltIdx]) {
3767
787
        // TODO - handle implicit truncation of inserted elements.
3768
787
        if (InVal.getScalarValueSizeInBits() != VTBits)
3769
28
          break;
3770
759
        Tmp = ComputeNumSignBits(InVal, Depth + 1);
3771
759
      }
3772
790
3773
790
      // If we demand the source vector then get its sign bits, and determine
3774
790
      // the minimum.
3775
790
      APInt VectorElts = DemandedElts;
3776
762
      VectorElts.clearBit(EltIdx);
3777
762
      if (!!VectorElts) {
3778
678
        Tmp2 = ComputeNumSignBits(InVec, VectorElts, Depth + 1);
3779
678
        Tmp = std::min(Tmp, Tmp2);
3780
678
      }
3781
762
    } else {
3782
8
      // Unknown element index, so ignore DemandedElts and demand them all.
3783
8
      Tmp = ComputeNumSignBits(InVec, Depth + 1);
3784
8
      Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
3785
8
      Tmp = std::min(Tmp, Tmp2);
3786
8
    }
3787
798
    assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3788
770
    return Tmp;
3789
798
  }
3790
35.2k
  case ISD::EXTRACT_VECTOR_ELT: {
3791
35.2k
    SDValue InVec = Op.getOperand(0);
3792
35.2k
    SDValue EltNo = Op.getOperand(1);
3793
35.2k
    EVT VecVT = InVec.getValueType();
3794
35.2k
    const unsigned BitWidth = Op.getValueSizeInBits();
3795
35.2k
    const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
3796
35.2k
    const unsigned NumSrcElts = VecVT.getVectorNumElements();
3797
35.2k
3798
35.2k
    // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
3799
35.2k
    // anything about sign bits. But if the sizes match we can derive knowledge
3800
35.2k
    // about sign bits from the vector operand.
3801
35.2k
    if (BitWidth != EltBitWidth)
3802
10.0k
      break;
3803
25.2k
3804
25.2k
    // If we know the element index, just demand that vector element, else for
3805
25.2k
    // an unknown element index, ignore DemandedElts and demand them all.
3806
25.2k
    APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
3807
25.2k
    ConstantSDNode *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3808
25.2k
    if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3809
25.2k
      DemandedSrcElts =
3810
25.2k
          APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3811
25.2k
3812
25.2k
    return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
3813
25.2k
  }
3814
42.3k
  case ISD::EXTRACT_SUBVECTOR: {
3815
42.3k
    // If we know the element index, just demand that subvector elements,
3816
42.3k
    // otherwise demand them all.
3817
42.3k
    SDValue Src = Op.getOperand(0);
3818
42.3k
    ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
3819
42.3k
    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3820
42.3k
    if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
3821
42.3k
      // Offset the demanded elts by the subvector index.
3822
42.3k
      uint64_t Idx = SubIdx->getZExtValue();
3823
42.3k
      APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
3824
42.3k
      return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
3825
42.3k
    }
3826
0
    return ComputeNumSignBits(Src, Depth + 1);
3827
0
  }
3828
6.80k
  case ISD::CONCAT_VECTORS: {
3829
6.80k
    // Determine the minimum number of sign bits across all demanded
3830
6.80k
    // elts of the input vectors. Early out if the result is already 1.
3831
6.80k
    Tmp = std::numeric_limits<unsigned>::max();
3832
6.80k
    EVT SubVectorVT = Op.getOperand(0).getValueType();
3833
6.80k
    unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3834
6.80k
    unsigned NumSubVectors = Op.getNumOperands();
3835
18.0k
    for (unsigned i = 0; (i < NumSubVectors) && 
(Tmp > 1)14.0k
;
++i11.2k
) {
3836
11.2k
      APInt DemandedSub = DemandedElts.lshr(i * NumSubVectorElts);
3837
11.2k
      DemandedSub = DemandedSub.trunc(NumSubVectorElts);
3838
11.2k
      if (!DemandedSub)
3839
2.95k
        continue;
3840
8.25k
      Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
3841
8.25k
      Tmp = std::min(Tmp, Tmp2);
3842
8.25k
    }
3843
6.80k
    assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3844
6.80k
    return Tmp;
3845
0
  }
3846
1.89k
  case ISD::INSERT_SUBVECTOR: {
3847
1.89k
    // If we know the element index, demand any elements from the subvector and
3848
1.89k
    // the remainder from the src its inserted into, otherwise demand them all.
3849
1.89k
    SDValue Src = Op.getOperand(0);
3850
1.89k
    SDValue Sub = Op.getOperand(1);
3851
1.89k
    auto *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3852
1.89k
    unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3853
1.89k
    if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
3854
1.89k
      Tmp = std::numeric_limits<unsigned>::max();
3855
1.89k
      uint64_t Idx = SubIdx->getZExtValue();
3856
1.89k
      APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3857
1.89k
      if (!!DemandedSubElts) {
3858
1.68k
        Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
3859
1.68k
        if (Tmp == 1) 
return 11.17k
; // early-out
3860
720
      }
3861
720
      APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
3862
720
      APInt DemandedSrcElts = DemandedElts & ~SubMask;
3863
720
      if (!!DemandedSrcElts) {
3864
364
        Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
3865
364
        Tmp = std::min(Tmp, Tmp2);
3866
364
      }
3867
720
      assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3868
720
      return Tmp;
3869
720
    }
3870
0
3871
0
    // Not able to determine the index so just assume worst case.
3872
0
    Tmp = ComputeNumSignBits(Sub, Depth + 1);
3873
0
    if (Tmp == 1) return 1; // early-out
3874
0
    Tmp2 = ComputeNumSignBits(Src, Depth + 1);
3875
0
    Tmp = std::min(Tmp, Tmp2);
3876
0
    assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
3877
0
    return Tmp;
3878
0
  }
3879
1.10M
  }
3880
1.10M
3881
1.10M
  // If we are looking at the loaded value of the SDNode.
3882
1.10M
  if (Op.getResNo() == 0) {
3883
1.08M
    // Handle LOADX separately here. EXTLOAD case will fallthrough.
3884
1.08M
    if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
3885
313k
      unsigned ExtType = LD->getExtensionType();
3886
313k
      switch (ExtType) {
3887
313k
      
default: break80.9k
;
3888
313k
      case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
3889
18.1k
        Tmp = LD->getMemoryVT().getScalarSizeInBits();
3890
18.1k
        return VTBits - Tmp + 1;
3891
313k
      case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
3892
27.1k
        Tmp = LD->getMemoryVT().getScalarSizeInBits();
3893
27.1k
        return VTBits - Tmp;
3894
313k
      case ISD::NON_EXTLOAD:
3895
187k
        if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
3896
592
          // We only need to handle vectors - computeKnownBits should handle
3897
592
          // scalar cases.
3898
592
          Type *CstTy = Cst->getType();
3899
592
          if (CstTy->isVectorTy() &&
3900
592
              (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) {
3901
592
            Tmp = VTBits;
3902
9.70k
            for (unsigned i = 0; i != NumElts; 
++i9.11k
) {
3903
9.13k
              if (!DemandedElts[i])
3904
1
                continue;
3905
9.13k
              if (Constant *Elt = Cst->getAggregateElement(i)) {
3906
9.13k
                if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3907
9.10k
                  const APInt &Value = CInt->getValue();
3908
9.10k
                  Tmp = std::min(Tmp, Value.getNumSignBits());
3909
9.10k
                  continue;
3910
9.10k
                }
3911
22
                if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3912
2
                  APInt Value = CFP->getValueAPF().bitcastToAPInt();
3913
2
                  Tmp = std::min(Tmp, Value.getNumSignBits());
3914
2
                  continue;
3915
2
                }
3916
20
              }
3917
20
              // Unknown type. Conservatively assume no bits match sign bit.
3918
20
              return 1;
3919
20
            }
3920
592
            
return Tmp572
;
3921
187k
          }
3922
592
        }
3923
187k
        break;
3924
313k
      }
3925
313k
    }
3926
1.08M
  }
3927
1.05M
3928
1.05M
  // Allow the target to implement this method for its nodes.
3929
1.05M
  if (Opcode >= ISD::BUILTIN_OP_END ||
3930
1.05M
      
Opcode == ISD::INTRINSIC_WO_CHAIN948k
||
3931
1.05M
      
Opcode == ISD::INTRINSIC_W_CHAIN946k
||
3932
1.05M
      
Opcode == ISD::INTRINSIC_VOID934k
) {
3933
124k
    unsigned NumBits =
3934
124k
        TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
3935
124k
    if (NumBits > 1)
3936
46.2k
      FirstAnswer = std::max(FirstAnswer, NumBits);
3937
124k
  }
3938
1.05M
3939
1.05M
  // Finally, if we can prove that the top bits of the result are 0's or 1's,
3940
1.05M
  // use this information.
3941
1.05M
  KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
3942
1.05M
3943
1.05M
  APInt Mask;
3944
1.05M
  if (Known.isNonNegative()) {        // sign bit is 0
3945
127k
    Mask = Known.Zero;
3946
932k
  } else if (Known.isNegative()) {  // sign bit is 1;
3947
293
    Mask = Known.One;
3948
932k
  } else {
3949
932k
    // Nothing known.
3950
932k
    return FirstAnswer;
3951
932k
  }
3952
127k
3953
127k
  // Okay, we know that the sign bit in Mask is set.  Use CLZ to determine
3954
127k
  // the number of identical bits in the top of the input value.
3955
127k
  Mask = ~Mask;
3956
127k
  Mask <<= Mask.getBitWidth()-VTBits;
3957
127k
  // Return # leading zeros.  We use 'min' here in case Val was zero before
3958
127k
  // shifting.  We don't want to return '64' as for an i32 "0".
3959
127k
  return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros()));
3960
127k
}
3961
3962
6.43M
bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
3963
6.43M
  if ((Op.getOpcode() != ISD::ADD && 
Op.getOpcode() != ISD::OR1.83M
) ||
3964
6.43M
      
!isa<ConstantSDNode>(Op.getOperand(1))4.91M
)
3965
1.82M
    return false;
3966
4.61M
3967
4.61M
  if (Op.getOpcode() == ISD::OR &&
3968
4.61M
      
!MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1))311k
)
3969
243
    return false;
3970
4.61M
3971
4.61M
  return true;
3972
4.61M
}
3973
3974
4.57k
bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
3975
4.57k
  // If we're told that NaNs won't happen, assume they won't.
3976
4.57k
  if (getTarget().Options.NoNaNsFPMath || 
Op->getFlags().hasNoNaNs()3.56k
)
3977
1.19k
    return true;
3978
3.37k
3979
3.37k
  if (Depth == 6)
3980
0
    return false; // Limit search depth.
3981
3.37k
3982
3.37k
  // TODO: Handle vectors.
3983
3.37k
  // If the value is a constant, we can obviously see if it is a NaN or not.
3984
3.37k
  if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
3985
374
    return !C->getValueAPF().isNaN() ||
3986
374
           
(4
SNaN4
&&
!C->getValueAPF().isSignaling()0
);
3987
374
  }
3988
3.00k
3989
3.00k
  unsigned Opcode = Op.getOpcode();
3990
3.00k
  switch (Opcode) {
3991
3.00k
  case ISD::FADD:
3992
89
  case ISD::FSUB:
3993
89
  case ISD::FMUL:
3994
89
  case ISD::FDIV:
3995
89
  case ISD::FREM:
3996
89
  case ISD::FSIN:
3997
89
  case ISD::FCOS: {
3998
89
    if (SNaN)
3999
58
      return true;
4000
31
    // TODO: Need isKnownNeverInfinity
4001
31
    return false;
4002
31
  }
4003
37
  case ISD::FCANONICALIZE:
4004
37
  case ISD::FEXP:
4005
37
  case ISD::FEXP2:
4006
37
  case ISD::FTRUNC:
4007
37
  case ISD::FFLOOR:
4008
37
  case ISD::FCEIL:
4009
37
  case ISD::FROUND:
4010
37
  case ISD::FRINT:
4011
37
  case ISD::FNEARBYINT: {
4012
37
    if (SNaN)
4013
37
      return true;
4014
0
    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4015
0
  }
4016
259
  case ISD::FABS:
4017
259
  case ISD::FNEG:
4018
259
  case ISD::FCOPYSIGN: {
4019
259
    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4020
259
  }
4021
259
  case ISD::SELECT:
4022
7
    return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
4023
7
           
isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1)4
;
4024
259
  case ISD::FP_EXTEND:
4025
2
  case ISD::FP_ROUND: {
4026
2
    if (SNaN)
4027
2
      return true;
4028
0
    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4029
0
  }
4030
3
  case ISD::SINT_TO_FP:
4031
3
  case ISD::UINT_TO_FP:
4032
3
    return true;
4033
9
  case ISD::FMA:
4034
9
  case ISD::FMAD: {
4035
9
    if (SNaN)
4036
1
      return true;
4037
8
    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4038
8
           
isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1)0
&&
4039
8
           
isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1)0
;
4040
8
  }
4041
8
  case ISD::FSQRT: // Need is known positive
4042
0
  case ISD::FLOG:
4043
0
  case ISD::FLOG2:
4044
0
  case ISD::FLOG10:
4045
0
  case ISD::FPOWI:
4046
0
  case ISD::FPOW: {
4047
0
    if (SNaN)
4048
0
      return true;
4049
0
    // TODO: Refine on operand
4050
0
    return false;
4051
0
  }
4052
183
  case ISD::FMINNUM:
4053
183
  case ISD::FMAXNUM: {
4054
183
    // Only one needs to be known not-nan, since it will be returned if the
4055
183
    // other ends up being one.
4056
183
    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
4057
183
           
isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1)98
;
4058
183
  }
4059
183
  case ISD::FMINNUM_IEEE:
4060
10
  case ISD::FMAXNUM_IEEE: {
4061
10
    if (SNaN)
4062
10
      return true;
4063
0
    // This can return a NaN if either operand is an sNaN, or if both operands
4064
0
    // are NaN.
4065
0
    return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
4066
0
            isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
4067
0
           (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
4068
0
            isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
4069
0
  }
4070
0
  case ISD::FMINIMUM:
4071
0
  case ISD::FMAXIMUM: {
4072
0
    // TODO: Does this quiet or return the origina NaN as-is?
4073
0
    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
4074
0
           isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
4075
0
  }
4076
332
  case ISD::EXTRACT_VECTOR_ELT: {
4077
332
    return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
4078
0
  }
4079
2.07k
  default:
4080
2.07k
    if (Opcode >= ISD::BUILTIN_OP_END ||
4081
2.07k
        
Opcode == ISD::INTRINSIC_WO_CHAIN2.05k
||
4082
2.07k
        
Opcode == ISD::INTRINSIC_W_CHAIN2.02k
||
4083
2.07k
        
Opcode == ISD::INTRINSIC_VOID2.02k
) {
4084
45
      return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
4085
45
    }
4086
2.02k
4087
2.02k
    return false;
4088
3.00k
  }
4089
3.00k
}
4090
4091
164
bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
4092
164
  assert(Op.getValueType().isFloatingPoint() &&
4093
164
         "Floating point type expected");
4094
164
4095
164
  // If the value is a constant, we can obviously see if it is a zero or not.
4096
164
  // TODO: Add BuildVector support.
4097
164
  if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op))
4098
26
    return !C->isZero();
4099
138
  return false;
4100
138
}
4101
4102
8.54k
bool SelectionDAG::isKnownNeverZero(SDValue Op) const {
4103
8.54k
  assert(!Op.getValueType().isFloatingPoint() &&
4104
8.54k
         "Floating point types unsupported - use isKnownNeverZeroFloat");
4105
8.54k
4106
8.54k
  // If the value is a constant, we can obviously see if it is a zero or not.
4107
8.54k
  if (ISD::matchUnaryPredicate(
4108
8.54k
          Op, [](ConstantSDNode *C) 
{ return !C->isNullValue(); }4.55k
))
4109
1.37k
    return true;
4110
7.17k
4111
7.17k
  // TODO: Recognize more cases here.
4112
7.17k
  switch (Op.getOpcode()) {
4113
7.17k
  
default: break7.12k
;
4114
7.17k
  case ISD::OR:
4115
45
    if (isKnownNeverZero(Op.getOperand(1)) ||
4116
45
        
isKnownNeverZero(Op.getOperand(0))15
)
4117
30
      return true;
4118
15
    break;
4119
7.14k
  }
4120
7.14k
4121
7.14k
  return false;
4122
7.14k
}
4123
4124
35.3k
bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
4125
35.3k
  // Check the obvious case.
4126
35.3k
  if (A == B) 
return true9.60k
;
4127
25.7k
4128
25.7k
  // For for negative and positive zero.
4129
25.7k
  if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
4130
1.31k
    if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
4131
89
      if (CA->isZero() && 
CB->isZero()9
)
return true4
;
4132
25.7k
4133
25.7k
  // Otherwise they may not be equal.
4134
25.7k
  return false;
4135
25.7k
}
4136
4137
// FIXME: unify with llvm::haveNoCommonBitsSet.
4138
// FIXME: could also handle masked merge pattern (X & ~M) op (Y & M)
4139
3.71M
bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
4140
3.71M
  assert(A.getValueType() == B.getValueType() &&
4141
3.71M
         "Values must have the same type");
4142
3.71M
  return (computeKnownBits(A).Zero | computeKnownBits(B).Zero).isAllOnesValue();
4143
3.71M
}
4144
4145
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
4146
                                ArrayRef<SDValue> Ops,
4147
563k
                                SelectionDAG &DAG) {
4148
563k
  int NumOps = Ops.size();
4149
563k
  assert(NumOps != 0 && "Can't build an empty vector!");
4150
563k
  assert(VT.getVectorNumElements() == (unsigned)NumOps &&
4151
563k
         "Incorrect element count in BUILD_VECTOR!");
4152
563k
4153
563k
  // BUILD_VECTOR of UNDEFs is UNDEF.
4154
604k
  if (
llvm::all_of(Ops, [](SDValue Op) 563k
{ return Op.isUndef(); }))
4155
1.55k
    return DAG.getUNDEF(VT);
4156
562k
4157
562k
  // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
4158
562k
  SDValue IdentitySrc;
4159
562k
  bool IsIdentity = true;
4160
584k
  for (int i = 0; i != NumOps; 
++i22.3k
) {
4161
580k
    if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4162
580k
        
Ops[i].getOperand(0).getValueType() != VT68.7k
||
4163
580k
        
(36.5k
IdentitySrc36.5k
&&
Ops[i].getOperand(0) != IdentitySrc14.9k
) ||
4164
580k
        
!isa<ConstantSDNode>(Ops[i].getOperand(1))36.0k
||
4165
580k
        
cast<ConstantSDNode>(Ops[i].getOperand(1))->getAPIntValue() != i33.7k
) {
4166
558k
      IsIdentity = false;
4167
558k
      break;
4168
558k
    }
4169
22.3k
    IdentitySrc = Ops[i].getOperand(0);
4170
22.3k
  }
4171
562k
  if (IsIdentity)
4172
3.89k
    return IdentitySrc;
4173
558k
4174
558k
  return SDValue();
4175
558k
}
4176
4177
/// Try to simplify vector concatenation to an input value, undef, or build
4178
/// vector.
4179
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
4180
                                  ArrayRef<SDValue> Ops,
4181
94.4k
                                  SelectionDAG &DAG) {
4182
94.4k
  assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
4183
94.4k
  assert(llvm::all_of(Ops,
4184
94.4k
                      [Ops](SDValue Op) {
4185
94.4k
                        return Ops[0].getValueType() == Op.getValueType();
4186
94.4k
                      }) &&
4187
94.4k
         "Concatenation of vectors with inconsistent value types!");
4188
94.4k
  assert((Ops.size() * Ops[0].getValueType().getVectorNumElements()) ==
4189
94.4k
             VT.getVectorNumElements() &&
4190
94.4k
         "Incorrect element count in vector concatenation!");
4191
94.4k
4192
94.4k
  if (Ops.size() == 1)
4193
0
    return Ops[0];
4194
94.4k
4195
94.4k
  // Concat of UNDEFs is UNDEF.
4196
98.9k
  
if (94.4k
llvm::all_of(Ops, [](SDValue Op) 94.4k
{ return Op.isUndef(); }))
4197
1.90k
    return DAG.getUNDEF(VT);
4198
92.4k
4199
92.4k
  // Scan the operands and look for extract operations from a single source
4200
92.4k
  // that correspond to insertion at the same location via this concatenation:
4201
92.4k
  // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
4202
92.4k
  SDValue IdentitySrc;
4203
92.4k
  bool IsIdentity = true;
4204
94.2k
  for (unsigned i = 0, e = Ops.size(); i != e; 
++i1.73k
) {
4205
94.2k
    SDValue Op = Ops[i];
4206
94.2k
    unsigned IdentityIndex = i * Op.getValueType().getVectorNumElements();
4207
94.2k
    if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
4208
94.2k
        
Op.getOperand(0).getValueType() != VT3.37k
||
4209
94.2k
        
(2.60k
IdentitySrc2.60k
&&
Op.getOperand(0) != IdentitySrc155
) ||
4210
94.2k
        
!isa<ConstantSDNode>(Op.getOperand(1))2.50k
||
4211
94.2k
        
Op.getConstantOperandVal(1) != IdentityIndex2.50k
) {
4212
92.4k
      IsIdentity = false;
4213
92.4k
      break;
4214
92.4k
    }
4215
1.73k
    assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
4216
1.73k
           "Unexpected identity source vector for concat of extracts");
4217
1.73k
    IdentitySrc = Op.getOperand(0);
4218
1.73k
  }
4219
92.4k
  if (IsIdentity) {
4220
17
    assert(IdentitySrc && "Failed to set source vector of extracts");
4221
17
    return IdentitySrc;
4222
17
  }
4223
92.4k
4224
92.4k
  // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
4225
92.4k
  // simplified to one big BUILD_VECTOR.
4226
92.4k
  // FIXME: Add support for SCALAR_TO_VECTOR as well.
4227
92.4k
  EVT SVT = VT.getScalarType();
4228
92.4k
  SmallVector<SDValue, 16> Elts;
4229
96.2k
  for (SDValue Op : Ops) {
4230
96.2k
    EVT OpVT = Op.getValueType();
4231
96.2k
    if (Op.isUndef())
4232
1.08k
      Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
4233
95.1k
    else if (Op.getOpcode() == ISD::BUILD_VECTOR)
4234
4.10k
      Elts.append(Op->op_begin(), Op->op_end());
4235
91.0k
    else
4236
91.0k
      return SDValue();
4237
96.2k
  }
4238
92.4k
4239
92.4k
  // BUILD_VECTOR requires all inputs to be of the same type, find the
4240
92.4k
  // maximum type and extend them all.
4241
92.4k
  
for (SDValue Op : Elts)1.46k
4242
22.1k
    SVT = (SVT.bitsLT(Op.getValueType()) ? 
Op.getValueType()703
:
SVT21.4k
);
4243
1.46k
4244
1.46k
  if (SVT.bitsGT(VT.getScalarType()))
4245
703
    for (SDValue &Op : Elts)
4246
13.7k
      Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
4247
13.7k
               ? 
DAG.getZExtOrTrunc(Op, DL, SVT)0
4248
13.7k
               : DAG.getSExtOrTrunc(Op, DL, SVT);
4249
1.46k
4250
1.46k
  SDValue V = DAG.getBuildVector(VT, DL, Elts);
4251
1.46k
  NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
4252
1.46k
  return V;
4253
92.4k
}
4254
4255
/// Gets or creates the specified node.
4256
4.49M
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
4257
4.49M
  FoldingSetNodeID ID;
4258
4.49M
  AddNodeIDNode(ID, Opcode, getVTList(VT), None);
4259
4.49M
  void *IP = nullptr;
4260
4.49M
  if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
4261
3.56M
    return SDValue(E, 0);
4262
927k
4263
927k
  auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
4264
927k
                              getVTList(VT));
4265
927k
  CSEMap.InsertNode(N, IP);
4266
927k
4267
927k
  InsertNode(N);
4268
927k
  SDValue V = SDValue(N, 0);
4269
927k
  NewSDValueDbgMsg(V, "Creating new node: ", this);
4270
927k
  return V;
4271
927k
}
4272
4273
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
4274
9.96M
                              SDValue Operand, const SDNodeFlags Flags) {
4275
9.96M
  // Constant fold unary operations with an integer constant operand. Even
4276
9.96M
  // opaque constant will be folded, because the folding of unary operations
4277
9.96M
  // doesn't create new constants with different values. Nevertheless, the
4278
9.96M
  // opaque flag is preserved during folding to prevent future folding with
4279
9.96M
  // other constants.
4280
9.96M
  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand)) {
4281
1.29M
    const APInt &Val = C->getAPIntValue();
4282
1.29M
    switch (Opcode) {
4283
1.29M
    
default: break98.0k
;
4284
1.29M
    case ISD::SIGN_EXTEND:
4285
195k
      return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
4286
195k
                         C->isTargetOpcode(), C->isOpaque());
4287
1.29M
    case ISD::TRUNCATE:
4288
841k
      if (C->isOpaque())
4289
754
        break;
4290
840k
      LLVM_FALLTHROUGH;
4291
954k
    case ISD::ANY_EXTEND:
4292
954k
    case ISD::ZERO_EXTEND:
4293
954k
      return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
4294
954k
                         C->isTargetOpcode(), C->isOpaque());
4295
954k
    case ISD::UINT_TO_FP:
4296
615
    case ISD::SINT_TO_FP: {
4297
615
      APFloat apf(EVTToAPFloatSemantics(VT),
4298
615
                  APInt::getNullValue(VT.getSizeInBits()));
4299
615
      (void)apf.convertFromAPInt(Val,
4300
615
                                 Opcode==ISD::SINT_TO_FP,
4301
615
                                 APFloat::rmNearestTiesToEven);
4302
615
      return getConstantFP(apf, DL, VT);
4303
615
    }
4304
21.9k
    case ISD::BITCAST:
4305
21.9k
      if (VT == MVT::f16 && 
C->getValueType(0) == MVT::i1615
)
4306
15
        return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
4307
21.9k
      if (VT == MVT::f32 && 
C->getValueType(0) == MVT::i32427
)
4308
427
        return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
4309
21.5k
      if (VT == MVT::f64 && 
C->getValueType(0) == MVT::i64343
)
4310
343
        return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
4311
21.1k
      if (VT == MVT::f128 && 
C->getValueType(0) == MVT::i1280
)
4312
0
        return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
4313
21.1k
      break;
4314
21.1k
    case ISD::ABS:
4315
102
      return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
4316
102
                         C->isOpaque());
4317
21.1k
    case ISD::BITREVERSE:
4318
452
      return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
4319
452
                         C->isOpaque());
4320
21.1k
    case ISD::BSWAP:
4321
214
      return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
4322
214
                         C->isOpaque());
4323
21.1k
    case ISD::CTPOP:
4324
660
      return getConstant(Val.countPopulation(), DL, VT, C->isTargetOpcode(),
4325
660
                         C->isOpaque());
4326
21.1k
    case ISD::CTLZ:
4327
16.5k
    case ISD::CTLZ_ZERO_UNDEF:
4328
16.5k
      return getConstant(Val.countLeadingZeros(), DL, VT, C->isTargetOpcode(),
4329
16.5k
                         C->isOpaque());
4330
16.5k
    case ISD::CTTZ:
4331
3.83k
    case ISD::CTTZ_ZERO_UNDEF:
4332
3.83k
      return getConstant(Val.countTrailingZeros(), DL, VT, C->isTargetOpcode(),
4333
3.83k
                         C->isOpaque());
4334
3.83k
    case ISD::FP16_TO_FP: {
4335
836
      bool Ignored;
4336
836
      APFloat FPV(APFloat::IEEEhalf(),
4337
836
                  (Val.getBitWidth() == 16) ? Val : 
Val.trunc(16)0
);
4338
836
4339
836
      // This can return overflow, underflow, or inexact; we don't care.
4340
836
      // FIXME need to be more flexible about rounding mode.
4341
836
      (void)FPV.convert(EVTToAPFloatSemantics(VT),
4342
836
                        APFloat::rmNearestTiesToEven, &Ignored);
4343
836
      return getConstantFP(FPV, DL, VT);
4344
8.78M
    }
4345
1.29M
    }
4346
1.29M
  }
4347
8.78M
4348
8.78M
  // Constant fold unary operations with a floating point constant operand.
4349
8.78M
  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand)) {
4350
16.9k
    APFloat V = C->getValueAPF();    // make copy
4351
16.9k
    switch (Opcode) {
4352
16.9k
    case ISD::FNEG:
4353
418
      V.changeSign();
4354
418
      return getConstantFP(V, DL, VT);
4355
16.9k
    case ISD::FABS:
4356
157
      V.clearSign();
4357
157
      return getConstantFP(V, DL, VT);
4358
16.9k
    case ISD::FCEIL: {
4359
28
      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
4360
28
      if (fs == APFloat::opOK || 
fs == APFloat::opInexact0
)
4361
28
        return getConstantFP(V, DL, VT);
4362
0
      break;
4363
0
    }
4364
25
    case ISD::FTRUNC: {
4365
25
      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
4366
25
      if (fs == APFloat::opOK || 
fs == APFloat::opInexact0
)
4367
25
        return getConstantFP(V, DL, VT);
4368
0
      break;
4369
0
    }
4370
25
    case ISD::FFLOOR: {
4371
25
      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
4372
25
      if (fs == APFloat::opOK || 
fs == APFloat::opInexact0
)
4373
25
        return getConstantFP(V, DL, VT);
4374
0
      break;
4375
0
    }
4376
158
    case ISD::FP_EXTEND: {
4377
158
      bool ignored;
4378
158
      // This can return overflow, underflow, or inexact; we don't care.
4379
158
      // FIXME need to be more flexible about rounding mode.
4380
158
      (void)V.convert(EVTToAPFloatSemantics(VT),
4381
158
                      APFloat::rmNearestTiesToEven, &ignored);
4382
158
      return getConstantFP(V, DL, VT);
4383
0
    }
4384
1.21k
    case ISD::FP_TO_SINT:
4385
1.21k
    case ISD::FP_TO_UINT: {
4386
1.21k
      bool ignored;
4387
1.21k
      APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
4388
1.21k
      // FIXME need to be more flexible about rounding mode.
4389
1.21k
      APFloat::opStatus s =
4390
1.21k
          V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
4391
1.21k
      if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
4392
39
        break;
4393
1.17k
      return getConstant(IntVal, DL, VT);
4394
1.17k
    }
4395
13.0k
    case ISD::BITCAST:
4396
13.0k
      if (VT == MVT::i16 && 
C->getValueType(0) == MVT::f16314
)
4397
314
        return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4398
12.7k
      else if (VT == MVT::i32 && 
C->getValueType(0) == MVT::f324.30k
)
4399
4.30k
        return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL, VT);
4400
8.44k
      else if (VT == MVT::i64 && 
C->getValueType(0) == MVT::f648.18k
)
4401
8.18k
        return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
4402
267
      break;
4403
267
    case ISD::FP_TO_FP16: {
4404
71
      bool Ignored;
4405
71
      // This can return overflow, underflow, or inexact; we don't care.
4406
71
      // FIXME need to be more flexible about rounding mode.
4407
71
      (void)V.convert(APFloat::IEEEhalf(),
4408
71
                      APFloat::rmNearestTiesToEven, &Ignored);
4409
71
      return getConstant(V.bitcastToAPInt(), DL, VT);
4410
8.77M
    }
4411
16.9k
    }
4412
16.9k
  }
4413
8.77M
4414
8.77M
  // Constant fold unary operations with a vector integer or float operand.
4415
8.77M
  if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Operand)) {
4416
195k
    if (BV->isConstant()) {
4417
45.9k
      switch (Opcode) {
4418
45.9k
      default:
4419
41.0k
        // FIXME: Entirely reasonable to perform folding of other unary
4420
41.0k
        // operations here as the need arises.
4421
41.0k
        break;
4422
45.9k
      case ISD::FNEG:
4423
4.94k
      case ISD::FABS:
4424
4.94k
      case ISD::FCEIL:
4425
4.94k
      case ISD::FTRUNC:
4426
4.94k
      case ISD::FFLOOR:
4427
4.94k
      case ISD::FP_EXTEND:
4428
4.94k
      case ISD::FP_TO_SINT:
4429
4.94k
      case ISD::FP_TO_UINT:
4430
4.94k
      case ISD::TRUNCATE:
4431
4.94k
      case ISD::ANY_EXTEND:
4432
4.94k
      case ISD::ZERO_EXTEND:
4433
4.94k
      case ISD::SIGN_EXTEND:
4434
4.94k
      case ISD::UINT_TO_FP:
4435
4.94k
      case ISD::SINT_TO_FP:
4436
4.94k
      case ISD::ABS:
4437
4.94k
      case ISD::BITREVERSE:
4438
4.94k
      case ISD::BSWAP:
4439
4.94k
      case ISD::CTLZ:
4440
4.94k
      case ISD::CTLZ_ZERO_UNDEF:
4441
4.94k
      case ISD::CTTZ:
4442
4.94k
      case ISD::CTTZ_ZERO_UNDEF:
4443
4.94k
      case ISD::CTPOP: {
4444
4.94k
        SDValue Ops = { Operand };
4445
4.94k
        if (SDValue Fold = FoldConstantVectorArithmetic(Opcode, DL, VT, Ops))
4446
4.94k
          return Fold;
4447
8.76M
      }
4448
45.9k
      }
4449
45.9k
    }
4450
195k
  }
4451
8.76M
4452
8.76M
  unsigned OpOpcode = Operand.getNode()->getOpcode();
4453
8.76M
  switch (Opcode) {
4454
8.76M
  case ISD::TokenFactor:
4455
4.06M
  case ISD::MERGE_VALUES:
4456
4.06M
  case ISD::CONCAT_VECTORS:
4457
4.06M
    return Operand;         // Factor, merge or concat of one node?  No need.
4458
4.06M
  case ISD::BUILD_VECTOR: {
4459
17.1k
    // Attempt to simplify BUILD_VECTOR.
4460
17.1k
    SDValue Ops[] = {Operand};
4461
17.1k
    if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
4462
295
      return V;
4463
16.8k
    break;
4464
16.8k
  }
4465
16.8k
  
case ISD::FP_ROUND: 0
llvm_unreachable0
("Invalid method to make FP_ROUND node");
4466
16.8k
  case ISD::FP_EXTEND:
4467
15.4k
    assert(VT.isFloatingPoint() &&
4468
15.4k
           Operand.getValueType().isFloatingPoint() && "Invalid FP cast!");
4469
15.4k
    if (Operand.getValueType() == VT) 
return Operand2.67k
; // noop conversion.
4470
12.7k
    assert((!VT.isVector() ||
4471
12.7k
            VT.getVectorNumElements() ==
4472
12.7k
            Operand.getValueType().getVectorNumElements()) &&
4473
12.7k
           "Vector element count mismatch!");
4474
12.7k
    assert(Operand.getValueType().bitsLT(VT) &&
4475
12.7k
           "Invalid fpext node, dst < src!");
4476
12.7k
    if (Operand.isUndef())
4477
29
      return getUNDEF(VT);
4478
12.7k
    break;
4479
15.1k
  case ISD::FP_TO_SINT:
4480
15.1k
  case ISD::FP_TO_UINT:
4481
15.1k
    if (Operand.isUndef())
4482
16
      return getUNDEF(VT);
4483
15.1k
    break;
4484
94.8k
  case ISD::SINT_TO_FP:
4485
94.8k
  case ISD::UINT_TO_FP:
4486
94.8k
    // [us]itofp(undef) = 0, because the result value is bounded.
4487
94.8k
    if (Operand.isUndef())
4488
256
      return getConstantFP(0.0, DL, VT);
4489
94.5k
    break;
4490
135k
  case ISD::SIGN_EXTEND:
4491
135k
    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4492
135k
           "Invalid SIGN_EXTEND!");
4493
135k
    assert(VT.isVector() == Operand.getValueType().isVector() &&
4494
135k
           "SIGN_EXTEND result type type should be vector iff the operand "
4495
135k
           "type is vector!");
4496
135k
    if (Operand.getValueType() == VT) 
return Operand4.19k
; // noop extension
4497
131k
    assert((!VT.isVector() ||
4498
131k
            VT.getVectorNumElements() ==
4499
131k
            Operand.getValueType().getVectorNumElements()) &&
4500
131k
           "Vector element count mismatch!");
4501
131k
    assert(Operand.getValueType().bitsLT(VT) &&
4502
131k
           "Invalid sext node, dst < src!");
4503
131k
    if (OpOpcode == ISD::SIGN_EXTEND || 
OpOpcode == ISD::ZERO_EXTEND131k
)
4504
495
      return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4505
131k
    else if (OpOpcode == ISD::UNDEF)
4506
1.12k
      // sext(undef) = 0, because the top bits will all be the same.
4507
1.12k
      return getConstant(0, DL, VT);
4508
130k
    break;
4509
481k
  case ISD::ZERO_EXTEND:
4510
481k
    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4511
481k
           "Invalid ZERO_EXTEND!");
4512
481k
    assert(VT.isVector() == Operand.getValueType().isVector() &&
4513
481k
           "ZERO_EXTEND result type type should be vector iff the operand "
4514
481k
           "type is vector!");
4515
481k
    if (Operand.getValueType() == VT) 
return Operand13.1k
; // noop extension
4516
468k
    assert((!VT.isVector() ||
4517
468k
            VT.getVectorNumElements() ==
4518
468k
            Operand.getValueType().getVectorNumElements()) &&
4519
468k
           "Vector element count mismatch!");
4520
468k
    assert(Operand.getValueType().bitsLT(VT) &&
4521
468k
           "Invalid zext node, dst < src!");
4522
468k
    if (OpOpcode == ISD::ZERO_EXTEND)   // (zext (zext x)) -> (zext x)
4523
6.29k
      return getNode(ISD::ZERO_EXTEND, DL, VT, Operand.getOperand(0));
4524
462k
    else if (OpOpcode == ISD::UNDEF)
4525
2.15k
      // zext(undef) = 0, because the top bits will be zero.
4526
2.15k
      return getConstant(0, DL, VT);
4527
460k
    break;
4528
524k
  case ISD::ANY_EXTEND:
4529
524k
    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4530
524k
           "Invalid ANY_EXTEND!");
4531
524k
    assert(VT.isVector() == Operand.getValueType().isVector() &&
4532
524k
           "ANY_EXTEND result type type should be vector iff the operand "
4533
524k
           "type is vector!");
4534
524k
    if (Operand.getValueType() == VT) 
return Operand338k
; // noop extension
4535
186k
    assert((!VT.isVector() ||
4536
186k
            VT.getVectorNumElements() ==
4537
186k
            Operand.getValueType().getVectorNumElements()) &&
4538
186k
           "Vector element count mismatch!");
4539
186k
    assert(Operand.getValueType().bitsLT(VT) &&
4540
186k
           "Invalid anyext node, dst < src!");
4541
186k
4542
186k
    if (OpOpcode == ISD::ZERO_EXTEND || 
OpOpcode == ISD::SIGN_EXTEND185k
||
4543
186k
        
OpOpcode == ISD::ANY_EXTEND184k
)
4544
20.3k
      // (ext (zext x)) -> (zext x)  and  (ext (sext x)) -> (sext x)
4545
20.3k
      return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4546
165k
    else if (OpOpcode == ISD::UNDEF)
4547
603
      return getUNDEF(VT);
4548
165k
4549
165k
    // (ext (trunc x)) -> x
4550
165k
    if (OpOpcode == ISD::TRUNCATE) {
4551
17.0k
      SDValue OpOp = Operand.getOperand(0);
4552
17.0k
      if (OpOp.getValueType() == VT) {
4553
14.2k
        transferDbgValues(Operand, OpOp);
4554
14.2k
        return OpOp;
4555
14.2k
      }
4556
151k
    }
4557
151k
    break;
4558
1.60M
  case ISD::TRUNCATE:
4559
1.60M
    assert(VT.isInteger() && Operand.getValueType().isInteger() &&
4560
1.60M
           "Invalid TRUNCATE!");
4561
1.60M
    assert(VT.isVector() == Operand.getValueType().isVector() &&
4562
1.60M
           "TRUNCATE result type type should be vector iff the operand "
4563
1.60M
           "type is vector!");
4564
1.60M
    if (Operand.getValueType() == VT) 
return Operand902k
; // noop truncate
4565
697k
    assert((!VT.isVector() ||
4566
697k
            VT.getVectorNumElements() ==
4567
697k
            Operand.getValueType().getVectorNumElements()) &&
4568
697k
           "Vector element count mismatch!");
4569
697k
    assert(Operand.getValueType().bitsGT(VT) &&
4570
697k
           "Invalid truncate node, src < dst!");
4571
697k
    if (OpOpcode == ISD::TRUNCATE)
4572
56.8k
      return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4573
640k
    if (OpOpcode == ISD::ZERO_EXTEND || 
OpOpcode == ISD::SIGN_EXTEND634k
||
4574
640k
        
OpOpcode == ISD::ANY_EXTEND632k
) {
4575
16.3k
      // If the source is smaller than the dest, we still need an extend.
4576
16.3k
      if (Operand.getOperand(0).getValueType().getScalarType()
4577
16.3k
            .bitsLT(VT.getScalarType()))
4578
4.39k
        return getNode(OpOpcode, DL, VT, Operand.getOperand(0));
4579
11.9k
      if (Operand.getOperand(0).getValueType().bitsGT(VT))
4580
1.14k
        return getNode(ISD::TRUNCATE, DL, VT, Operand.getOperand(0));
4581
10.8k
      return Operand.getOperand(0);
4582
10.8k
    }
4583
624k
    if (OpOpcode == ISD::UNDEF)
4584
20.4k
      return getUNDEF(VT);
4585
604k
    break;
4586
604k
  case ISD::ANY_EXTEND_VECTOR_INREG:
4587
10.6k
  case ISD::ZERO_EXTEND_VECTOR_INREG:
4588
10.6k
  case ISD::SIGN_EXTEND_VECTOR_INREG:
4589
10.6k
    assert(VT.isVector() && "This DAG node is restricted to vector types.");
4590
10.6k
    assert(Operand.getValueType().bitsLE(VT) &&
4591
10.6k
           "The input must be the same size or smaller than the result.");
4592
10.6k
    assert(VT.getVectorNumElements() <
4593
10.6k
             Operand.getValueType().getVectorNumElements() &&
4594
10.6k
           "The destination vector type must have fewer lanes than the input.");
4595
10.6k
    break;
4596
10.6k
  case ISD::ABS:
4597
1.69k
    assert(VT.isInteger() && VT == Operand.getValueType() &&
4598
1.69k
           "Invalid ABS!");
4599
1.69k
    if (OpOpcode == ISD::UNDEF)
4600
1
      return getUNDEF(VT);
4601
1.69k
    break;
4602
1.69k
  case ISD::BSWAP:
4603
1.65k
    assert(VT.isInteger() && VT == Operand.getValueType() &&
4604
1.65k
           "Invalid BSWAP!");
4605
1.65k
    assert((VT.getScalarSizeInBits() % 16 == 0) &&
4606
1.65k
           "BSWAP types must be a multiple of 16 bits!");
4607
1.65k
    if (OpOpcode == ISD::UNDEF)
4608
49
      return getUNDEF(VT);
4609
1.60k
    break;
4610
1.60k
  case ISD::BITREVERSE:
4611
981
    assert(VT.isInteger() && VT == Operand.getValueType() &&
4612
981
           "Invalid BITREVERSE!");
4613
981
    if (OpOpcode == ISD::UNDEF)
4614
13
      return getUNDEF(VT);
4615
968
    break;
4616
1.04M
  case ISD::BITCAST:
4617
1.04M
    // Basic sanity checking.
4618
1.04M
    assert(VT.getSizeInBits() == Operand.getValueSizeInBits() &&
4619
1.04M
           "Cannot BITCAST between types of different sizes!");
4620
1.04M
    if (VT == Operand.getValueType()) 
return Operand267k
; // noop conversion.
4621
773k
    if (OpOpcode == ISD::BITCAST)  // bitconv(bitconv(x)) -> bitconv(x)
4622
48.5k
      return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0));
4623
724k
    if (OpOpcode == ISD::UNDEF)
4624
22.6k
      return getUNDEF(VT);
4625
702k
    break;
4626
702k
  case ISD::SCALAR_TO_VECTOR:
4627
50.0k
    assert(VT.isVector() && !Operand.getValueType().isVector() &&
4628
50.0k
           (VT.getVectorElementType() == Operand.getValueType() ||
4629
50.0k
            (VT.getVectorElementType().isInteger() &&
4630
50.0k
             Operand.getValueType().isInteger() &&
4631
50.0k
             VT.getVectorElementType().bitsLE(Operand.getValueType()))) &&
4632
50.0k
           "Illegal SCALAR_TO_VECTOR node!");
4633
50.0k
    if (OpOpcode == ISD::UNDEF)
4634
9
      return getUNDEF(VT);
4635
50.0k
    // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
4636
50.0k
    if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
4637
50.0k
        
isa<ConstantSDNode>(Operand.getOperand(1))15.4k
&&
4638
50.0k
        
Operand.getConstantOperandVal(1) == 015.1k
&&
4639
50.0k
        
Operand.getOperand(0).getValueType() == VT6.59k
)
4640
2.40k
      return Operand.getOperand(0);
4641
47.6k
    break;
4642
47.6k
  case ISD::FNEG:
4643
8.43k
    // Negation of an unknown bag of bits is still completely undefined.
4644
8.43k
    if (OpOpcode == ISD::UNDEF)
4645
19
      return getUNDEF(VT);
4646
8.41k
4647
8.41k
    // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
4648
8.41k
    if ((getTarget().Options.UnsafeFPMath || 
Flags.hasNoSignedZeros()7.73k
) &&
4649
8.41k
        
OpOpcode == ISD::FSUB731
)
4650
42
      return getNode(ISD::FSUB, DL, VT, Operand.getOperand(1),
4651
42
                     Operand.getOperand(0), Flags);
4652
8.37k
    if (OpOpcode == ISD::FNEG)  // --X -> X
4653
9
      return Operand.getOperand(0);
4654
8.36k
    break;
4655
8.36k
  case ISD::FABS:
4656
3.14k
    if (OpOpcode == ISD::FNEG)  // abs(-X) -> abs(X)
4657
18
      return getNode(ISD::FABS, DL, VT, Operand.getOperand(0));
4658
3.12k
    break;
4659
2.96M
  }
4660
2.96M
4661
2.96M
  SDNode *N;
4662
2.96M
  SDVTList VTs = getVTList(VT);
4663
2.96M
  SDValue Ops[] = {Operand};
4664
2.96M
  if (VT != MVT::Glue) { // Don't CSE flag producing nodes
4665
2.95M
    FoldingSetNodeID ID;
4666
2.95M
    AddNodeIDNode(ID, Opcode, VTs, Ops);
4667
2.95M
    void *IP = nullptr;
4668
2.95M
    if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
4669
241k
      E->intersectFlagsWith(Flags);
4670
241k
      return SDValue(E, 0);
4671
241k
    }
4672
2.71M
4673
2.71M
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4674
2.71M
    N->setFlags(Flags);
4675
2.71M
    createOperands(N, Ops);
4676
2.71M
    CSEMap.InsertNode(N, IP);
4677
2.71M
  } else {
4678
1.40k
    N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
4679
1.40k
    createOperands(N, Ops);
4680
1.40k
  }
4681
2.96M
4682
2.96M
  InsertNode(N);
4683
2.71M
  SDValue V = SDValue(N, 0);
4684
2.71M
  NewSDValueDbgMsg(V, "Creating new node: ", this);
4685
2.71M
  return V;
4686
2.96M
}
4687
4688
static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
4689
937k
                                        const APInt &C2) {
4690
937k
  switch (Opcode) {
4691
937k
  
case ISD::ADD: return std::make_pair(C1 + C2, true)593k
;
4692
937k
  
case ISD::SUB: return std::make_pair(C1 - C2, true)47.1k
;
4693
937k
  
case ISD::MUL: return std::make_pair(C1 * C2, true)13.6k
;
4694
937k
  
case ISD::AND: return std::make_pair(C1 & C2, true)120k
;
4695
937k
  
case ISD::OR: return std::make_pair(C1 | C2, true)3.04k
;
4696
937k
  
case ISD::XOR: return std::make_pair(C1 ^ C2, true)6.53k
;
4697
937k
  
case ISD::SHL: return std::make_pair(C1 << C2, true)23.0k
;
4698
937k
  
case ISD::SRL: return std::make_pair(C1.lshr(C2), true)8.02k
;
4699
937k
  
case ISD::SRA: return std::make_pair(C1.ashr(C2), true)153
;
4700
937k
  
case ISD::ROTL: return std::make_pair(C1.rotl(C2), true)16
;
4701
937k
  
case ISD::ROTR: return std::make_pair(C1.rotr(C2), true)0
;
4702
937k
  
case ISD::SMIN: return std::make_pair(C1.sle(C2) 896
?
C1476
:
C2420
, true);
4703
937k
  
case ISD::SMAX: return std::make_pair(C1.sge(C2) 900
?
C1516
:
C2384
, true);
4704
937k
  
case ISD::UMIN: return std::make_pair(C1.ule(C2) 896
?
C1467
:
C2429
, true);
4705
937k
  
case ISD::UMAX: return std::make_pair(C1.uge(C2) 897
?
C1513
:
C2384
, true);
4706
937k
  
case ISD::SADDSAT: return std::make_pair(C1.sadd_sat(C2), true)114
;
4707
937k
  
case ISD::UADDSAT: return std::make_pair(C1.uadd_sat(C2), true)98
;
4708
937k
  
case ISD::SSUBSAT: return std::make_pair(C1.ssub_sat(C2), true)114
;
4709
937k
  
case ISD::USUBSAT: return std::make_pair(C1.usub_sat(C2), true)98
;
4710
937k
  case ISD::UDIV:
4711
38
    if (!C2.getBoolValue())
4712
0
      break;
4713
38
    return std::make_pair(C1.udiv(C2), true);
4714
20.3k
  case ISD::UREM:
4715
20.3k
    if (!C2.getBoolValue())
4716
0
      break;
4717
20.3k
    return std::make_pair(C1.urem(C2), true);
4718
20.3k
  case ISD::SDIV:
4719
110
    if (!C2.getBoolValue())
4720
0
      break;
4721
110
    return std::make_pair(C1.sdiv(C2), true);
4722
110
  case ISD::SREM:
4723
91
    if (!C2.getBoolValue())
4724
0
      break;
4725
91
    return std::make_pair(C1.srem(C2), true);
4726
97.4k
  }
4727
97.4k
  return std::make_pair(APInt(1, 0), false);
4728
97.4k
}
4729
4730
SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4731
                                             EVT VT, const ConstantSDNode *C1,
4732
943k
                                             const ConstantSDNode *C2) {
4733
943k
  if (C1->isOpaque() || 
C2->isOpaque()937k
)
4734
5.54k
    return SDValue();
4735
937k
4736
937k
  std::pair<APInt, bool> Folded = FoldValue(Opcode, C1->getAPIntValue(),
4737
937k
                                            C2->getAPIntValue());
4738
937k
  if (!Folded.second)
4739
97.4k
    return SDValue();
4740
840k
  return getConstant(Folded.first, DL, VT);
4741
840k
}
4742
4743
SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
4744
                                       const GlobalAddressSDNode *GA,
4745
91.5k
                                       const SDNode *N2) {
4746
91.5k
  if (GA->getOpcode() != ISD::GlobalAddress)
4747
574
    return SDValue();
4748
90.9k
  if (!TLI->isOffsetFoldingLegal(GA))
4749
84.7k
    return SDValue();
4750
6.18k
  auto *C2 = dyn_cast<ConstantSDNode>(N2);
4751
6.18k
  if (!C2)
4752
1.60k
    return SDValue();
4753
4.57k
  int64_t Offset = C2->getSExtValue();
4754
4.57k
  switch (Opcode) {
4755
4.57k
  
case ISD::ADD: break4.56k
;
4756
4.57k
  
case ISD::SUB: Offset = -uint64_t(Offset); break1
;
4757
4.57k
  
default: return SDValue()14
;
4758
4.56k
  }
4759
4.56k
  return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
4760
4.56k
                          GA->getOffset() + uint64_t(Offset));
4761
4.56k
}
4762
4763
10.1M
bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
4764
10.1M
  switch (Opcode) {
4765
10.1M
  case ISD::SDIV:
4766
58.6k
  case ISD::UDIV:
4767
58.6k
  case ISD::SREM:
4768
58.6k
  case ISD::UREM: {
4769
58.6k
    // If a divisor is zero/undef or any element of a divisor vector is
4770
58.6k
    // zero/undef, the whole op is undef.
4771
58.6k
    assert(Ops.size() == 2 && "Div/rem should have 2 operands");
4772
58.6k
    SDValue Divisor = Ops[1];
4773
58.6k
    if (Divisor.isUndef() || 
isNullConstant(Divisor)58.6k
)
4774
37
      return true;
4775
58.6k
4776
58.6k
    return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
4777
58.6k
           llvm::any_of(Divisor->op_values(),
4778
74.7k
                        [](SDValue V) { return V.isUndef() ||
4779
74.7k
                                        
isNullConstant(V)74.7k
; });
4780
58.6k
    // TODO: Handle signed overflow.
4781
58.6k
  }
4782
58.6k
  // TODO: Handle oversized shifts.
4783
10.0M
  default:
4784
10.0M
    return false;
4785
10.1M
  }
4786
10.1M
}
4787
4788
SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
4789
8.94M
                                             EVT VT, SDNode *N1, SDNode *N2) {
4790
8.94M
  // If the opcode is a target-specific ISD node, there's nothing we can
4791
8.94M
  // do here and the operand rules may not line up with the below, so
4792
8.94M
  // bail early.
4793
8.94M
  if (Opcode >= ISD::BUILTIN_OP_END)
4794
780k
    return SDValue();
4795
8.16M
4796
8.16M
  if (isUndef(Opcode, {SDValue(N1, 0), SDValue(N2, 0)}))
4797
102
    return getUNDEF(VT);
4798
8.16M
4799
8.16M
  // Handle the case of two scalars.
4800
8.16M
  if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
4801
995k
    if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) {
4802
936k
      SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, C1, C2);
4803
936k
      assert((!Folded || !VT.isVector()) &&
4804
936k
             "Can't fold vectors ops with scalar operands");
4805
936k
      return Folded;
4806
936k
    }
4807
7.22M
  }
4808
7.22M
4809
7.22M
  // fold (add Sym, c) -> Sym+c
4810
7.22M
  if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1))
4811
90.1k
    return FoldSymbolOffset(Opcode, VT, GA, N2);
4812
7.13M
  if (TLI->isCommutativeBinOp(Opcode))
4813
3.52M
    if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
4814
1.38k
      return FoldSymbolOffset(Opcode, VT, GA, N1);
4815
7.13M
4816
7.13M
  // For vectors, extract each constant element and fold them individually.
4817
7.13M
  // Either input may be an undef value.
4818
7.13M
  auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
4819
7.13M
  if (!BV1 && 
!N1->isUndef()7.06M
)
4820
7.01M
    return SDValue();
4821
120k
  auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
4822
120k
  if (!BV2 && 
!N2->isUndef()93.3k
)
4823
84.7k
    return SDValue();
4824
35.9k
  // If both operands are undef, that's handled the same way as scalars.
4825
35.9k
  if (!BV1 && 
!BV26.12k
)
4826
6.05k
    return SDValue();
4827
29.9k
4828
29.9k
  assert((!BV1 || !BV2 || BV1->getNumOperands() == BV2->getNumOperands()) &&
4829
29.9k
         "Vector binop with different number of elements in operands?");
4830
29.9k
4831
29.9k
  EVT SVT = VT.getScalarType();
4832
29.9k
  EVT LegalSVT = SVT;
4833
29.9k
  if (NewNodesMustHaveLegalTypes && 
LegalSVT.isInteger()9.91k
) {
4834
9.91k
    LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4835
9.91k
    if (LegalSVT.bitsLT(SVT))
4836
0
      return SDValue();
4837
29.9k
  }
4838
29.9k
  SmallVector<SDValue, 4> Outputs;
4839
29.9k
  unsigned NumOps = BV1 ? 
BV1->getNumOperands()29.8k
:
BV2->getNumOperands()73
;
4840
190k
  for (unsigned I = 0; I != NumOps; 
++I160k
) {
4841
177k
    SDValue V1 = BV1 ? 
BV1->getOperand(I)177k
:
getUNDEF(SVT)311
;
4842
177k
    SDValue V2 = BV2 ? 
BV2->getOperand(I)121k
:
getUNDEF(SVT)56.0k
;
4843
177k
    if (SVT.isInteger()) {
4844
174k
      if (V1->getValueType(0).bitsGT(SVT))
4845
3.30k
        V1 = getNode(ISD::TRUNCATE, DL, SVT, V1);
4846
174k
      if (V2->getValueType(0).bitsGT(SVT))
4847
1.40k
        V2 = getNode(ISD::TRUNCATE, DL, SVT, V2);
4848
174k
    }
4849
177k
4850
177k
    if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT)
4851
0
      return SDValue();
4852
177k
4853
177k
    // Fold one vector element.
4854
177k
    SDValue ScalarResult = getNode(Opcode, DL, SVT, V1, V2);
4855
177k
    if (LegalSVT != SVT)
4856
790
      ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4857
177k
4858
177k
    // Scalar folding only succeeded if the result is a constant or UNDEF.
4859
177k
    if (!ScalarResult.isUndef() && 
ScalarResult.getOpcode() != ISD::Constant170k
&&
4860
177k
        
ScalarResult.getOpcode() != ISD::ConstantFP19.4k
)
4861
17.2k
      return SDValue();
4862
160k
    Outputs.push_back(ScalarResult);
4863
160k
  }
4864
29.9k
4865
29.9k
  assert(VT.getVectorNumElements() == Outputs.size() &&
4866
12.7k
         "Vector size mismatch!");
4867
12.7k
4868
12.7k
  // We may have a vector type but a scalar result. Create a splat.
4869
12.7k
  Outputs.resize(VT.getVectorNumElements(), Outputs.back());
4870
12.7k
4871
12.7k
  // Build a big vector out of the scalar elements we generated.
4872
12.7k
  return getBuildVector(VT, SDLoc(), Outputs);
4873
29.9k
}
4874
4875
// TODO: Merge with FoldConstantArithmetic
4876
SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode,
4877
                                                   const SDLoc &DL, EVT VT,
4878
                                                   ArrayRef<SDValue> Ops,
4879
1.95M
                                                   const SDNodeFlags Flags) {
4880
1.95M
  // If the opcode is a target-specific ISD node, there's nothing we can
4881
1.95M
  // do here and the operand rules may not line up with the below, so
4882
1.95M
  // bail early.
4883
1.95M
  if (Opcode >= ISD::BUILTIN_OP_END)
4884
0
    return SDValue();
4885
1.95M
4886
1.95M
  if (isUndef(Opcode, Ops))
4887
2
    return getUNDEF(VT);
4888
1.95M
4889
1.95M
  // We can only fold vectors - maybe merge with FoldConstantArithmetic someday?
4890
1.95M
  if (!VT.isVector())
4891
1.27M
    return SDValue();
4892
680k
4893
680k
  unsigned NumElts = VT.getVectorNumElements();
4894
680k
4895
680k
  auto IsScalarOrSameVectorSize = [&](const SDValue &Op) {
4896
11.1k
    return !Op.getValueType().isVector() ||
4897
11.1k
           
Op.getValueType().getVectorNumElements() == NumElts9.54k
;
4898
11.1k
  };
4899
680k
4900
722k
  auto IsConstantBuildVectorOrUndef = [&](const SDValue &Op) {
4901
722k
    BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op);
4902
722k
    return (Op.isUndef()) || 
(Op.getOpcode() == ISD::CONDCODE)722k
||
4903
722k
           
(721k
BV721k
&&
BV->isConstant()68.5k
);
4904
722k
  };
4905
680k
4906
680k
  // All operands must be vector types with the same number of elements as
4907
680k
  // the result type and must be either UNDEF or a build vector of constant
4908
680k
  // or UNDEF scalars.
4909
680k
  if (!llvm::all_of(Ops, IsConstantBuildVectorOrUndef) ||
4910
680k
      
!llvm::all_of(Ops, IsScalarOrSameVectorSize)7.24k
)
4911
673k
    return SDValue();
4912
7.24k
4913
7.24k
  // If we are comparing vectors, then the result needs to be a i1 boolean
4914
7.24k
  // that is then sign-extended back to the legal result type.
4915
7.24k
  EVT SVT = (Opcode == ISD::SETCC ? 
MVT::i11.60k
:
VT.getScalarType()5.64k
);
4916
7.24k
4917
7.24k
  // Find legal integer scalar type for constant promotion and
4918
7.24k
  // ensure that its scalar size is at least as large as source.
4919
7.24k
  EVT LegalSVT = VT.getScalarType();
4920
7.24k
  if (NewNodesMustHaveLegalTypes && 
LegalSVT.isInteger()2.46k
) {
4921
2.46k
    LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
4922
2.46k
    if (LegalSVT.bitsLT(VT.getScalarType()))
4923
0
      return SDValue();
4924
7.24k
  }
4925
7.24k
4926
7.24k
  // Constant fold each scalar lane separately.
4927
7.24k
  SmallVector<SDValue, 4> ScalarResults;
4928
81.4k
  for (unsigned i = 0; i != NumElts; 
i++74.1k
) {
4929
74.2k
    SmallVector<SDValue, 4> ScalarOps;
4930
113k
    for (SDValue Op : Ops) {
4931
113k
      EVT InSVT = Op.getValueType().getScalarType();
4932
113k
      BuildVectorSDNode *InBV = dyn_cast<BuildVectorSDNode>(Op);
4933
113k
      if (!InBV) {
4934
16.4k
        // We've checked that this is UNDEF or a constant of some kind.
4935
16.4k
        if (Op.isUndef())
4936
68
          ScalarOps.push_back(getUNDEF(InSVT));
4937
16.3k
        else
4938
16.3k
          ScalarOps.push_back(Op);
4939
16.4k
        continue;
4940
16.4k
      }
4941
97.1k
4942
97.1k
      SDValue ScalarOp = InBV->getOperand(i);
4943
97.1k
      EVT ScalarVT = ScalarOp.getValueType();
4944
97.1k
4945
97.1k
      // Build vector (integer) scalar operands may need implicit
4946
97.1k
      // truncation - do this before constant folding.
4947
97.1k
      if (ScalarVT.isInteger() && 
ScalarVT.bitsGT(InSVT)95.3k
)
4948
1.24k
        ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
4949
97.1k
4950
97.1k
      ScalarOps.push_back(ScalarOp);
4951
97.1k
    }
4952
74.2k
4953
74.2k
    // Constant fold the scalar operands.
4954
74.2k
    SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps, Flags);
4955
74.2k
4956
74.2k
    // Legalize the (integer) scalar constant if necessary.
4957
74.2k
    if (LegalSVT != SVT)
4958
9.48k
      ScalarResult = getNode(ISD::SIGN_EXTEND, DL, LegalSVT, ScalarResult);
4959
74.2k
4960
74.2k
    // Scalar folding only succeeded if the result is a constant or UNDEF.
4961
74.2k
    if (!ScalarResult.isUndef() && 
ScalarResult.getOpcode() != ISD::Constant74.0k
&&
4962
74.2k
        
ScalarResult.getOpcode() != ISD::ConstantFP782
)
4963
10
      return SDValue();
4964
74.1k
    ScalarResults.push_back(ScalarResult);
4965
74.1k
  }
4966
7.24k
4967
7.24k
  SDValue V = getBuildVector(VT, DL, ScalarResults);
4968
7.23k
  NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
4969
7.23k
  return V;
4970
7.24k
}
4971
4972
SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
4973
8.09M
                                         EVT VT, SDValue N1, SDValue N2) {
4974
8.09M
  // TODO: We don't do any constant folding for strict FP opcodes here, but we
4975
8.09M
  //       should. That will require dealing with a potentially non-default
4976
8.09M
  //       rounding mode, checking the "opStatus" return value from the APFloat
4977
8.09M
  //       math calculations, and possibly other variations.
4978
8.09M
  auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode());
4979
8.09M
  auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode());
4980
8.09M
  if (N1CFP && 
N2CFP18.2k
) {
4981
13.2k
    APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF();
4982
13.2k
    switch (Opcode) {
4983
13.2k
    case ISD::FADD:
4984
401
      C1.add(C2, APFloat::rmNearestTiesToEven);
4985
401
      return getConstantFP(C1, DL, VT);
4986
13.2k
    case ISD::FSUB:
4987
63
      C1.subtract(C2, APFloat::rmNearestTiesToEven);
4988
63
      return getConstantFP(C1, DL, VT);
4989
13.2k
    case ISD::FMUL:
4990
347
      C1.multiply(C2, APFloat::rmNearestTiesToEven);
4991
347
      return getConstantFP(C1, DL, VT);
4992
13.2k
    case ISD::FDIV:
4993
76
      C1.divide(C2, APFloat::rmNearestTiesToEven);
4994
76
      return getConstantFP(C1, DL, VT);
4995
13.2k
    case ISD::FREM:
4996
30
      C1.mod(C2);
4997
30
      return getConstantFP(C1, DL, VT);
4998
13.2k
    case ISD::FCOPYSIGN:
4999
4
      C1.copySign(C2);
5000
4
      return getConstantFP(C1, DL, VT);
5001
13.2k
    
default: break12.3k
;
5002
8.09M
    }
5003
8.09M
  }
5004
8.09M
  if (N1CFP && 
Opcode == ISD::FP_ROUND17.2k
) {
5005
8
    APFloat C1 = N1CFP->getValueAPF();    // make copy
5006
8
    bool Unused;
5007
8
    // This can return overflow, underflow, or inexact; we don't care.
5008
8
    // FIXME need to be more flexible about rounding mode.
5009
8
    (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven,
5010
8
                      &Unused);
5011
8
    return getConstantFP(C1, DL, VT);
5012
8
  }
5013
8.09M
5014
8.09M
  switch (Opcode) {
5015
8.09M
  case ISD::FADD:
5016
108k
  case ISD::FSUB:
5017
108k
  case ISD::FMUL:
5018
108k
  case ISD::FDIV:
5019
108k
  case ISD::FREM:
5020
108k
    // If both operands are undef, the result is undef. If 1 operand is undef,
5021
108k
    // the result is NaN. This should match the behavior of the IR optimizer.
5022
108k
    if (N1.isUndef() && 
N2.isUndef()3.40k
)
5023
1.43k
      return getUNDEF(VT);
5024
106k
    if (N1.isUndef() || 
N2.isUndef()104k
)
5025
2.36k
      return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT);
5026
8.08M
  }
5027
8.08M
  return SDValue();
5028
8.08M
}
5029
5030
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5031
10.3M
                              SDValue N1, SDValue N2, const SDNodeFlags Flags) {
5032
10.3M
  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
5033
10.3M
  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
5034
10.3M
  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
5035
10.3M
  ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
5036
10.3M
5037
10.3M
  // Canonicalize constant to RHS if commutative.
5038
10.3M
  if (TLI->isCommutativeBinOp(Opcode)) {
5039
5.25M
    if (N1C && 
!N2C375k
) {
5040
75.0k
      std::swap(N1C, N2C);
5041
75.0k
      std::swap(N1, N2);
5042
5.18M
    } else if (N1CFP && 
!N2CFP3.94k
) {
5043
3.12k
      std::swap(N1CFP, N2CFP);
5044
3.12k
      std::swap(N1, N2);
5045
3.12k
    }
5046
5.25M
  }
5047
10.3M