Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file defines the interfaces that RISCV uses to lower LLVM code into a
10
// selection DAG.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "RISCVISelLowering.h"
15
#include "RISCV.h"
16
#include "RISCVMachineFunctionInfo.h"
17
#include "RISCVRegisterInfo.h"
18
#include "RISCVSubtarget.h"
19
#include "RISCVTargetMachine.h"
20
#include "Utils/RISCVMatInt.h"
21
#include "llvm/ADT/SmallSet.h"
22
#include "llvm/ADT/Statistic.h"
23
#include "llvm/CodeGen/CallingConvLower.h"
24
#include "llvm/CodeGen/MachineFrameInfo.h"
25
#include "llvm/CodeGen/MachineFunction.h"
26
#include "llvm/CodeGen/MachineInstrBuilder.h"
27
#include "llvm/CodeGen/MachineRegisterInfo.h"
28
#include "llvm/CodeGen/SelectionDAGISel.h"
29
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30
#include "llvm/CodeGen/ValueTypes.h"
31
#include "llvm/IR/DiagnosticInfo.h"
32
#include "llvm/IR/DiagnosticPrinter.h"
33
#include "llvm/Support/Debug.h"
34
#include "llvm/Support/ErrorHandling.h"
35
#include "llvm/Support/raw_ostream.h"
36
37
using namespace llvm;
38
39
#define DEBUG_TYPE "riscv-lower"
40
41
STATISTIC(NumTailCalls, "Number of tail calls");
42
43
RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
44
                                         const RISCVSubtarget &STI)
45
373
    : TargetLowering(TM), Subtarget(STI) {
46
373
47
373
  if (Subtarget.isRV32E())
48
1
    report_fatal_error("Codegen not yet implemented for RV32E");
49
372
50
372
  RISCVABI::ABI ABI = Subtarget.getTargetABI();
51
372
  assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
52
372
53
372
  switch (ABI) {
54
372
  default:
55
1
    report_fatal_error("Don't know how to lower this ABI");
56
372
  case RISCVABI::ABI_ILP32:
57
371
  case RISCVABI::ABI_ILP32F:
58
371
  case RISCVABI::ABI_ILP32D:
59
371
  case RISCVABI::ABI_LP64:
60
371
  case RISCVABI::ABI_LP64F:
61
371
  case RISCVABI::ABI_LP64D:
62
371
    break;
63
371
  }
64
371
65
371
  MVT XLenVT = Subtarget.getXLenVT();
66
371
67
371
  // Set up the register classes.
68
371
  addRegisterClass(XLenVT, &RISCV::GPRRegClass);
69
371
70
371
  if (Subtarget.hasStdExtF())
71
124
    addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
72
371
  if (Subtarget.hasStdExtD())
73
67
    addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
74
371
75
371
  // Compute derived properties from the register classes.
76
371
  computeRegisterProperties(STI.getRegisterInfo());
77
371
78
371
  setStackPointerRegisterToSaveRestore(RISCV::X2);
79
371
80
371
  for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
81
1.11k
    setLoadExtAction(N, XLenVT, MVT::i1, Promote);
82
371
83
371
  // TODO: add all necessary setOperationAction calls.
84
371
  setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
85
371
86
371
  setOperationAction(ISD::BR_JT, MVT::Other, Expand);
87
371
  setOperationAction(ISD::BR_CC, XLenVT, Expand);
88
371
  setOperationAction(ISD::SELECT, XLenVT, Custom);
89
371
  setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
90
371
91
371
  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
92
371
  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
93
371
94
371
  setOperationAction(ISD::VASTART, MVT::Other, Custom);
95
371
  setOperationAction(ISD::VAARG, MVT::Other, Expand);
96
371
  setOperationAction(ISD::VACOPY, MVT::Other, Expand);
97
371
  setOperationAction(ISD::VAEND, MVT::Other, Expand);
98
371
99
371
  for (auto VT : {MVT::i1, MVT::i8, MVT::i16})
100
1.11k
    setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
101
371
102
371
  if (Subtarget.is64Bit()) {
103
147
    setOperationAction(ISD::SHL, MVT::i32, Custom);
104
147
    setOperationAction(ISD::SRA, MVT::i32, Custom);
105
147
    setOperationAction(ISD::SRL, MVT::i32, Custom);
106
147
  }
107
371
108
371
  if (!Subtarget.hasStdExtM()) {
109
362
    setOperationAction(ISD::MUL, XLenVT, Expand);
110
362
    setOperationAction(ISD::MULHS, XLenVT, Expand);
111
362
    setOperationAction(ISD::MULHU, XLenVT, Expand);
112
362
    setOperationAction(ISD::SDIV, XLenVT, Expand);
113
362
    setOperationAction(ISD::UDIV, XLenVT, Expand);
114
362
    setOperationAction(ISD::SREM, XLenVT, Expand);
115
362
    setOperationAction(ISD::UREM, XLenVT, Expand);
116
362
  }
117
371
118
371
  if (Subtarget.is64Bit() && 
Subtarget.hasStdExtM()147
) {
119
4
    setOperationAction(ISD::SDIV, MVT::i32, Custom);
120
4
    setOperationAction(ISD::UDIV, MVT::i32, Custom);
121
4
    setOperationAction(ISD::UREM, MVT::i32, Custom);
122
4
  }
123
371
124
371
  setOperationAction(ISD::SDIVREM, XLenVT, Expand);
125
371
  setOperationAction(ISD::UDIVREM, XLenVT, Expand);
126
371
  setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
127
371
  setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
128
371
129
371
  setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
130
371
  setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
131
371
  setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
132
371
133
371
  setOperationAction(ISD::ROTL, XLenVT, Expand);
134
371
  setOperationAction(ISD::ROTR, XLenVT, Expand);
135
371
  setOperationAction(ISD::BSWAP, XLenVT, Expand);
136
371
  setOperationAction(ISD::CTTZ, XLenVT, Expand);
137
371
  setOperationAction(ISD::CTLZ, XLenVT, Expand);
138
371
  setOperationAction(ISD::CTPOP, XLenVT, Expand);
139
371
140
371
  ISD::CondCode FPCCToExtend[] = {
141
371
      ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
142
371
      ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
143
371
      ISD::SETGE,  ISD::SETNE};
144
371
145
371
  ISD::NodeType FPOpToExtend[] = {
146
371
      ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM};
147
371
148
371
  if (Subtarget.hasStdExtF()) {
149
124
    setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
150
124
    setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
151
124
    for (auto CC : FPCCToExtend)
152
1.48k
      setCondCodeAction(CC, MVT::f32, Expand);
153
124
    setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
154
124
    setOperationAction(ISD::SELECT, MVT::f32, Custom);
155
124
    setOperationAction(ISD::BR_CC, MVT::f32, Expand);
156
124
    for (auto Op : FPOpToExtend)
157
620
      setOperationAction(Op, MVT::f32, Expand);
158
124
  }
159
371
160
371
  if (Subtarget.hasStdExtF() && 
Subtarget.is64Bit()124
)
161
54
    setOperationAction(ISD::BITCAST, MVT::i32, Custom);
162
371
163
371
  if (Subtarget.hasStdExtD()) {
164
67
    setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
165
67
    setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
166
67
    for (auto CC : FPCCToExtend)
167
804
      setCondCodeAction(CC, MVT::f64, Expand);
168
67
    setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
169
67
    setOperationAction(ISD::SELECT, MVT::f64, Custom);
170
67
    setOperationAction(ISD::BR_CC, MVT::f64, Expand);
171
67
    setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
172
67
    setTruncStoreAction(MVT::f64, MVT::f32, Expand);
173
67
    for (auto Op : FPOpToExtend)
174
335
      setOperationAction(Op, MVT::f64, Expand);
175
67
  }
176
371
177
371
  setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
178
371
  setOperationAction(ISD::BlockAddress, XLenVT, Custom);
179
371
  setOperationAction(ISD::ConstantPool, XLenVT, Custom);
180
371
181
371
  setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
182
371
183
371
  // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
184
371
  // Unfortunately this can't be determined just from the ISA naming string.
185
371
  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
186
371
                     Subtarget.is64Bit() ? 
Legal147
:
Custom224
);
187
371
188
371
  if (Subtarget.hasStdExtA()) {
189
9
    setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
190
9
    setMinCmpXchgSizeInBits(32);
191
362
  } else {
192
362
    setMaxAtomicSizeInBitsSupported(0);
193
362
  }
194
371
195
371
  setBooleanContents(ZeroOrOneBooleanContent);
196
371
197
371
  // Function alignments (log2).
198
371
  unsigned FunctionAlignment = Subtarget.hasStdExtC() ? 
14
:
2367
;
199
371
  setMinFunctionAlignment(FunctionAlignment);
200
371
  setPrefFunctionAlignment(FunctionAlignment);
201
371
202
371
  // Effectively disable jump table generation.
203
371
  setMinimumJumpTableEntries(INT_MAX);
204
371
}
205
206
EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
207
2.87k
                                            EVT VT) const {
208
2.87k
  if (!VT.isVector())
209
2.87k
    return getPointerTy(DL);
210
1
  return VT.changeVectorElementTypeToInteger();
211
1
}
212
213
bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
214
                                             const CallInst &I,
215
                                             MachineFunction &MF,
216
200
                                             unsigned Intrinsic) const {
217
200
  switch (Intrinsic) {
218
200
  default:
219
100
    return false;
220
200
  case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
221
100
  case Intrinsic::riscv_masked_atomicrmw_add_i32:
222
100
  case Intrinsic::riscv_masked_atomicrmw_sub_i32:
223
100
  case Intrinsic::riscv_masked_atomicrmw_nand_i32:
224
100
  case Intrinsic::riscv_masked_atomicrmw_max_i32:
225
100
  case Intrinsic::riscv_masked_atomicrmw_min_i32:
226
100
  case Intrinsic::riscv_masked_atomicrmw_umax_i32:
227
100
  case Intrinsic::riscv_masked_atomicrmw_umin_i32:
228
100
  case Intrinsic::riscv_masked_cmpxchg_i32:
229
100
    PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
230
100
    Info.opc = ISD::INTRINSIC_W_CHAIN;
231
100
    Info.memVT = MVT::getVT(PtrTy->getElementType());
232
100
    Info.ptrVal = I.getArgOperand(0);
233
100
    Info.offset = 0;
234
100
    Info.align = 4;
235
100
    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
236
100
                 MachineMemOperand::MOVolatile;
237
100
    return true;
238
200
  }
239
200
}
240
241
bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
242
                                                const AddrMode &AM, Type *Ty,
243
                                                unsigned AS,
244
2.60k
                                                Instruction *I) const {
245
2.60k
  // No global is ever allowed as a base.
246
2.60k
  if (AM.BaseGV)
247
223
    return false;
248
2.38k
249
2.38k
  // Require a 12-bit signed offset.
250
2.38k
  if (!isInt<12>(AM.BaseOffs))
251
92
    return false;
252
2.29k
253
2.29k
  switch (AM.Scale) {
254
2.29k
  case 0: // "r+i" or just "i", depending on HasBaseReg.
255
2.23k
    break;
256
2.29k
  case 1:
257
32
    if (!AM.HasBaseReg) // allow "r+i".
258
0
      break;
259
32
    return false; // disallow "r+r" or "r+r+i".
260
32
  default:
261
22
    return false;
262
2.23k
  }
263
2.23k
264
2.23k
  return true;
265
2.23k
}
266
267
1.08k
bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
268
1.08k
  return isInt<12>(Imm);
269
1.08k
}
270
271
1.35k
bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
272
1.35k
  return isInt<12>(Imm);
273
1.35k
}
274
275
// On RV32, 64-bit integers are split into their high and low parts and held
276
// in two different registers, so the trunc is free since the low register can
277
// just be used.
278
385
bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
279
385
  if (Subtarget.is64Bit() || 
!SrcTy->isIntegerTy()104
||
!DstTy->isIntegerTy()104
)
280
281
    return false;
281
104
  unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
282
104
  unsigned DestBits = DstTy->getPrimitiveSizeInBits();
283
104
  return (SrcBits == 64 && 
DestBits == 3241
);
284
104
}
285
286
4.86k
bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
287
4.86k
  if (Subtarget.is64Bit() || 
SrcVT.isVector()906
||
DstVT.isVector()906
||
288
4.86k
      
!SrcVT.isInteger()906
||
!DstVT.isInteger()906
)
289
3.96k
    return false;
290
906
  unsigned SrcBits = SrcVT.getSizeInBits();
291
906
  unsigned DestBits = DstVT.getSizeInBits();
292
906
  return (SrcBits == 64 && 
DestBits == 320
);
293
906
}
294
295
592
bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
296
592
  // Zexts are free if they can be combined with a load.
297
592
  if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
298
186
    EVT MemVT = LD->getMemoryVT();
299
186
    if ((MemVT == MVT::i8 || 
MemVT == MVT::i16145
||
300
186
         
(104
Subtarget.is64Bit()104
&&
MemVT == MVT::i3241
)) &&
301
186
        
(103
LD->getExtensionType() == ISD::NON_EXTLOAD103
||
302
103
         
LD->getExtensionType() == ISD::ZEXTLOAD0
))
303
103
      return true;
304
489
  }
305
489
306
489
  return TargetLowering::isZExtFree(Val, VT2);
307
489
}
308
309
150
bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
310
150
  return Subtarget.is64Bit() && 
SrcVT == MVT::i3284
&&
DstVT == MVT::i6436
;
311
150
}
312
313
1.12k
bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
314
1.12k
  return (VT == MVT::f32 && 
Subtarget.hasStdExtF()594
) ||
315
1.12k
         
(548
VT == MVT::f64548
&&
Subtarget.hasStdExtD()534
);
316
1.12k
}
317
318
// Changes the condition code and swaps operands if necessary, so the SetCC
319
// operation matches one of the comparisons supported directly in the RISC-V
320
// ISA.
321
242
static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
322
242
  switch (CC) {
323
242
  default:
324
94
    break;
325
242
  case ISD::SETGT:
326
148
  case ISD::SETLE:
327
148
  case ISD::SETUGT:
328
148
  case ISD::SETULE:
329
148
    CC = ISD::getSetCCSwappedOperands(CC);
330
148
    std::swap(LHS, RHS);
331
148
    break;
332
242
  }
333
242
}
334
335
// Return the RISC-V branch opcode that matches the given DAG integer
336
// condition code. The CondCode must be one of those supported by the RISC-V
337
// ISA (see normaliseSetCC).
338
348
static unsigned getBranchOpcodeForIntCondCode(ISD::CondCode CC) {
339
348
  switch (CC) {
340
348
  default:
341
0
    llvm_unreachable("Unsupported CondCode");
342
348
  case ISD::SETEQ:
343
57
    return RISCV::BEQ;
344
348
  case ISD::SETNE:
345
124
    return RISCV::BNE;
346
348
  case ISD::SETLT:
347
50
    return RISCV::BLT;
348
348
  case ISD::SETGE:
349
39
    return RISCV::BGE;
350
348
  case ISD::SETULT:
351
39
    return RISCV::BLTU;
352
348
  case ISD::SETUGE:
353
39
    return RISCV::BGEU;
354
348
  }
355
348
}
356
357
SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
358
2.06k
                                            SelectionDAG &DAG) const {
359
2.06k
  switch (Op.getOpcode()) {
360
2.06k
  default:
361
0
    report_fatal_error("unimplemented operand");
362
2.06k
  case ISD::GlobalAddress:
363
1.45k
    return lowerGlobalAddress(Op, DAG);
364
2.06k
  case ISD::BlockAddress:
365
3
    return lowerBlockAddress(Op, DAG);
366
2.06k
  case ISD::ConstantPool:
367
90
    return lowerConstantPool(Op, DAG);
368
2.06k
  case ISD::GlobalTLSAddress:
369
16
    return lowerGlobalTLSAddress(Op, DAG);
370
2.06k
  case ISD::SELECT:
371
403
    return lowerSELECT(Op, DAG);
372
2.06k
  case ISD::VASTART:
373
81
    return lowerVASTART(Op, DAG);
374
2.06k
  case ISD::FRAMEADDR:
375
6
    return lowerFRAMEADDR(Op, DAG);
376
2.06k
  case ISD::RETURNADDR:
377
4
    return lowerRETURNADDR(Op, DAG);
378
2.06k
  case ISD::SHL_PARTS:
379
3
    return lowerShiftLeftParts(Op, DAG);
380
2.06k
  case ISD::SRA_PARTS:
381
3
    return lowerShiftRightParts(Op, DAG, true);
382
2.06k
  case ISD::SRL_PARTS:
383
3
    return lowerShiftRightParts(Op, DAG, false);
384
2.06k
  case ISD::BITCAST: {
385
4
    assert(Subtarget.is64Bit() && Subtarget.hasStdExtF() &&
386
4
           "Unexpected custom legalisation");
387
4
    SDLoc DL(Op);
388
4
    SDValue Op0 = Op.getOperand(0);
389
4
    if (Op.getValueType() != MVT::f32 || Op0.getValueType() != MVT::i32)
390
0
      return SDValue();
391
4
    SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
392
4
    SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
393
4
    return FPConv;
394
4
  }
395
2.06k
  }
396
2.06k
}
397
398
static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
399
2.89k
                             SelectionDAG &DAG, unsigned Flags) {
400
2.89k
  return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
401
2.89k
}
402
403
static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
404
5
                             SelectionDAG &DAG, unsigned Flags) {
405
5
  return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
406
5
                                   Flags);
407
5
}
408
409
static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
410
179
                             SelectionDAG &DAG, unsigned Flags) {
411
179
  return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
412
179
                                   N->getOffset(), Flags);
413
179
}
414
415
template <class NodeTy>
416
SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
417
1.54k
                                     bool IsLocal) const {
418
1.54k
  SDLoc DL(N);
419
1.54k
  EVT Ty = getPointerTy(DAG.getDataLayout());
420
1.54k
421
1.54k
  if (isPositionIndependent()) {
422
4
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
423
4
    if (IsLocal)
424
2
      // Use PC-relative addressing to access the symbol. This generates the
425
2
      // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
426
2
      // %pcrel_lo(auipc)).
427
2
      return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
428
2
429
2
    // Use PC-relative addressing to access the GOT for this symbol, then load
430
2
    // the address from the GOT. This generates the pattern (PseudoLA sym),
431
2
    // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
432
2
    return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
433
2
  }
434
1.53k
435
1.53k
  switch (getTargetMachine().getCodeModel()) {
436
1.53k
  default:
437
0
    report_fatal_error("Unsupported code model for lowering");
438
1.53k
  case CodeModel::Small: {
439
1.53k
    // Generate a sequence for accessing addresses within the first 2 GiB of
440
1.53k
    // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
441
1.53k
    SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
442
1.53k
    SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
443
1.53k
    SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
444
1.53k
    return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
445
1.53k
  }
446
1.53k
  case CodeModel::Medium: {
447
4
    // Generate a sequence for accessing addresses within any 2GiB range within
448
4
    // the address space. This generates the pattern (PseudoLLA sym), which
449
4
    // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
450
4
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
451
4
    return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
452
1.53k
  }
453
1.53k
  }
454
1.53k
}
llvm::SDValue llvm::RISCVTargetLowering::getAddr<llvm::GlobalAddressSDNode>(llvm::GlobalAddressSDNode*, llvm::SelectionDAG&, bool) const
Line
Count
Source
417
1.45k
                                     bool IsLocal) const {
418
1.45k
  SDLoc DL(N);
419
1.45k
  EVT Ty = getPointerTy(DAG.getDataLayout());
420
1.45k
421
1.45k
  if (isPositionIndependent()) {
422
4
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
423
4
    if (IsLocal)
424
2
      // Use PC-relative addressing to access the symbol. This generates the
425
2
      // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
426
2
      // %pcrel_lo(auipc)).
427
2
      return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
428
2
429
2
    // Use PC-relative addressing to access the GOT for this symbol, then load
430
2
    // the address from the GOT. This generates the pattern (PseudoLA sym),
431
2
    // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
432
2
    return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
433
2
  }
434
1.44k
435
1.44k
  switch (getTargetMachine().getCodeModel()) {
436
1.44k
  default:
437
0
    report_fatal_error("Unsupported code model for lowering");
438
1.44k
  case CodeModel::Small: {
439
1.44k
    // Generate a sequence for accessing addresses within the first 2 GiB of
440
1.44k
    // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
441
1.44k
    SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
442
1.44k
    SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
443
1.44k
    SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
444
1.44k
    return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
445
1.44k
  }
446
1.44k
  case CodeModel::Medium: {
447
2
    // Generate a sequence for accessing addresses within any 2GiB range within
448
2
    // the address space. This generates the pattern (PseudoLLA sym), which
449
2
    // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
450
2
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
451
2
    return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
452
1.44k
  }
453
1.44k
  }
454
1.44k
}
llvm::SDValue llvm::RISCVTargetLowering::getAddr<llvm::BlockAddressSDNode>(llvm::BlockAddressSDNode*, llvm::SelectionDAG&, bool) const
Line
Count
Source
417
3
                                     bool IsLocal) const {
418
3
  SDLoc DL(N);
419
3
  EVT Ty = getPointerTy(DAG.getDataLayout());
420
3
421
3
  if (isPositionIndependent()) {
422
0
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
423
0
    if (IsLocal)
424
0
      // Use PC-relative addressing to access the symbol. This generates the
425
0
      // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
426
0
      // %pcrel_lo(auipc)).
427
0
      return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
428
0
429
0
    // Use PC-relative addressing to access the GOT for this symbol, then load
430
0
    // the address from the GOT. This generates the pattern (PseudoLA sym),
431
0
    // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
432
0
    return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
433
0
  }
434
3
435
3
  switch (getTargetMachine().getCodeModel()) {
436
3
  default:
437
0
    report_fatal_error("Unsupported code model for lowering");
438
3
  case CodeModel::Small: {
439
2
    // Generate a sequence for accessing addresses within the first 2 GiB of
440
2
    // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
441
2
    SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
442
2
    SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
443
2
    SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
444
2
    return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
445
3
  }
446
3
  case CodeModel::Medium: {
447
1
    // Generate a sequence for accessing addresses within any 2GiB range within
448
1
    // the address space. This generates the pattern (PseudoLLA sym), which
449
1
    // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
450
1
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
451
1
    return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
452
3
  }
453
3
  }
454
3
}
llvm::SDValue llvm::RISCVTargetLowering::getAddr<llvm::ConstantPoolSDNode>(llvm::ConstantPoolSDNode*, llvm::SelectionDAG&, bool) const
Line
Count
Source
417
90
                                     bool IsLocal) const {
418
90
  SDLoc DL(N);
419
90
  EVT Ty = getPointerTy(DAG.getDataLayout());
420
90
421
90
  if (isPositionIndependent()) {
422
0
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
423
0
    if (IsLocal)
424
0
      // Use PC-relative addressing to access the symbol. This generates the
425
0
      // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
426
0
      // %pcrel_lo(auipc)).
427
0
      return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
428
0
429
0
    // Use PC-relative addressing to access the GOT for this symbol, then load
430
0
    // the address from the GOT. This generates the pattern (PseudoLA sym),
431
0
    // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
432
0
    return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
433
0
  }
434
90
435
90
  switch (getTargetMachine().getCodeModel()) {
436
90
  default:
437
0
    report_fatal_error("Unsupported code model for lowering");
438
90
  case CodeModel::Small: {
439
89
    // Generate a sequence for accessing addresses within the first 2 GiB of
440
89
    // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
441
89
    SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
442
89
    SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
443
89
    SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
444
89
    return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
445
90
  }
446
90
  case CodeModel::Medium: {
447
1
    // Generate a sequence for accessing addresses within any 2GiB range within
448
1
    // the address space. This generates the pattern (PseudoLLA sym), which
449
1
    // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
450
1
    SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
451
1
    return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
452
90
  }
453
90
  }
454
90
}
455
456
SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
457
1.45k
                                                SelectionDAG &DAG) const {
458
1.45k
  SDLoc DL(Op);
459
1.45k
  EVT Ty = Op.getValueType();
460
1.45k
  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
461
1.45k
  int64_t Offset = N->getOffset();
462
1.45k
  MVT XLenVT = Subtarget.getXLenVT();
463
1.45k
464
1.45k
  const GlobalValue *GV = N->getGlobal();
465
1.45k
  bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
466
1.45k
  SDValue Addr = getAddr(N, DAG, IsLocal);
467
1.45k
468
1.45k
  // In order to maximise the opportunity for common subexpression elimination,
469
1.45k
  // emit a separate ADD node for the global address offset instead of folding
470
1.45k
  // it in the global address node. Later peephole optimisations may choose to
471
1.45k
  // fold it back in when profitable.
472
1.45k
  if (Offset != 0)
473
1.28k
    return DAG.getNode(ISD::ADD, DL, Ty, Addr,
474
1.28k
                       DAG.getConstant(Offset, DL, XLenVT));
475
161
  return Addr;
476
161
}
477
478
SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
479
3
                                               SelectionDAG &DAG) const {
480
3
  BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
481
3
482
3
  return getAddr(N, DAG);
483
3
}
484
485
SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
486
90
                                               SelectionDAG &DAG) const {
487
90
  ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
488
90
489
90
  return getAddr(N, DAG);
490
90
}
491
492
SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
493
                                              SelectionDAG &DAG,
494
12
                                              bool UseGOT) const {
495
12
  SDLoc DL(N);
496
12
  EVT Ty = getPointerTy(DAG.getDataLayout());
497
12
  const GlobalValue *GV = N->getGlobal();
498
12
  MVT XLenVT = Subtarget.getXLenVT();
499
12
500
12
  if (UseGOT) {
501
2
    // Use PC-relative addressing to access the GOT for this TLS symbol, then
502
2
    // load the address from the GOT and add the thread pointer. This generates
503
2
    // the pattern (PseudoLA_TLS_IE sym), which expands to
504
2
    // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
505
2
    SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
506
2
    SDValue Load =
507
2
        SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
508
2
509
2
    // Add the thread pointer.
510
2
    SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
511
2
    return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
512
2
  }
513
10
514
10
  // Generate a sequence for accessing the address relative to the thread
515
10
  // pointer, with the appropriate adjustment for the thread pointer offset.
516
10
  // This generates the pattern
517
10
  // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
518
10
  SDValue AddrHi =
519
10
      DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
520
10
  SDValue AddrAdd =
521
10
      DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
522
10
  SDValue AddrLo =
523
10
      DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
524
10
525
10
  SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
526
10
  SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
527
10
  SDValue MNAdd = SDValue(
528
10
      DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
529
10
      0);
530
10
  return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
531
10
}
532
533
SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
534
4
                                               SelectionDAG &DAG) const {
535
4
  SDLoc DL(N);
536
4
  EVT Ty = getPointerTy(DAG.getDataLayout());
537
4
  IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
538
4
  const GlobalValue *GV = N->getGlobal();
539
4
540
4
  // Use a PC-relative addressing mode to access the global dynamic GOT address.
541
4
  // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
542
4
  // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
543
4
  SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
544
4
  SDValue Load =
545
4
      SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
546
4
547
4
  // Prepare argument list to generate call.
548
4
  ArgListTy Args;
549
4
  ArgListEntry Entry;
550
4
  Entry.Node = Load;
551
4
  Entry.Ty = CallTy;
552
4
  Args.push_back(Entry);
553
4
554
4
  // Setup call to __tls_get_addr.
555
4
  TargetLowering::CallLoweringInfo CLI(DAG);
556
4
  CLI.setDebugLoc(DL)
557
4
      .setChain(DAG.getEntryNode())
558
4
      .setLibCallee(CallingConv::C, CallTy,
559
4
                    DAG.getExternalSymbol("__tls_get_addr", Ty),
560
4
                    std::move(Args));
561
4
562
4
  return LowerCallTo(CLI).first;
563
4
}
564
565
SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
566
16
                                                   SelectionDAG &DAG) const {
567
16
  SDLoc DL(Op);
568
16
  EVT Ty = Op.getValueType();
569
16
  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
570
16
  int64_t Offset = N->getOffset();
571
16
  MVT XLenVT = Subtarget.getXLenVT();
572
16
573
16
  // Non-PIC TLS lowering should always use the LocalExec model.
574
16
  TLSModel::Model Model = isPositionIndependent()
575
16
                              ? 
getTargetMachine().getTLSModel(N->getGlobal())8
576
16
                              : 
TLSModel::LocalExec8
;
577
16
578
16
  SDValue Addr;
579
16
  switch (Model) {
580
16
  case TLSModel::LocalExec:
581
10
    Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
582
10
    break;
583
16
  case TLSModel::InitialExec:
584
2
    Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
585
2
    break;
586
16
  case TLSModel::LocalDynamic:
587
4
  case TLSModel::GeneralDynamic:
588
4
    Addr = getDynamicTLSAddr(N, DAG);
589
4
    break;
590
16
  }
591
16
592
16
  // In order to maximise the opportunity for common subexpression elimination,
593
16
  // emit a separate ADD node for the global address offset instead of folding
594
16
  // it in the global address node. Later peephole optimisations may choose to
595
16
  // fold it back in when profitable.
596
16
  if (Offset != 0)
597
0
    return DAG.getNode(ISD::ADD, DL, Ty, Addr,
598
0
                       DAG.getConstant(Offset, DL, XLenVT));
599
16
  return Addr;
600
16
}
601
602
403
SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
603
403
  SDValue CondV = Op.getOperand(0);
604
403
  SDValue TrueV = Op.getOperand(1);
605
403
  SDValue FalseV = Op.getOperand(2);
606
403
  SDLoc DL(Op);
607
403
  MVT XLenVT = Subtarget.getXLenVT();
608
403
609
403
  // If the result type is XLenVT and CondV is the output of a SETCC node
610
403
  // which also operated on XLenVT inputs, then merge the SETCC node into the
611
403
  // lowered RISCVISD::SELECT_CC to take advantage of the integer
612
403
  // compare+branch instructions. i.e.:
613
403
  // (select (setcc lhs, rhs, cc), truev, falsev)
614
403
  // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
615
403
  if (Op.getSimpleValueType() == XLenVT && 
CondV.getOpcode() == ISD::SETCC341
&&
616
403
      
CondV.getOperand(0).getSimpleValueType() == XLenVT246
) {
617
242
    SDValue LHS = CondV.getOperand(0);
618
242
    SDValue RHS = CondV.getOperand(1);
619
242
    auto CC = cast<CondCodeSDNode>(CondV.getOperand(2));
620
242
    ISD::CondCode CCVal = CC->get();
621
242
622
242
    normaliseSetCC(LHS, RHS, CCVal);
623
242
624
242
    SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT);
625
242
    SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
626
242
    SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
627
242
    return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
628
242
  }
629
161
630
161
  // Otherwise:
631
161
  // (select condv, truev, falsev)
632
161
  // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
633
161
  SDValue Zero = DAG.getConstant(0, DL, XLenVT);
634
161
  SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT);
635
161
636
161
  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
637
161
  SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
638
161
639
161
  return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
640
161
}
641
642
81
SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
643
81
  MachineFunction &MF = DAG.getMachineFunction();
644
81
  RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
645
81
646
81
  SDLoc DL(Op);
647
81
  SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
648
81
                                 getPointerTy(MF.getDataLayout()));
649
81
650
81
  // vastart just stores the address of the VarArgsFrameIndex slot into the
651
81
  // memory location argument.
652
81
  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
653
81
  return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
654
81
                      MachinePointerInfo(SV));
655
81
}
656
657
SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
658
8
                                            SelectionDAG &DAG) const {
659
8
  const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
660
8
  MachineFunction &MF = DAG.getMachineFunction();
661
8
  MachineFrameInfo &MFI = MF.getFrameInfo();
662
8
  MFI.setFrameAddressIsTaken(true);
663
8
  unsigned FrameReg = RI.getFrameRegister(MF);
664
8
  int XLenInBytes = Subtarget.getXLen() / 8;
665
8
666
8
  EVT VT = Op.getValueType();
667
8
  SDLoc DL(Op);
668
8
  SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
669
8
  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
670
22
  while (Depth--) {
671
14
    int Offset = -(XLenInBytes * 2);
672
14
    SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
673
14
                              DAG.getIntPtrConstant(Offset, DL));
674
14
    FrameAddr =
675
14
        DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
676
14
  }
677
8
  return FrameAddr;
678
8
}
679
680
SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
681
4
                                             SelectionDAG &DAG) const {
682
4
  const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
683
4
  MachineFunction &MF = DAG.getMachineFunction();
684
4
  MachineFrameInfo &MFI = MF.getFrameInfo();
685
4
  MFI.setReturnAddressIsTaken(true);
686
4
  MVT XLenVT = Subtarget.getXLenVT();
687
4
  int XLenInBytes = Subtarget.getXLen() / 8;
688
4
689
4
  if (verifyReturnAddressArgumentIsConstant(Op, DAG))
690
0
    return SDValue();
691
4
692
4
  EVT VT = Op.getValueType();
693
4
  SDLoc DL(Op);
694
4
  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
695
4
  if (Depth) {
696
2
    int Off = -XLenInBytes;
697
2
    SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
698
2
    SDValue Offset = DAG.getConstant(Off, DL, VT);
699
2
    return DAG.getLoad(VT, DL, DAG.getEntryNode(),
700
2
                       DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
701
2
                       MachinePointerInfo());
702
2
  }
703
2
704
2
  // Return the value of the return address register, marking it an implicit
705
2
  // live-in.
706
2
  unsigned Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
707
2
  return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
708
2
}
709
710
SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
711
3
                                                 SelectionDAG &DAG) const {
712
3
  SDLoc DL(Op);
713
3
  SDValue Lo = Op.getOperand(0);
714
3
  SDValue Hi = Op.getOperand(1);
715
3
  SDValue Shamt = Op.getOperand(2);
716
3
  EVT VT = Lo.getValueType();
717
3
718
3
  // if Shamt-XLEN < 0: // Shamt < XLEN
719
3
  //   Lo = Lo << Shamt
720
3
  //   Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
721
3
  // else:
722
3
  //   Lo = 0
723
3
  //   Hi = Lo << (Shamt-XLEN)
724
3
725
3
  SDValue Zero = DAG.getConstant(0, DL, VT);
726
3
  SDValue One = DAG.getConstant(1, DL, VT);
727
3
  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
728
3
  SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
729
3
  SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
730
3
  SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
731
3
732
3
  SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
733
3
  SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
734
3
  SDValue ShiftRightLo =
735
3
      DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
736
3
  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
737
3
  SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
738
3
  SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
739
3
740
3
  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
741
3
742
3
  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
743
3
  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
744
3
745
3
  SDValue Parts[2] = {Lo, Hi};
746
3
  return DAG.getMergeValues(Parts, DL);
747
3
}
748
749
SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
750
6
                                                  bool IsSRA) const {
751
6
  SDLoc DL(Op);
752
6
  SDValue Lo = Op.getOperand(0);
753
6
  SDValue Hi = Op.getOperand(1);
754
6
  SDValue Shamt = Op.getOperand(2);
755
6
  EVT VT = Lo.getValueType();
756
6
757
6
  // SRA expansion:
758
6
  //   if Shamt-XLEN < 0: // Shamt < XLEN
759
6
  //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
760
6
  //     Hi = Hi >>s Shamt
761
6
  //   else:
762
6
  //     Lo = Hi >>s (Shamt-XLEN);
763
6
  //     Hi = Hi >>s (XLEN-1)
764
6
  //
765
6
  // SRL expansion:
766
6
  //   if Shamt-XLEN < 0: // Shamt < XLEN
767
6
  //     Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
768
6
  //     Hi = Hi >>u Shamt
769
6
  //   else:
770
6
  //     Lo = Hi >>u (Shamt-XLEN);
771
6
  //     Hi = 0;
772
6
773
6
  unsigned ShiftRightOp = IsSRA ? 
ISD::SRA3
:
ISD::SRL3
;
774
6
775
6
  SDValue Zero = DAG.getConstant(0, DL, VT);
776
6
  SDValue One = DAG.getConstant(1, DL, VT);
777
6
  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
778
6
  SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
779
6
  SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
780
6
  SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
781
6
782
6
  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
783
6
  SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
784
6
  SDValue ShiftLeftHi =
785
6
      DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
786
6
  SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
787
6
  SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
788
6
  SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
789
6
  SDValue HiFalse =
790
6
      IsSRA ? 
DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1)3
:
Zero3
;
791
6
792
6
  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
793
6
794
6
  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
795
6
  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
796
6
797
6
  SDValue Parts[2] = {Lo, Hi};
798
6
  return DAG.getMergeValues(Parts, DL);
799
6
}
800
801
// Returns the opcode of the target-specific SDNode that implements the 32-bit
802
// form of the given Opcode.
803
542
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
804
542
  switch (Opcode) {
805
542
  default:
806
0
    llvm_unreachable("Unexpected opcode");
807
542
  case ISD::SHL:
808
289
    return RISCVISD::SLLW;
809
542
  case ISD::SRA:
810
29
    return RISCVISD::SRAW;
811
542
  case ISD::SRL:
812
140
    return RISCVISD::SRLW;
813
542
  case ISD::SDIV:
814
28
    return RISCVISD::DIVW;
815
542
  case ISD::UDIV:
816
28
    return RISCVISD::DIVUW;
817
542
  case ISD::UREM:
818
28
    return RISCVISD::REMUW;
819
542
  }
820
542
}
821
822
// Converts the given 32-bit operation to a target-specific SelectionDAG node.
823
// Because i32 isn't a legal type for RV64, these operations would otherwise
824
// be promoted to i64, making it difficult to select the SLLW/DIVUW/.../*W
825
// later one because the fact the operation was originally of type i32 is
826
// lost.
827
542
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG) {
828
542
  SDLoc DL(N);
829
542
  RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
830
542
  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
831
542
  SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
832
542
  SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
833
542
  // ReplaceNodeResults requires we maintain the same type for the return value.
834
542
  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
835
542
}
836
837
void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
838
                                             SmallVectorImpl<SDValue> &Results,
839
592
                                             SelectionDAG &DAG) const {
840
592
  SDLoc DL(N);
841
592
  switch (N->getOpcode()) {
842
592
  default:
843
0
    llvm_unreachable("Don't know how to custom type legalize this operation!");
844
592
  case ISD::READCYCLECOUNTER: {
845
1
    assert(!Subtarget.is64Bit() &&
846
1
           "READCYCLECOUNTER only has custom type legalization on riscv32");
847
1
848
1
    SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
849
1
    SDValue RCW =
850
1
        DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
851
1
852
1
    Results.push_back(RCW);
853
1
    Results.push_back(RCW.getValue(1));
854
1
    Results.push_back(RCW.getValue(2));
855
1
    break;
856
592
  }
857
592
  case ISD::SHL:
858
501
  case ISD::SRA:
859
501
  case ISD::SRL:
860
501
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
861
501
           "Unexpected custom legalisation");
862
501
    if (N->getOperand(1).getOpcode() == ISD::Constant)
863
43
      return;
864
458
    Results.push_back(customLegalizeToWOp(N, DAG));
865
458
    break;
866
458
  case ISD::SDIV:
867
86
  case ISD::UDIV:
868
86
  case ISD::UREM:
869
86
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
870
86
           Subtarget.hasStdExtM() && "Unexpected custom legalisation");
871
86
    if (N->getOperand(0).getOpcode() == ISD::Constant ||
872
86
        N->getOperand(1).getOpcode() == ISD::Constant)
873
2
      return;
874
84
    Results.push_back(customLegalizeToWOp(N, DAG));
875
84
    break;
876
84
  case ISD::BITCAST: {
877
4
    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
878
4
           Subtarget.hasStdExtF() && "Unexpected custom legalisation");
879
4
    SDLoc DL(N);
880
4
    SDValue Op0 = N->getOperand(0);
881
4
    if (Op0.getValueType() != MVT::f32)
882
0
      return;
883
4
    SDValue FPConv =
884
4
        DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
885
4
    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
886
4
    break;
887
4
  }
888
592
  }
889
592
}
890
891
SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
892
14.4k
                                               DAGCombinerInfo &DCI) const {
893
14.4k
  SelectionDAG &DAG = DCI.DAG;
894
14.4k
895
14.4k
  switch (N->getOpcode()) {
896
14.4k
  default:
897
13.0k
    break;
898
14.4k
  case RISCVISD::SplitF64: {
899
212
    SDValue Op0 = N->getOperand(0);
900
212
    // If the input to SplitF64 is just BuildPairF64 then the operation is
901
212
    // redundant. Instead, use BuildPairF64's operands directly.
902
212
    if (Op0->getOpcode() == RISCVISD::BuildPairF64)
903
48
      return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
904
164
905
164
    SDLoc DL(N);
906
164
907
164
    // It's cheaper to materialise two 32-bit integers than to load a double
908
164
    // from the constant pool and transfer it to integer registers through the
909
164
    // stack.
910
164
    if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
911
17
      APInt V = C->getValueAPF().bitcastToAPInt();
912
17
      SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
913
17
      SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
914
17
      return DCI.CombineTo(N, Lo, Hi);
915
17
    }
916
147
917
147
    // This is a target-specific version of a DAGCombine performed in
918
147
    // DAGCombiner::visitBITCAST. It performs the equivalent of:
919
147
    // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
920
147
    // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
921
147
    if (!(Op0.getOpcode() == ISD::FNEG || 
Op0.getOpcode() == ISD::FABS146
) ||
922
147
        
!Op0.getNode()->hasOneUse()3
)
923
144
      break;
924
3
    SDValue NewSplitF64 =
925
3
        DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
926
3
                    Op0.getOperand(0));
927
3
    SDValue Lo = NewSplitF64.getValue(0);
928
3
    SDValue Hi = NewSplitF64.getValue(1);
929
3
    APInt SignBit = APInt::getSignMask(32);
930
3
    if (Op0.getOpcode() == ISD::FNEG) {
931
1
      SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
932
1
                                  DAG.getConstant(SignBit, DL, MVT::i32));
933
1
      return DCI.CombineTo(N, Lo, NewHi);
934
1
    }
935
2
    assert(Op0.getOpcode() == ISD::FABS);
936
2
    SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
937
2
                                DAG.getConstant(~SignBit, DL, MVT::i32));
938
2
    return DCI.CombineTo(N, Lo, NewHi);
939
2
  }
940
974
  case RISCVISD::SLLW:
941
974
  case RISCVISD::SRAW:
942
974
  case RISCVISD::SRLW: {
943
974
    // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
944
974
    SDValue LHS = N->getOperand(0);
945
974
    SDValue RHS = N->getOperand(1);
946
974
    APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
947
974
    APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 5);
948
974
    if ((SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI)) ||
949
974
        (SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI)))
950
58
      return SDValue();
951
916
    break;
952
916
  }
953
916
  case RISCVISD::FMV_X_ANYEXTW_RV64: {
954
283
    SDLoc DL(N);
955
283
    SDValue Op0 = N->getOperand(0);
956
283
    // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
957
283
    // conversion is unnecessary and can be replaced with an ANY_EXTEND
958
283
    // of the FMV_W_X_RV64 operand.
959
283
    if (Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) {
960
70
      SDValue AExtOp =
961
70
          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0.getOperand(0));
962
70
      return DCI.CombineTo(N, AExtOp);
963
70
    }
964
213
965
213
    // This is a target-specific version of a DAGCombine performed in
966
213
    // DAGCombiner::visitBITCAST. It performs the equivalent of:
967
213
    // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
968
213
    // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
969
213
    if (!(Op0.getOpcode() == ISD::FNEG || 
Op0.getOpcode() == ISD::FABS212
) ||
970
213
        
!Op0.getNode()->hasOneUse()4
)
971
209
      break;
972
4
    SDValue NewFMV = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64,
973
4
                                 Op0.getOperand(0));
974
4
    APInt SignBit = APInt::getSignMask(32).sext(64);
975
4
    if (Op0.getOpcode() == ISD::FNEG) {
976
1
      return DCI.CombineTo(N,
977
1
                           DAG.getNode(ISD::XOR, DL, MVT::i64, NewFMV,
978
1
                                       DAG.getConstant(SignBit, DL, MVT::i64)));
979
1
    }
980
3
    assert(Op0.getOpcode() == ISD::FABS);
981
3
    return DCI.CombineTo(N,
982
3
                         DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV,
983
3
                                     DAG.getConstant(~SignBit, DL, MVT::i64)));
984
3
  }
985
14.2k
  }
986
14.2k
987
14.2k
  return SDValue();
988
14.2k
}
989
990
bool RISCVTargetLowering::isDesirableToCommuteWithShift(
991
274
    const SDNode *N, CombineLevel Level) const {
992
274
  // The following folds are only desirable if `(OP _, c1 << c2)` can be
993
274
  // materialised in fewer instructions than `(OP _, c1)`:
994
274
  //
995
274
  //   (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
996
274
  //   (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
997
274
  SDValue N0 = N->getOperand(0);
998
274
  EVT Ty = N0.getValueType();
999
274
  if (Ty.isScalarInteger() &&
1000
274
      (N0.getOpcode() == ISD::ADD || 
N0.getOpcode() == ISD::OR262
)) {
1001
12
    auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
1002
12
    auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
1003
12
    if (C1 && C2) {
1004
12
      APInt C1Int = C1->getAPIntValue();
1005
12
      APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
1006
12
1007
12
      // We can materialise `c1 << c2` into an add immediate, so it's "free",
1008
12
      // and the combine should happen, to potentially allow further combines
1009
12
      // later.
1010
12
      if (isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
1011
0
        return true;
1012
12
1013
12
      // We can materialise `c1` in an add immediate, so it's "free", and the
1014
12
      // combine should be prevented.
1015
12
      if (isLegalAddImmediate(C1Int.getSExtValue()))
1016
8
        return false;
1017
4
1018
4
      // Neither constant will fit into an immediate, so find materialisation
1019
4
      // costs.
1020
4
      int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
1021
4
                                              Subtarget.is64Bit());
1022
4
      int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
1023
4
          ShiftedC1Int, Ty.getSizeInBits(), Subtarget.is64Bit());
1024
4
1025
4
      // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
1026
4
      // combine should be prevented.
1027
4
      if (C1Cost < ShiftedC1Cost)
1028
2
        return false;
1029
264
    }
1030
12
  }
1031
264
  return true;
1032
264
}
1033
1034
unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
1035
    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1036
283
    unsigned Depth) const {
1037
283
  switch (Op.getOpcode()) {
1038
283
  default:
1039
4
    break;
1040
283
  case RISCVISD::SLLW:
1041
279
  case RISCVISD::SRAW:
1042
279
  case RISCVISD::SRLW:
1043
279
  case RISCVISD::DIVW:
1044
279
  case RISCVISD::DIVUW:
1045
279
  case RISCVISD::REMUW:
1046
279
    // TODO: As the result is sign-extended, this is conservatively correct. A
1047
279
    // more precise answer could be calculated for SRAW depending on known
1048
279
    // bits in the shift amount.
1049
279
    return 33;
1050
4
  }
1051
4
1052
4
  return 1;
1053
4
}
1054
1055
MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
1056
1
                                           MachineBasicBlock *BB) {
1057
1
  assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
1058
1
1059
1
  // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
1060
1
  // Should the count have wrapped while it was being read, we need to try
1061
1
  // again.
1062
1
  // ...
1063
1
  // read:
1064
1
  // rdcycleh x3 # load high word of cycle
1065
1
  // rdcycle  x2 # load low word of cycle
1066
1
  // rdcycleh x4 # load high word of cycle
1067
1
  // bne x3, x4, read # check if high word reads match, otherwise try again
1068
1
  // ...
1069
1
1070
1
  MachineFunction &MF = *BB->getParent();
1071
1
  const BasicBlock *LLVM_BB = BB->getBasicBlock();
1072
1
  MachineFunction::iterator It = ++BB->getIterator();
1073
1
1074
1
  MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
1075
1
  MF.insert(It, LoopMBB);
1076
1
1077
1
  MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
1078
1
  MF.insert(It, DoneMBB);
1079
1
1080
1
  // Transfer the remainder of BB and its successor edges to DoneMBB.
1081
1
  DoneMBB->splice(DoneMBB->begin(), BB,
1082
1
                  std::next(MachineBasicBlock::iterator(MI)), BB->end());
1083
1
  DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
1084
1
1085
1
  BB->addSuccessor(LoopMBB);
1086
1
1087
1
  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1088
1
  unsigned ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1089
1
  unsigned LoReg = MI.getOperand(0).getReg();
1090
1
  unsigned HiReg = MI.getOperand(1).getReg();
1091
1
  DebugLoc DL = MI.getDebugLoc();
1092
1
1093
1
  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1094
1
  BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
1095
1
      .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
1096
1
      .addReg(RISCV::X0);
1097
1
  BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
1098
1
      .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
1099
1
      .addReg(RISCV::X0);
1100
1
  BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
1101
1
      .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
1102
1
      .addReg(RISCV::X0);
1103
1
1104
1
  BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
1105
1
      .addReg(HiReg)
1106
1
      .addReg(ReadAgainReg)
1107
1
      .addMBB(LoopMBB);
1108
1
1109
1
  LoopMBB->addSuccessor(LoopMBB);
1110
1
  LoopMBB->addSuccessor(DoneMBB);
1111
1
1112
1
  MI.eraseFromParent();
1113
1
1114
1
  return DoneMBB;
1115
1
}
1116
1117
static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
1118
55
                                             MachineBasicBlock *BB) {
1119
55
  assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
1120
55
1121
55
  MachineFunction &MF = *BB->getParent();
1122
55
  DebugLoc DL = MI.getDebugLoc();
1123
55
  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1124
55
  const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1125
55
  unsigned LoReg = MI.getOperand(0).getReg();
1126
55
  unsigned HiReg = MI.getOperand(1).getReg();
1127
55
  unsigned SrcReg = MI.getOperand(2).getReg();
1128
55
  const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
1129
55
  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
1130
55
1131
55
  TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
1132
55
                          RI);
1133
55
  MachineMemOperand *MMO =
1134
55
      MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
1135
55
                              MachineMemOperand::MOLoad, 8, 8);
1136
55
  BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
1137
55
      .addFrameIndex(FI)
1138
55
      .addImm(0)
1139
55
      .addMemOperand(MMO);
1140
55
  BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
1141
55
      .addFrameIndex(FI)
1142
55
      .addImm(4)
1143
55
      .addMemOperand(MMO);
1144
55
  MI.eraseFromParent(); // The pseudo instruction is gone now.
1145
55
  return BB;
1146
55
}
1147
1148
static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
1149
175
                                                 MachineBasicBlock *BB) {
1150
175
  assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
1151
175
         "Unexpected instruction");
1152
175
1153
175
  MachineFunction &MF = *BB->getParent();
1154
175
  DebugLoc DL = MI.getDebugLoc();
1155
175
  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1156
175
  const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1157
175
  unsigned DstReg = MI.getOperand(0).getReg();
1158
175
  unsigned LoReg = MI.getOperand(1).getReg();
1159
175
  unsigned HiReg = MI.getOperand(2).getReg();
1160
175
  const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
1161
175
  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
1162
175
1163
175
  MachineMemOperand *MMO =
1164
175
      MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
1165
175
                              MachineMemOperand::MOStore, 8, 8);
1166
175
  BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
1167
175
      .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
1168
175
      .addFrameIndex(FI)
1169
175
      .addImm(0)
1170
175
      .addMemOperand(MMO);
1171
175
  BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
1172
175
      .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
1173
175
      .addFrameIndex(FI)
1174
175
      .addImm(4)
1175
175
      .addMemOperand(MMO);
1176
175
  TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
1177
175
  MI.eraseFromParent(); // The pseudo instruction is gone now.
1178
175
  return BB;
1179
175
}
1180
1181
1.61k
static bool isSelectPseudo(MachineInstr &MI) {
1182
1.61k
  switch (MI.getOpcode()) {
1183
1.61k
  default:
1184
786
    return false;
1185
1.61k
  case RISCV::Select_GPR_Using_CC_GPR:
1186
826
  case RISCV::Select_FPR32_Using_CC_GPR:
1187
826
  case RISCV::Select_FPR64_Using_CC_GPR:
1188
826
    return true;
1189
1.61k
  }
1190
1.61k
}
1191
1192
static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
1193
348
                                           MachineBasicBlock *BB) {
1194
348
  // To "insert" Select_* instructions, we actually have to insert the triangle
1195
348
  // control-flow pattern.  The incoming instructions know the destination vreg
1196
348
  // to set, the condition code register to branch on, the true/false values to
1197
348
  // select between, and the condcode to use to select the appropriate branch.
1198
348
  //
1199
348
  // We produce the following control flow:
1200
348
  //     HeadMBB
1201
348
  //     |  \
1202
348
  //     |  IfFalseMBB
1203
348
  //     | /
1204
348
  //    TailMBB
1205
348
  //
1206
348
  // When we find a sequence of selects we attempt to optimize their emission
1207
348
  // by sharing the control flow. Currently we only handle cases where we have
1208
348
  // multiple selects with the exact same condition (same LHS, RHS and CC).
1209
348
  // The selects may be interleaved with other instructions if the other
1210
348
  // instructions meet some requirements we deem safe:
1211
348
  // - They are debug instructions. Otherwise,
1212
348
  // - They do not have side-effects, do not access memory and their inputs do
1213
348
  //   not depend on the results of the select pseudo-instructions.
1214
348
  // The TrueV/FalseV operands of the selects cannot depend on the result of
1215
348
  // previous selects in the sequence.
1216
348
  // These conditions could be further relaxed. See the X86 target for a
1217
348
  // related approach and more information.
1218
348
  unsigned LHS = MI.getOperand(1).getReg();
1219
348
  unsigned RHS = MI.getOperand(2).getReg();
1220
348
  auto CC = static_cast<ISD::CondCode>(MI.getOperand(3).getImm());
1221
348
1222
348
  SmallVector<MachineInstr *, 4> SelectDebugValues;
1223
348
  SmallSet<unsigned, 4> SelectDests;
1224
348
  SelectDests.insert(MI.getOperand(0).getReg());
1225
348
1226
348
  MachineInstr *LastSelectPseudo = &MI;
1227
348
1228
348
  for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
1229
1.17k
       SequenceMBBI != E; 
++SequenceMBBI831
) {
1230
1.17k
    if (SequenceMBBI->isDebugInstr())
1231
6
      continue;
1232
1.17k
    else if (isSelectPseudo(*SequenceMBBI)) {
1233
415
      if (SequenceMBBI->getOperand(1).getReg() != LHS ||
1234
415
          
SequenceMBBI->getOperand(2).getReg() != RHS413
||
1235
415
          
SequenceMBBI->getOperand(3).getImm() != CC413
||
1236
415
          
SelectDests.count(SequenceMBBI->getOperand(4).getReg())413
||
1237
415
          
SelectDests.count(SequenceMBBI->getOperand(5).getReg())411
)
1238
4
        break;
1239
411
      LastSelectPseudo = &*SequenceMBBI;
1240
411
      SequenceMBBI->collectDebugValues(SelectDebugValues);
1241
411
      SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
1242
758
    } else {
1243
758
      if (SequenceMBBI->hasUnmodeledSideEffects() ||
1244
758
          SequenceMBBI->mayLoadOrStore())
1245
201
        break;
1246
1.20k
      
if (557
llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) 557
{
1247
1.20k
            return MO.isReg() && 
MO.isUse()842
&&
SelectDests.count(MO.getReg())412
;
1248
1.20k
          }))
1249
143
        break;
1250
557
    }
1251
1.17k
  }
1252
348
1253
348
  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
1254
348
  const BasicBlock *LLVM_BB = BB->getBasicBlock();
1255
348
  DebugLoc DL = MI.getDebugLoc();
1256
348
  MachineFunction::iterator I = ++BB->getIterator();
1257
348
1258
348
  MachineBasicBlock *HeadMBB = BB;
1259
348
  MachineFunction *F = BB->getParent();
1260
348
  MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
1261
348
  MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
1262
348
1263
348
  F->insert(I, IfFalseMBB);
1264
348
  F->insert(I, TailMBB);
1265
348
1266
348
  // Transfer debug instructions associated with the selects to TailMBB.
1267
348
  for (MachineInstr *DebugInstr : SelectDebugValues) {
1268
4
    TailMBB->push_back(DebugInstr->removeFromParent());
1269
4
  }
1270
348
1271
348
  // Move all instructions after the sequence to TailMBB.
1272
348
  TailMBB->splice(TailMBB->end(), HeadMBB,
1273
348
                  std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
1274
348
  // Update machine-CFG edges by transferring all successors of the current
1275
348
  // block to the new block which will contain the Phi nodes for the selects.
1276
348
  TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
1277
348
  // Set the successors for HeadMBB.
1278
348
  HeadMBB->addSuccessor(IfFalseMBB);
1279
348
  HeadMBB->addSuccessor(TailMBB);
1280
348
1281
348
  // Insert appropriate branch.
1282
348
  unsigned Opcode = getBranchOpcodeForIntCondCode(CC);
1283
348
1284
348
  BuildMI(HeadMBB, DL, TII.get(Opcode))
1285
348
    .addReg(LHS)
1286
348
    .addReg(RHS)
1287
348
    .addMBB(TailMBB);
1288
348
1289
348
  // IfFalseMBB just falls through to TailMBB.
1290
348
  IfFalseMBB->addSuccessor(TailMBB);
1291
348
1292
348
  // Create PHIs for all of the select pseudo-instructions.
1293
348
  auto SelectMBBI = MI.getIterator();
1294
348
  auto SelectEnd = std::next(LastSelectPseudo->getIterator());
1295
348
  auto InsertionPoint = TailMBB->begin();
1296
787
  while (SelectMBBI != SelectEnd) {
1297
439
    auto Next = std::next(SelectMBBI);
1298
439
    if (isSelectPseudo(*SelectMBBI)) {
1299
411
      // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
1300
411
      BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
1301
411
              TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
1302
411
          .addReg(SelectMBBI->getOperand(4).getReg())
1303
411
          .addMBB(HeadMBB)
1304
411
          .addReg(SelectMBBI->getOperand(5).getReg())
1305
411
          .addMBB(IfFalseMBB);
1306
411
      SelectMBBI->eraseFromParent();
1307
411
    }
1308
439
    SelectMBBI = Next;
1309
439
  }
1310
348
1311
348
  F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
1312
348
  return TailMBB;
1313
348
}
1314
1315
MachineBasicBlock *
1316
RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
1317
579
                                                 MachineBasicBlock *BB) const {
1318
579
  switch (MI.getOpcode()) {
1319
579
  default:
1320
0
    llvm_unreachable("Unexpected instr type to insert");
1321
579
  case RISCV::ReadCycleWide:
1322
1
    assert(!Subtarget.is64Bit() &&
1323
1
           "ReadCycleWrite is only to be used on riscv32");
1324
1
    return emitReadCycleWidePseudo(MI, BB);
1325
579
  case RISCV::Select_GPR_Using_CC_GPR:
1326
348
  case RISCV::Select_FPR32_Using_CC_GPR:
1327
348
  case RISCV::Select_FPR64_Using_CC_GPR:
1328
348
    return emitSelectPseudo(MI, BB);
1329
348
  case RISCV::BuildPairF64Pseudo:
1330
175
    return emitBuildPairF64Pseudo(MI, BB);
1331
348
  case RISCV::SplitF64Pseudo:
1332
55
    return emitSplitF64Pseudo(MI, BB);
1333
579
  }
1334
579
}
1335
1336
// Calling Convention Implementation.
1337
// The expectations for frontend ABI lowering vary from target to target.
1338
// Ideally, an LLVM frontend would be able to avoid worrying about many ABI
1339
// details, but this is a longer term goal. For now, we simply try to keep the
1340
// role of the frontend as simple and well-defined as possible. The rules can
1341
// be summarised as:
1342
// * Never split up large scalar arguments. We handle them here.
1343
// * If a hardfloat calling convention is being used, and the struct may be
1344
// passed in a pair of registers (fp+fp, int+fp), and both registers are
1345
// available, then pass as two separate arguments. If either the GPRs or FPRs
1346
// are exhausted, then pass according to the rule below.
1347
// * If a struct could never be passed in registers or directly in a stack
1348
// slot (as it is larger than 2*XLEN and the floating point rules don't
1349
// apply), then pass it using a pointer with the byval attribute.
1350
// * If a struct is less than 2*XLEN, then coerce to either a two-element
1351
// word-sized array or a 2*XLEN scalar (depending on alignment).
1352
// * The frontend can determine whether a struct is returned by reference or
1353
// not based on its size and fields. If it will be returned by reference, the
1354
// frontend must modify the prototype so a pointer with the sret annotation is
1355
// passed as the first argument. This is not necessary for large scalar
1356
// returns.
1357
// * Struct return values and varargs should be coerced to structs containing
1358
// register-size fields in the same situations they would be for fixed
1359
// arguments.
1360
1361
static const MCPhysReg ArgGPRs[] = {
1362
  RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
1363
  RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
1364
};
1365
static const MCPhysReg ArgFPR32s[] = {
1366
  RISCV::F10_32, RISCV::F11_32, RISCV::F12_32, RISCV::F13_32,
1367
  RISCV::F14_32, RISCV::F15_32, RISCV::F16_32, RISCV::F17_32
1368
};
1369
static const MCPhysReg ArgFPR64s[] = {
1370
  RISCV::F10_64, RISCV::F11_64, RISCV::F12_64, RISCV::F13_64,
1371
  RISCV::F14_64, RISCV::F15_64, RISCV::F16_64, RISCV::F17_64
1372
};
1373
1374
// Pass a 2*XLEN argument that has been split into two XLEN values through
1375
// registers or the stack as necessary.
1376
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
1377
                                ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
1378
                                MVT ValVT2, MVT LocVT2,
1379
738
                                ISD::ArgFlagsTy ArgFlags2) {
1380
738
  unsigned XLenInBytes = XLen / 8;
1381
738
  if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
1382
710
    // At least one half can be passed via register.
1383
710
    State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
1384
710
                                     VA1.getLocVT(), CCValAssign::Full));
1385
710
  } else {
1386
28
    // Both halves must be passed on the stack, with proper alignment.
1387
28
    unsigned StackAlign = std::max(XLenInBytes, ArgFlags1.getOrigAlign());
1388
28
    State.addLoc(
1389
28
        CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
1390
28
                            State.AllocateStack(XLenInBytes, StackAlign),
1391
28
                            VA1.getLocVT(), CCValAssign::Full));
1392
28
    State.addLoc(CCValAssign::getMem(
1393
28
        ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1394
28
        CCValAssign::Full));
1395
28
    return false;
1396
28
  }
1397
710
1398
710
  if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
1399
692
    // The second half can also be passed via register.
1400
692
    State.addLoc(
1401
692
        CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
1402
692
  } else {
1403
18
    // The second half is passed via the stack, without additional alignment.
1404
18
    State.addLoc(CCValAssign::getMem(
1405
18
        ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1406
18
        CCValAssign::Full));
1407
18
  }
1408
710
1409
710
  return false;
1410
710
}
1411
1412
// Implements the RISC-V calling convention. Returns true upon failure.
1413
static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
1414
                     MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
1415
                     ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
1416
19.0k
                     bool IsRet, Type *OrigTy) {
1417
19.0k
  unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
1418
19.0k
  assert(XLen == 32 || XLen == 64);
1419
19.0k
  MVT XLenVT = XLen == 32 ? 
MVT::i3210.9k
:
MVT::i648.12k
;
1420
19.0k
1421
19.0k
  // Any return value split in to more than two values can't be returned
1422
19.0k
  // directly.
1423
19.0k
  if (IsRet && 
ValNo > 18.55k
)
1424
35
    return true;
1425
19.0k
1426
19.0k
  // UseGPRForF32 if targeting one of the soft-float ABIs, if passing a
1427
19.0k
  // variadic argument, or if no F32 argument registers are available.
1428
19.0k
  bool UseGPRForF32 = true;
1429
19.0k
  // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
1430
19.0k
  // variadic argument, or if no F64 argument registers are available.
1431
19.0k
  bool UseGPRForF64 = true;
1432
19.0k
1433
19.0k
  switch (ABI) {
1434
19.0k
  default:
1435
0
    llvm_unreachable("Unexpected ABI");
1436
19.0k
  case RISCVABI::ABI_ILP32:
1437
17.0k
  case RISCVABI::ABI_LP64:
1438
17.0k
    break;
1439
17.0k
  case RISCVABI::ABI_ILP32F:
1440
963
  case RISCVABI::ABI_LP64F:
1441
963
    UseGPRForF32 = !IsFixed;
1442
963
    break;
1443
963
  case RISCVABI::ABI_ILP32D:
1444
956
  case RISCVABI::ABI_LP64D:
1445
956
    UseGPRForF32 = !IsFixed;
1446
956
    UseGPRForF64 = !IsFixed;
1447
956
    break;
1448
19.0k
  }
1449
19.0k
1450
19.0k
  if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s))
1451
14
    UseGPRForF32 = true;
1452
19.0k
  if (State.getFirstUnallocated(ArgFPR64s) == array_lengthof(ArgFPR64s))
1453
14
    UseGPRForF64 = true;
1454
19.0k
1455
19.0k
  // From this point on, rely on UseGPRForF32, UseGPRForF64 and similar local
1456
19.0k
  // variables rather than directly checking against the target ABI.
1457
19.0k
1458
19.0k
  if (UseGPRForF32 && 
ValVT == MVT::f3217.1k
) {
1459
1.16k
    LocVT = XLenVT;
1460
1.16k
    LocInfo = CCValAssign::BCvt;
1461
17.8k
  } else if (UseGPRForF64 && 
XLen == 6416.9k
&&
ValVT == MVT::f647.31k
) {
1462
428
    LocVT = MVT::i64;
1463
428
    LocInfo = CCValAssign::BCvt;
1464
428
  }
1465
19.0k
1466
19.0k
  // If this is a variadic argument, the RISC-V calling convention requires
1467
19.0k
  // that it is assigned an 'even' or 'aligned' register if it has 8-byte
1468
19.0k
  // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
1469
19.0k
  // be used regardless of whether the original argument was split during
1470
19.0k
  // legalisation or not. The argument will not be passed by registers if the
1471
19.0k
  // original type is larger than 2*XLEN, so the register alignment rule does
1472
19.0k
  // not apply.
1473
19.0k
  unsigned TwoXLenInBytes = (2 * XLen) / 8;
1474
19.0k
  if (!IsFixed && 
ArgFlags.getOrigAlign() == TwoXLenInBytes164
&&
1475
19.0k
      
DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes29
) {
1476
29
    unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
1477
29
    // Skip 'odd' register if necessary.
1478
29
    if (RegIdx != array_lengthof(ArgGPRs) && 
RegIdx % 2 == 124
)
1479
20
      State.AllocateReg(ArgGPRs);
1480
29
  }
1481
19.0k
1482
19.0k
  SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
1483
19.0k
  SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
1484
19.0k
      State.getPendingArgFlags();
1485
19.0k
1486
19.0k
  assert(PendingLocs.size() == PendingArgFlags.size() &&
1487
19.0k
         "PendingLocs and PendingArgFlags out of sync");
1488
19.0k
1489
19.0k
  // Handle passing f64 on RV32D with a soft float ABI or when floating point
1490
19.0k
  // registers are exhausted.
1491
19.0k
  if (UseGPRForF64 && 
XLen == 3218.0k
&&
ValVT == MVT::f6410.2k
) {
1492
470
    assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
1493
470
           "Can't lower f64 if it is split");
1494
470
    // Depending on available argument GPRS, f64 may be passed in a pair of
1495
470
    // GPRs, split between a GPR and the stack, or passed completely on the
1496
470
    // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
1497
470
    // cases.
1498
470
    unsigned Reg = State.AllocateReg(ArgGPRs);
1499
470
    LocVT = MVT::i32;
1500
470
    if (!Reg) {
1501
9
      unsigned StackOffset = State.AllocateStack(8, 8);
1502
9
      State.addLoc(
1503
9
          CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1504
9
      return false;
1505
9
    }
1506
461
    if (!State.AllocateReg(ArgGPRs))
1507
4
      State.AllocateStack(4, 4);
1508
461
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1509
461
    return false;
1510
461
  }
1511
18.5k
1512
18.5k
  // Split arguments might be passed indirectly, so keep track of the pending
1513
18.5k
  // values.
1514
18.5k
  if (ArgFlags.isSplit() || 
!PendingLocs.empty()17.6k
) {
1515
1.96k
    LocVT = XLenVT;
1516
1.96k
    LocInfo = CCValAssign::Indirect;
1517
1.96k
    PendingLocs.push_back(
1518
1.96k
        CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
1519
1.96k
    PendingArgFlags.push_back(ArgFlags);
1520
1.96k
    if (!ArgFlags.isSplitEnd()) {
1521
1.10k
      return false;
1522
1.10k
    }
1523
17.4k
  }
1524
17.4k
1525
17.4k
  // If the split argument only had two elements, it should be passed directly
1526
17.4k
  // in registers or on the stack.
1527
17.4k
  if (ArgFlags.isSplitEnd() && 
PendingLocs.size() <= 2859
) {
1528
738
    assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
1529
738
    // Apply the normal calling convention rules to the first half of the
1530
738
    // split argument.
1531
738
    CCValAssign VA = PendingLocs[0];
1532
738
    ISD::ArgFlagsTy AF = PendingArgFlags[0];
1533
738
    PendingLocs.clear();
1534
738
    PendingArgFlags.clear();
1535
738
    return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
1536
738
                               ArgFlags);
1537
738
  }
1538
16.7k
1539
16.7k
  // Allocate to a register if possible, or else a stack slot.
1540
16.7k
  unsigned Reg;
1541
16.7k
  if (ValVT == MVT::f32 && 
!UseGPRForF321.24k
)
1542
80
    Reg = State.AllocateReg(ArgFPR32s, ArgFPR64s);
1543
16.6k
  else if (ValVT == MVT::f64 && 
!UseGPRForF64484
)
1544
56
    Reg = State.AllocateReg(ArgFPR64s, ArgFPR32s);
1545
16.5k
  else
1546
16.5k
    Reg = State.AllocateReg(ArgGPRs);
1547
16.7k
  unsigned StackOffset = Reg ? 
016.4k
:
State.AllocateStack(XLen / 8, XLen / 8)260
;
1548
16.7k
1549
16.7k
  // If we reach this point and PendingLocs is non-empty, we must be at the
1550
16.7k
  // end of a split argument that must be passed indirectly.
1551
16.7k
  if (!PendingLocs.empty()) {
1552
121
    assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
1553
121
    assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
1554
121
1555
484
    for (auto &It : PendingLocs) {
1556
484
      if (Reg)
1557
412
        It.convertToReg(Reg);
1558
72
      else
1559
72
        It.convertToMem(StackOffset);
1560
484
      State.addLoc(It);
1561
484
    }
1562
121
    PendingLocs.clear();
1563
121
    PendingArgFlags.clear();
1564
121
    return false;
1565
121
  }
1566
16.5k
1567
16.5k
  assert((!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) &&
1568
16.5k
         "Expected an XLenVT at this stage");
1569
16.5k
1570
16.5k
  if (Reg) {
1571
16.3k
    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1572
16.3k
    return false;
1573
16.3k
  }
1574
242
1575
242
  // When an f32 or f64 is passed on the stack, no bit-conversion is needed.
1576
242
  if (ValVT == MVT::f32 || 
ValVT == MVT::f64230
) {
1577
14
    LocVT = ValVT;
1578
14
    LocInfo = CCValAssign::Full;
1579
14
  }
1580
242
  State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1581
242
  return false;
1582
242
}
1583
1584
void RISCVTargetLowering::analyzeInputArgs(
1585
    MachineFunction &MF, CCState &CCInfo,
1586
4.69k
    const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const {
1587
4.69k
  unsigned NumArgs = Ins.size();
1588
4.69k
  FunctionType *FType = MF.getFunction().getFunctionType();
1589
4.69k
1590
12.0k
  for (unsigned i = 0; i != NumArgs; 
++i7.40k
) {
1591
7.40k
    MVT ArgVT = Ins[i].VT;
1592
7.40k
    ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
1593
7.40k
1594
7.40k
    Type *ArgTy = nullptr;
1595
7.40k
    if (IsRet)
1596
1.23k
      ArgTy = FType->getReturnType();
1597
6.16k
    else if (Ins[i].isOrigArg())
1598
6.15k
      ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
1599
7.40k
1600
7.40k
    RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1601
7.40k
    if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1602
7.40k
                 ArgFlags, CCInfo, /*IsRet=*/true, IsRet, ArgTy)) {
1603
0
      LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
1604
0
                        << EVT(ArgVT).getEVTString() << '\n');
1605
0
      llvm_unreachable(nullptr);
1606
0
    }
1607
7.40k
  }
1608
4.69k
}
1609
1610
void RISCVTargetLowering::analyzeOutputArgs(
1611
    MachineFunction &MF, CCState &CCInfo,
1612
    const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
1613
4.69k
    CallLoweringInfo *CLI) const {
1614
4.69k
  unsigned NumArgs = Outs.size();
1615
4.69k
1616
12.0k
  for (unsigned i = 0; i != NumArgs; 
i++7.30k
) {
1617
7.30k
    MVT ArgVT = Outs[i].VT;
1618
7.30k
    ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
1619
7.30k
    Type *OrigTy = CLI ? 
CLI->getArgs()[Outs[i].OrigArgIndex].Ty4.32k
:
nullptr2.97k
;
1620
7.30k
1621
7.30k
    RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1622
7.30k
    if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1623
7.30k
                 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) {
1624
0
      LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
1625
0
                        << EVT(ArgVT).getEVTString() << "\n");
1626
0
      llvm_unreachable(nullptr);
1627
0
    }
1628
7.30k
  }
1629
4.69k
}
1630
1631
// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
1632
// values.
1633
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
1634
6.82k
                                   const CCValAssign &VA, const SDLoc &DL) {
1635
6.82k
  switch (VA.getLocInfo()) {
1636
6.82k
  default:
1637
0
    llvm_unreachable("Unexpected CCValAssign::LocInfo");
1638
6.82k
  case CCValAssign::Full:
1639
6.02k
    break;
1640
6.82k
  case CCValAssign::BCvt:
1641
800
    if (VA.getLocVT() == MVT::i64 && 
VA.getValVT() == MVT::f32511
) {
1642
287
      Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
1643
287
      break;
1644
287
    }
1645
513
    Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
1646
513
    break;
1647
6.82k
  }
1648
6.82k
  return Val;
1649
6.82k
}
1650
1651
// The caller is responsible for loading the full value if the argument is
1652
// passed with CCValAssign::Indirect.
1653
static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
1654
5.63k
                                const CCValAssign &VA, const SDLoc &DL) {
1655
5.63k
  MachineFunction &MF = DAG.getMachineFunction();
1656
5.63k
  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1657
5.63k
  EVT LocVT = VA.getLocVT();
1658
5.63k
  SDValue Val;
1659
5.63k
  const TargetRegisterClass *RC;
1660
5.63k
1661
5.63k
  switch (LocVT.getSimpleVT().SimpleTy) {
1662
5.63k
  default:
1663
0
    llvm_unreachable("Unexpected register type");
1664
5.63k
  case MVT::i32:
1665
5.57k
  case MVT::i64:
1666
5.57k
    RC = &RISCV::GPRRegClass;
1667
5.57k
    break;
1668
5.57k
  case MVT::f32:
1669
36
    RC = &RISCV::FPR32RegClass;
1670
36
    break;
1671
5.57k
  case MVT::f64:
1672
26
    RC = &RISCV::FPR64RegClass;
1673
26
    break;
1674
5.63k
  }
1675
5.63k
1676
5.63k
  unsigned VReg = RegInfo.createVirtualRegister(RC);
1677
5.63k
  RegInfo.addLiveIn(VA.getLocReg(), VReg);
1678
5.63k
  Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1679
5.63k
1680
5.63k
  if (VA.getLocInfo() == CCValAssign::Indirect)
1681
49
    return Val;
1682
5.59k
1683
5.59k
  return convertLocVTToValVT(DAG, Val, VA, DL);
1684
5.59k
}
1685
1686
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
1687
6.92k
                                   const CCValAssign &VA, const SDLoc &DL) {
1688
6.92k
  EVT LocVT = VA.getLocVT();
1689
6.92k
1690
6.92k
  switch (VA.getLocInfo()) {
1691
6.92k
  default:
1692
0
    llvm_unreachable("Unexpected CCValAssign::LocInfo");
1693
6.92k
  case CCValAssign::Full:
1694
6.52k
    break;
1695
6.92k
  case CCValAssign::BCvt:
1696
397
    if (VA.getLocVT() == MVT::i64 && 
VA.getValVT() == MVT::f32248
) {
1697
143
      Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
1698
143
      break;
1699
143
    }
1700
254
    Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
1701
254
    break;
1702
6.92k
  }
1703
6.92k
  return Val;
1704
6.92k
}
1705
1706
// The caller is responsible for loading the full value if the argument is
1707
// passed with CCValAssign::Indirect.
1708
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
1709
143
                                const CCValAssign &VA, const SDLoc &DL) {
1710
143
  MachineFunction &MF = DAG.getMachineFunction();
1711
143
  MachineFrameInfo &MFI = MF.getFrameInfo();
1712
143
  EVT LocVT = VA.getLocVT();
1713
143
  EVT ValVT = VA.getValVT();
1714
143
  EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
1715
143
  int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
1716
143
                                 VA.getLocMemOffset(), /*Immutable=*/true);
1717
143
  SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1718
143
  SDValue Val;
1719
143
1720
143
  ISD::LoadExtType ExtType;
1721
143
  switch (VA.getLocInfo()) {
1722
143
  default:
1723
0
    llvm_unreachable("Unexpected CCValAssign::LocInfo");
1724
143
  case CCValAssign::Full:
1725
143
  case CCValAssign::Indirect:
1726
143
  case CCValAssign::BCvt:
1727
143
    ExtType = ISD::NON_EXTLOAD;
1728
143
    break;
1729
143
  }
1730
143
  Val = DAG.getExtLoad(
1731
143
      ExtType, DL, LocVT, Chain, FIN,
1732
143
      MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
1733
143
  return Val;
1734
143
}
1735
1736
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
1737
212
                                       const CCValAssign &VA, const SDLoc &DL) {
1738
212
  assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
1739
212
         "Unexpected VA");
1740
212
  MachineFunction &MF = DAG.getMachineFunction();
1741
212
  MachineFrameInfo &MFI = MF.getFrameInfo();
1742
212
  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1743
212
1744
212
  if (VA.isMemLoc()) {
1745
3
    // f64 is passed on the stack.
1746
3
    int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*Immutable=*/true);
1747
3
    SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1748
3
    return DAG.getLoad(MVT::f64, DL, Chain, FIN,
1749
3
                       MachinePointerInfo::getFixedStack(MF, FI));
1750
3
  }
1751
209
1752
209
  assert(VA.isRegLoc() && "Expected register VA assignment");
1753
209
1754
209
  unsigned LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1755
209
  RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
1756
209
  SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
1757
209
  SDValue Hi;
1758
209
  if (VA.getLocReg() == RISCV::X17) {
1759
2
    // Second half of f64 is passed on the stack.
1760
2
    int FI = MFI.CreateFixedObject(4, 0, /*Immutable=*/true);
1761
2
    SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1762
2
    Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
1763
2
                     MachinePointerInfo::getFixedStack(MF, FI));
1764
207
  } else {
1765
207
    // Second half of f64 is passed in another GPR.
1766
207
    unsigned HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1767
207
    RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
1768
207
    Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
1769
207
  }
1770
209
  return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
1771
209
}
1772
1773
// Transform physical registers into virtual registers.
1774
SDValue RISCVTargetLowering::LowerFormalArguments(
1775
    SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1776
    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1777
3.36k
    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1778
3.36k
1779
3.36k
  switch (CallConv) {
1780
3.36k
  default:
1781
0
    report_fatal_error("Unsupported calling convention");
1782
3.36k
  case CallingConv::C:
1783
3.36k
  case CallingConv::Fast:
1784
3.36k
    break;
1785
3.36k
  }
1786
3.36k
1787
3.36k
  MachineFunction &MF = DAG.getMachineFunction();
1788
3.36k
1789
3.36k
  const Function &Func = MF.getFunction();
1790
3.36k
  if (Func.hasFnAttribute("interrupt")) {
1791
56
    if (!Func.arg_empty())
1792
2
      report_fatal_error(
1793
2
        "Functions with the interrupt attribute cannot have arguments!");
1794
54
1795
54
    StringRef Kind =
1796
54
      MF.getFunction().getFnAttribute("interrupt").getValueAsString();
1797
54
1798
54
    if (!(Kind == "user" || 
Kind == "supervisor"46
||
Kind == "machine"40
))
1799
2
      report_fatal_error(
1800
2
        "Function interrupt attribute argument not supported!");
1801
3.36k
  }
1802
3.36k
1803
3.36k
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
1804
3.36k
  MVT XLenVT = Subtarget.getXLenVT();
1805
3.36k
  unsigned XLenInBytes = Subtarget.getXLen() / 8;
1806
3.36k
  // Used with vargs to acumulate store chains.
1807
3.36k
  std::vector<SDValue> OutChains;
1808
3.36k
1809
3.36k
  // Assign locations to all of the incoming arguments.
1810
3.36k
  SmallVector<CCValAssign, 16> ArgLocs;
1811
3.36k
  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1812
3.36k
  analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false);
1813
3.36k
1814
9.35k
  for (unsigned i = 0, e = ArgLocs.size(); i != e; 
++i5.99k
) {
1815
5.99k
    CCValAssign &VA = ArgLocs[i];
1816
5.99k
    SDValue ArgValue;
1817
5.99k
    // Passing f64 on RV32D with a soft float ABI must be handled as a special
1818
5.99k
    // case.
1819
5.99k
    if (VA.getLocVT() == MVT::i32 && 
VA.getValVT() == MVT::f643.07k
)
1820
212
      ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
1821
5.78k
    else if (VA.isRegLoc())
1822
5.63k
      ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL);
1823
143
    else
1824
143
      ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
1825
5.99k
1826
5.99k
    if (VA.getLocInfo() == CCValAssign::Indirect) {
1827
58
      // If the original argument was split and passed by reference (e.g. i128
1828
58
      // on RV32), we need to load all parts of it here (using the same
1829
58
      // address).
1830
58
      InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1831
58
                                   MachinePointerInfo()));
1832
58
      unsigned ArgIndex = Ins[i].OrigArgIndex;
1833
58
      assert(Ins[i].PartOffset == 0);
1834
232
      while (i + 1 != e && 
Ins[i + 1].OrigArgIndex == ArgIndex208
) {
1835
174
        CCValAssign &PartVA = ArgLocs[i + 1];
1836
174
        unsigned PartOffset = Ins[i + 1].PartOffset;
1837
174
        SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1838
174
                                      DAG.getIntPtrConstant(PartOffset, DL));
1839
174
        InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1840
174
                                     MachinePointerInfo()));
1841
174
        ++i;
1842
174
      }
1843
58
      continue;
1844
58
    }
1845
5.93k
    InVals.push_back(ArgValue);
1846
5.93k
  }
1847
3.36k
1848
3.36k
  if (IsVarArg) {
1849
81
    ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
1850
81
    unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
1851
81
    const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1852
81
    MachineFrameInfo &MFI = MF.getFrameInfo();
1853
81
    MachineRegisterInfo &RegInfo = MF.getRegInfo();
1854
81
    RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1855
81
1856
81
    // Offset of the first variable argument from stack pointer, and size of
1857
81
    // the vararg save area. For now, the varargs save area is either zero or
1858
81
    // large enough to hold a0-a7.
1859
81
    int VaArgOffset, VarArgsSaveSize;
1860
81
1861
81
    // If all registers are allocated, then all varargs must be passed on the
1862
81
    // stack and we don't need to save any argregs.
1863
81
    if (ArgRegs.size() == Idx) {
1864
0
      VaArgOffset = CCInfo.getNextStackOffset();
1865
0
      VarArgsSaveSize = 0;
1866
81
    } else {
1867
81
      VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
1868
81
      VaArgOffset = -VarArgsSaveSize;
1869
81
    }
1870
81
1871
81
    // Record the frame index of the first variable argument
1872
81
    // which is a value necessary to VASTART.
1873
81
    int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1874
81
    RVFI->setVarArgsFrameIndex(FI);
1875
81
1876
81
    // If saving an odd number of registers then create an extra stack slot to
1877
81
    // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
1878
81
    // offsets to even-numbered registered remain 2*XLEN-aligned.
1879
81
    if (Idx % 2) {
1880
64
      FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes,
1881
64
                                 true);
1882
64
      VarArgsSaveSize += XLenInBytes;
1883
64
    }
1884
81
1885
81
    // Copy the integer registers that may have been used for passing varargs
1886
81
    // to the vararg save area.
1887
629
    for (unsigned I = Idx; I < ArgRegs.size();
1888
548
         ++I, VaArgOffset += XLenInBytes) {
1889
548
      const unsigned Reg = RegInfo.createVirtualRegister(RC);
1890
548
      RegInfo.addLiveIn(ArgRegs[I], Reg);
1891
548
      SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
1892
548
      FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1893
548
      SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1894
548
      SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
1895
548
                                   MachinePointerInfo::getFixedStack(MF, FI));
1896
548
      cast<StoreSDNode>(Store.getNode())
1897
548
          ->getMemOperand()
1898
548
          ->setValue((Value *)nullptr);
1899
548
      OutChains.push_back(Store);
1900
548
    }
1901
81
    RVFI->setVarArgsSaveSize(VarArgsSaveSize);
1902
81
  }
1903
3.36k
1904
3.36k
  // All stores are grouped in one node to allow the matching between
1905
3.36k
  // the size of Ins and InVals. This only happens for vararg functions.
1906
3.36k
  if (!OutChains.empty()) {
1907
81
    OutChains.push_back(Chain);
1908
81
    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
1909
81
  }
1910
3.36k
1911
3.36k
  return Chain;
1912
3.36k
}
1913
1914
/// isEligibleForTailCallOptimization - Check whether the call is eligible
1915
/// for tail call optimization.
1916
/// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
1917
bool RISCVTargetLowering::isEligibleForTailCallOptimization(
1918
    CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
1919
53
    const SmallVector<CCValAssign, 16> &ArgLocs) const {
1920
53
1921
53
  auto &Callee = CLI.Callee;
1922
53
  auto CalleeCC = CLI.CallConv;
1923
53
  auto IsVarArg = CLI.IsVarArg;
1924
53
  auto &Outs = CLI.Outs;
1925
53
  auto &Caller = MF.getFunction();
1926
53
  auto CallerCC = Caller.getCallingConv();
1927
53
1928
53
  // Do not tail call opt functions with "disable-tail-calls" attribute.
1929
53
  if (Caller.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
1930
15
    return false;
1931
38
1932
38
  // Exception-handling functions need a special set of instructions to
1933
38
  // indicate a return to the hardware. Tail-calling another function would
1934
38
  // probably break this.
1935
38
  // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
1936
38
  // should be expanded as new function attributes are introduced.
1937
38
  if (Caller.hasFnAttribute("interrupt"))
1938
2
    return false;
1939
36
1940
36
  // Do not tail call opt functions with varargs.
1941
36
  if (IsVarArg)
1942
2
    return false;
1943
34
1944
34
  // Do not tail call opt if the stack is used to pass parameters.
1945
34
  if (CCInfo.getNextStackOffset() != 0)
1946
2
    return false;
1947
32
1948
32
  // Do not tail call opt if any parameters need to be passed indirectly.
1949
32
  // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
1950
32
  // passed indirectly. So the address of the value will be passed in a
1951
32
  // register, or if not available, then the address is put on the stack. In
1952
32
  // order to pass indirectly, space on the stack often needs to be allocated
1953
32
  // in order to store the value. In this case the CCInfo.getNextStackOffset()
1954
32
  // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
1955
32
  // are passed CCValAssign::Indirect.
1956
32
  for (auto &VA : ArgLocs)
1957
30
    if (VA.getLocInfo() == CCValAssign::Indirect)
1958
2
      return false;
1959
32
1960
32
  // Do not tail call opt if either caller or callee uses struct return
1961
32
  // semantics.
1962
32
  auto IsCallerStructRet = Caller.hasStructRetAttr();
1963
30
  auto IsCalleeStructRet = Outs.empty() ? 
false6
:
Outs[0].Flags.isSRet()24
;
1964
30
  if (IsCallerStructRet || 
IsCalleeStructRet24
)
1965
8
    return false;
1966
22
1967
22
  // Externally-defined functions with weak linkage should not be
1968
22
  // tail-called. The behaviour of branch instructions in this situation (as
1969
22
  // used for tail calls) is implementation-defined, so we cannot rely on the
1970
22
  // linker replacing the tail call with a return.
1971
22
  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1972
18
    const GlobalValue *GV = G->getGlobal();
1973
18
    if (GV->hasExternalWeakLinkage())
1974
2
      return false;
1975
20
  }
1976
20
1977
20
  // The callee has to preserve all registers the caller needs to preserve.
1978
20
  const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
1979
20
  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
1980
20
  if (CalleeCC != CallerCC) {
1981
0
    const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
1982
0
    if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
1983
0
      return false;
1984
20
  }
1985
20
1986
20
  // Byval parameters hand the function a pointer directly into the stack area
1987
20
  // we want to reuse during a tail call. Working around this *is* possible
1988
20
  // but less efficient and uglier in LowerCall.
1989
20
  for (auto &Arg : Outs)
1990
22
    if (Arg.Flags.isByVal())
1991
2
      return false;
1992
20
1993
20
  
return true18
;
1994
20
}
1995
1996
// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
1997
// and output parameter nodes.
1998
SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
1999
1.34k
                                       SmallVectorImpl<SDValue> &InVals) const {
2000
1.34k
  SelectionDAG &DAG = CLI.DAG;
2001
1.34k
  SDLoc &DL = CLI.DL;
2002
1.34k
  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2003
1.34k
  SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2004
1.34k
  SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2005
1.34k
  SDValue Chain = CLI.Chain;
2006
1.34k
  SDValue Callee = CLI.Callee;
2007
1.34k
  bool &IsTailCall = CLI.IsTailCall;
2008
1.34k
  CallingConv::ID CallConv = CLI.CallConv;
2009
1.34k
  bool IsVarArg = CLI.IsVarArg;
2010
1.34k
  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2011
1.34k
  MVT XLenVT = Subtarget.getXLenVT();
2012
1.34k
2013
1.34k
  MachineFunction &MF = DAG.getMachineFunction();
2014
1.34k
2015
1.34k
  // Analyze the operands of the call, assigning locations to each operand.
2016
1.34k
  SmallVector<CCValAssign, 16> ArgLocs;
2017
1.34k
  CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2018
1.34k
  analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI);
2019
1.34k
2020
1.34k
  // Check if it's really possible to do a tail call.
2021
1.34k
  if (IsTailCall)
2022
53
    IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
2023
1.34k
2024
1.34k
  if (IsTailCall)
2025
18
    ++NumTailCalls;
2026
1.33k
  else if (CLI.CS && 
CLI.CS.isMustTailCall()1.10k
)
2027
4
    report_fatal_error("failed to perform tail call elimination on a call "
2028
4
                       "site marked musttail");
2029
1.34k
2030
1.34k
  // Get a count of how many bytes are to be pushed on the stack.
2031
1.34k
  unsigned NumBytes = ArgCCInfo.getNextStackOffset();
2032
1.34k
2033
1.34k
  // Create local copies for byval args
2034
1.34k
  SmallVector<SDValue, 8> ByValArgs;
2035
5.66k
  for (unsigned i = 0, e = Outs.size(); i != e; 
++i4.32k
) {
2036
4.32k
    ISD::ArgFlagsTy Flags = Outs[i].Flags;
2037
4.32k
    if (!Flags.isByVal())
2038
4.31k
      continue;
2039
12
2040
12
    SDValue Arg = OutVals[i];
2041
12
    unsigned Size = Flags.getByValSize();
2042
12
    unsigned Align = Flags.getByValAlign();
2043
12
2044
12
    int FI = MF.getFrameInfo().CreateStackObject(Size, Align, /*isSS=*/false);
2045
12
    SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2046
12
    SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
2047
12
2048
12
    Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Align,
2049
12
                          /*IsVolatile=*/false,
2050
12
                          /*AlwaysInline=*/false,
2051
12
                          IsTailCall, MachinePointerInfo(),
2052
12
                          MachinePointerInfo());
2053
12
    ByValArgs.push_back(FIPtr);
2054
12
  }
2055
1.34k
2056
1.34k
  if (!IsTailCall)
2057
1.32k
    Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
2058
1.34k
2059
1.34k
  // Copy argument values to their designated locations.
2060
1.34k
  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2061
1.34k
  SmallVector<SDValue, 8> MemOpChains;
2062
1.34k
  SDValue StackPtr;
2063
5.47k
  for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; 
++i4.13k
) {
2064
4.13k
    CCValAssign &VA = ArgLocs[i];
2065
4.13k
    SDValue ArgValue = OutVals[i];
2066
4.13k
    ISD::ArgFlagsTy Flags = Outs[i].Flags;
2067
4.13k
2068
4.13k
    // Handle passing f64 on RV32D with a soft float ABI as a special case.
2069
4.13k
    bool IsF64OnRV32DSoftABI =
2070
4.13k
        VA.getLocVT() == MVT::i32 && 
VA.getValVT() == MVT::f642.57k
;
2071
4.13k
    if (IsF64OnRV32DSoftABI && 
VA.isRegLoc()45
) {
2072
39
      SDValue SplitF64 = DAG.getNode(
2073
39
          RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
2074
39
      SDValue Lo = SplitF64.getValue(0);
2075
39
      SDValue Hi = SplitF64.getValue(1);
2076
39
2077
39
      unsigned RegLo = VA.getLocReg();
2078
39
      RegsToPass.push_back(std::make_pair(RegLo, Lo));
2079
39
2080
39
      if (RegLo == RISCV::X17) {
2081
2
        // Second half of f64 is passed on the stack.
2082
2
        // Work out the address of the stack slot.
2083
2
        if (!StackPtr.getNode())
2084
2
          StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
2085
2
        // Emit the store.
2086
2
        MemOpChains.push_back(
2087
2
            DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
2088
37
      } else {
2089
37
        // Second half of f64 is passed in another GPR.
2090
37
        unsigned RegHigh = RegLo + 1;
2091
37
        RegsToPass.push_back(std::make_pair(RegHigh, Hi));
2092
37
      }
2093
39
      continue;
2094
39
    }
2095
4.09k
2096
4.09k
    // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
2097
4.09k
    // as any other MemLoc.
2098
4.09k
2099
4.09k
    // Promote the value if needed.
2100
4.09k
    // For now, only handle fully promoted and indirect arguments.
2101
4.09k
    if (VA.getLocInfo() == CCValAssign::Indirect) {
2102
63
      // Store the argument in a stack slot and pass its address.
2103
63
      SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
2104
63
      int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
2105
63
      MemOpChains.push_back(
2106
63
          DAG.getStore(Chain, DL, ArgValue, SpillSlot,
2107
63
                       MachinePointerInfo::getFixedStack(MF, FI)));
2108
63
      // If the original argument was split (e.g. i128), we need
2109
63
      // to store all parts of it here (and pass just one address).
2110
63
      unsigned ArgIndex = Outs[i].OrigArgIndex;
2111
63
      assert(Outs[i].PartOffset == 0);
2112
252
      while (i + 1 != e && 
Outs[i + 1].OrigArgIndex == ArgIndex228
) {
2113
189
        SDValue PartValue = OutVals[i + 1];
2114
189
        unsigned PartOffset = Outs[i + 1].PartOffset;
2115
189
        SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
2116
189
                                      DAG.getIntPtrConstant(PartOffset, DL));
2117
189
        MemOpChains.push_back(
2118
189
            DAG.getStore(Chain, DL, PartValue, Address,
2119
189
                         MachinePointerInfo::getFixedStack(MF, FI)));
2120
189
        ++i;
2121
189
      }
2122
63
      ArgValue = SpillSlot;
2123
4.03k
    } else {
2124
4.03k
      ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL);
2125
4.03k
    }
2126
4.09k
2127
4.09k
    // Use local copy if it is a byval arg.
2128
4.09k
    if (Flags.isByVal())
2129
12
      ArgValue = ByValArgs[j++];
2130
4.09k
2131
4.09k
    if (VA.isRegLoc()) {
2132
3.89k
      // Queue up the argument copies and emit them at the end.
2133
3.89k
      RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
2134
3.89k
    } else {
2135
197
      assert(VA.isMemLoc() && "Argument not register or memory");
2136
197
      assert(!IsTailCall && "Tail call not allowed if stack is used "
2137
197
                            "for passing parameters");
2138
197
2139
197
      // Work out the address of the stack slot.
2140
197
      if (!StackPtr.getNode())
2141
60
        StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
2142
197
      SDValue Address =
2143
197
          DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
2144
197
                      DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
2145
197
2146
197
      // Emit the store.
2147
197
      MemOpChains.push_back(
2148
197
          DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
2149
197
    }
2150
4.09k
  }
2151
1.34k
2152
1.34k
  // Join the stores, which are independent of one another.
2153
1.34k
  if (!MemOpChains.empty())
2154
80
    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2155
1.34k
2156
1.34k
  SDValue Glue;
2157
1.34k
2158
1.34k
  // Build a sequence of copy-to-reg nodes, chained and glued together.
2159
3.97k
  for (auto &Reg : RegsToPass) {
2160
3.97k
    Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
2161
3.97k
    Glue = Chain.getValue(1);
2162
3.97k
  }
2163
1.34k
2164
1.34k
  // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
2165
1.34k
  // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
2166
1.34k
  // split it and then direct call can be matched by PseudoCALL.
2167
1.34k
  if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
2168
1.11k
    const GlobalValue *GV = S->getGlobal();
2169
1.11k
2170
1.11k
    unsigned OpFlags = RISCVII::MO_CALL;
2171
1.11k
    if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
2172
17
      OpFlags = RISCVII::MO_PLT;
2173
1.11k
2174
1.11k
    Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
2175
1.11k
  } else 
if (ExternalSymbolSDNode *230
S230
= dyn_cast<ExternalSymbolSDNode>(Callee)) {
2176
226
    unsigned OpFlags = RISCVII::MO_CALL;
2177
226
2178
226
    if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
2179
226
                                                 nullptr))
2180
4
      OpFlags = RISCVII::MO_PLT;
2181
226
2182
226
    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
2183
226
  }
2184
1.34k
2185
1.34k
  // The first call operand is the chain and the second is the target address.
2186
1.34k
  SmallVector<SDValue, 8> Ops;
2187
1.34k
  Ops.push_back(Chain);
2188
1.34k
  Ops.push_back(Callee);
2189
1.34k
2190
1.34k
  // Add argument registers to the end of the list so that they are
2191
1.34k
  // known live into the call.
2192
1.34k
  for (auto &Reg : RegsToPass)
2193
3.97k
    Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
2194
1.34k
2195
1.34k
  if (!IsTailCall) {
2196
1.32k
    // Add a register mask operand representing the call-preserved registers.
2197
1.32k
    const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2198
1.32k
    const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2199
1.32k
    assert(Mask && "Missing call preserved mask for calling convention");
2200
1.32k
    Ops.push_back(DAG.getRegisterMask(Mask));
2201
1.32k
  }
2202
1.34k
2203
1.34k
  // Glue the call to the argument copies, if any.
2204
1.34k
  if (Glue.getNode())
2205
1.16k
    Ops.push_back(Glue);
2206
1.34k
2207
1.34k
  // Emit the call.
2208
1.34k
  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2209
1.34k
2210
1.34k
  if (IsTailCall) {
2211
18
    MF.getFrameInfo().setHasTailCall();
2212
18
    return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
2213
18
  }
2214
1.32k
2215
1.32k
  Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
2216
1.32k
  Glue = Chain.getValue(1);
2217
1.32k
2218
1.32k
  // Mark the end of the call, which is glued to the call itself.
2219
1.32k
  Chain = DAG.getCALLSEQ_END(Chain,
2220
1.32k
                             DAG.getConstant(NumBytes, DL, PtrVT, true),
2221
1.32k
                             DAG.getConstant(0, DL, PtrVT, true),
2222
1.32k
                             Glue, DL);
2223
1.32k
  Glue = Chain.getValue(1);
2224
1.32k
2225
1.32k
  // Assign locations to each value returned by this call.
2226
1.32k
  SmallVector<CCValAssign, 16> RVLocs;
2227
1.32k
  CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
2228
1.32k
  analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true);
2229
1.32k
2230
1.32k
  // Copy all of the result registers out of their specified physreg.
2231
1.32k
  for (auto &VA : RVLocs) {
2232
1.23k
    // Copy the value out
2233
1.23k
    SDValue RetValue =
2234
1.23k
        DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
2235
1.23k
    // Glue the RetValue to the end of the call sequence
2236
1.23k
    Chain = RetValue.getValue(1);
2237
1.23k
    Glue = RetValue.getValue(2);
2238
1.23k
2239
1.23k
    if (VA.getLocVT() == MVT::i32 && 
VA.getValVT() == MVT::f64790
) {
2240
25
      assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
2241
25
      SDValue RetValue2 =
2242
25
          DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
2243
25
      Chain = RetValue2.getValue(1);
2244
25
      Glue = RetValue2.getValue(2);
2245
25
      RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
2246
25
                             RetValue2);
2247
25
    }
2248
1.23k
2249
1.23k
    RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);
2250
1.23k
2251
1.23k
    InVals.push_back(RetValue);
2252
1.23k
  }
2253
1.32k
2254
1.32k
  return Chain;
2255
1.32k
}
2256
2257
bool RISCVTargetLowering::CanLowerReturn(
2258
    CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2259
4.71k
    const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
2260
4.71k
  SmallVector<CCValAssign, 16> RVLocs;
2261
4.71k
  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
2262
9.02k
  for (unsigned i = 0, e = Outs.size(); i != e; 
++i4.30k
) {
2263
4.34k
    MVT VT = Outs[i].VT;
2264
4.34k
    ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
2265
4.34k
    RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
2266
4.34k
    if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
2267
4.34k
                 ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr))
2268
35
      return false;
2269
4.34k
  }
2270
4.71k
  
return true4.68k
;
2271
4.71k
}
2272
2273
SDValue
2274
RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2275
                                 bool IsVarArg,
2276
                                 const SmallVectorImpl<ISD::OutputArg> &Outs,
2277
                                 const SmallVectorImpl<SDValue> &OutVals,
2278
3.35k
                                 const SDLoc &DL, SelectionDAG &DAG) const {
2279
3.35k
  // Stores the assignment of the return value to a location.
2280
3.35k
  SmallVector<CCValAssign, 16> RVLocs;
2281
3.35k
2282
3.35k
  // Info about the registers and stack slot.
2283
3.35k
  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
2284
3.35k
                 *DAG.getContext());
2285
3.35k
2286
3.35k
  analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
2287
3.35k
                    nullptr);
2288
3.35k
2289
3.35k
  SDValue Glue;
2290
3.35k
  SmallVector<SDValue, 4> RetOps(1, Chain);
2291
3.35k
2292
3.35k
  // Copy the result values into the output registers.
2293
6.32k
  for (unsigned i = 0, e = RVLocs.size(); i < e; 
++i2.97k
) {
2294
2.97k
    SDValue Val = OutVals[i];
2295
2.97k
    CCValAssign &VA = RVLocs[i];
2296
2.97k
    assert(VA.isRegLoc() && "Can only return in registers!");
2297
2.97k
2298
2.97k
    if (VA.getLocVT() == MVT::i32 && 
VA.getValVT() == MVT::f641.56k
) {
2299
82
      // Handle returning f64 on RV32D with a soft float ABI.
2300
82
      assert(VA.isRegLoc() && "Expected return via registers");
2301
82
      SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
2302
82
                                     DAG.getVTList(MVT::i32, MVT::i32), Val);
2303
82
      SDValue Lo = SplitF64.getValue(0);
2304
82
      SDValue Hi = SplitF64.getValue(1);
2305
82
      unsigned RegLo = VA.getLocReg();
2306
82
      unsigned RegHi = RegLo + 1;
2307
82
      Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
2308
82
      Glue = Chain.getValue(1);
2309
82
      RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
2310
82
      Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
2311
82
      Glue = Chain.getValue(1);
2312
82
      RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
2313
2.89k
    } else {
2314
2.89k
      // Handle a 'normal' return.
2315
2.89k
      Val = convertValVTToLocVT(DAG, Val, VA, DL);
2316
2.89k
      Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
2317
2.89k
2318
2.89k
      // Guarantee that all emitted copies are stuck together.
2319
2.89k
      Glue = Chain.getValue(1);
2320
2.89k
      RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2321
2.89k
    }
2322
2.97k
  }
2323
3.35k
2324
3.35k
  RetOps[0] = Chain; // Update chain.
2325
3.35k
2326
3.35k
  // Add the glue node if we have it.
2327
3.35k
  if (Glue.getNode()) {
2328
2.72k
    RetOps.push_back(Glue);
2329
2.72k
  }
2330
3.35k
2331
3.35k
  // Interrupt service routines use different return instructions.
2332
3.35k
  const Function &Func = DAG.getMachineFunction().getFunction();
2333
3.35k
  if (Func.hasFnAttribute("interrupt")) {
2334
52
    if (!Func.getReturnType()->isVoidTy())
2335
2
      report_fatal_error(
2336
2
          "Functions with the interrupt attribute must have void return type!");
2337
50
2338
50
    MachineFunction &MF = DAG.getMachineFunction();
2339
50
    StringRef Kind =
2340
50
      MF.getFunction().getFnAttribute("interrupt").getValueAsString();
2341
50
2342
50
    unsigned RetOpc;
2343
50
    if (Kind == "user")
2344
6
      RetOpc = RISCVISD::URET_FLAG;
2345
44
    else if (Kind == "supervisor")
2346
6
      RetOpc = RISCVISD::SRET_FLAG;
2347
38
    else
2348
38
      RetOpc = RISCVISD::MRET_FLAG;
2349
50
2350
50
    return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
2351
50
  }
2352
3.29k
2353
3.29k
  return DAG.getNode(RISCVISD::RET_FLAG, DL, MVT::Other, RetOps);
2354
3.29k
}
2355
2356
0
const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
2357
0
  switch ((RISCVISD::NodeType)Opcode) {
2358
0
  case RISCVISD::FIRST_NUMBER:
2359
0
    break;
2360
0
  case RISCVISD::RET_FLAG:
2361
0
    return "RISCVISD::RET_FLAG";
2362
0
  case RISCVISD::URET_FLAG:
2363
0
    return "RISCVISD::URET_FLAG";
2364
0
  case RISCVISD::SRET_FLAG:
2365
0
    return "RISCVISD::SRET_FLAG";
2366
0
  case RISCVISD::MRET_FLAG:
2367
0
    return "RISCVISD::MRET_FLAG";
2368
0
  case RISCVISD::CALL:
2369
0
    return "RISCVISD::CALL";
2370
0
  case RISCVISD::SELECT_CC:
2371
0
    return "RISCVISD::SELECT_CC";
2372
0
  case RISCVISD::BuildPairF64:
2373
0
    return "RISCVISD::BuildPairF64";
2374
0
  case RISCVISD::SplitF64:
2375
0
    return "RISCVISD::SplitF64";
2376
0
  case RISCVISD::TAIL:
2377
0
    return "RISCVISD::TAIL";
2378
0
  case RISCVISD::SLLW:
2379
0
    return "RISCVISD::SLLW";
2380
0
  case RISCVISD::SRAW:
2381
0
    return "RISCVISD::SRAW";
2382
0
  case RISCVISD::SRLW:
2383
0
    return "RISCVISD::SRLW";
2384
0
  case RISCVISD::DIVW:
2385
0
    return "RISCVISD::DIVW";
2386
0
  case RISCVISD::DIVUW:
2387
0
    return "RISCVISD::DIVUW";
2388
0
  case RISCVISD::REMUW:
2389
0
    return "RISCVISD::REMUW";
2390
0
  case RISCVISD::FMV_W_X_RV64:
2391
0
    return "RISCVISD::FMV_W_X_RV64";
2392
0
  case RISCVISD::FMV_X_ANYEXTW_RV64:
2393
0
    return "RISCVISD::FMV_X_ANYEXTW_RV64";
2394
0
  case RISCVISD::READ_CYCLE_WIDE:
2395
0
    return "RISCVISD::READ_CYCLE_WIDE";
2396
0
  }
2397
0
  return nullptr;
2398
0
}
2399
2400
std::pair<unsigned, const TargetRegisterClass *>
2401
RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
2402
                                                  StringRef Constraint,
2403
128
                                                  MVT VT) const {
2404
128
  // First, see if this is a constraint that directly corresponds to a
2405
128
  // RISCV register class.
2406
128
  if (Constraint.size() == 1) {
2407
122
    switch (Constraint[0]) {
2408
122
    case 'r':
2409
97
      return std::make_pair(0U, &RISCV::GPRRegClass);
2410
122
    default:
2411
25
      break;
2412
31
    }
2413
31
  }
2414
31
2415
31
  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
2416
31
}
2417
2418
void RISCVTargetLowering::LowerAsmOperandForConstraint(
2419
    SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2420
29
    SelectionDAG &DAG) const {
2421
29
  // Currently only support length 1 constraints.
2422
29
  if (Constraint.length() == 1) {
2423
29
    switch (Constraint[0]) {
2424
29
    case 'I':
2425
8
      // Validate & create a 12-bit signed immediate operand.
2426
8
      if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2427
8
        uint64_t CVal = C->getSExtValue();
2428
8
        if (isInt<12>(CVal))
2429
4
          Ops.push_back(
2430
4
              DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2431
8
      }
2432
8
      return;
2433
29
    case 'J':
2434
4
      // Validate & create an integer zero operand.
2435
4
      if (auto *C = dyn_cast<ConstantSDNode>(Op))
2436
4
        if (C->getZExtValue() == 0)
2437
2
          Ops.push_back(
2438
2
              DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
2439
4
      return;
2440
29
    case 'K':
2441
8
      // Validate & create a 5-bit unsigned immediate operand.
2442
8
      if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2443
8
        uint64_t CVal = C->getZExtValue();
2444
8
        if (isUInt<5>(CVal))
2445
4
          Ops.push_back(
2446
4
              DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2447
8
      }
2448
8
      return;
2449
29
    default:
2450
9
      break;
2451
9
    }
2452
9
  }
2453
9
  TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
2454
9
}
2455
2456
Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
2457
                                                   Instruction *Inst,
2458
28
                                                   AtomicOrdering Ord) const {
2459
28
  if (isa<LoadInst>(Inst) && 
Ord == AtomicOrdering::SequentiallyConsistent14
)
2460
7
    return Builder.CreateFence(Ord);
2461
21
  if (isa<StoreInst>(Inst) && 
isReleaseOrStronger(Ord)14
)
2462
14
    return Builder.CreateFence(AtomicOrdering::Release);
2463
7
  return nullptr;
2464
7
}
2465
2466
Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
2467
                                                    Instruction *Inst,
2468
28
                                                    AtomicOrdering Ord) const {
2469
28
  if (isa<LoadInst>(Inst) && 
isAcquireOrStronger(Ord)14
)
2470
14
    return Builder.CreateFence(AtomicOrdering::Acquire);
2471
14
  return nullptr;
2472
14
}
2473
2474
TargetLowering::AtomicExpansionKind
2475
385
RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
2476
385
  // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
2477
385
  // point operations can't be used in an lr/sc sequence without breaking the
2478
385
  // forward-progress guarantee.
2479
385
  if (AI->isFloatingPointOperation())
2480
0
    return AtomicExpansionKind::CmpXChg;
2481
385
2482
385
  unsigned Size = AI->getType()->getPrimitiveSizeInBits();
2483
385
  if (Size == 8 || 
Size == 16305
)
2484
160
    return AtomicExpansionKind::MaskedIntrinsic;
2485
225
  return AtomicExpansionKind::None;
2486
225
}
2487
2488
static Intrinsic::ID
2489
160
getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
2490
160
  if (XLen == 32) {
2491
80
    switch (BinOp) {
2492
80
    default:
2493
0
      llvm_unreachable("Unexpected AtomicRMW BinOp");
2494
80
    case AtomicRMWInst::Xchg:
2495
10
      return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
2496
80
    case AtomicRMWInst::Add:
2497
10
      return Intrinsic::riscv_masked_atomicrmw_add_i32;
2498
80
    case AtomicRMWInst::Sub:
2499
10
      return Intrinsic::riscv_masked_atomicrmw_sub_i32;
2500
80
    case AtomicRMWInst::Nand:
2501
10
      return Intrinsic::riscv_masked_atomicrmw_nand_i32;
2502
80
    case AtomicRMWInst::Max:
2503
10
      return Intrinsic::riscv_masked_atomicrmw_max_i32;
2504
80
    case AtomicRMWInst::Min:
2505
10
      return Intrinsic::riscv_masked_atomicrmw_min_i32;
2506
80
    case AtomicRMWInst::UMax:
2507
10
      return Intrinsic::riscv_masked_atomicrmw_umax_i32;
2508
80
    case AtomicRMWInst::UMin:
2509
10
      return Intrinsic::riscv_masked_atomicrmw_umin_i32;
2510
80
    }
2511
80
  }
2512
80
2513
80
  if (XLen == 64) {
2514
80
    switch (BinOp) {
2515
80
    default:
2516
0
      llvm_unreachable("Unexpected AtomicRMW BinOp");
2517
80
    case AtomicRMWInst::Xchg:
2518
10
      return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
2519
80
    case AtomicRMWInst::Add:
2520
10
      return Intrinsic::riscv_masked_atomicrmw_add_i64;
2521
80
    case AtomicRMWInst::Sub:
2522
10
      return Intrinsic::riscv_masked_atomicrmw_sub_i64;
2523
80
    case AtomicRMWInst::Nand:
2524
10
      return Intrinsic::riscv_masked_atomicrmw_nand_i64;
2525
80
    case AtomicRMWInst::Max:
2526
10
      return Intrinsic::riscv_masked_atomicrmw_max_i64;
2527
80
    case AtomicRMWInst::Min:
2528
10
      return Intrinsic::riscv_masked_atomicrmw_min_i64;
2529
80
    case AtomicRMWInst::UMax:
2530
10
      return Intrinsic::riscv_masked_atomicrmw_umax_i64;
2531
80
    case AtomicRMWInst::UMin:
2532
10
      return Intrinsic::riscv_masked_atomicrmw_umin_i64;
2533
0
    }
2534
0
  }
2535
0
2536
0
  llvm_unreachable("Unexpected XLen\n");
2537
0
}
2538
2539
Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
2540
    IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
2541
160
    Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
2542
160
  unsigned XLen = Subtarget.getXLen();
2543
160
  Value *Ordering =
2544
160
      Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
2545
160
  Type *Tys[] = {AlignedAddr->getType()};
2546
160
  Function *LrwOpScwLoop = Intrinsic::getDeclaration(
2547
160
      AI->getModule(),
2548
160
      getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
2549
160
2550
160
  if (XLen == 64) {
2551
80
    Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
2552
80
    Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2553
80
    ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
2554
80
  }
2555
160
2556
160
  Value *Result;
2557
160
2558
160
  // Must pass the shift amount needed to sign extend the loaded value prior
2559
160
  // to performing a signed comparison for min/max. ShiftAmt is the number of
2560
160
  // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
2561
160
  // is the number of bits to left+right shift the value in order to
2562
160
  // sign-extend.
2563
160
  if (AI->getOperation() == AtomicRMWInst::Min ||
2564
160
      
AI->getOperation() == AtomicRMWInst::Max140
) {
2565
40
    const DataLayout &DL = AI->getModule()->getDataLayout();
2566
40
    unsigned ValWidth =
2567
40
        DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
2568
40
    Value *SextShamt =
2569
40
        Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
2570
40
    Result = Builder.CreateCall(LrwOpScwLoop,
2571
40
                                {AlignedAddr, Incr, Mask, SextShamt, Ordering});
2572
120
  } else {
2573
120
    Result =
2574
120
        Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
2575
120
  }
2576
160
2577
160
  if (XLen == 64)
2578
80
    Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2579
160
  return Result;
2580
160
}
2581
2582
TargetLowering::AtomicExpansionKind
2583
RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
2584
71
    AtomicCmpXchgInst *CI) const {
2585
71
  unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
2586
71
  if (Size == 8 || 
Size == 1651
)
2587
40
    return AtomicExpansionKind::MaskedIntrinsic;
2588
31
  return AtomicExpansionKind::None;
2589
31
}
2590
2591
Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
2592
    IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
2593
40
    Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
2594
40
  unsigned XLen = Subtarget.getXLen();
2595
40
  Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
2596
40
  Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
2597
40
  if (XLen == 64) {
2598
20
    CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
2599
20
    NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
2600
20
    Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2601
20
    CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
2602
20
  }
2603
40
  Type *Tys[] = {AlignedAddr->getType()};
2604
40
  Function *MaskedCmpXchg =
2605
40
      Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
2606
40
  Value *Result = Builder.CreateCall(
2607
40
      MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
2608
40
  if (XLen == 64)
2609
20
    Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2610
40
  return Result;
2611
40
}
2612
2613
unsigned RISCVTargetLowering::getExceptionPointerRegister(
2614
20
    const Constant *PersonalityFn) const {
2615
20
  return RISCV::X10;
2616
20
}
2617
2618
unsigned RISCVTargetLowering::getExceptionSelectorRegister(
2619
10
    const Constant *PersonalityFn) const {
2620
10
  return RISCV::X11;
2621
10
}